deflake-rs 0.1.0

use clap::{Parser, ValueEnum};
use glob::glob;
use std::{
    collections::{HashMap, HashSet},
    fs::File,
    io::{BufReader, Read},
};
use test_interface::{TestId, TestResult, TestResultOutput, TestResults};

mod ast;
mod cli;
mod git;
mod test_interface;

#[derive(Parser, Debug)]
#[command(version, about, long_about = None)]
pub struct Cli {
    #[arg(short = 'p', long = "project", help = "the project to analyse")]
    pub project: String,

    #[arg(short = 'f', long = "flakies", help = "file with flaky test ids")]
    pub flakies: String,
}

fn main() {
    let args = Cli::parse();
    println!("Analysing");

    let mut file = File::open(args.flakies).unwrap();
    let mut flakies = String::new();
    file.read_to_string(&mut flakies);
    println!("{}", flakies);

    let mut flaky_ids = HashSet::new();
    flakies.lines().for_each(|l| {
        flaky_ids.insert(l.to_string());
    });

    let glob_pattern = &format!("./uploads/rq3-{}-*", args.project);
    //dbg!(glob_pattern);

    // Need
    // - was the flaky tests execued at all?
    // - was the flaky marked as flaky
    // - was it marked as failed
    // - did it parse
    // - any other tests marked as flaky or failed?

    // expected_result
    let mut flaky_passed: HashMap<String, HashMap<String, u32>> = HashMap::new();
    let mut flaky_flaky: HashMap<String, HashMap<String, u32>> = HashMap::new();

    let mut flaky_failed: HashMap<String, HashMap<String, u32>> = HashMap::new();
    let mut non_flaky: HashMap<String, HashMap<String, u32>> = HashMap::new();
    let mut non_failed: HashMap<String, HashMap<String, u32>> = HashMap::new();

    // commit -> test -> result
    let mut info: HashMap<String, HashMap<String, HashSet<String>>> = HashMap::new();

    let mut commits = HashSet::new();
    let mut runs = 0;

    for item in glob(glob_pattern.as_str()).expect("Failed to get files") {
        if let Ok(path) = item {
            let file_name: Vec<&str> = path.as_os_str().to_str().unwrap().split("-").collect();
            let commit = file_name.get(file_name.len() - 2).unwrap().to_string();
            commits.insert(commit.clone());

            runs += 1;
            let info_c = info.entry(commit.clone()).or_insert(HashMap::new());

            let file = File::open(path.clone()).unwrap();
            let reader = BufReader::new(file);
            let results: TestResultOutput =
                serde_json::from_reader(reader).expect("Invalid results file");

            let mut this_run = HashSet::new();
            results.tests.into_iter().for_each(|(id, res)| {
                if flaky_ids.contains(id.as_str()) {
                    this_run.insert(id.clone());

                    let info_test = info_c.entry(id.clone()).or_insert(HashSet::new());
                    info_test.insert(res.to_string());

                    match res {
                        TestResult::Flaky => {
                            if flaky_flaky.get(&commit).is_none() {
                                flaky_flaky.insert(commit.clone(), HashMap::new());
                            }
                            let c = flaky_flaky.get_mut(&commit).unwrap();
                            let v = match c.get(id.as_str()) {
                                Some(v) => v,
                                None => &0,
                            };
                            c.insert(id.clone(), v + 1);
                        }
                        TestResult::Error | TestResult::Timeout => {
                            if flaky_failed.get(&commit).is_none() {
                                flaky_failed.insert(commit.clone(), HashMap::new());
                            }
                            let c = flaky_failed.get_mut(&commit).unwrap();
                            let v = match c.get(id.as_str()) {
                                Some(v) => v,
                                None => &0,
                            };
                            c.insert(id.clone(), v + 1);
                            println!("Test {} marked as fail, but flaky", id);
                        }
                        TestResult::Success => {
                            if flaky_passed.get(&commit).is_none() {
                                flaky_passed.insert(commit.clone(), HashMap::new());
                            }
                            let c = flaky_passed.get_mut(&commit).unwrap();
                            let v = match c.get(id.as_str()) {
                                Some(v) => v,
                                None => &0,
                            };
                            c.insert(id.clone(), v + 1);
                        }
                    }
                } else {
                    match res {
                        TestResult::Flaky => {
                            //println!("New flake {}", id);
                            if non_flaky.get(&commit).is_none() {
                                non_flaky.insert(commit.clone(), HashMap::new());
                            }
                            let c = non_flaky.get_mut(&commit).unwrap();
                            let v = match c.get(id.as_str()) {
                                Some(v) => v,
                                None => &0,
                            };
                            c.insert(id.clone(), v + 1);
                        }
                        TestResult::Error | TestResult::Timeout => {
                            if non_failed.get(&commit).is_none() {
                                non_failed.insert(commit.clone(), HashMap::new());
                            }
                            let c = non_failed.get_mut(&commit).unwrap();
                            let v = match c.get(id.as_str()) {
                                Some(v) => v,
                                None => &0,
                            };
                            c.insert(id.clone(), v + 1);
                        }
                        TestResult::Success => {}
                    }
                }
            });
            let all_flakes_run = this_run.is_superset(&flaky_ids);
            if !all_flakes_run {
                println!("WARN: not all flaky tests executed");
            }
        }
    }

    println!("Flaky passed: {:?}", flaky_passed);
    println!("Flaky correct: {:?}", flaky_flaky);
    println!("Flaky incorrect: {:?}", flaky_failed);
    println!("New flakes: {:?}", non_flaky);
    println!("Failed correct: {:?}", non_failed);

    // TODO for each hashmap combine so counts of each commit
    fn process_result(hm: HashMap<String, HashMap<String, u32>>) -> u32 {
        let mut t = 0;
        hm.iter().for_each(|(commit, ids)| {
            ids.iter().for_each(|(id, i)| {
                if *i > 0 {
                    t += 1;
                }
            });
        });

        t
    }

    println!("Runs: {}", runs);
    println!("Commits: {}", commits.len());
    println!("Flakies: {}", flaky_ids.len());
    println!("Flaky passed: {:?}", process_result(flaky_passed));
    println!("Flaky correct: {:?}", process_result(flaky_flaky));
    println!("Flaky incorrect: {:?}", process_result(flaky_failed));
    println!("New flakes: {:?}", process_result(non_flaky));
    println!("Failed correct: {:?}", process_result(non_failed));
}