evaltor 0.1.0

A tool for evaluating the evaluatees on benchmarks
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108

use clap::Parser;
use std::{collections::HashMap, fs, path::PathBuf, sync::Arc};

use crate::{
    bench::{BenchIF, Benchmark, MultiBenchmark},
    evaluatees::{Evaluatee, EvaluateeIF},
};

/// Evaltor
#[derive(Parser, Debug, Clone)]
#[command(
    version,
    about,
    after_help = "Copyright (C) 2023 - Present, Yuheng Su <gipsyh.icu@gmail.com>. All rights reserved."
)]
pub struct Options {
    /// benchmark configuration file in toml format
    pub bench_config: PathBuf,

    /// benchmark
    #[arg(short, long, value_delimiter = ',', required = true)]
    pub bench: Vec<String>,

    /// evaluatee configuration file in toml format
    pub evaluatee_config: PathBuf,

    /// evaluatee
    #[arg(short, long, value_delimiter = ',', required = true)]
    pub evaluatee: Vec<String>,

    /// time limit
    #[arg(short, long, default_value_t = 3600)]
    pub timeout: u64,

    /// memory limit
    #[arg(short, long, default_value_t = 16)]
    pub memout: usize,

    /// certify
    #[arg(short, long, default_value_t = false)]
    pub certify: bool,

    /// number of workers
    #[arg(short = 'w', long)]
    pub num_worker: Option<usize>,

    /// number of fuzz tests
    #[arg(short = 'f', long, default_value_t = 1000)]
    pub num_fuzz: usize,
}

impl Options {
    pub fn parse_bench(&self) -> Result<Box<dyn BenchIF>, Box<dyn std::error::Error>> {
        let bench_config: HashMap<String, HashMap<String, PathBuf>> =
            toml::de::from_str(&fs::read_to_string(&self.bench_config).unwrap())?;
        let mut benchs: Vec<Box<dyn BenchIF>> = Vec::new();
        for b in self.bench.iter() {
            let parts: Vec<&str> = b.split('.').collect();
            assert!(parts.len() == 2);
            let name = parts[0];
            let format = parts[1];
            let path = &bench_config[name][format];
            benchs.push(Box::new(Benchmark::new(name, path, format)));
        }
        let bench: Box<dyn BenchIF> = if benchs.len() == 1 {
            benchs.into_iter().next().unwrap()
        } else {
            let mut bench = MultiBenchmark::new();
            for b in benchs {
                bench = bench.add(b);
            }
            Box::new(bench)
        };
        Ok(bench)
    }

    pub fn parse_evaluatee(&self) -> Result<Vec<Arc<dyn EvaluateeIF>>, Box<dyn std::error::Error>> {
        #[derive(serde::Deserialize, Debug)]
        struct TomlEvaluatee {
            cmd: PathBuf,
            args: Vec<String>,
        }
        #[derive(serde::Deserialize, Debug)]
        struct TomlEvaluatees {
            name: String,
            exit_code: HashMap<String, String>,
            #[serde(flatten)]
            evaluatees: HashMap<String, TomlEvaluatee>,
        }

        let config: TomlEvaluatees =
            toml::de::from_str(&fs::read_to_string(&self.evaluatee_config).unwrap())?;
        let exit_code: HashMap<i64, String> = config
            .exit_code
            .into_iter()
            .map(|(k, v)| (k.parse::<i64>().unwrap(), v))
            .collect();
        let mut evaluatees: Vec<Arc<dyn EvaluateeIF>> = Vec::new();
        for e in self.evaluatee.iter() {
            let te = config.evaluatees.get(e).unwrap();
            let mut evaluatee = Evaluatee::new(&config.name, &te.cmd, &te.args);
            evaluatee.version = e.to_string();
            evaluatee.exit_code = exit_code.clone();
            evaluatees.push(Arc::new(evaluatee));
        }
        Ok(evaluatees)
    }
}