evaltor 0.1.0

A tool for evaluating the evaluatees on benchmarks
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

import os
import re
from typing import Dict, Optional, Union


class Evaluatee:
    TIMEOUT = 3600

    def __init__(self, file: os.PathLike, name: Optional[str] = None):
        self.name = (
            name if name is not None else os.path.splitext(os.path.basename(file))[0]
        )
        self.data: Dict[str, float] = {}
        self.timeout = set()
        self.memout = set()
        with open(file, "r") as file:
            for line in file:
                case, time = line.strip().split()
                case = case.rsplit("/", 1)[-1]
                case = case.rsplit(".", 1)[0] if case.endswith((".aig", ".btor", ".btor2")) else case
                if time == "Timeout":
                    self.timeout.add(case)
                elif time == "Failed":
                    self.memout.add(case)
                else:
                    match = re.match(r'([a-zA-Z]+)\(([-+]?\d*\.\d+|\d+)\)', time)
                    assert match
                    tag = match.group(1)
                    time = float(match.group(2))
                    if time > self.TIMEOUT:
                        self.timeout.add(case)
                    else:
                        self.data[case] = time

    def __getitem__(self, key):
        return self.data.get(key)

    def cases(self) -> list[str]:
        return list(self.data.keys() | self.timeout | self.memout)

    def num_solved(self) -> int:
        return len(self.data)

    def num_failed(self) -> int:
        return len(self.timeout) + len(self.memout)

    def num_total(self) -> int:
        return self.num_solved() + self.num_failed()

    def par2(self) -> float:
        return round(
            (sum(self.data.values()) + self.num_failed() * 2 * self.TIMEOUT)
            / self.num_total(),
            2,
        )

    def summary(self):
        print(
            self.name,
            "{}/{}".format(self.num_solved(), self.num_total()),
            "par2: {:.2f}".format(self.par2()),
        )