evaltor 0.1.0

A tool for evaluating the evaluatees on benchmarks
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106

import matplotlib.pyplot as plt
from scipy.stats import gmean
from evaluatee import Evaluatee
import math
import numpy as np


def cal_ticks(max_val):
    loc = [0.1, 1]
    lab = ["0.1", "1"]
    exp = 1
    now = 10
    while now < max_val:
        lab.append(r"$10^" + str(exp) + "$")
        loc.append(now)
        now *= 10
        exp += 1
    lab.append("TO")
    loc.append(max_val)
    return loc, lab


def scatter_single(fg, x: Evaluatee, y: Evaluatee):
    X = []
    Y = []
    XG = []
    YG = []
    num_x = 0
    num_y = 0
    speedup = []
    for case in x.cases():
        xt, yt = x[case], y[case]
        if xt is None and yt is None:
            continue
        bsdp = xt is None or yt is None
        if xt is None and yt is not None:
            YG.append(case)
        elif yt is None and xt is not None:
            XG.append(case)
        elif xt is not None and yt is not None:
            if xt > yt * 5:
                YG.append(case)
            elif yt > xt * 5:
                XG.append(case)

        xt = x.TIMEOUT if xt is None else max(xt, 0.1)
        yt = y.TIMEOUT if yt is None else max(yt, 0.1)
        X.append(xt)
        Y.append(yt)

        if xt <= 5 and yt <= 5:
            continue
        if not bsdp:
            speedup.append(yt / xt)
        if xt < yt:
            num_x += 1
        elif xt > yt:
            num_y += 1

    print("xgood:")
    for case in XG:
        print(case)
    print()
    print("ygood:")
    for case in YG:
        print(case)

    print(num_x, num_y)
    gm = gmean(speedup)
    print(1 / gm)
    fg.scatter(X, Y, marker="x", s=25, linewidths=1)
    fg.set_aspect("equal")
    fg.set_xscale("log")
    fg.set_yscale("log")
    fg.set_xlabel(x.name, fontsize=12)
    fg.set_ylabel(y.name, fontsize=12)
    fg.yaxis.set_label_coords(-0.11, 0.5)
    fg.set_xlim(0.1, x.TIMEOUT * 1.2)
    fg.set_ylim(0.1, y.TIMEOUT * 1.2)
    fg.plot([0, x.TIMEOUT], [0, y.TIMEOUT], color="grey", linestyle="dashed")
    fg.plot([0, x.TIMEOUT], [0, y.TIMEOUT * gm], linestyle="dashed", color="#721454")
    loc, lab = cal_ticks(x.TIMEOUT)
    fg.set_xticks(loc, lab)
    fg.set_yticks(loc, lab)


def scatter(evaluatee: list[Evaluatee], plot_x=4):
    plot_x = 3
    plot_y = int(math.ceil((len(evaluatee) - 1) / plot_x))
    fig, ax = plt.subplots(plot_y, plot_x, figsize=(4 * plot_x, 4 * plot_y))
    if isinstance(ax[0], list) or isinstance(ax[0], np.ndarray):
        ax = [x for sub_ax in ax for x in sub_ax]
    for e, subax in zip(evaluatee[1:], ax):
        scatter_single(subax, evaluatee[0], e)
    plt.tight_layout()
    fig.show()
    fig.savefig("scatter.png", dpi=500)


if __name__ == "__main__":
    import sys

    evaluatee = []
    for file in sys.argv[1:]:
        evaluatee.append(Evaluatee(file))
    scatter(evaluatee)