evaltor 0.1.0

A tool for evaluating the evaluatees on benchmarks
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

import sys
from evaluatee import Evaluatee

def merge(evaluatee: list[Evaluatee]):
    result = {}
    for case in evaluatee[0].cases():
        result[case] = [e[case] for e in evaluatee]
    return result

if __name__ == "__main__":
    evaluatee = []
    for file in sys.argv[1:]:
        evaluatee.append(Evaluatee(file))
    merged = merge(evaluatee)
    with open("merge-result", 'w') as result:
        keys = sorted(merged.keys())
        for case in merged:
            output = f"{case} "
            output += " ".join(str(time) if time is not None else "Timeout" for time in merged[case])
            result.write(output + '\n')