import importlib.util
from pathlib import Path
_spec = importlib.util.spec_from_file_location(
"pr_bench",
str(Path("benchmarks/workflow/pr_bench.py").resolve()),
)
pr_bench = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(pr_bench)
config.setdefault("base", "HEAD~1")
config.setdefault("head", "HEAD")
AA_RUNS = pr_bench.AA_RUNS
TIME_IDS = [f"{i:02d}" for i in range(1, AA_RUNS + 1)]
MEM_ROUNDS = [f"{i:02d}" for i in range(1, pr_bench.MEM_AA_ROUNDS + 1)]
CMP_ROUNDS = [f"{i:02d}" for i in range(1, pr_bench.COMPARE_ROUNDS + 1)]
AA_CONF = "asv.aa.conf.json"
CMP_CONF = "asv.cmp.conf.json"
MEM_CONF = "asv.mem.conf.json"
rule all:
input:
"results/comparison.txt",
"results/metadata.txt",
"results/row-bounds.txt",
"results/regression-threshold-time.txt",
"results/regression-threshold-memory.txt",
"results/commenter-decision.txt",
"results/aa-noise.json",
"results/asv-continuous.txt",
"results/calls.tsv",
"results/check.ok",
rule distinct_commits:
priority: 100
output:
touch("results/distinct-commits.ok"),
params:
base=config["base"],
head=config["head"],
shell:
rule derive_conf:
input:
"asv.conf.json",
output:
aa=AA_CONF,
cmp=CMP_CONF,
mem=MEM_CONF,
shell:
rule machine:
input:
AA_CONF,
output:
touch(".asv/machine-pr-bench.ok"),
params:
machine=pr_bench.MACHINE,
shell:
rule aa_time:
input:
conf=AA_CONF,
machine=".asv/machine-pr-bench.ok",
output:
expand("results/aa/time-{i}.json", i=TIME_IDS),
params:
base=config["base"],
repeat=pr_bench.TIME_REPEAT,
rounds=pr_bench.TIME_AA_ROUNDS,
machine=pr_bench.MACHINE,
ids=" ".join(TIME_IDS),
shell:
rule aa_memory:
input:
conf=AA_CONF,
machine=".asv/machine-pr-bench.ok",
time=expand("results/aa/time-{i}.json", i=TIME_IDS),
compared="results/asv-continuous.txt",
output:
expand(
"results/aa/memory-{rep}-{rnd}.json",
rep=TIME_IDS,
rnd=MEM_ROUNDS,
),
params:
base=config["base"],
machine=pr_bench.MACHINE,
reps=" ".join(TIME_IDS),
rounds=" ".join(MEM_ROUNDS),
shell:
rule noise_time:
input:
expand("results/aa/time-{i}.json", i=TIME_IDS),
output:
noise="results/aa-time-noise.json",
factor="results/asv-factor.txt",
time_threshold="results/regression-threshold-time.txt",
shell:
rule noise:
input:
expand("results/aa/time-{i}.json", i=TIME_IDS),
expand(
"results/aa/memory-{rep}-{rnd}.json",
rep=TIME_IDS,
rnd=MEM_ROUNDS,
),
output:
noise="results/aa-noise.json",
memory_threshold="results/regression-threshold-memory.txt",
shell:
rule compare_time:
input:
conf=CMP_CONF,
factor="results/asv-factor.txt",
machine=".asv/machine-pr-bench.ok",
distinct="results/distinct-commits.ok",
output:
"results/asv-continuous.txt",
params:
base=config["base"],
head=config["head"],
rounds=pr_bench.COMPARE_ROUNDS,
repeat=pr_bench.TIME_REPEAT,
machine=pr_bench.MACHINE,
shell:
rule compare_memory:
input:
conf=MEM_CONF,
machine=".asv/machine-pr-bench.ok",
noise="results/aa-noise.json",
time="results/asv-continuous.txt",
distinct="results/distinct-commits.ok",
output:
touch("results/cmp-mem.ok"),
params:
base=config["base"],
head=config["head"],
machine=pr_bench.MACHINE,
rounds=" ".join(CMP_ROUNDS),
shell:
rule report:
input:
noise="results/aa-noise.json",
continuous="results/asv-continuous.txt",
mem="results/cmp-mem.ok",
output:
comparison="results/comparison.txt",
metadata="results/metadata.txt",
calls="results/calls.tsv",
check_status="results/check-status.txt",
bounds="results/row-bounds.txt",
decision="results/commenter-decision.txt",
params:
base=config["base"],
head=config["head"],
shell:
rule check:
input:
"results/comparison.txt",
"results/calls.tsv",
"results/check-status.txt",
output:
touch("results/check.ok"),
shell:
rule h2h_prepare:
output:
touch("bench/h2h/work/prepare.ok"),
shell:
rule h2h_readcon_c:
input:
"bench/h2h/work/prepare.ok",
output:
"bench/h2h/logs/campaign/c-read-tiny10.stdout",
"bench/h2h/logs/campaign/c-read-frames-10000x100.stdout",
"bench/h2h/logs/campaign/c-read-atoms-1000000.stdout",
"bench/h2h/logs/campaign/c-write-atoms-1000000.stdout",
shell:
rule h2h_readcon_fortran:
input:
"bench/h2h/work/prepare.ok",
output:
"bench/h2h/logs/campaign/fortran-read-tiny10.stdout",
"bench/h2h/logs/campaign/fortran-read-frames-10000x100.stdout",
"bench/h2h/logs/campaign/fortran-read-atoms-1000000.stdout",
"bench/h2h/logs/campaign/fortran-write-atoms-1000000.stdout",
shell:
rule h2h_readcon_python:
input:
"bench/h2h/work/prepare.ok",
output:
"bench/h2h/logs/campaign/python-read-tiny10.stdout",
"bench/h2h/logs/campaign/python-read-frames-10000x100.stdout",
"bench/h2h/logs/campaign/python-read-atoms-1000000.stdout",
"bench/h2h/logs/campaign/python-write-atoms-1000000.stdout",
"bench/h2h/logs/campaign/python-read-atoms-1000000-fresh.stdout",
shell:
rule h2h_eon_cpp:
input:
"bench/h2h/work/prepare.ok",
output:
"bench/h2h/logs/campaign/eon-cpp-read-tiny10.stdout",
"bench/h2h/logs/campaign/eon-cpp-read-atoms-1000000.stdout",
shell:
rule h2h_eon_python:
input:
"bench/h2h/work/prepare.ok",
output:
"bench/h2h/logs/campaign/eon-python-read-atoms-1000000.stdout",
shell:
rule h2h_ase:
input:
"bench/h2h/work/prepare.ok",
output:
"bench/h2h/logs/campaign/ase-eon-read-atoms-1000000.stdout",
shell:
rule h2h_chemfiles:
input:
"bench/h2h/work/prepare.ok",
output:
"bench/h2h/logs/campaign/chemfiles-extxyz-read-atoms-1000000.stdout",
shell:
rule h2h_mdanalysis:
input:
"bench/h2h/work/prepare.ok",
output:
"bench/h2h/logs/campaign/mdanalysis-pdb-read-atoms-1000000.stdout",
shell:
rule h2h_ovito:
input:
"bench/h2h/work/prepare.ok",
output:
"bench/h2h/logs/campaign/ovito-extxyz-read-atoms-1000000.stdout",
shell:
rule h2h_pymatgen:
input:
"bench/h2h/work/prepare.ok",
output:
"bench/h2h/logs/campaign/pymatgen-poscar-read-atoms-1000000.stdout",
shell:
rule h2h_numpy:
input:
"bench/h2h/work/prepare.ok",
output:
"bench/h2h/logs/campaign/numpy-npy-read-atoms-1000000.stdout",
shell:
rule h2h_ase_traj:
input:
"bench/h2h/work/prepare.ok",
output:
"bench/h2h/logs/campaign/ase-extxyz-read-atoms-1000000.stdout",
shell:
rule h2h_report:
input:
"bench/h2h/logs/campaign/c-read-tiny10.stdout",
"bench/h2h/logs/campaign/c-read-frames-10000x100.stdout",
"bench/h2h/logs/campaign/c-read-atoms-1000000.stdout",
"bench/h2h/logs/campaign/c-write-atoms-1000000.stdout",
"bench/h2h/logs/campaign/fortran-read-tiny10.stdout",
"bench/h2h/logs/campaign/fortran-read-frames-10000x100.stdout",
"bench/h2h/logs/campaign/fortran-read-atoms-1000000.stdout",
"bench/h2h/logs/campaign/fortran-write-atoms-1000000.stdout",
"bench/h2h/logs/campaign/python-read-tiny10.stdout",
"bench/h2h/logs/campaign/python-read-frames-10000x100.stdout",
"bench/h2h/logs/campaign/python-read-atoms-1000000.stdout",
"bench/h2h/logs/campaign/python-write-atoms-1000000.stdout",
"bench/h2h/logs/campaign/python-read-atoms-1000000-fresh.stdout",
"bench/h2h/logs/campaign/eon-cpp-read-tiny10.stdout",
"bench/h2h/logs/campaign/eon-cpp-read-atoms-1000000.stdout",
"bench/h2h/logs/campaign/eon-python-read-atoms-1000000.stdout",
"bench/h2h/logs/campaign/ase-eon-read-atoms-1000000.stdout",
"bench/h2h/logs/campaign/chemfiles-extxyz-read-atoms-1000000.stdout",
"bench/h2h/logs/campaign/mdanalysis-pdb-read-atoms-1000000.stdout",
"bench/h2h/logs/campaign/ovito-extxyz-read-atoms-1000000.stdout",
"bench/h2h/logs/campaign/pymatgen-poscar-read-atoms-1000000.stdout",
"bench/h2h/logs/campaign/numpy-npy-read-atoms-1000000.stdout",
"bench/h2h/logs/campaign/ase-extxyz-read-atoms-1000000.stdout",
output:
"bench/h2h/results.json",
shell:
rule h2h:
input:
"bench/h2h/results.json",
rule fidelity:
output:
"bench/fidelity/results/fidelity.json",
"bench/fidelity/results/fidelity.csv",
"bench/fidelity/results/timing.json",
"bench/fidelity/results/timing.csv",
"bench/fidelity/logs/timing.log",
shell:
rule h2h_parallel_frames:
output:
"bench/h2h/logs/parallel-frames/campaign.log",
shell: