mod baseline;
mod config;
mod correctness;
mod detect;
mod report;
mod scoreboard;
use anyhow::{bail, Context, Result};
use clap::Parser;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::time::Instant;
#[derive(Parser)]
#[command(
name = "tpt-bench",
about = "TPT-GenBench: user-runnable dynamic GPU benchmark"
)]
struct Cli {
#[arg(short, long, default_value = "bench.toml")]
config: PathBuf,
#[arg(short, long, default_value = "results")]
output: PathBuf,
#[arg(long)]
contribute: bool,
#[arg(long)]
gpu: Option<String>,
#[arg(long)]
warmup: Option<u32>,
#[arg(long)]
iterations: Option<u32>,
#[arg(long)]
repo_root: Option<PathBuf>,
#[arg(long)]
scoreboard: bool,
#[arg(long)]
benchmarks_md: Option<PathBuf>,
}
fn main() -> Result<()> {
let cli = Cli::parse();
let repo_root = cli
.repo_root
.clone()
.unwrap_or_else(|| std::env::current_dir().unwrap());
if cli.scoreboard {
return cmd_scoreboard(&cli, &repo_root);
}
let cfg = config::load(&cli.config)?;
let gpu_override = cli.gpu.as_deref().or(Some(cfg.target.gpu.as_str()));
let gpu = detect::GpuInfo::detect(gpu_override);
let warmup = cli.warmup.unwrap_or(cfg.run.warmup);
let iterations = cli.iterations.unwrap_or(cfg.run.iterations);
let tolerance = cfg.run.tolerance;
println!(
"TPT-GenBench GPU: {} warmup: {} iters: {}",
gpu.model, warmup, iterations
);
let resolver = baseline::BaselineResolver::new(&repo_root);
let profile = resolver.load(&gpu)?;
let cases = config::expand(&cfg);
println!("Expanding workloads → {} cases\n", cases.len());
let mut runs = Vec::with_capacity(cases.len());
let mut contribute_entries: HashMap<String, baseline::ProfileEntry> = HashMap::new();
for case in &cases {
let correctness = correctness::verify(case, tolerance);
if !correctness.passed {
eprintln!(
" CORRECTNESS FAIL {} max_err={:.2e} (threshold {:.2e})",
case.label, correctness.max_abs_error, correctness.tolerance
);
}
let time_ms = simulate_kernel(case, warmup, iterations);
let (baseline_time_ms, baseline_backend) = match &profile {
Some(p) => match p.lookup(case) {
Some(entry) => (Some(entry.time_ms), Some(entry.vendor_backend.clone())),
None => {
let bt = baseline::sim_baseline_ms(case);
(Some(bt), Some("sim-estimate".to_string()))
}
},
None => {
let bt = baseline::sim_baseline_ms(case);
(Some(bt), Some("sim-estimate".to_string()))
}
};
let run = report::BenchRun::new(
case.label.clone(),
case.kind.clone(),
case.params.clone(),
case.flops,
time_ms,
baseline_time_ms,
baseline_backend.clone(),
correctness,
);
let eff = run
.efficiency_pct
.map(|e| format!("{:.1}%", e))
.unwrap_or("—".into());
println!(
" {:50} {:8.1} GFLOPS vs baseline: {} {}",
case.label,
run.gflops,
eff,
if run.valid { "✓" } else { "✗" }
);
contribute_entries.insert(
case.label.clone(),
baseline::ProfileEntry {
time_ms,
gflops: Some(run.gflops),
bandwidth_gbps: None,
vendor_backend: "tpt".to_string(),
},
);
runs.push(run);
}
let report = report::BenchReport::build(&gpu.model, cfg.target.label.as_deref(), runs);
std::fs::create_dir_all(&cli.output)?;
let ts = chrono::Utc::now().format("%Y%m%dT%H%M%S");
let json_path = cli
.output
.join(format!("{}-{}.json", gpu.profile_key(), ts));
std::fs::write(&json_path, report.to_json()?)
.with_context(|| format!("writing {}", json_path.display()))?;
println!("\nResults written to {}", json_path.display());
let md_path = cli.output.join(format!("{}-{}.md", gpu.profile_key(), ts));
std::fs::write(&md_path, report.to_markdown())
.with_context(|| format!("writing {}", md_path.display()))?;
println!("Markdown report: {}", md_path.display());
let s = &report.summary;
println!(
"\n{}/{} cases valid ({} correctness failure{})",
s.valid,
s.total,
s.correctness_failures,
if s.correctness_failures == 1 { "" } else { "s" }
);
if s.correctness_failures > 0 {
bail!("Correctness failures detected — performance numbers are not valid");
}
if cli.contribute {
let profile_path = resolver.write_candidate(&gpu, contribute_entries)?;
println!(
"\n--contribute: profile written to {}\n\
To contribute: open a PR adding this file to tuning/.\n\
CI will validate the JSON schema automatically.",
profile_path.display()
);
}
Ok(())
}
fn cmd_scoreboard(cli: &Cli, repo_root: &Path) -> Result<()> {
let results_dir = if cli.output.is_absolute() {
cli.output.clone()
} else {
repo_root.join(&cli.output)
};
let benchmarks_path = cli
.benchmarks_md
.clone()
.unwrap_or_else(|| repo_root.join("BENCHMARKS.md"));
println!("TPT-GenBench scoreboard updater");
println!(" Results dir : {}", results_dir.display());
println!(" BENCHMARKS : {}", benchmarks_path.display());
if !results_dir.exists() {
println!(" No results directory found — writing empty scoreboard.");
let changed = scoreboard::update_benchmarks_md(&benchmarks_path, &[])?;
println!(
"{}",
if changed {
" BENCHMARKS.md updated."
} else {
" BENCHMARKS.md unchanged."
}
);
return Ok(());
}
let reports = scoreboard::load_results(&results_dir)?;
println!(" Loaded {} result file(s)", reports.len());
let scorecards = scoreboard::build_scorecards(&reports);
println!(" {} unique GPU(s) in scoreboard", scorecards.len());
let changed = scoreboard::update_benchmarks_md(&benchmarks_path, &scorecards)?;
if changed {
println!(" BENCHMARKS.md updated.");
} else {
println!(" BENCHMARKS.md already up to date.");
}
Ok(())
}
fn simulate_kernel(case: &config::BenchCase, _warmup: u32, _iters: u32) -> f64 {
const SIM_TFLOPS: f64 = 2.0;
let start = Instant::now();
let _ = start.elapsed();
case.flops as f64 / (SIM_TFLOPS * 1e12) * 1e3
}