use anyhow::{Context, Result};
use std::collections::HashMap;
use std::path::Path;
use crate::report::BenchReport;
#[derive(Debug, Clone)]
pub struct KindStats {
pub avg_gflops: f64,
pub peak_gflops: f64,
pub avg_efficiency_pct: Option<f64>,
#[allow(dead_code)]
pub case_count: usize,
}
#[derive(Debug, Clone)]
pub struct GpuScorecard {
pub gpu_model: String,
pub run_count: usize,
pub latest_timestamp: String,
pub total_cases: usize,
pub valid_cases: usize,
pub by_kind: Vec<(String, KindStats)>,
}
pub fn load_results(dir: &Path) -> Result<Vec<BenchReport>> {
let entries = std::fs::read_dir(dir)
.with_context(|| format!("opening results directory: {}", dir.display()))?;
let mut paths: Vec<_> = entries
.filter_map(|e| e.ok())
.map(|e| e.path())
.filter(|p| p.extension().map(|x| x == "json").unwrap_or(false))
.collect();
paths.sort();
let mut reports = Vec::with_capacity(paths.len());
for path in &paths {
let raw =
std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?;
match serde_json::from_str::<BenchReport>(&raw) {
Ok(r) => reports.push(r),
Err(e) => eprintln!("scoreboard: skipping {} ({})", path.display(), e),
}
}
Ok(reports)
}
pub fn build_scorecards(reports: &[BenchReport]) -> Vec<GpuScorecard> {
type KindAccum = (f64, f64, f64, usize, usize);
type GpuAccum = (usize, String, usize, usize, HashMap<String, KindAccum>);
let mut by_gpu: HashMap<String, GpuAccum> = HashMap::new();
for report in reports {
let entry = by_gpu
.entry(report.gpu_model.clone())
.or_insert_with(|| (0, String::new(), 0, 0, HashMap::new()));
entry.0 += 1;
if report.timestamp > entry.1 {
entry.1 = report.timestamp.clone();
}
entry.2 += report.summary.total;
entry.3 += report.summary.valid;
for run in &report.runs {
let k: &mut KindAccum = entry.4.entry(run.kind.clone()).or_default();
k.0 += run.gflops;
if run.gflops > k.1 {
k.1 = run.gflops;
}
if let Some(eff) = run.efficiency_pct {
k.2 += eff;
k.4 += 1;
}
k.3 += 1;
}
}
let mut scorecards: Vec<GpuScorecard> = by_gpu
.into_iter()
.map(|(gpu_model, (run_count, latest_ts, total, valid, kinds))| {
let mut by_kind: Vec<(String, KindStats)> = kinds
.into_iter()
.map(|(kind, (gsum, peak, esum, cnt, ecnt))| {
(
kind,
KindStats {
avg_gflops: if cnt > 0 { gsum / cnt as f64 } else { 0.0 },
peak_gflops: peak,
avg_efficiency_pct: if ecnt > 0 {
Some(esum / ecnt as f64)
} else {
None
},
case_count: cnt,
},
)
})
.collect();
by_kind.sort_by(|a, b| a.0.cmp(&b.0));
GpuScorecard {
gpu_model,
run_count,
latest_timestamp: latest_ts,
total_cases: total,
valid_cases: valid,
by_kind,
}
})
.collect();
scorecards.sort_by(|a, b| {
let ag = peak_kind(a, "gemm");
let bg = peak_kind(b, "gemm");
bg.partial_cmp(&ag)
.unwrap_or(std::cmp::Ordering::Equal)
.then(a.gpu_model.cmp(&b.gpu_model))
});
scorecards
}
fn peak_kind(c: &GpuScorecard, kind: &str) -> f64 {
c.by_kind
.iter()
.find(|(k, _)| k == kind)
.map(|(_, s)| s.peak_gflops)
.unwrap_or(0.0)
}
pub fn render_scoreboard(scorecards: &[GpuScorecard], updated_at: &str) -> String {
let mut out = String::new();
out.push_str("## Community GPU Scoreboard\n\n");
out.push_str(&format!("_Last updated: {}_ \n", updated_at));
out.push_str(&format!("_GPUs reported: {}_\n\n", scorecards.len()));
if scorecards.is_empty() {
out.push_str(
"No results submitted yet. \n\
Run `cargo run -p tpt-gpu-bench -- --contribute` on your machine to be the first!\n",
);
return out;
}
let gemm_cards: Vec<_> = scorecards
.iter()
.filter(|c| c.by_kind.iter().any(|(k, _)| k == "gemm"))
.collect();
if !gemm_cards.is_empty() {
out.push_str("### GEMM Leaderboard\n\n");
out.push_str("| GPU | Avg GFLOPS | Peak GFLOPS | Avg vs cuBLAS | Submissions |\n");
out.push_str("|-----|:----------:|:-----------:|:-------------:|:-----------:|\n");
for c in &gemm_cards {
if let Some((_, s)) = c.by_kind.iter().find(|(k, _)| k == "gemm") {
let eff = fmt_eff(s.avg_efficiency_pct);
out.push_str(&format!(
"| {} | {:.0} | {:.0} | {} | {} |\n",
c.gpu_model, s.avg_gflops, s.peak_gflops, eff, c.run_count
));
}
}
out.push('\n');
}
let mut attn_cards: Vec<_> = scorecards
.iter()
.filter(|c| c.by_kind.iter().any(|(k, _)| k == "attention"))
.collect();
if !attn_cards.is_empty() {
attn_cards.sort_by(|a, b| {
peak_kind(b, "attention")
.partial_cmp(&peak_kind(a, "attention"))
.unwrap_or(std::cmp::Ordering::Equal)
});
out.push_str("### Attention Leaderboard\n\n");
out.push_str("| GPU | Avg GFLOPS | Peak GFLOPS | Avg vs FlashAttn v2 | Submissions |\n");
out.push_str("|-----|:----------:|:-----------:|:-------------------:|:-----------:|\n");
for c in &attn_cards {
if let Some((_, s)) = c.by_kind.iter().find(|(k, _)| k == "attention") {
let eff = fmt_eff(s.avg_efficiency_pct);
out.push_str(&format!(
"| {} | {:.0} | {:.0} | {} | {} |\n",
c.gpu_model, s.avg_gflops, s.peak_gflops, eff, c.run_count
));
}
}
out.push('\n');
}
out.push_str("### All Submissions\n\n");
out.push_str("| GPU | Kernel types | Cases | Valid | Last submitted |\n");
out.push_str("|-----|-------------|:-----:|:-----:|:--------------:|\n");
for c in scorecards {
let kinds: Vec<_> = c.by_kind.iter().map(|(k, _)| k.as_str()).collect();
let last = short_date(&c.latest_timestamp);
out.push_str(&format!(
"| {} | {} | {} | {} | {} |\n",
c.gpu_model,
kinds.join(", "),
c.total_cases,
c.valid_cases,
last
));
}
out.push('\n');
out.push_str(
"> **Contribute your GPU:** run `cargo run -p tpt-gpu-bench -- --contribute` then open a PR \
adding `results/<gpu>-<ts>.json` — CI updates this table automatically.\n",
);
out
}
fn fmt_eff(e: Option<f64>) -> String {
e.map(|v| format!("{:.1}%", v))
.unwrap_or_else(|| "—".into())
}
fn short_date(ts: &str) -> &str {
if ts.len() >= 10 {
&ts[..10]
} else {
ts
}
}
const MARKER_START: &str = "<!-- SCOREBOARD:START -->";
const MARKER_END: &str = "<!-- SCOREBOARD:END -->";
pub fn update_benchmarks_md(path: &Path, scorecards: &[GpuScorecard]) -> Result<bool> {
let updated_at = chrono::Utc::now().format("%Y-%m-%d %H:%M UTC").to_string();
let body = render_scoreboard(scorecards, &updated_at);
let block = format!("{}\n{}{}\n", MARKER_START, body, MARKER_END);
let existing = if path.exists() {
std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?
} else {
String::new()
};
let updated =
if let (Some(si), Some(ei)) = (existing.find(MARKER_START), existing.find(MARKER_END)) {
let after = ei + MARKER_END.len();
format!("{}{}{}", &existing[..si], block, &existing[after..])
} else {
let insert_at = existing.find("\n## Contributing").unwrap_or(existing.len());
format!(
"{}\n{}{}",
&existing[..insert_at],
block,
&existing[insert_at..]
)
};
if updated == existing {
return Ok(false);
}
std::fs::write(path, &updated).with_context(|| format!("writing {}", path.display()))?;
Ok(true)
}