use crate::record::Record;
use std::collections::{BTreeMap, BTreeSet};
pub const GATED_METRIC: &str = "instructions";
const WALL_METRIC: &str = "wall_min_ms";
pub type Key = (String, String, String);
#[derive(Debug, Clone, PartialEq)]
pub struct Change {
pub bench: String,
pub tool: String,
pub runner: String,
pub metric: String,
pub base: f64,
pub head: f64,
}
impl Change {
pub fn pct(&self) -> Option<f64> {
if self.base == 0.0 {
return None;
}
Some((self.head - self.base) / self.base * 100.0)
}
pub fn regressed(&self, gate_pct: f64) -> bool {
if self.metric != GATED_METRIC {
return false;
}
match self.pct() {
Some(p) => p > gate_pct,
None => self.head > 0.0,
}
}
}
#[derive(Debug, Default, PartialEq)]
pub struct Comparison {
pub changes: Vec<Change>,
pub added: Vec<Key>,
pub removed: Vec<Key>,
}
impl Comparison {
pub fn regressions(&self, gate_pct: f64) -> Vec<&Change> {
self.changes
.iter()
.filter(|c| c.regressed(gate_pct))
.collect()
}
pub fn is_empty(&self) -> bool {
self.changes.is_empty()
}
}
fn index(records: &[Record]) -> BTreeMap<(Key, String), f64> {
let mut out: BTreeMap<(Key, String), f64> = BTreeMap::new();
for r in records {
let key: Key = (r.bench.clone(), r.tool.clone(), r.runner.clone());
for (metric, value) in &r.metrics {
out.entry((key.clone(), metric.clone()))
.and_modify(|existing| {
if value < existing {
*existing = *value;
}
})
.or_insert(*value);
}
}
out
}
pub fn compare(base: &[Record], head: &[Record]) -> Comparison {
let (b, h) = (index(base), index(head));
let mut changes = Vec::new();
for ((key, metric), head_value) in &h {
if let Some(base_value) = b.get(&(key.clone(), metric.clone())) {
changes.push(Change {
bench: key.0.clone(),
tool: key.1.clone(),
runner: key.2.clone(),
metric: metric.clone(),
base: *base_value,
head: *head_value,
});
}
}
let base_keys: BTreeSet<Key> = b.keys().map(|(k, _)| k.clone()).collect();
let head_keys: BTreeSet<Key> = h.keys().map(|(k, _)| k.clone()).collect();
Comparison {
changes,
added: head_keys.difference(&base_keys).cloned().collect(),
removed: base_keys.difference(&head_keys).cloned().collect(),
}
}
pub type Trend = BTreeMap<Key, Vec<f64>>;
pub fn build_trend(
walked: &[(String, Vec<Record>)],
head_sha: &str,
head_records: &[Record],
) -> Trend {
let mut trend = Trend::new();
let mut head_in_history = false;
for (sha, records) in walked {
if sha == head_sha {
head_in_history = true;
add_point(&mut trend, head_records);
} else {
add_point(&mut trend, records);
}
}
if !head_in_history {
add_point(&mut trend, head_records);
}
trend
}
fn add_point(trend: &mut Trend, records: &[Record]) {
let mut lowest: BTreeMap<Key, f64> = BTreeMap::new();
for r in records {
if let Some(v) = r.metrics.get(GATED_METRIC) {
lowest
.entry((r.bench.clone(), r.tool.clone(), r.runner.clone()))
.and_modify(|e| {
if v < e {
*e = *v;
}
})
.or_insert(*v);
}
}
for (key, value) in lowest {
trend.entry(key).or_default().push(value);
}
}
const BARS: [char; 8] = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'];
pub fn sparkline(values: &[f64]) -> String {
if values.len() < 2 {
return String::new();
}
let min = values.iter().copied().fold(f64::INFINITY, f64::min);
let max = values.iter().copied().fold(f64::NEG_INFINITY, f64::max);
if max <= min {
return BARS[BARS.len() / 2].to_string().repeat(values.len());
}
values
.iter()
.map(|v| {
let t = (v - min) / (max - min);
let i = (t * (BARS.len() - 1) as f64).round() as usize;
BARS[i.min(BARS.len() - 1)]
})
.collect()
}
fn thousands(v: f64) -> String {
let n = format!("{:.0}", v.abs());
let mut out = String::new();
for (i, c) in n.chars().enumerate() {
if i > 0 && (n.len() - i) % 3 == 0 {
out.push(',');
}
out.push(c);
}
if v < 0.0 { format!("-{out}") } else { out }
}
fn signed_pct(p: Option<f64>) -> String {
match p {
Some(p) => format!("{}{:.2}%", if p >= 0.0 { "+" } else { "" }, p),
None => "new".to_string(),
}
}
const CREDIT: &str = "\n<sub>Measured by [tak](https://github.com/jdx/tak) — instruction-counted \
CLI benchmarks, stored in this repository's git notes.</sub>\n";
pub fn markdown(c: &Comparison, trend: &Trend, gate_pct: f64, credit: bool) -> String {
let mut out = String::new();
if c.is_empty() {
out.push_str(
"**Nothing was compared, and so nothing was gated.** No series \
appears on both sides: either the base has no measurements \
recorded, or the two were measured on different runner classes, \
which are deliberately not comparable — counts shift between \
machine types by more than a real regression does.\n",
);
} else {
out.push_str(&table(c, trend, gate_pct));
}
out.push_str(&outliers(c));
out.push_str(
"\n<sub>Only instruction counts gate. Wall clock is shown for context — \
on identical hardware it moves 4-20% run to run.</sub>\n",
);
if credit {
out.push_str(CREDIT);
}
out
}
fn table(c: &Comparison, trend: &Trend, gate_pct: f64) -> String {
let mut out = String::new();
let mut series: BTreeMap<Key, (Option<&Change>, Option<&Change>)> = BTreeMap::new();
for change in &c.changes {
let key = (
change.bench.clone(),
change.tool.clone(),
change.runner.clone(),
);
let slot = series.entry(key).or_insert((None, None));
match change.metric.as_str() {
GATED_METRIC => slot.0 = Some(change),
WALL_METRIC => slot.1 = Some(change),
_ => {}
}
}
let any_trend = series
.keys()
.any(|k| trend.get(k).is_some_and(|v| v.len() > 1));
if any_trend {
out.push_str("| benchmark | trend | instructions | Δ | wall (min) | Δ |\n");
out.push_str("|---|---|---:|---:|---:|---:|\n");
} else {
out.push_str("| benchmark | instructions | Δ | wall (min) | Δ |\n");
out.push_str("|---|---:|---:|---:|---:|\n");
}
for (key, (ins, wall)) in &series {
let (bench, tool, _runner) = key;
let name = if tool == "self" {
bench.clone()
} else {
format!("{bench} ({tool})")
};
let (ins_cell, ins_delta) = match ins {
Some(ch) => {
let flag = if ch.regressed(gate_pct) {
" ⚠️"
} else {
""
};
(
format!("{} → {}", thousands(ch.base), thousands(ch.head)),
format!("**{}**{flag}", signed_pct(ch.pct())),
)
}
None => ("—".into(), "—".into()),
};
let (wall_cell, wall_delta) = match wall {
Some(ch) => (
format!("{:.2} → {:.2}ms", ch.base, ch.head),
signed_pct(ch.pct()),
),
None => ("—".into(), "—".into()),
};
if any_trend {
let spark = trend
.get(key)
.map(|v| sparkline(v))
.filter(|s| !s.is_empty())
.map(|s| format!("`{s}`"))
.unwrap_or_else(|| "—".into());
out.push_str(&format!(
"| {name} | {spark} | {ins_cell} | {ins_delta} | {wall_cell} | {wall_delta} |\n"
));
} else {
out.push_str(&format!(
"| {name} | {ins_cell} | {ins_delta} | {wall_cell} | {wall_delta} |\n"
));
}
}
let regressions = c.regressions(gate_pct);
out.push('\n');
if regressions.is_empty() {
out.push_str(&format!(
"No instruction-count regression above {gate_pct}%.\n"
));
} else {
out.push_str(&format!(
"**{} benchmark(s) above the {gate_pct}% gate:** {}\n",
regressions.len(),
regressions
.iter()
.map(|c| format!("`{}` {}", c.bench, signed_pct(c.pct())))
.collect::<Vec<_>>()
.join(", ")
));
}
out
}
fn describe(key: &Key) -> String {
let (bench, tool, runner) = key;
if tool == "self" {
format!("`{bench}` on `{runner}`")
} else {
format!("`{bench}` ({tool}) on `{runner}`")
}
}
fn outliers(c: &Comparison) -> String {
let mut out = String::new();
if !c.added.is_empty() {
out.push_str(&format!(
"\nNew, nothing to compare against: {}\n",
c.added.iter().map(describe).collect::<Vec<_>>().join(", ")
));
}
if !c.removed.is_empty() {
out.push_str(&format!(
"\nMeasured on the base but not here — a benchmark that stops \
running also stops gating: {}\n",
c.removed
.iter()
.map(describe)
.collect::<Vec<_>>()
.join(", ")
));
}
out
}
#[cfg(test)]
mod tests {
use super::*;
fn key(bench: &str) -> Key {
(bench.to_string(), "self".to_string(), "gha".to_string())
}
fn rec(bench: &str, runner: &str, ins: f64, wall: f64) -> Record {
Record {
v: 1,
bench: bench.into(),
tool: "self".into(),
version: None,
runner: runner.into(),
ts: "2026-01-01T00:00:00Z".into(),
metrics: BTreeMap::from([
(GATED_METRIC.to_string(), ins),
(WALL_METRIC.to_string(), wall),
]),
}
}
#[test]
fn a_rise_beyond_the_gate_is_a_regression() {
let c = compare(
&[rec("a", "gha", 1_000_000.0, 10.0)],
&[rec("a", "gha", 1_020_000.0, 10.0)],
);
assert_eq!(c.regressions(1.0).len(), 1, "2% should trip a 1% gate");
assert!(c.regressions(5.0).is_empty(), "2% should not trip 5%");
}
#[test]
fn an_improvement_never_gates() {
let c = compare(
&[rec("a", "gha", 1_000_000.0, 10.0)],
&[rec("a", "gha", 500_000.0, 10.0)],
);
assert!(c.regressions(1.0).is_empty());
let ins = c.changes.iter().find(|c| c.metric == GATED_METRIC).unwrap();
assert_eq!(ins.pct().unwrap().round(), -50.0);
}
#[test]
fn wall_clock_never_gates_however_bad_it_looks() {
let c = compare(
&[rec("a", "gha", 1_000_000.0, 10.0)],
&[rec("a", "gha", 1_000_000.0, 100.0)],
);
assert!(c.regressions(0.001).is_empty());
let wall = c.changes.iter().find(|c| c.metric == WALL_METRIC).unwrap();
assert_eq!(wall.pct().unwrap().round(), 900.0);
}
#[test]
fn a_different_runner_is_not_a_comparison() {
let c = compare(
&[rec("a", "gha-linux", 1_000_000.0, 10.0)],
&[rec("a", "gha-macos", 2_000_000.0, 10.0)],
);
assert!(c.is_empty(), "nothing should line up");
assert_eq!(c.added.len(), 1);
assert_eq!(c.removed.len(), 1);
assert!(c.regressions(1.0).is_empty());
let md = markdown(&c, &Trend::new(), 1.0, false);
assert!(md.contains("nothing was gated"), "{md}");
assert!(md.contains("gha-macos"), "the new runner is unnamed: {md}");
assert!(md.contains("gha-linux"), "the old runner is unnamed: {md}");
}
#[test]
fn a_head_with_no_measurements_names_what_vanished() {
let c = compare(
&[
rec("a", "gha", 1_000_000.0, 10.0),
rec("b", "gha", 2.0, 2.0),
],
&[],
);
let md = markdown(&c, &Trend::new(), 1.0, false);
assert!(c.regressions(1.0).is_empty());
assert!(md.contains("nothing was gated"), "{md}");
assert!(md.contains("`a`") && md.contains("`b`"), "{md}");
assert!(md.contains("stops gating"), "{md}");
}
#[test]
fn the_gating_caveat_survives_an_empty_comparison() {
let md = markdown(&compare(&[], &[]), &Trend::new(), 1.0, false);
assert!(md.contains("Only instruction counts gate"), "{md}");
}
#[test]
fn a_new_benchmark_is_reported_but_not_gated() {
let c = compare(
&[rec("a", "gha", 1_000_000.0, 10.0)],
&[
rec("a", "gha", 1_000_000.0, 10.0),
rec("b", "gha", 9.9e9, 10.0),
],
);
assert_eq!(
c.added,
vec![("b".to_string(), "self".to_string(), "gha".to_string())]
);
assert!(c.regressions(1.0).is_empty());
}
#[test]
fn a_vanished_benchmark_is_surfaced() {
let c = compare(
&[
rec("a", "gha", 1_000_000.0, 10.0),
rec("b", "gha", 1.0, 1.0),
],
&[rec("a", "gha", 1_000_000.0, 10.0)],
);
assert_eq!(c.removed.len(), 1);
assert!(markdown(&c, &Trend::new(), 1.0, false).contains("stops gating"));
}
#[test]
fn duplicate_records_reduce_to_the_minimum() {
let c = compare(
&[rec("a", "gha", 1_000_000.0, 10.0)],
&[
rec("a", "gha", 3_000_000.0, 30.0),
rec("a", "gha", 1_000_000.0, 10.0),
],
);
assert!(c.regressions(1.0).is_empty(), "the clean sample should win");
}
#[test]
fn nothing_in_common_says_so_rather_than_passing_quietly() {
let c = compare(&[], &[rec("a", "gha", 1.0, 1.0)]);
assert!(c.is_empty());
assert!(markdown(&c, &Trend::new(), 1.0, false).contains("nothing was gated"));
}
#[test]
fn a_rise_from_a_zero_base_still_gates() {
let c = compare(
&[rec("a", "gha", 0.0, 10.0)],
&[rec("a", "gha", 5_000_000.0, 10.0)],
);
let ins = c.changes.iter().find(|c| c.metric == GATED_METRIC).unwrap();
assert_eq!(ins.pct(), None, "no percentage exists against zero");
assert_eq!(c.regressions(1.0).len(), 1, "it must still fail the gate");
assert!(markdown(&c, &Trend::new(), 1.0, false).contains("new"));
}
#[test]
fn zero_to_zero_is_not_a_regression() {
let c = compare(&[rec("a", "gha", 0.0, 1.0)], &[rec("a", "gha", 0.0, 1.0)]);
assert!(c.regressions(1.0).is_empty());
}
#[test]
fn outliers_keep_their_tool() {
let mut pnpm = rec("install", "gha", 1.0, 1.0);
pnpm.tool = "pnpm".into();
let c = compare(&[], &[rec("install", "gha", 1.0, 1.0), pnpm]);
let md = markdown(&c, &Trend::new(), 1.0, false);
assert!(md.contains("`install` on `gha`"), "{md}");
assert!(md.contains("`install` (pnpm) on `gha`"), "{md}");
}
#[test]
fn a_branch_head_is_appended_last() {
let walked = vec![
("c1".to_string(), vec![rec("a", "gha", 10.0, 1.0)]),
("c2".to_string(), vec![rec("a", "gha", 20.0, 1.0)]),
];
let t = build_trend(&walked, "branch", &[rec("a", "gha", 30.0, 1.0)]);
assert_eq!(t[&key("a")], vec![10.0, 20.0, 30.0]);
}
#[test]
fn a_head_inside_history_stays_in_its_place() {
let walked = vec![
("c1".to_string(), vec![rec("a", "gha", 10.0, 1.0)]),
("head".to_string(), vec![rec("a", "gha", 20.0, 1.0)]),
("c3".to_string(), vec![rec("a", "gha", 30.0, 1.0)]),
];
let t = build_trend(&walked, "head", &[rec("a", "gha", 99.0, 1.0)]);
assert_eq!(
t[&key("a")],
vec![10.0, 99.0, 30.0],
"the head's own measurement belongs where the head is, once"
);
}
#[test]
fn a_commit_contributes_one_point() {
let walked = vec![(
"c1".to_string(),
vec![rec("a", "gha", 50.0, 1.0), rec("a", "gha", 10.0, 1.0)],
)];
let t = build_trend(&walked, "branch", &[]);
assert_eq!(t[&key("a")], vec![10.0]);
}
#[test]
fn a_sparkline_spans_the_full_range() {
let s = sparkline(&[1.0, 2.0, 3.0, 4.0, 5.0]);
assert_eq!(s.chars().count(), 5);
assert_eq!(s.chars().next().unwrap(), '▁');
assert_eq!(s.chars().last().unwrap(), '█');
}
#[test]
fn a_flat_series_sits_mid_height() {
let s = sparkline(&[7.0, 7.0, 7.0]);
assert_eq!(s, "▅▅▅");
}
#[test]
fn one_point_draws_nothing() {
assert_eq!(sparkline(&[1.0]), "");
assert_eq!(sparkline(&[]), "");
}
#[test]
fn series_are_scaled_independently() {
let small = sparkline(&[7_000_000.0, 7_100_000.0]);
let large = sparkline(&[100_000_000.0, 101_000_000.0]);
assert_eq!(small, large, "both rise by the same shape");
}
#[test]
fn the_trend_column_appears_only_when_there_is_a_trend() {
let c = compare(
&[rec("a", "gha", 1_000_000.0, 10.0)],
&[rec("a", "gha", 1_000_000.0, 10.0)],
);
assert!(!markdown(&c, &Trend::new(), 1.0, false).contains("| trend |"));
let mut trend = Trend::new();
trend.insert(
("a".into(), "self".into(), "gha".into()),
vec![1.0, 2.0, 3.0],
);
let md = markdown(&c, &trend, 1.0, false);
assert!(md.contains("| trend |"), "{md}");
assert!(md.contains('█'), "{md}");
}
#[test]
fn thousands_separates() {
assert_eq!(thousands(1234567.0), "1,234,567");
assert_eq!(thousands(999.0), "999");
assert_eq!(thousands(1000.0), "1,000");
}
}