1use crate::record::Record;
13use std::collections::{BTreeMap, BTreeSet};
14
15pub const GATED_METRIC: &str = "instructions";
17
18const WALL_METRIC: &str = "wall_min_ms";
20
21pub type Key = (String, String, String);
29
30#[derive(Debug, Clone, PartialEq)]
32pub struct Change {
33 pub bench: String,
34 pub tool: String,
35 pub runner: String,
36 pub metric: String,
37 pub base: f64,
38 pub head: f64,
39}
40
41impl Change {
42 pub fn pct(&self) -> Option<f64> {
49 if self.base == 0.0 {
50 return None;
51 }
52 Some((self.head - self.base) / self.base * 100.0)
53 }
54
55 pub fn regressed(&self, gate_pct: f64) -> bool {
61 if self.metric != GATED_METRIC {
62 return false;
63 }
64 match self.pct() {
65 Some(p) => p > gate_pct,
66 None => self.head > 0.0,
67 }
68 }
69}
70
71#[derive(Debug, Default, PartialEq)]
72pub struct Comparison {
73 pub changes: Vec<Change>,
75 pub added: Vec<Key>,
79 pub removed: Vec<Key>,
83}
84
85impl Comparison {
86 pub fn regressions(&self, gate_pct: f64) -> Vec<&Change> {
87 self.changes
88 .iter()
89 .filter(|c| c.regressed(gate_pct))
90 .collect()
91 }
92
93 pub fn is_empty(&self) -> bool {
95 self.changes.is_empty()
96 }
97}
98
99fn index(records: &[Record]) -> BTreeMap<(Key, String), f64> {
107 let mut out: BTreeMap<(Key, String), f64> = BTreeMap::new();
108 for r in records {
109 let key: Key = (r.bench.clone(), r.tool.clone(), r.runner.clone());
110 for (metric, value) in &r.metrics {
111 out.entry((key.clone(), metric.clone()))
112 .and_modify(|existing| {
113 if value < existing {
114 *existing = *value;
115 }
116 })
117 .or_insert(*value);
118 }
119 }
120 out
121}
122
123pub fn compare(base: &[Record], head: &[Record]) -> Comparison {
125 let (b, h) = (index(base), index(head));
126
127 let mut changes = Vec::new();
128 for ((key, metric), head_value) in &h {
129 if let Some(base_value) = b.get(&(key.clone(), metric.clone())) {
130 changes.push(Change {
131 bench: key.0.clone(),
132 tool: key.1.clone(),
133 runner: key.2.clone(),
134 metric: metric.clone(),
135 base: *base_value,
136 head: *head_value,
137 });
138 }
139 }
140
141 let base_keys: BTreeSet<Key> = b.keys().map(|(k, _)| k.clone()).collect();
142 let head_keys: BTreeSet<Key> = h.keys().map(|(k, _)| k.clone()).collect();
143
144 Comparison {
145 changes,
146 added: head_keys.difference(&base_keys).cloned().collect(),
147 removed: base_keys.difference(&head_keys).cloned().collect(),
148 }
149}
150
151pub type Trend = BTreeMap<Key, Vec<f64>>;
153
154pub fn build_trend(
164 walked: &[(String, Vec<Record>)],
165 head_sha: &str,
166 head_records: &[Record],
167) -> Trend {
168 let mut trend = Trend::new();
169 let mut head_in_history = false;
170 for (sha, records) in walked {
171 if sha == head_sha {
172 head_in_history = true;
173 add_point(&mut trend, head_records);
175 } else {
176 add_point(&mut trend, records);
177 }
178 }
179 if !head_in_history {
180 add_point(&mut trend, head_records);
181 }
182 trend
183}
184
185fn add_point(trend: &mut Trend, records: &[Record]) {
192 let mut lowest: BTreeMap<Key, f64> = BTreeMap::new();
193 for r in records {
194 if let Some(v) = r.metrics.get(GATED_METRIC) {
195 lowest
196 .entry((r.bench.clone(), r.tool.clone(), r.runner.clone()))
197 .and_modify(|e| {
198 if v < e {
199 *e = *v;
200 }
201 })
202 .or_insert(*v);
203 }
204 }
205 for (key, value) in lowest {
206 trend.entry(key).or_default().push(value);
207 }
208}
209
210const BARS: [char; 8] = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'];
212
213pub fn sparkline(values: &[f64]) -> String {
224 if values.len() < 2 {
225 return String::new();
226 }
227 let min = values.iter().copied().fold(f64::INFINITY, f64::min);
228 let max = values.iter().copied().fold(f64::NEG_INFINITY, f64::max);
229 if max <= min {
230 return BARS[BARS.len() / 2].to_string().repeat(values.len());
231 }
232 values
233 .iter()
234 .map(|v| {
235 let t = (v - min) / (max - min);
236 let i = (t * (BARS.len() - 1) as f64).round() as usize;
237 BARS[i.min(BARS.len() - 1)]
238 })
239 .collect()
240}
241
242fn thousands(v: f64) -> String {
244 let n = format!("{:.0}", v.abs());
245 let mut out = String::new();
246 for (i, c) in n.chars().enumerate() {
247 if i > 0 && (n.len() - i) % 3 == 0 {
248 out.push(',');
249 }
250 out.push(c);
251 }
252 if v < 0.0 { format!("-{out}") } else { out }
253}
254
255fn signed_pct(p: Option<f64>) -> String {
257 match p {
258 Some(p) => format!("{}{:.2}%", if p >= 0.0 { "+" } else { "" }, p),
259 None => "new".to_string(),
260 }
261}
262
263const CREDIT: &str = "\n<sub>Measured by [tak](https://github.com/jdx/tak) — instruction-counted \
276 CLI benchmarks, stored in this repository's git notes.</sub>\n";
277
278pub fn markdown(c: &Comparison, trend: &Trend, gate_pct: f64, credit: bool) -> String {
279 let mut out = String::new();
280
281 if c.is_empty() {
282 out.push_str(
286 "**Nothing was compared, and so nothing was gated.** No series \
287 appears on both sides: either the base has no measurements \
288 recorded, or the two were measured on different runner classes, \
289 which are deliberately not comparable — counts shift between \
290 machine types by more than a real regression does.\n",
291 );
292 } else {
293 out.push_str(&table(c, trend, gate_pct));
294 }
295
296 out.push_str(&outliers(c));
297 out.push_str(
298 "\n<sub>Only instruction counts gate. Wall clock is shown for context — \
299 on identical hardware it moves 4-20% run to run.</sub>\n",
300 );
301 if credit {
302 out.push_str(CREDIT);
303 }
304 out
305}
306
307fn table(c: &Comparison, trend: &Trend, gate_pct: f64) -> String {
309 let mut out = String::new();
310 let mut series: BTreeMap<Key, (Option<&Change>, Option<&Change>)> = BTreeMap::new();
313 for change in &c.changes {
314 let key = (
315 change.bench.clone(),
316 change.tool.clone(),
317 change.runner.clone(),
318 );
319 let slot = series.entry(key).or_insert((None, None));
320 match change.metric.as_str() {
321 GATED_METRIC => slot.0 = Some(change),
322 WALL_METRIC => slot.1 = Some(change),
323 _ => {}
324 }
325 }
326
327 let any_trend = series
328 .keys()
329 .any(|k| trend.get(k).is_some_and(|v| v.len() > 1));
330 if any_trend {
331 out.push_str("| benchmark | trend | instructions | Δ | wall (min) | Δ |\n");
332 out.push_str("|---|---|---:|---:|---:|---:|\n");
333 } else {
334 out.push_str("| benchmark | instructions | Δ | wall (min) | Δ |\n");
335 out.push_str("|---|---:|---:|---:|---:|\n");
336 }
337 for (key, (ins, wall)) in &series {
338 let (bench, tool, _runner) = key;
339 let name = if tool == "self" {
340 bench.clone()
341 } else {
342 format!("{bench} ({tool})")
343 };
344 let (ins_cell, ins_delta) = match ins {
345 Some(ch) => {
346 let flag = if ch.regressed(gate_pct) {
347 " ⚠️"
348 } else {
349 ""
350 };
351 (
352 format!("{} → {}", thousands(ch.base), thousands(ch.head)),
353 format!("**{}**{flag}", signed_pct(ch.pct())),
354 )
355 }
356 None => ("—".into(), "—".into()),
357 };
358 let (wall_cell, wall_delta) = match wall {
359 Some(ch) => (
360 format!("{:.2} → {:.2}ms", ch.base, ch.head),
361 signed_pct(ch.pct()),
362 ),
363 None => ("—".into(), "—".into()),
364 };
365 if any_trend {
366 let spark = trend
367 .get(key)
368 .map(|v| sparkline(v))
369 .filter(|s| !s.is_empty())
370 .map(|s| format!("`{s}`"))
371 .unwrap_or_else(|| "—".into());
372 out.push_str(&format!(
373 "| {name} | {spark} | {ins_cell} | {ins_delta} | {wall_cell} | {wall_delta} |\n"
374 ));
375 } else {
376 out.push_str(&format!(
377 "| {name} | {ins_cell} | {ins_delta} | {wall_cell} | {wall_delta} |\n"
378 ));
379 }
380 }
381
382 let regressions = c.regressions(gate_pct);
383 out.push('\n');
384 if regressions.is_empty() {
385 out.push_str(&format!(
386 "No instruction-count regression above {gate_pct}%.\n"
387 ));
388 } else {
389 out.push_str(&format!(
390 "**{} benchmark(s) above the {gate_pct}% gate:** {}\n",
391 regressions.len(),
392 regressions
393 .iter()
394 .map(|c| format!("`{}` {}", c.bench, signed_pct(c.pct())))
395 .collect::<Vec<_>>()
396 .join(", ")
397 ));
398 }
399
400 out
401}
402
403fn describe(key: &Key) -> String {
410 let (bench, tool, runner) = key;
411 if tool == "self" {
412 format!("`{bench}` on `{runner}`")
413 } else {
414 format!("`{bench}` ({tool}) on `{runner}`")
415 }
416}
417
418fn outliers(c: &Comparison) -> String {
421 let mut out = String::new();
422 if !c.added.is_empty() {
423 out.push_str(&format!(
424 "\nNew, nothing to compare against: {}\n",
425 c.added.iter().map(describe).collect::<Vec<_>>().join(", ")
426 ));
427 }
428 if !c.removed.is_empty() {
429 out.push_str(&format!(
430 "\nMeasured on the base but not here — a benchmark that stops \
431 running also stops gating: {}\n",
432 c.removed
433 .iter()
434 .map(describe)
435 .collect::<Vec<_>>()
436 .join(", ")
437 ));
438 }
439 out
440}
441
442#[cfg(test)]
443mod tests {
444 use super::*;
445
446 fn key(bench: &str) -> Key {
447 (bench.to_string(), "self".to_string(), "gha".to_string())
448 }
449
450 fn rec(bench: &str, runner: &str, ins: f64, wall: f64) -> Record {
451 Record {
452 v: 1,
453 bench: bench.into(),
454 tool: "self".into(),
455 version: None,
456 runner: runner.into(),
457 ts: "2026-01-01T00:00:00Z".into(),
458 metrics: BTreeMap::from([
459 (GATED_METRIC.to_string(), ins),
460 (WALL_METRIC.to_string(), wall),
461 ]),
462 }
463 }
464
465 #[test]
466 fn a_rise_beyond_the_gate_is_a_regression() {
467 let c = compare(
468 &[rec("a", "gha", 1_000_000.0, 10.0)],
469 &[rec("a", "gha", 1_020_000.0, 10.0)],
470 );
471 assert_eq!(c.regressions(1.0).len(), 1, "2% should trip a 1% gate");
472 assert!(c.regressions(5.0).is_empty(), "2% should not trip 5%");
473 }
474
475 #[test]
476 fn an_improvement_never_gates() {
477 let c = compare(
478 &[rec("a", "gha", 1_000_000.0, 10.0)],
479 &[rec("a", "gha", 500_000.0, 10.0)],
480 );
481 assert!(c.regressions(1.0).is_empty());
482 let ins = c.changes.iter().find(|c| c.metric == GATED_METRIC).unwrap();
483 assert_eq!(ins.pct().unwrap().round(), -50.0);
484 }
485
486 #[test]
489 fn wall_clock_never_gates_however_bad_it_looks() {
490 let c = compare(
491 &[rec("a", "gha", 1_000_000.0, 10.0)],
492 &[rec("a", "gha", 1_000_000.0, 100.0)],
493 );
494 assert!(c.regressions(0.001).is_empty());
495 let wall = c.changes.iter().find(|c| c.metric == WALL_METRIC).unwrap();
496 assert_eq!(wall.pct().unwrap().round(), 900.0);
497 }
498
499 #[test]
502 fn a_different_runner_is_not_a_comparison() {
503 let c = compare(
504 &[rec("a", "gha-linux", 1_000_000.0, 10.0)],
505 &[rec("a", "gha-macos", 2_000_000.0, 10.0)],
506 );
507 assert!(c.is_empty(), "nothing should line up");
508 assert_eq!(c.added.len(), 1);
509 assert_eq!(c.removed.len(), 1);
510 assert!(c.regressions(1.0).is_empty());
511
512 let md = markdown(&c, &Trend::new(), 1.0, false);
515 assert!(md.contains("nothing was gated"), "{md}");
516 assert!(md.contains("gha-macos"), "the new runner is unnamed: {md}");
517 assert!(md.contains("gha-linux"), "the old runner is unnamed: {md}");
518 }
519
520 #[test]
524 fn a_head_with_no_measurements_names_what_vanished() {
525 let c = compare(
526 &[
527 rec("a", "gha", 1_000_000.0, 10.0),
528 rec("b", "gha", 2.0, 2.0),
529 ],
530 &[],
531 );
532 let md = markdown(&c, &Trend::new(), 1.0, false);
533 assert!(c.regressions(1.0).is_empty());
534 assert!(md.contains("nothing was gated"), "{md}");
535 assert!(md.contains("`a`") && md.contains("`b`"), "{md}");
536 assert!(md.contains("stops gating"), "{md}");
537 }
538
539 #[test]
542 fn the_gating_caveat_survives_an_empty_comparison() {
543 let md = markdown(&compare(&[], &[]), &Trend::new(), 1.0, false);
544 assert!(md.contains("Only instruction counts gate"), "{md}");
545 }
546
547 #[test]
548 fn a_new_benchmark_is_reported_but_not_gated() {
549 let c = compare(
550 &[rec("a", "gha", 1_000_000.0, 10.0)],
551 &[
552 rec("a", "gha", 1_000_000.0, 10.0),
553 rec("b", "gha", 9.9e9, 10.0),
554 ],
555 );
556 assert_eq!(
557 c.added,
558 vec![("b".to_string(), "self".to_string(), "gha".to_string())]
559 );
560 assert!(c.regressions(1.0).is_empty());
561 }
562
563 #[test]
566 fn a_vanished_benchmark_is_surfaced() {
567 let c = compare(
568 &[
569 rec("a", "gha", 1_000_000.0, 10.0),
570 rec("b", "gha", 1.0, 1.0),
571 ],
572 &[rec("a", "gha", 1_000_000.0, 10.0)],
573 );
574 assert_eq!(c.removed.len(), 1);
575 assert!(markdown(&c, &Trend::new(), 1.0, false).contains("stops gating"));
576 }
577
578 #[test]
581 fn duplicate_records_reduce_to_the_minimum() {
582 let c = compare(
583 &[rec("a", "gha", 1_000_000.0, 10.0)],
584 &[
585 rec("a", "gha", 3_000_000.0, 30.0),
586 rec("a", "gha", 1_000_000.0, 10.0),
587 ],
588 );
589 assert!(c.regressions(1.0).is_empty(), "the clean sample should win");
590 }
591
592 #[test]
593 fn nothing_in_common_says_so_rather_than_passing_quietly() {
594 let c = compare(&[], &[rec("a", "gha", 1.0, 1.0)]);
595 assert!(c.is_empty());
596 assert!(markdown(&c, &Trend::new(), 1.0, false).contains("nothing was gated"));
597 }
598
599 #[test]
603 fn a_rise_from_a_zero_base_still_gates() {
604 let c = compare(
605 &[rec("a", "gha", 0.0, 10.0)],
606 &[rec("a", "gha", 5_000_000.0, 10.0)],
607 );
608 let ins = c.changes.iter().find(|c| c.metric == GATED_METRIC).unwrap();
609 assert_eq!(ins.pct(), None, "no percentage exists against zero");
610 assert_eq!(c.regressions(1.0).len(), 1, "it must still fail the gate");
611 assert!(markdown(&c, &Trend::new(), 1.0, false).contains("new"));
612 }
613
614 #[test]
616 fn zero_to_zero_is_not_a_regression() {
617 let c = compare(&[rec("a", "gha", 0.0, 1.0)], &[rec("a", "gha", 0.0, 1.0)]);
618 assert!(c.regressions(1.0).is_empty());
619 }
620
621 #[test]
625 fn outliers_keep_their_tool() {
626 let mut pnpm = rec("install", "gha", 1.0, 1.0);
627 pnpm.tool = "pnpm".into();
628 let c = compare(&[], &[rec("install", "gha", 1.0, 1.0), pnpm]);
629 let md = markdown(&c, &Trend::new(), 1.0, false);
630 assert!(md.contains("`install` on `gha`"), "{md}");
631 assert!(md.contains("`install` (pnpm) on `gha`"), "{md}");
632 }
633
634 #[test]
637 fn a_branch_head_is_appended_last() {
638 let walked = vec![
639 ("c1".to_string(), vec![rec("a", "gha", 10.0, 1.0)]),
640 ("c2".to_string(), vec![rec("a", "gha", 20.0, 1.0)]),
641 ];
642 let t = build_trend(&walked, "branch", &[rec("a", "gha", 30.0, 1.0)]);
643 assert_eq!(t[&key("a")], vec![10.0, 20.0, 30.0]);
644 }
645
646 #[test]
650 fn a_head_inside_history_stays_in_its_place() {
651 let walked = vec![
652 ("c1".to_string(), vec![rec("a", "gha", 10.0, 1.0)]),
653 ("head".to_string(), vec![rec("a", "gha", 20.0, 1.0)]),
654 ("c3".to_string(), vec![rec("a", "gha", 30.0, 1.0)]),
655 ];
656 let t = build_trend(&walked, "head", &[rec("a", "gha", 99.0, 1.0)]);
657 assert_eq!(
658 t[&key("a")],
659 vec![10.0, 99.0, 30.0],
660 "the head's own measurement belongs where the head is, once"
661 );
662 }
663
664 #[test]
667 fn a_commit_contributes_one_point() {
668 let walked = vec![(
669 "c1".to_string(),
670 vec![rec("a", "gha", 50.0, 1.0), rec("a", "gha", 10.0, 1.0)],
671 )];
672 let t = build_trend(&walked, "branch", &[]);
673 assert_eq!(t[&key("a")], vec![10.0]);
674 }
675
676 #[test]
677 fn a_sparkline_spans_the_full_range() {
678 let s = sparkline(&[1.0, 2.0, 3.0, 4.0, 5.0]);
679 assert_eq!(s.chars().count(), 5);
680 assert_eq!(s.chars().next().unwrap(), '▁');
681 assert_eq!(s.chars().last().unwrap(), '█');
682 }
683
684 #[test]
687 fn a_flat_series_sits_mid_height() {
688 let s = sparkline(&[7.0, 7.0, 7.0]);
689 assert_eq!(s, "▅▅▅");
690 }
691
692 #[test]
695 fn one_point_draws_nothing() {
696 assert_eq!(sparkline(&[1.0]), "");
697 assert_eq!(sparkline(&[]), "");
698 }
699
700 #[test]
703 fn series_are_scaled_independently() {
704 let small = sparkline(&[7_000_000.0, 7_100_000.0]);
705 let large = sparkline(&[100_000_000.0, 101_000_000.0]);
706 assert_eq!(small, large, "both rise by the same shape");
707 }
708
709 #[test]
710 fn the_trend_column_appears_only_when_there_is_a_trend() {
711 let c = compare(
712 &[rec("a", "gha", 1_000_000.0, 10.0)],
713 &[rec("a", "gha", 1_000_000.0, 10.0)],
714 );
715 assert!(!markdown(&c, &Trend::new(), 1.0, false).contains("| trend |"));
716
717 let mut trend = Trend::new();
718 trend.insert(
719 ("a".into(), "self".into(), "gha".into()),
720 vec![1.0, 2.0, 3.0],
721 );
722 let md = markdown(&c, &trend, 1.0, false);
723 assert!(md.contains("| trend |"), "{md}");
724 assert!(md.contains('█'), "{md}");
725 }
726
727 #[test]
728 fn thousands_separates() {
729 assert_eq!(thousands(1234567.0), "1,234,567");
730 assert_eq!(thousands(999.0), "999");
731 assert_eq!(thousands(1000.0), "1,000");
732 }
733}