frink-cli 0.26.0

//! `frink bench --render`: turn the receipts on disk into the engine
//! table in [`benchmarks/RESULTS.md`](../../../benchmarks/RESULTS.md).
//!
//! Split out of `bench_suite` because running the suite and publishing
//! it are two concepts, and because `bench_suite` was 903 lines when
//! this table's shape needed changing. Rendering reads receipts and
//! writes markdown; it never measures anything.
//!
//! # One row per model
//!
//! The table used to carry one row per (model, test), so a ten-model
//! host was twenty rows and a single model's prefill and decode sat
//! ten rows apart. They are now one row: the two verdicts first,
//! because that is what a reader scans for, then the four raw tok/s
//! numbers for anyone who wants them.
//!
//! Rows sort by model name rather than worst-gap-first. With prefill
//! and decode on one line there is no single gap to sort on, and the
//! generated `### Summary` above already carries each host's range. A
//! stable alphabetical order also puts the same model in the same place
//! in every host's table.
//!
//! # What this file must never do
//!
//! Invent, adjust, or omit a number. Every cell comes from a receipt.
//! A hand-edited ledger is not a measurement, which is why the table
//! lives between markers and this is the only thing that writes it.

use std::path::{Path, PathBuf};

use crate::bench_suite::{engine_receipt_dir, host_identity, load_suite};

pub const BEGIN: &str = "<!-- BEGIN ENGINE TABLE (generated by `frink bench --render`) -->";
pub const END: &str = "<!-- END ENGINE TABLE -->";

/// One measured (host, backend, model, test) cell.
#[derive(Clone)]
struct Row {
    /// Which machine produced this row. Rows never merge across hosts;
    /// see the grouping below.
    host: String,
    model: String,
    backend: String,
    test: String,
    frink: Option<f64>,
    llama: Option<f64>,
    gap: Option<f64>,
}

/// One model's prefill and decode on one host and backend: the shape
/// the table actually prints.
#[derive(Default, Clone)]
struct ModelRow {
    model: String,
    pp_test: String,
    pp_frink: Option<f64>,
    pp_llama: Option<f64>,
    pp_gap: Option<f64>,
    tg_test: String,
    tg_frink: Option<f64>,
    tg_llama: Option<f64>,
    tg_gap: Option<f64>,
}

fn is_prefill(test: &str) -> bool {
    test.starts_with("pp")
}

fn is_decode(test: &str) -> bool {
    test.starts_with("tg")
}

/// Collapse per-test rows into one row per model, preserving every
/// number. A test that is neither `pp*` nor `tg*` is dropped from this
/// view rather than silently filed under one of them.
fn pivot(rows: &[&Row]) -> Vec<ModelRow> {
    use std::collections::BTreeMap;
    let mut by: BTreeMap<String, ModelRow> = BTreeMap::new();
    for r in rows {
        let e = by.entry(r.model.clone()).or_default();
        e.model.clone_from(&r.model);
        if is_prefill(&r.test) {
            e.pp_test.clone_from(&r.test);
            e.pp_frink = r.frink;
            e.pp_llama = r.llama;
            e.pp_gap = r.gap;
        } else if is_decode(&r.test) {
            e.tg_test.clone_from(&r.test);
            e.tg_frink = r.frink;
            e.tg_llama = r.llama;
            e.tg_gap = r.gap;
        }
    }
    by.into_values().collect()
}

/// tok/s for the table, at a precision that cannot contradict the gap
/// beside it.
///
/// The gap column comes from the receipt, not from these two cells, so
/// a reader who divides the printed numbers must land on the printed
/// gap. Rounding everything to whole tok/s breaks that at small values
/// (149 ÷ 151 reads 0.99 against a printed 0.98), and carrying two
/// decimals to 3891.58 is noise nobody uses. One decimal below 100,
/// none above, keeps both ends honest.
fn tps(v: Option<f64>) -> String {
    v.map(|v| {
        if v >= 100.0 {
            format!("{v:.0}")
        } else {
            format!("{v:.1}")
        }
    })
    .unwrap_or_else(|| "—".into())
}

pub fn render(bench_dir: &Path) -> anyhow::Result<()> {
    let dir = engine_receipt_dir(bench_dir);
    let mut receipts: Vec<serde_json::Value> = Vec::new();
    if dir.is_dir() {
        let mut paths: Vec<PathBuf> = std::fs::read_dir(&dir)?
            .filter_map(|e| e.ok().map(|e| e.path()))
            .filter(|p| p.extension().is_some_and(|e| e == "json"))
            .collect();
        paths.sort();
        for p in paths {
            if let Ok(text) = std::fs::read_to_string(&p) {
                if let Ok(v) = serde_json::from_str::<serde_json::Value>(&text) {
                    receipts.push(v);
                }
            }
        }
    }

    // No receipts at all means there is nothing to render. Writing the
    // table anyway would replace a real ledger with an empty one and
    // report success, which is worse than doing nothing.
    if receipts.is_empty() {
        anyhow::bail!(
            "no engine receipts under {}, so there is nothing to render. \
             Run `frink bench --suite` first.",
            dir.display()
        );
    }

    let suite = load_suite(bench_dir).unwrap_or_default();
    let name_of = |id: &str| {
        suite
            .iter()
            .find(|e| e.id == id)
            .map(|e| e.name.clone())
            .unwrap_or_else(|| id.to_string())
    };

    // Rows from two machines are not one table. `render` reads every
    // receipt in the directory, so the moment a second host writes one,
    // its numbers would sort in beside this one's under a single
    // heading with nothing saying so. A reader comparing a 5.06x row
    // against a 1.41x row would be comparing two computers.
    let mut hosts: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
    for r in &receipts {
        hosts.insert(host_identity(r));
    }

    let mut rows: Vec<Row> = Vec::new();
    for r in &receipts {
        let host_label = host_identity(r);
        let id = r.get("id").and_then(|v| v.as_str()).unwrap_or("?");
        let backend = r
            .get("backend")
            .and_then(|v| v.as_str())
            .unwrap_or("?")
            .to_string();
        let Some(tests) = r.get("tests").and_then(|v| v.as_array()) else {
            continue;
        };
        for t in tests {
            rows.push(Row {
                host: host_label.clone(),
                model: name_of(id),
                backend: backend.clone(),
                test: t
                    .get("test")
                    .and_then(|v| v.as_str())
                    .unwrap_or("?")
                    .to_string(),
                frink: t.get("frink_tps").and_then(|v| v.as_f64()),
                llama: t.get("llama_tps").and_then(|v| v.as_f64()),
                gap: t.get("gap").and_then(|v| v.as_f64()),
            });
        }
    }

    let mut table = String::new();
    table.push_str(BEGIN);
    table.push_str("\n\n# frink vs llama.cpp\n\n");
    table.push_str(
        "Same machine, same GGUF, same backend. `pp` is prefill, `tg` is decode.\n\n\
         **Gap = llama.cpp ÷ frink. Below 1.00 means frink is faster.**\n\
         🟢 faster · ⚪ within 5% · 🔴 slower\n\n",
    );
    // Which machine, stated in the generated block rather than in prose
    // above it, so it cannot drift away from the numbers it describes.
    {
        let n = hosts.len();
        if n == 1 {
            table.push_str(&format!(
                "Measured on: **{}**\n\n",
                hosts.iter().next().cloned().unwrap_or_default()
            ));
        } else if n > 1 {
            table.push_str(&format!(
                "Measured on **{n} machines**, one section each. \
                 A gap only means something against the machine it was measured on, \
                 so rows are never compared across sections.\n\n"
            ));
        }
    }

    // A summary table, not prose. Generated from the same receipts as
    // the detail rows below, so it cannot drift away from them the way
    // a hand-written headline does.
    {
        use std::collections::BTreeMap;
        let mut by: BTreeMap<(String, String, bool), Vec<f64>> = BTreeMap::new();
        for r in &rows {
            if let Some(g) = r.gap {
                by.entry((r.host.clone(), r.backend.clone(), is_prefill(&r.test)))
                    .or_default()
                    .push(g);
            }
        }
        if !by.is_empty() {
            table.push_str("### At a glance\n\n");
            table.push_str("| Machine | Backend | Prefill | Decode |\n");
            table.push_str("|---|---|---|---|\n");
            let mut seen: Vec<(String, String)> =
                by.keys().map(|(h, b, _)| (h.clone(), b.clone())).collect();
            seen.dedup();
            for (host, backend) in seen {
                let fmt = |pp: bool| -> String {
                    match by.get(&(host.clone(), backend.clone(), pp)) {
                        Some(v) if !v.is_empty() => {
                            let lo = v.iter().cloned().fold(f64::INFINITY, f64::min);
                            let hi = v.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
                            if (hi - lo).abs() < 0.005 {
                                gap_cell(lo)
                            } else {
                                format!("{} to {}", gap_cell(lo), gap_cell(hi))
                            }
                        }
                        _ => "—".to_string(),
                    }
                };
                table.push_str(&format!(
                    "| {host} | {} | {} | {} |\n",
                    backend.to_uppercase(),
                    fmt(true),
                    fmt(false)
                ));
            }
            table.push('\n');
        }
    }

    fn push_section_at(table: &mut String, depth: &str, title: &str, rows: &[&Row]) {
        if rows.is_empty() {
            return;
        }
        let models = pivot(rows);
        if models.is_empty() {
            return;
        }
        // Column headers name the workloads actually measured rather
        // than hard-coding pp512/tg128, so a suite run at another size
        // cannot be mislabelled by this file.
        let pp = models
            .iter()
            .map(|m| m.pp_test.as_str())
            .find(|s| !s.is_empty())
            .unwrap_or("pp")
            .to_string();
        let tg = models
            .iter()
            .map(|m| m.tg_test.as_str())
            .find(|s| !s.is_empty())
            .unwrap_or("tg")
            .to_string();
        table.push_str(&format!("{depth} {title}\n\n"));
        table.push_str(&format!(
            "| Model | Prefill | Decode | frink {pp} | llama.cpp {pp} | frink {tg} | llama.cpp {tg} |\n"
        ));
        table.push_str("|---|---|---|---:|---:|---:|---:|\n");
        for m in &models {
            table.push_str(&format!(
                "| {} | {} | {} | {} | {} | {} | {} |\n",
                m.model,
                m.pp_gap.map(gap_cell).unwrap_or_else(|| "—".into()),
                m.tg_gap.map(gap_cell).unwrap_or_else(|| "—".into()),
                tps(m.pp_frink),
                tps(m.pp_llama),
                tps(m.tg_frink),
                tps(m.tg_llama),
            ));
        }
        table.push('\n');
    }

    let metal: Vec<&Row> = rows.iter().filter(|r| r.backend == "metal").collect();
    let cuda: Vec<&Row> = rows.iter().filter(|r| r.backend == "cuda").collect();
    let cpu: Vec<&Row> = rows.iter().filter(|r| r.backend == "cpu").collect();
    let other: Vec<&Row> = rows
        .iter()
        .filter(|r| !matches!(r.backend.as_str(), "metal" | "cuda" | "cpu"))
        .collect();

    if rows.is_empty() {
        table.push_str("| _no engine receipts yet_ | | | | | |\n\n");
    } else if hosts.len() <= 1 {
        push_section_at(&mut table, "###", "Metal", &metal);
        push_section_at(&mut table, "###", "CUDA", &cuda);
        push_section_at(&mut table, "###", "CPU", &cpu);
        push_section_at(&mut table, "###", "Other backends", &other);
    } else {
        // One section per machine. A reader scanning for a gap sees the
        // host before the number, which is the only order in which the
        // number means anything.
        for host in &hosts {
            let here: Vec<&Row> = rows.iter().filter(|r| &r.host == host).collect();
            table.push_str(&format!("### {host}\n\n"));
            for (title, backend) in [("Metal", "metal"), ("CUDA", "cuda"), ("CPU", "cpu")] {
                let sub: Vec<&Row> = here
                    .iter()
                    .copied()
                    .filter(|r| r.backend == backend)
                    .collect();
                push_section_at(&mut table, "####", title, &sub);
            }
            let sub: Vec<&Row> = here
                .iter()
                .copied()
                .filter(|r| !matches!(r.backend.as_str(), "metal" | "cuda" | "cpu"))
                .collect();
            push_section_at(&mut table, "####", "Other backends", &sub);
        }
    }

    // The footer is generated too. Anything a reader needs in order to
    // read the table has to be inside the markers, or the next render
    // publishes a page whose legend has drifted from its numbers.
    table.push_str(
        "---\n\n\
         Generated by `frink bench --render` from [`receipts/engine/`](receipts/engine/). \
         Do not hand-edit: the next render overwrites it.\n\
         How the numbers are taken, and the traps they have fallen into: \
         [`README.md`](README.md). \
         Older measurements and before/after studies: [`HISTORY.md`](HISTORY.md).\n\n",
    );
    table.push_str(END);

    let results = bench_dir.join("RESULTS.md");
    let existing = std::fs::read_to_string(&results).unwrap_or_default();
    let updated = splice(&existing, &table);
    std::fs::write(&results, updated)?;
    eprintln!("frink bench: engine table written to {}", results.display());
    Ok(())
}

fn gap_cell(g: f64) -> String {
    let marker = if g < 0.95 {
        "🟢"
    } else if g <= 1.05 {
        "⚪"
    } else {
        "🔴"
    };
    format!("{marker} **{g:.2}×**")
}

/// Replaces the marked block, or appends it if the markers are absent.
fn splice(existing: &str, block: &str) -> String {
    if let (Some(start), Some(end)) = (existing.find(BEGIN), existing.find(END)) {
        let mut out = String::with_capacity(existing.len() + block.len());
        out.push_str(&existing[..start]);
        out.push_str(block);
        out.push_str(&existing[end + END.len()..]);
        return out;
    }
    let mut out = existing.to_string();
    if !out.ends_with('\n') {
        out.push('\n');
    }
    out.push('\n');
    out.push_str(block);
    out.push('\n');
    out
}

#[cfg(test)]
mod tests {
    use super::*;

    /// Writes one engine receipt for `host` into `dir`, with whatever
    /// tests the caller names.
    fn receipt_with(dir: &Path, id: &str, host: &str, backend: &str, tests: &[(&str, f64, f64)]) {
        let tests: Vec<serde_json::Value> = tests
            .iter()
            .map(|(test, frink, llama)| {
                serde_json::json!({
                    "test": test, "frink_tps": frink, "llama_tps": llama,
                    "gap": llama / frink,
                })
            })
            .collect();
        let r = serde_json::json!({
            "schema": 2, "kind": "engine", "id": id,
            "backend": backend, "backend_active": backend,
            "host_spec": {"label": host},
            "tests": tests,
        });
        std::fs::write(
            dir.join(format!("{id}_{backend}.json")),
            serde_json::to_string(&r).expect("json"),
        )
        .expect("write receipt");
    }

    fn receipt(dir: &Path, id: &str, host: &str, backend: &str, frink: f64, llama: f64) {
        receipt_with(dir, id, host, backend, &[("tg128", frink, llama)]);
    }

    fn scratch(tag: &str) -> PathBuf {
        let dir = std::env::temp_dir().join(format!(
            "frink_{tag}_{}_{:?}",
            std::process::id(),
            std::thread::current().id()
        ));
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(dir.join("receipts").join("engine")).expect("mkdir");
        std::fs::write(
            dir.join("RESULTS.md"),
            format!("head\n{BEGIN}\nold\n{END}\ntail\n"),
        )
        .expect("seed");
        dir
    }

    /// A model's prefill and decode land on ONE row, and every number
    /// from both receipts survives the pivot.
    ///
    /// The table used to print one row per (model, test), so a model's
    /// two halves sat ten rows apart and a reader comparing them had to
    /// find both. The risk in collapsing them is dropping a cell
    /// silently, so this asserts all four tok/s values and both gaps.
    ///
    /// Sabotage: drop the `e.tg_frink = r.frink` assignment in
    /// `pivot`; the decode column goes to `—` and this goes red.
    #[test]
    fn a_models_prefill_and_decode_share_one_row() {
        let dir = scratch("pivot");
        let engine = dir.join("receipts").join("engine");
        receipt_with(
            &engine,
            "m1",
            "Box One",
            "metal",
            &[("pp512", 100.0, 200.0), ("tg128", 50.0, 25.0)],
        );

        render(&dir).expect("render");
        let out = std::fs::read_to_string(dir.join("RESULTS.md")).expect("read");

        let row = out
            .lines()
            .find(|l| l.starts_with("| m1 |"))
            .unwrap_or_else(|| panic!("no row for the model:\n{out}"));
        // Prefill gap 2.00 (slower), decode gap 0.50 (faster), then the
        // four raw numbers in header order.
        assert!(row.contains("🔴 **2.00×**"), "prefill gap missing: {row}");
        assert!(row.contains("🟢 **0.50×**"), "decode gap missing: {row}");
        for cell in ["| 100 |", "| 200 |", "| 50.0 |", "| 25.0 |"] {
            assert!(row.contains(cell), "{cell} missing from {row}");
        }
        // One row, not two: the old shape printed the model twice.
        assert_eq!(
            out.lines().filter(|l| l.starts_with("| m1 |")).count(),
            1,
            "the model must appear on exactly one row:\n{out}"
        );
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// Column headers name the workloads that were actually measured.
    ///
    /// Hard-coding `pp512` / `tg128` would mislabel a suite run at any
    /// other size, which is a wrong number in the ledger with no way to
    /// tell from the file.
    ///
    /// Sabotage: replace the `find` with a literal `"pp512"`.
    #[test]
    fn the_column_headers_name_the_workload_that_ran() {
        let dir = scratch("headers");
        let engine = dir.join("receipts").join("engine");
        receipt_with(
            &engine,
            "m1",
            "Box One",
            "metal",
            &[("pp2048", 10.0, 10.0), ("tg64", 10.0, 10.0)],
        );

        render(&dir).expect("render");
        let out = std::fs::read_to_string(dir.join("RESULTS.md")).expect("read");
        assert!(out.contains("frink pp2048"), "header not derived:\n{out}");
        assert!(out.contains("llama.cpp tg64"), "header not derived:\n{out}");
        assert!(!out.contains("pp512"), "a workload nobody ran:\n{out}");
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// A model measured on only one of the two workloads still gets a
    /// row, with the missing half marked rather than dropped.
    ///
    /// Gemma-4-E2B is exactly this: `llama-bench` has no `gemma4` arch,
    /// so its llama.cpp column is blank on both tests. A pivot that
    /// required both halves would delete the row and the ledger would
    /// silently stop mentioning a model frink runs.
    ///
    /// Sabotage: make `pivot` skip a `ModelRow` whose `tg_gap` is None.
    #[test]
    fn a_model_measured_on_one_workload_is_not_dropped() {
        let dir = scratch("halfrow");
        let engine = dir.join("receipts").join("engine");
        receipt_with(
            &engine,
            "only_pp",
            "Box One",
            "metal",
            &[("pp512", 7.0, 9.0)],
        );

        render(&dir).expect("render");
        let out = std::fs::read_to_string(dir.join("RESULTS.md")).expect("read");
        let row = out
            .lines()
            .find(|l| l.starts_with("| only_pp |"))
            .unwrap_or_else(|| panic!("row dropped:\n{out}"));
        assert!(row.contains("—"), "the missing half must be marked: {row}");
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// Two machines render as two sections, and neither is dropped.
    ///
    /// This used to be a hard refusal ("one table cannot describe
    /// them"), which meant the ledger could only ever describe the
    /// laptop that happened to run the suite: it claimed a CPU gap of
    /// 1.41x to 5.06x while having no x86 or CUDA row at all. The
    /// refusal's REASON was right, so rows are separated rather than
    /// merged, and this pins that.
    #[test]
    fn two_hosts_render_as_two_sections_rather_than_an_error() {
        let dir = scratch("render");
        let engine = dir.join("receipts").join("engine");
        receipt(&engine, "m1", "Apple M2 Pro", "metal", 100.0, 90.0);
        receipt(&engine, "m1", "Rented Xeon", "cpu", 10.0, 20.0);

        render(&dir).expect("two hosts must render, not refuse");

        let out = std::fs::read_to_string(dir.join("RESULTS.md")).expect("read");
        assert!(out.contains("Apple M2 Pro"), "first host missing:\n{out}");
        assert!(out.contains("Rented Xeon"), "second host missing:\n{out}");
        assert!(
            out.contains("2 machines"),
            "the reader is not told there are two machines:\n{out}"
        );
        // The whole point: a row is never presented without its host.
        let xeon = out.find("Rented Xeon").expect("host heading");
        let cpu_row = out.find("| 10.0 |");
        if let Some(cpu_row) = cpu_row {
            assert!(
                cpu_row > xeon,
                "the Xeon's row appears before its host heading"
            );
        }
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// The summary is GENERATED, not written by hand.
    ///
    /// It replaced a hand-written headline table, which is a thing that
    /// drifts: the numbers above the fold stop matching the receipts
    /// below it and nobody notices, because nothing compares them.
    #[test]
    fn the_summary_is_derived_from_the_same_rows_as_the_detail_tables() {
        let dir = scratch("summary");
        let engine = dir.join("receipts").join("engine");
        // Two gaps on one host+backend, so the summary must show a range.
        receipt(&engine, "a", "Box One", "cuda", 10.0, 20.0);
        receipt(&engine, "b", "Box One", "cuda", 10.0, 100.0);

        render(&dir).expect("render");
        let out = std::fs::read_to_string(dir.join("RESULTS.md")).expect("read");

        assert!(out.contains("### At a glance"), "no summary table:\n{out}");
        let summary = &out[out.find("### At a glance").expect("summary")..];
        let first_detail = summary.find("\n### ").unwrap_or(summary.len());
        let summary = &summary[..first_detail];
        assert!(
            summary.contains("2.00×") && summary.contains("10.00×"),
            "the summary must span the rows it describes:\n{summary}"
        );
        assert!(
            summary.contains("Box One"),
            "the summary must name the host:\n{summary}"
        );
        let _ = std::fs::remove_dir_all(&dir);
    }

    #[test]
    fn gap_cell_colours_match_the_ledger_convention() {
        assert!(gap_cell(0.80).starts_with("🟢"));
        assert!(gap_cell(1.00).starts_with("⚪"));
        assert!(gap_cell(0.96).starts_with("⚪"));
        assert!(gap_cell(1.40).starts_with("🔴"));
    }

    #[test]
    fn splice_replaces_an_existing_block_and_keeps_the_surrounding_text() {
        let doc = format!("before\n{BEGIN}\nold\n{END}\nafter\n");
        let out = splice(&doc, &format!("{BEGIN}\nnew\n{END}"));
        assert!(out.contains("before"), "text before the block must survive");
        assert!(out.contains("after"), "text after the block must survive");
        assert!(out.contains("new"));
        assert!(!out.contains("old"), "the old block must be gone");
    }

    #[test]
    fn splice_appends_when_the_markers_are_missing() {
        let out = splice("just some prose\n", &format!("{BEGIN}\nfresh\n{END}"));
        assert!(out.starts_with("just some prose"));
        assert!(out.contains("fresh"));
    }

    /// A render with no receipts must refuse rather than publish an
    /// empty table over a real one.
    #[test]
    fn rendering_nothing_refuses_instead_of_emptying_the_ledger() {
        let dir = std::env::temp_dir().join(format!("frink-render-guard-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(dir.join("receipts").join("engine")).unwrap();
        let results = dir.join("RESULTS.md");
        let original = "# Results\n\nreal numbers live here\n";
        std::fs::write(&results, original).unwrap();

        let err = render(&dir).unwrap_err().to_string();
        assert!(
            err.contains("nothing to render"),
            "expected a refusal naming the empty receipt dir, got: {err}"
        );
        assert_eq!(
            std::fs::read_to_string(&results).unwrap(),
            original,
            "the existing ledger must survive a render that had no receipts"
        );
        let _ = std::fs::remove_dir_all(&dir);
    }

    #[test]
    fn splice_does_not_duplicate_the_block_on_a_second_render() {
        let block = format!("{BEGIN}\nv1\n{END}");
        let once = splice("doc\n", &block);
        let twice = splice(&once, &format!("{BEGIN}\nv2\n{END}"));
        assert_eq!(twice.matches(BEGIN).count(), 1, "exactly one engine block");
        assert!(twice.contains("v2") && !twice.contains("v1"));
    }
}