persistant_core/
bench.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2//! The kind-agnostic half of backend benchmarking stage 3 (speed): the environment a figure was taken
3//! in, timing statistics, the data file, and the comparison that never crosses machines. The kinds
4//! (`persistant_kv::speed`, `persistant_blob::speed`) own their workloads and how they are timed; they
5//! produce [`SpeedRow`]s and nothing here knows what a key-value store or a blob store is.
6//!
7//! - **Speed ranks only what passes the contract.** A row is either [`Outcome::Measured`] or
8//!   [`Outcome::Excluded`] with the leaf and reason; an excluded backend is listed, never ranked.
9//! - **No constant rankings.** Every row carries its [`Environment`] and the command that re-runs it;
10//!   [`compare`] orders backends only *within one environment*.
11//! - **Results are data files.** [`SpeedTable`] reads and writes a tab-separated file.
12
13use crate::{Config, Reason};
14use std::collections::BTreeMap;
15use std::fmt;
16
17/// Where and when a figure was measured. Keep every field honest: a row without its environment is a
18/// claim about the present.
19#[derive(Debug, Clone, PartialEq, Eq)]
20pub struct Environment {
21    /// CPU model and logical core count, as the OS reports them.
22    pub machine: String,
23    /// `std::env::consts::OS`.
24    pub os: String,
25    /// `std::env::consts::ARCH`.
26    pub arch: String,
27    /// UTC date, `YYYY-MM-DD`.
28    pub date: String,
29    /// The server image digest, when the caller ran one (`sha256:...`).
30    pub digest: Option<String>,
31    /// The command that reproduces this run.
32    pub rerun: String,
33}
34
35impl Environment {
36    /// Read the machine, OS, architecture and date; the caller supplies the re-run command and the
37    /// image digest, if any.
38    pub fn detect(rerun: impl Into<String>, digest: Option<String>) -> Self {
39        Environment {
40            machine: machine(),
41            os: std::env::consts::OS.to_string(),
42            arch: std::env::consts::ARCH.to_string(),
43            date: today_utc(),
44            digest,
45            rerun: clean(&rerun.into(), true),
46        }
47    }
48
49    /// What decides whether two rows may be compared: same machine, OS and architecture.
50    pub fn comparable_key(&self) -> String {
51        format!("{} | {} | {}", self.machine, self.os, self.arch)
52    }
53
54    fn to_config(&self) -> Config {
55        let mut c = Config::new()
56            .with("arch", &self.arch)
57            .with("date", &self.date)
58            .with("machine", clean(&self.machine, false))
59            .with("os", &self.os);
60        if let Some(d) = &self.digest {
61            c = c.with("digest", clean(d, false));
62        }
63        c
64    }
65}
66
67/// Make a value safe for the data file: no tab or newline, and (inside a `k=v;k=v` field) no `;`
68/// or `=`.
69fn clean(s: &str, free_text: bool) -> String {
70    s.chars()
71        .map(|c| match c {
72            '\t' | '\n' | '\r' => ' ',
73            ';' | '=' if !free_text => ' ',
74            c => c,
75        })
76        .collect::<String>()
77        .trim()
78        .to_string()
79}
80
81fn machine() -> String {
82    let cores = std::thread::available_parallelism().map_or(0, std::num::NonZero::get);
83    let model = std::env::var("PROCESSOR_IDENTIFIER")
84        .ok()
85        .or_else(|| {
86            std::fs::read_to_string("/proc/cpuinfo").ok().and_then(|t| {
87                t.lines()
88                    .find(|l| l.starts_with("model name"))
89                    .and_then(|l| l.split_once(':'))
90                    .map(|(_, v)| v.trim().to_string())
91            })
92        })
93        .unwrap_or_else(|| "unknown cpu".to_string());
94    format!("{} x{cores}", clean(&model, false))
95}
96
97fn today_utc() -> String {
98    let secs = std::time::SystemTime::now()
99        .duration_since(std::time::UNIX_EPOCH)
100        .map_or(0, |d| d.as_secs());
101    let (y, m, d) = civil_from_days(i64::try_from(secs / 86_400).unwrap_or(0));
102    format!("{y:04}-{m:02}-{d:02}")
103}
104
105/// Days since 1970-01-01 to a proleptic Gregorian date (Howard Hinnant's algorithm).
106fn civil_from_days(z: i64) -> (i64, u32, u32) {
107    let z = z + 719_468;
108    let era = z.div_euclid(146_097);
109    let doe = z.rem_euclid(146_097);
110    let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
111    let y = yoe + era * 400;
112    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
113    let mp = (5 * doy + 2) / 153;
114    let d = u32::try_from(doy - (153 * mp + 2) / 5 + 1).unwrap_or(1);
115    let m = u32::try_from(if mp < 10 { mp + 3 } else { mp - 9 }).unwrap_or(1);
116    (if m <= 2 { y + 1 } else { y }, m, d)
117}
118
119/// Timing of one workload: nearest-rank percentiles of per-operation wall-clock time, in
120/// nanoseconds. The clock's own resolution is coarser on some platforms; a figure near it means little.
121#[derive(Debug, Clone, Copy, PartialEq, Eq)]
122pub struct Stats {
123    /// Operations timed.
124    pub n: u32,
125    /// Median, nanoseconds.
126    pub p50_ns: u64,
127    /// 95th percentile, nanoseconds.
128    pub p95_ns: u64,
129    /// 99th percentile, nanoseconds.
130    pub p99_ns: u64,
131    /// Slowest, nanoseconds.
132    pub max_ns: u64,
133    /// Operations per second over the summed operation time (not wall-clock including setup).
134    pub ops_per_sec: u64,
135}
136
137impl Stats {
138    /// From per-operation durations in nanoseconds. `None` for no samples.
139    pub fn from_samples(mut us: Vec<u64>) -> Option<Self> {
140        if us.is_empty() {
141            return None;
142        }
143        us.sort_unstable();
144        let n = us.len();
145        let at = |p: usize| us[(n * p).div_ceil(100).saturating_sub(1).min(n - 1)];
146        let total: u64 = us.iter().sum();
147        Some(Stats {
148            n: u32::try_from(n).unwrap_or(u32::MAX),
149            p50_ns: at(50),
150            p95_ns: at(95),
151            p99_ns: at(99),
152            max_ns: us[n - 1],
153            ops_per_sec: u64::try_from(n)
154                .unwrap_or(u64::MAX)
155                .saturating_mul(1_000_000_000)
156                .checked_div(total)
157                .unwrap_or(0),
158        })
159    }
160}
161
162/// A figure, or the reason there is none.
163#[derive(Debug, Clone, PartialEq, Eq)]
164pub enum Outcome {
165    /// Timed. Only a backend that passed the contract for the workload gets this.
166    Measured(Stats),
167    /// Failed the contract for the workload's leaf: listed, never ranked.
168    Excluded {
169        /// The leaf that decided.
170        leaf: String,
171        /// `Missing`, `Wrong` or `Untested`.
172        reason: Reason,
173        /// The test's own words.
174        note: String,
175    },
176}
177
178/// One recorded fact: a workload on a backend in an environment.
179#[derive(Debug, Clone, PartialEq, Eq)]
180pub struct SpeedRow {
181    /// [`Workload::id`].
182    pub workload: String,
183    /// Backend name, as in the capability matrix.
184    pub backend: String,
185    /// Backend version, as in the capability matrix.
186    pub version: String,
187    /// Backend configuration, as in the capability matrix.
188    pub config: Config,
189    /// Where and when.
190    pub env: Environment,
191    /// The figure or the exclusion.
192    pub outcome: Outcome,
193}
194
195const HEADER: &str = "workload\tbackend\tversion\tconfig\tstatus\tn\tp50_ns\tp95_ns\tp99_ns\tmax_ns\tops_per_sec\tnote\tenv\trerun";
196
197/// Why a speed file did not parse.
198#[derive(Debug, Clone, PartialEq, Eq)]
199pub struct SpeedParseError(pub String);
200
201impl fmt::Display for SpeedParseError {
202    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
203        write!(f, "speed file: {}", self.0)
204    }
205}
206
207impl std::error::Error for SpeedParseError {}
208
209impl SpeedRow {
210    fn to_line(&self) -> String {
211        let (status, nums, note) = match &self.outcome {
212            Outcome::Measured(s) => (
213                "measured",
214                [
215                    s.n.to_string(),
216                    s.p50_ns.to_string(),
217                    s.p95_ns.to_string(),
218                    s.p99_ns.to_string(),
219                    s.max_ns.to_string(),
220                    s.ops_per_sec.to_string(),
221                ],
222                String::new(),
223            ),
224            Outcome::Excluded { leaf, reason, note } => (
225                "excluded",
226                Default::default(),
227                format!("{leaf}: {}: {}", reason.as_str(), clean(note, true)),
228            ),
229        };
230        let mut cols: Vec<String> = vec![
231            self.workload.clone(),
232            self.backend.clone(),
233            self.version.clone(),
234            self.config.to_string(),
235            status.to_string(),
236        ];
237        cols.extend(nums);
238        cols.push(note);
239        cols.push(self.env.to_config().to_string());
240        cols.push(self.env.rerun.clone());
241        cols.join("\t")
242    }
243
244    fn from_line(line: &str) -> Result<Self, SpeedParseError> {
245        let c: Vec<&str> = line.split('\t').collect();
246        if c.len() != 14 {
247            return Err(SpeedParseError(format!(
248                "{} columns, want 14: {line:?}",
249                c.len()
250            )));
251        }
252        let bad = |what: &str| SpeedParseError(format!("{what} in {line:?}"));
253        let env_cfg = Config::parse(c[12]).map_err(|e| SpeedParseError(e.to_string()))?;
254        let get = |k: &str| env_cfg.get(k).map(str::to_string).ok_or_else(|| bad(k));
255        let env = Environment {
256            machine: get("machine")?,
257            os: get("os")?,
258            arch: get("arch")?,
259            date: get("date")?,
260            digest: env_cfg.get("digest").map(str::to_string),
261            rerun: c[13].to_string(),
262        };
263        let num = |i: usize| c[i].parse::<u64>().map_err(|_| bad("number"));
264        let outcome = match c[4] {
265            "measured" => Outcome::Measured(Stats {
266                n: u32::try_from(num(5)?).map_err(|_| bad("n"))?,
267                p50_ns: num(6)?,
268                p95_ns: num(7)?,
269                p99_ns: num(8)?,
270                max_ns: num(9)?,
271                ops_per_sec: num(10)?,
272            }),
273            "excluded" => {
274                let mut parts = c[11].splitn(3, ": ");
275                let (leaf, reason, note) = (
276                    parts.next().unwrap_or(""),
277                    parts.next().unwrap_or(""),
278                    parts.next().unwrap_or(""),
279                );
280                let reason = match reason {
281                    "missing" => Reason::Missing,
282                    "wrong" => Reason::Wrong,
283                    "untested" => Reason::Untested,
284                    _ => return Err(bad("reason")),
285                };
286                Outcome::Excluded {
287                    leaf: leaf.to_string(),
288                    reason,
289                    note: note.to_string(),
290                }
291            }
292            _ => return Err(bad("status")),
293        };
294        Ok(SpeedRow {
295            workload: c[0].to_string(),
296            backend: c[1].to_string(),
297            version: c[2].to_string(),
298            config: Config::parse(c[3]).map_err(|e| SpeedParseError(e.to_string()))?,
299            env,
300            outcome,
301        })
302    }
303}
304
305/// A set of rows, read from and written to one tab-separated file.
306#[derive(Debug, Clone, Default, PartialEq, Eq)]
307pub struct SpeedTable {
308    rows: Vec<SpeedRow>,
309}
310
311impl SpeedTable {
312    /// A table of `rows`.
313    pub fn from_rows(rows: Vec<SpeedRow>) -> Self {
314        SpeedTable { rows }
315    }
316
317    /// The rows.
318    pub fn rows(&self) -> &[SpeedRow] {
319        &self.rows
320    }
321
322    /// Add the rows of another file (a different backend, maybe a different machine).
323    pub fn merge(&mut self, other: SpeedTable) {
324        self.rows.extend(other.rows);
325    }
326
327    /// The file text.
328    pub fn to_tsv(&self) -> String {
329        let mut out = String::from(HEADER);
330        out.push('\n');
331        for r in &self.rows {
332            out.push_str(&r.to_line());
333            out.push('\n');
334        }
335        out
336    }
337
338    /// Parse a file written by [`to_tsv`](Self::to_tsv).
339    pub fn parse(text: &str) -> Result<Self, SpeedParseError> {
340        let mut lines = text.lines();
341        if lines.next() != Some(HEADER) {
342            return Err(SpeedParseError("missing or wrong header".to_string()));
343        }
344        let rows = lines
345            .filter(|l| !l.is_empty())
346            .map(SpeedRow::from_line)
347            .collect::<Result<_, _>>()?;
348        Ok(SpeedTable { rows })
349    }
350}
351
352/// A backend that has a number for a workload.
353#[derive(Debug, Clone, PartialEq, Eq)]
354pub struct Ranked {
355    /// Backend name.
356    pub backend: String,
357    /// Backend version.
358    pub version: String,
359    /// Backend configuration.
360    pub config: Config,
361    /// The measurement.
362    pub stats: Stats,
363    /// The date of the measurement.
364    pub date: String,
365    /// The command that reproduces it.
366    pub rerun: String,
367}
368
369/// A backend that has no number for a workload, and why.
370#[derive(Debug, Clone, PartialEq, Eq)]
371pub struct Left {
372    /// Backend name.
373    pub backend: String,
374    /// Backend version.
375    pub version: String,
376    /// The leaf that excluded it.
377    pub leaf: String,
378    /// Why.
379    pub reason: Reason,
380}
381
382/// One workload in one environment: the eligible backends fastest first by median, then the excluded
383/// ones. The environment is part of the answer.
384#[derive(Debug, Clone, PartialEq, Eq)]
385pub struct Comparison {
386    /// [`Workload::id`].
387    pub workload: String,
388    /// [`Environment::comparable_key`]: this ordering holds here and nowhere else.
389    pub environment: String,
390    /// Backends with a number, ascending by `p50_ns` (ties by name).
391    pub ranked: Vec<Ranked>,
392    /// Backends listed without a number.
393    pub excluded: Vec<Left>,
394}
395
396/// Group rows by workload and environment and order the measured ones. Rows from different machines
397/// never share a [`Comparison`].
398pub fn compare(table: &SpeedTable) -> Vec<Comparison> {
399    let mut groups: BTreeMap<(String, String), Comparison> = BTreeMap::new();
400    for r in table.rows() {
401        let key = (r.workload.clone(), r.env.comparable_key());
402        let g = groups.entry(key.clone()).or_insert_with(|| Comparison {
403            workload: key.0.clone(),
404            environment: key.1.clone(),
405            ranked: Vec::new(),
406            excluded: Vec::new(),
407        });
408        match &r.outcome {
409            Outcome::Measured(s) => g.ranked.push(Ranked {
410                backend: r.backend.clone(),
411                version: r.version.clone(),
412                config: r.config.clone(),
413                stats: *s,
414                date: r.env.date.clone(),
415                rerun: r.env.rerun.clone(),
416            }),
417            Outcome::Excluded { leaf, reason, .. } => g.excluded.push(Left {
418                backend: r.backend.clone(),
419                version: r.version.clone(),
420                leaf: leaf.clone(),
421                reason: *reason,
422            }),
423        }
424    }
425    let mut out: Vec<Comparison> = groups.into_values().collect();
426    for g in &mut out {
427        g.ranked.sort_by(|a, b| {
428            (a.stats.p50_ns, &a.backend, &a.version).cmp(&(b.stats.p50_ns, &b.backend, &b.version))
429        });
430    }
431    out
432}
433
434#[cfg(test)]
435mod tests {
436    use super::*;
437
438    fn env(machine: &str) -> Environment {
439        Environment {
440            machine: machine.to_string(),
441            os: "linux".to_string(),
442            arch: "x86_64".to_string(),
443            date: "2026-10-16".to_string(),
444            digest: Some("sha256:abc".to_string()),
445            rerun: "cargo run --example speed".to_string(),
446        }
447    }
448
449    fn measured(backend: &str, machine: &str, p50: u64) -> SpeedRow {
450        SpeedRow {
451            workload: "speed.kv.get".to_string(),
452            backend: backend.to_string(),
453            version: "1".to_string(),
454            config: Config::new(),
455            env: env(machine),
456            outcome: Outcome::Measured(Stats::from_samples(vec![p50; 10]).unwrap()),
457        }
458    }
459
460    #[test]
461    fn percentiles_are_nearest_rank_and_an_empty_sample_has_none() {
462        assert!(Stats::from_samples(vec![]).is_none());
463        let s = Stats::from_samples((1..=100).collect()).unwrap();
464        assert_eq!(
465            (s.n, s.p50_ns, s.p95_ns, s.p99_ns, s.max_ns),
466            (100, 50, 95, 99, 100)
467        );
468        let one = Stats::from_samples(vec![7]).unwrap();
469        assert_eq!((one.p50_ns, one.p99_ns, one.max_ns), (7, 7, 7));
470    }
471
472    #[test]
473    fn dates_are_civil_utc() {
474        assert_eq!(civil_from_days(0), (1970, 1, 1));
475        assert_eq!(civil_from_days(19_000), (2022, 1, 8));
476        assert_eq!(civil_from_days(20_742), (2026, 10, 16));
477    }
478
479    #[test]
480    fn a_table_round_trips_measured_and_excluded_rows() {
481        let mut rows = vec![measured("a", "m", 5)];
482        rows.push(SpeedRow {
483            outcome: Outcome::Excluded {
484                leaf: "kv.get".to_string(),
485                reason: Reason::Wrong,
486                note: "lost a write".to_string(),
487            },
488            ..measured("b", "m", 1)
489        });
490        let t = SpeedTable::from_rows(rows);
491        assert_eq!(SpeedTable::parse(&t.to_tsv()).unwrap(), t);
492    }
493
494    #[test]
495    fn a_malformed_file_is_an_error_not_an_empty_table() {
496        assert!(SpeedTable::parse("nope").is_err());
497        assert!(SpeedTable::parse(&format!("{HEADER}\ntoo\tfew")).is_err());
498    }
499
500    #[test]
501    fn rankings_never_cross_machines_and_excluded_backends_are_listed_not_ranked() {
502        let mut rows = vec![
503            measured("slow", "laptop", 900),
504            measured("fast", "laptop", 20),
505            // On another machine the order is the opposite: it must be a separate comparison.
506            measured("slow", "server", 5),
507            measured("fast", "server", 50),
508        ];
509        rows.push(SpeedRow {
510            outcome: Outcome::Excluded {
511                leaf: "kv.get".to_string(),
512                reason: Reason::Wrong,
513                note: "lost a write".to_string(),
514            },
515            ..measured("broken", "laptop", 1)
516        });
517        let cmp = compare(&SpeedTable::from_rows(rows));
518        assert_eq!(cmp.len(), 2, "one comparison per environment");
519        let laptop = cmp
520            .iter()
521            .find(|c| c.environment.starts_with("laptop"))
522            .unwrap();
523        let names: Vec<_> = laptop.ranked.iter().map(|r| r.backend.as_str()).collect();
524        assert_eq!(names, ["fast", "slow"]);
525        assert_eq!(laptop.excluded.len(), 1);
526        assert_eq!(laptop.excluded[0].backend, "broken");
527        let server = cmp
528            .iter()
529            .find(|c| c.environment.starts_with("server"))
530            .unwrap();
531        let names: Vec<_> = server.ranked.iter().map(|r| r.backend.as_str()).collect();
532        assert_eq!(names, ["slow", "fast"]);
533    }
534}
persistant_core/bench.rs

persistant_core/
bench.rs
