aver/bench/
manifest.rs

1//! Scenario manifest: TOML on disk, struct in memory.
2//!
3//! ```toml
4//! # bench/scenarios/factorial.toml
5//! name  = "factorial"             # optional, defaults to file stem
6//! entry = "/tmp/factorial.av"     # absolute or relative to manifest dir
7//! iterations = 50
8//! warmup     = 5
9//! args       = []                 # CLI args passed to `main`, default empty
10//!
11//! [expected]
12//! # response_bytes = 23           # exact match (optional)
13//! # response_bytes_min = 100      # range match (optional)
14//! # response_bytes_max = 200
15//! ```
16//!
17//! Keep the format dumb: TOML, flat, no env vars, no string templates.
18//! A scenario is meant to be the simplest reproducible thing that pins
19//! a measurement, not a generic test runner. If a scenario needs more
20//! than this, that's a sign to write a small Aver harness fn in the
21//! `entry` file rather than to grow this struct.
22
23use std::path::{Path, PathBuf};
24
25#[derive(Debug)]
26pub enum ManifestError {
27    Read(String),
28    Parse(String),
29    Validate(String),
30}
31
32impl std::fmt::Display for ManifestError {
33    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
34        match self {
35            Self::Read(m) | Self::Parse(m) | Self::Validate(m) => f.write_str(m),
36        }
37    }
38}
39
40#[derive(Debug, Clone)]
41pub struct Manifest {
42    pub name: String,
43    /// Absolute path to the entry `.av` file.
44    pub entry: PathBuf,
45    pub iterations: usize,
46    pub warmup: usize,
47    pub args: Vec<String>,
48    pub expected: ExpectedShape,
49    pub tolerance: Tolerance,
50}
51
52/// Bench targets — picks which backend runs the scenario.
53#[derive(Debug, Clone, Copy, PartialEq, Eq)]
54pub enum BenchTarget {
55    /// In-process VM via `vm::compile_program_with_modules`.
56    Vm,
57    /// In-process WASM via wasmtime (compiled once, instantiated +
58    /// invoked per iteration). Requires the `wasm` feature.
59    WasmLocal,
60    /// Native Rust binary produced by `aver compile --target rust` +
61    /// `cargo build --release`. Subprocess spawn per iteration.
62    Rust,
63    /// In-process wasm-gc backend via wasmtime with the GC + tail-call
64    /// proposals enabled. Compiles once with `aver compile --target
65    /// wasm-gc`, instantiates + invokes per iteration. wasmtime's GC
66    /// heap is the ceiling for alloc-heavy scenarios — the V8 column
67    /// (below) is the production-relevant number for any wasm-gc
68    /// deploy that lands in a browser, Cloudflare Workers, Node, Bun,
69    /// or Deno. Keep both around so the gap is visible.
70    WasmGc,
71    /// Same wasm-gc bytes (`aver compile --target wasm-gc`), executed
72    /// under V8 via Node 22+ and `tools/wasm-gc-bench-v8.mjs`. Spawns
73    /// one Node process per scenario, parses raw sample timings off
74    /// stdout, then stitches the full `BenchReport` Rust-side. Memory
75    /// `project_v8_vs_wasmtime_gc.md` documents the engine gap that
76    /// motivates the dual column (string_interp is 2300× faster on V8
77    /// than on wasmtime as of 0.17.2). Requires `node` v22+ on PATH
78    /// or under `~/.nvm/versions/node/v22.*` — earlier Nodes ship a
79    /// V8 that rejects packed `i8` arrays.
80    WasmGcV8,
81}
82
83impl BenchTarget {
84    pub fn parse(s: &str) -> Result<Self, String> {
85        match s {
86            "vm" => Ok(Self::Vm),
87            "wasm-local" => Ok(Self::WasmLocal),
88            "wasm-gc" => Ok(Self::WasmGc),
89            "wasm-gc-v8" => Ok(Self::WasmGcV8),
90            "rust" => Ok(Self::Rust),
91            other => Err(format!(
92                "unknown bench target '{}'; expected one of: vm, wasm-local, wasm-gc, wasm-gc-v8, rust",
93                other
94            )),
95        }
96    }
97
98    pub const fn name(self) -> &'static str {
99        match self {
100            Self::Vm => "vm",
101            Self::WasmLocal => "wasm-local",
102            Self::WasmGc => "wasm-gc",
103            Self::WasmGcV8 => "wasm-gc-v8",
104            Self::Rust => "rust",
105        }
106    }
107}
108
109/// Per-metric regression tolerances used by `--compare baseline.json`.
110/// Defaults are deliberately loose for 0.15.1 — the bench harness is new
111/// and machines vary. Tighten per-scenario in TOML once a baseline is
112/// stable on the target machine.
113#[derive(Debug, Clone, Copy)]
114pub struct Tolerance {
115    /// Maximum allowed `p50_ms` increase as a percentage of baseline.
116    /// Default 20.0 (i.e. +20% over baseline counts as a regression).
117    pub wall_time_p50_pct: f64,
118    /// Maximum allowed `p95_ms` increase, percentage. Default 30.0 —
119    /// p95 is noisier than p50 by nature, so the bar is looser.
120    pub wall_time_p95_pct: f64,
121}
122
123impl Default for Tolerance {
124    fn default() -> Self {
125        Self {
126            wall_time_p50_pct: 20.0,
127            wall_time_p95_pct: 30.0,
128        }
129    }
130}
131
132#[derive(Debug, Clone, Default)]
133pub struct ExpectedShape {
134    pub response_bytes: Option<usize>,
135    pub response_bytes_min: Option<usize>,
136    pub response_bytes_max: Option<usize>,
137}
138
139impl Manifest {
140    pub fn load(path: &Path) -> Result<Self, ManifestError> {
141        let raw = std::fs::read_to_string(path)
142            .map_err(|e| ManifestError::Read(format!("cannot read '{}': {}", path.display(), e)))?;
143        let table: toml::Value = raw.parse().map_err(|e: toml::de::Error| {
144            ManifestError::Parse(format!("{}: {}", path.display(), e))
145        })?;
146        let manifest_dir = path.parent().unwrap_or_else(|| Path::new("."));
147        Self::from_toml(&table, path, manifest_dir)
148    }
149
150    fn from_toml(
151        table: &toml::Value,
152        manifest_path: &Path,
153        manifest_dir: &Path,
154    ) -> Result<Self, ManifestError> {
155        let entry_raw = table.get("entry").and_then(|v| v.as_str()).ok_or_else(|| {
156            ManifestError::Validate(format!(
157                "{}: missing required `entry = \"...\"`",
158                manifest_path.display()
159            ))
160        })?;
161        let entry_path = Path::new(entry_raw);
162        let entry = if entry_path.is_absolute() {
163            entry_path.to_path_buf()
164        } else {
165            manifest_dir.join(entry_path)
166        };
167        if !entry.exists() {
168            return Err(ManifestError::Validate(format!(
169                "{}: entry '{}' does not exist (resolved to '{}')",
170                manifest_path.display(),
171                entry_raw,
172                entry.display()
173            )));
174        }
175
176        let name = table
177            .get("name")
178            .and_then(|v| v.as_str())
179            .map(|s| s.to_string())
180            .unwrap_or_else(|| {
181                manifest_path
182                    .file_stem()
183                    .and_then(|s| s.to_str())
184                    .unwrap_or("scenario")
185                    .to_string()
186            });
187
188        let iterations = table
189            .get("iterations")
190            .and_then(|v| v.as_integer())
191            .unwrap_or(10) as usize;
192        let warmup = table
193            .get("warmup")
194            .and_then(|v| v.as_integer())
195            .unwrap_or(1) as usize;
196
197        if iterations == 0 {
198            return Err(ManifestError::Validate(format!(
199                "{}: `iterations` must be > 0",
200                manifest_path.display()
201            )));
202        }
203
204        let args = table
205            .get("args")
206            .and_then(|v| v.as_array())
207            .map(|arr| {
208                arr.iter()
209                    .filter_map(|v| v.as_str().map(|s| s.to_string()))
210                    .collect()
211            })
212            .unwrap_or_default();
213
214        let expected = match table.get("expected") {
215            Some(toml::Value::Table(t)) => ExpectedShape {
216                response_bytes: t
217                    .get("response_bytes")
218                    .and_then(|v| v.as_integer())
219                    .map(|n| n as usize),
220                response_bytes_min: t
221                    .get("response_bytes_min")
222                    .and_then(|v| v.as_integer())
223                    .map(|n| n as usize),
224                response_bytes_max: t
225                    .get("response_bytes_max")
226                    .and_then(|v| v.as_integer())
227                    .map(|n| n as usize),
228            },
229            _ => ExpectedShape::default(),
230        };
231
232        let tolerance = match table.get("tolerance") {
233            Some(toml::Value::Table(t)) => {
234                let mut tol = Tolerance::default();
235                if let Some(v) = t.get("wall_time_p50_pct").and_then(|v| v.as_float()) {
236                    tol.wall_time_p50_pct = v;
237                }
238                if let Some(v) = t.get("wall_time_p95_pct").and_then(|v| v.as_float()) {
239                    tol.wall_time_p95_pct = v;
240                }
241                tol
242            }
243            _ => Tolerance::default(),
244        };
245
246        Ok(Manifest {
247            name,
248            entry,
249            iterations,
250            warmup,
251            args,
252            expected,
253            tolerance,
254        })
255    }
256}
aver/bench/manifest.rs

aver/bench/
manifest.rs
