harn_kernel/
benchmark.rs

1//! Deterministic aggregation for benchmark receipts.
2//!
3//! Hosts own clocks and sample capture. Keeping aggregation here gives native,
4//! browser, and future edge adapters one definition of the receipt statistics
5//! without giving the portable kernel clock authority.
6
7use std::fmt;
8
9use serde::{Deserialize, Serialize};
10
11use crate::{opcode_abi_fingerprint, semantic_abi_fingerprint_hex, DataValue, ARTIFACT_VERSION};
12
13mod schema;
14pub use schema::portable_benchmark_json_schema;
15
16pub const PORTABLE_BENCHMARK_SCHEMA_VERSION: &str = "harn.portable_kernel.benchmark.v1";
17pub const PORTABLE_MAX_DISPATCH_ITERATIONS: usize = 1_000_000;
18pub const PORTABLE_MAX_COMPILE_ITERATIONS: usize = 100_000;
19pub const PORTABLE_MAX_WORKERS: usize = 256;
20
21/// Closed, shared receipt emitted by native and browser benchmark adapters.
22/// Hosts own clocks and sample capture; the kernel owns field names,
23/// aggregation, provenance, bounds, and serialization.
24#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
25#[serde(rename_all = "camelCase", deny_unknown_fields)]
26pub struct PortableBenchmarkReceipt {
27    pub schema_version: String,
28    pub target: BenchmarkTarget,
29    pub source: String,
30    pub entry: String,
31    pub entry_kind: BenchmarkEntryKind,
32    pub artifact_bytes: usize,
33    pub artifact_digest: String,
34    pub iterations: usize,
35    pub workers: usize,
36    pub provenance: BenchmarkProvenance,
37    pub initialization_ms: Option<f64>,
38    pub compile: CompileMeasurements,
39    pub decode: Option<BenchmarkStatistics>,
40    pub dispatch: DispatchMeasurements,
41    pub terminal_digest: String,
42}
43
44#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
45#[serde(rename_all = "snake_case")]
46pub enum BenchmarkTarget {
47    Native,
48    Browser,
49}
50
51#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
52#[serde(rename_all = "snake_case")]
53pub enum BenchmarkEntryKind {
54    Function,
55    Pipeline,
56}
57
58#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
59#[serde(rename_all = "snake_case")]
60pub enum BenchmarkBuildProfile {
61    Debug,
62    Release,
63}
64
65#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
66#[serde(rename_all = "camelCase", deny_unknown_fields)]
67pub struct BenchmarkProvenance {
68    pub harn_version: String,
69    pub kernel_version: String,
70    pub artifact_format_version: u16,
71    pub semantic_abi_fingerprint: String,
72    pub opcode_abi_fingerprint: String,
73    pub build_profile: BenchmarkBuildProfile,
74    pub os: String,
75    pub arch: String,
76}
77
78impl BenchmarkProvenance {
79    pub fn current(
80        harn_version: impl Into<String>,
81        build_profile: BenchmarkBuildProfile,
82        os: impl Into<String>,
83        arch: impl Into<String>,
84    ) -> Self {
85        Self {
86            harn_version: harn_version.into(),
87            kernel_version: crate::KERNEL_VERSION.to_string(),
88            artifact_format_version: ARTIFACT_VERSION,
89            semantic_abi_fingerprint: semantic_abi_fingerprint_hex(),
90            opcode_abi_fingerprint: hex::encode(opcode_abi_fingerprint()),
91            build_profile,
92            os: os.into(),
93            arch: arch.into(),
94        }
95    }
96}
97
98#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
99#[serde(rename_all = "camelCase", deny_unknown_fields)]
100pub struct CompileMeasurements {
101    pub first_ms: f64,
102    pub repeated: BenchmarkStatistics,
103}
104
105#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
106#[serde(rename_all = "camelCase", deny_unknown_fields)]
107pub struct DispatchMeasurements {
108    pub first_ms: f64,
109    pub repeated: BenchmarkStatistics,
110    pub batch_wall_ms: f64,
111    pub throughput_per_second: f64,
112}
113
114impl PortableBenchmarkReceipt {
115    /// Validate the closed cross-host receipt contract before persistence.
116    pub fn validate(&self) -> Result<(), String> {
117        if self.schema_version != PORTABLE_BENCHMARK_SCHEMA_VERSION {
118            return Err("portable benchmark schema version is not supported".to_string());
119        }
120        if self.source.is_empty() || self.entry.is_empty() || self.artifact_bytes == 0 {
121            return Err("portable benchmark identity fields must not be empty".to_string());
122        }
123        if self.iterations == 0 || self.iterations > PORTABLE_MAX_DISPATCH_ITERATIONS {
124            return Err(
125                "portable benchmark dispatch iteration count is outside limits".to_string(),
126            );
127        }
128        if self.workers == 0 || self.workers > PORTABLE_MAX_WORKERS {
129            return Err("portable benchmark worker count is outside limits".to_string());
130        }
131        if self.workers > self.iterations {
132            return Err(
133                "portable benchmark workers must not exceed dispatch iterations".to_string(),
134            );
135        }
136        if self.compile.repeated.iterations == 0
137            || self.compile.repeated.iterations > PORTABLE_MAX_COMPILE_ITERATIONS
138        {
139            return Err("portable benchmark compile iteration count is outside limits".to_string());
140        }
141        if self.dispatch.repeated.iterations != self.iterations {
142            return Err(
143                "portable benchmark dispatch statistics do not match iterations".to_string(),
144            );
145        }
146        if let Some(decode) = self.decode {
147            if decode.iterations != self.compile.repeated.iterations {
148                return Err(
149                    "portable benchmark decode statistics do not match compilation samples"
150                        .to_string(),
151                );
152            }
153        }
154        match self.target {
155            BenchmarkTarget::Native
156                if self.initialization_ms.is_some() || self.decode.is_none() =>
157            {
158                return Err("native portable benchmarks require decode samples and no adapter initialization".to_string());
159            }
160            BenchmarkTarget::Browser
161                if self.initialization_ms.is_none() || self.decode.is_some() =>
162            {
163                return Err("browser portable benchmarks require adapter initialization and include decode in dispatch".to_string());
164            }
165            BenchmarkTarget::Native | BenchmarkTarget::Browser => {}
166        }
167        if !is_digest(&self.artifact_digest)
168            || !is_digest(&self.terminal_digest)
169            || !is_digest(&self.provenance.semantic_abi_fingerprint)
170            || !is_digest(&self.provenance.opcode_abi_fingerprint)
171        {
172            return Err("portable benchmark digests must be lowercase 32-byte hex".to_string());
173        }
174        if self.provenance.harn_version.is_empty()
175            || self.provenance.kernel_version.is_empty()
176            || self.provenance.os.is_empty()
177            || self.provenance.arch.is_empty()
178            || self.provenance.artifact_format_version == 0
179        {
180            return Err("portable benchmark provenance is incomplete".to_string());
181        }
182        for value in [
183            self.initialization_ms.unwrap_or(0.0),
184            self.compile.first_ms,
185            self.dispatch.first_ms,
186            self.dispatch.batch_wall_ms,
187            self.dispatch.throughput_per_second,
188        ] {
189            if !value.is_finite() || value < 0.0 {
190                return Err(
191                    "portable benchmark measurements must be finite and non-negative".to_string(),
192                );
193            }
194        }
195        for statistics in [
196            &self.compile.repeated,
197            &self.dispatch.repeated,
198            self.decode.as_ref().unwrap_or(&self.compile.repeated),
199        ] {
200            if !valid_statistics(statistics) {
201                return Err(
202                    "portable benchmark statistics must be finite and non-negative".to_string(),
203                );
204            }
205        }
206        if self.dispatch.batch_wall_ms == 0.0 || self.dispatch.throughput_per_second == 0.0 {
207            return Err("portable benchmark batch measurements must be positive".to_string());
208        }
209        Ok(())
210    }
211}
212
213fn is_digest(value: &str) -> bool {
214    value.len() == 64
215        && value
216            .bytes()
217            .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
218}
219
220fn valid_statistics(statistics: &BenchmarkStatistics) -> bool {
221    statistics.iterations > 0
222        && [
223            statistics.min_ms,
224            statistics.mean_ms,
225            statistics.p50_ms,
226            statistics.p95_ms,
227            statistics.max_ms,
228            statistics.stddev_ms,
229            statistics.total_ms,
230        ]
231        .into_iter()
232        .all(|value| value.is_finite() && value >= 0.0)
233}
234
235/// Digest the canonical tagged-JSON representation used in benchmark receipts.
236///
237/// Native, browser, and future adapters call this projection instead of
238/// choosing their own JSON normalization or hash input.
239pub fn benchmark_terminal_digest(value: &DataValue) -> String {
240    blake3::hash(value.to_json().to_string().as_bytes())
241        .to_hex()
242        .to_string()
243}
244
245/// Aggregate wall-time statistics used by portable benchmark receipts.
246///
247/// Percentiles use R-7 linear interpolation, the default used by common
248/// statistical tools. Inputs must be finite, non-negative millisecond values.
249#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
250#[non_exhaustive]
251pub struct BenchmarkStatistics {
252    pub iterations: usize,
253    pub min_ms: f64,
254    pub mean_ms: f64,
255    pub p50_ms: f64,
256    pub p95_ms: f64,
257    pub max_ms: f64,
258    pub stddev_ms: f64,
259    pub total_ms: f64,
260}
261
262impl BenchmarkStatistics {
263    /// Aggregate a finite, non-empty set of elapsed-time samples.
264    ///
265    /// The population standard deviation is reported because a benchmark
266    /// receipt describes every observed iteration rather than estimating a
267    /// larger population from a sample.
268    pub fn from_samples(
269        samples: impl IntoIterator<Item = f64>,
270    ) -> Result<Self, BenchmarkStatisticsError> {
271        let mut sorted = samples.into_iter().collect::<Vec<_>>();
272        if sorted.is_empty() {
273            return Err(BenchmarkStatisticsError::Empty);
274        }
275        for (index, sample) in sorted.iter_mut().enumerate() {
276            if !sample.is_finite() {
277                return Err(BenchmarkStatisticsError::NonFinite { index });
278            }
279            if *sample < 0.0 {
280                return Err(BenchmarkStatisticsError::Negative { index });
281            }
282            // Equal elapsed times must have one byte-level representation on
283            // every host; JavaScript and native clocks can otherwise surface
284            // signed zero differently in receipts.
285            if *sample == 0.0 {
286                *sample = 0.0;
287            }
288        }
289
290        sorted.sort_by(f64::total_cmp);
291        let iterations = sorted.len();
292        let total_ms = compensated_sum(sorted.iter().copied());
293        if !total_ms.is_finite() {
294            return Err(BenchmarkStatisticsError::AggregateOverflow);
295        }
296        let mean_ms = total_ms / iterations as f64;
297        let variance = compensated_sum(sorted.iter().map(|sample| {
298            let delta = sample - mean_ms;
299            delta * delta
300        })) / iterations as f64;
301        if !variance.is_finite() {
302            return Err(BenchmarkStatisticsError::AggregateOverflow);
303        }
304
305        Ok(Self {
306            iterations,
307            min_ms: sorted[0],
308            mean_ms,
309            p50_ms: percentile_r7(&sorted, 0.50),
310            p95_ms: percentile_r7(&sorted, 0.95),
311            max_ms: sorted[iterations - 1],
312            stddev_ms: variance.sqrt(),
313            total_ms,
314        })
315    }
316}
317
318/// Deterministic reason benchmark samples could not be aggregated.
319#[derive(Debug, Clone, Copy, PartialEq, Eq)]
320#[non_exhaustive]
321pub enum BenchmarkStatisticsError {
322    Empty,
323    NonFinite { index: usize },
324    Negative { index: usize },
325    AggregateOverflow,
326}
327
328impl BenchmarkStatisticsError {
329    pub const fn code(self) -> &'static str {
330        match self {
331            Self::Empty => "benchmark_samples_empty",
332            Self::NonFinite { .. } => "benchmark_sample_non_finite",
333            Self::Negative { .. } => "benchmark_sample_negative",
334            Self::AggregateOverflow => "benchmark_aggregate_overflow",
335        }
336    }
337}
338
339impl fmt::Display for BenchmarkStatisticsError {
340    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
341        match self {
342            Self::Empty => formatter.write_str("benchmark samples must not be empty"),
343            Self::NonFinite { index } => {
344                write!(formatter, "benchmark sample {index} must be finite")
345            }
346            Self::Negative { index } => {
347                write!(formatter, "benchmark sample {index} must not be negative")
348            }
349            Self::AggregateOverflow => {
350                formatter.write_str("benchmark sample aggregate exceeds finite range")
351            }
352        }
353    }
354}
355
356impl std::error::Error for BenchmarkStatisticsError {}
357
358fn percentile_r7(sorted: &[f64], probability: f64) -> f64 {
359    if sorted.len() == 1 {
360        return sorted[0];
361    }
362    let rank = probability * (sorted.len() - 1) as f64;
363    let lower = rank.floor() as usize;
364    let upper = rank.ceil() as usize;
365    if lower == upper {
366        sorted[lower]
367    } else {
368        let weight = rank - lower as f64;
369        sorted[lower] * (1.0 - weight) + sorted[upper] * weight
370    }
371}
372
373fn compensated_sum(values: impl IntoIterator<Item = f64>) -> f64 {
374    let mut sum = 0.0;
375    let mut compensation = 0.0;
376    for value in values {
377        let corrected = value - compensation;
378        let next = sum + corrected;
379        compensation = (next - sum) - corrected;
380        sum = next;
381    }
382    sum
383}
384
385#[cfg(test)]
386mod tests {
387    use super::*;
388
389    #[test]
390    fn aggregates_unsorted_samples_with_r7_percentiles() {
391        let stats = BenchmarkStatistics::from_samples([30.0, 10.0, 40.0, 20.0]).unwrap();
392
393        assert_eq!(stats.iterations, 4);
394        assert_eq!(stats.min_ms, 10.0);
395        assert_eq!(stats.mean_ms, 25.0);
396        assert_eq!(stats.p50_ms, 25.0);
397        assert_eq!(stats.p95_ms, 38.5);
398        assert_eq!(stats.max_ms, 40.0);
399        assert_eq!(stats.stddev_ms, 125.0_f64.sqrt());
400        assert_eq!(stats.total_ms, 100.0);
401    }
402
403    #[test]
404    fn single_sample_has_zero_variance_and_exact_percentiles() {
405        let stats = BenchmarkStatistics::from_samples([1.25]).unwrap();
406
407        assert_eq!(stats.p50_ms, 1.25);
408        assert_eq!(stats.p95_ms, 1.25);
409        assert_eq!(stats.stddev_ms, 0.0);
410    }
411
412    #[test]
413    fn normalizes_signed_zero_for_cross_host_receipts() {
414        let stats = BenchmarkStatistics::from_samples([-0.0, 0.0]).unwrap();
415
416        assert_eq!(stats.min_ms.to_bits(), 0.0_f64.to_bits());
417        assert_eq!(stats.max_ms.to_bits(), 0.0_f64.to_bits());
418    }
419
420    #[test]
421    fn rejects_empty_or_invalid_elapsed_times() {
422        assert_eq!(
423            BenchmarkStatistics::from_samples([]).unwrap_err(),
424            BenchmarkStatisticsError::Empty
425        );
426        assert_eq!(
427            BenchmarkStatistics::from_samples([1.0, f64::NAN]).unwrap_err(),
428            BenchmarkStatisticsError::NonFinite { index: 1 }
429        );
430        assert_eq!(
431            BenchmarkStatistics::from_samples([1.0, f64::INFINITY]).unwrap_err(),
432            BenchmarkStatisticsError::NonFinite { index: 1 }
433        );
434        assert_eq!(
435            BenchmarkStatistics::from_samples([1.0, -0.1]).unwrap_err(),
436            BenchmarkStatisticsError::Negative { index: 1 }
437        );
438        assert_eq!(
439            BenchmarkStatistics::from_samples([f64::MAX, f64::MAX]).unwrap_err(),
440            BenchmarkStatisticsError::AggregateOverflow
441        );
442    }
443
444    #[test]
445    fn serialized_field_names_match_the_receipt_contract() {
446        let stats = BenchmarkStatistics::from_samples([10.0, 20.0]).unwrap();
447        let value = serde_json::to_value(stats).unwrap();
448
449        assert_eq!(value["iterations"], 2);
450        assert_eq!(value["p50_ms"], 15.0);
451        assert_eq!(value["p95_ms"], 19.5);
452    }
453
454    #[test]
455    fn terminal_digest_uses_canonical_tagged_json() {
456        let left = DataValue::Record(std::collections::BTreeMap::from([
457            ("z".to_string(), DataValue::Float(f64::NAN)),
458            ("a".to_string(), DataValue::Int(i64::MAX)),
459        ]));
460        let right = DataValue::from_json(left.to_json()).unwrap();
461
462        assert_eq!(
463            benchmark_terminal_digest(&left),
464            benchmark_terminal_digest(&right)
465        );
466    }
467}
harn_kernel/benchmark.rs

harn_kernel/
benchmark.rs
