harn_kernel/
benchmark.rs

1//! Deterministic aggregation for benchmark receipts.
2//!
3//! Hosts own clocks and sample capture. Keeping aggregation here gives native,
4//! browser, and future edge adapters one definition of the receipt statistics
5//! without giving the portable kernel clock authority.
6
7use std::fmt;
8
9use serde::{Deserialize, Serialize};
10
11use crate::{opcode_abi_fingerprint, semantic_abi_fingerprint_hex, DataValue, ARTIFACT_VERSION};
12
13mod schema;
14pub use schema::portable_benchmark_json_schema;
15
16pub const PORTABLE_BENCHMARK_SCHEMA_VERSION: &str = "harn.portable_kernel.benchmark.v1";
17pub const PORTABLE_MAX_DISPATCH_ITERATIONS: usize = 1_000_000;
18pub const PORTABLE_MAX_COMPILE_ITERATIONS: usize = 100_000;
19pub const PORTABLE_MAX_WORKERS: usize = 256;
20
21/// Closed, shared receipt emitted by native and browser benchmark adapters.
22/// Hosts own clocks and sample capture; the kernel owns field names,
23/// aggregation, provenance, bounds, and serialization.
24#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
25#[serde(rename_all = "camelCase", deny_unknown_fields)]
26pub struct PortableBenchmarkReceipt {
27    pub schema_version: String,
28    pub target: BenchmarkTarget,
29    pub source: String,
30    pub entry: String,
31    pub entry_kind: BenchmarkEntryKind,
32    pub artifact_bytes: usize,
33    pub artifact_digest: String,
34    pub iterations: usize,
35    pub workers: usize,
36    pub provenance: BenchmarkProvenance,
37    pub initialization_ms: Option<f64>,
38    pub compile: CompileMeasurements,
39    pub decode: Option<BenchmarkStatistics>,
40    pub dispatch: DispatchMeasurements,
41    pub terminal_digest: String,
42}
43
44#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
45#[serde(rename_all = "snake_case")]
46pub enum BenchmarkTarget {
47    Native,
48    Browser,
49}
50
51#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
52#[serde(rename_all = "snake_case")]
53pub enum BenchmarkEntryKind {
54    Function,
55    Pipeline,
56}
57
58#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
59#[serde(rename_all = "snake_case")]
60pub enum BenchmarkBuildProfile {
61    Debug,
62    Release,
63}
64
65#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
66#[serde(rename_all = "camelCase", deny_unknown_fields)]
67pub struct BenchmarkProvenance {
68    pub harn_version: String,
69    pub kernel_version: String,
70    pub artifact_format_version: u16,
71    pub semantic_abi_fingerprint: String,
72    pub opcode_abi_fingerprint: String,
73    pub build_profile: BenchmarkBuildProfile,
74    pub os: String,
75    pub arch: String,
76}
77
78impl BenchmarkProvenance {
79    pub fn current(
80        harn_version: impl Into<String>,
81        build_profile: BenchmarkBuildProfile,
82        os: impl Into<String>,
83        arch: impl Into<String>,
84    ) -> Self {
85        Self {
86            harn_version: harn_version.into(),
87            kernel_version: crate::KERNEL_VERSION.to_string(),
88            artifact_format_version: ARTIFACT_VERSION,
89            semantic_abi_fingerprint: semantic_abi_fingerprint_hex(),
90            opcode_abi_fingerprint: opcode_abi_fingerprint()
91                .iter()
92                .map(|byte| format!("{byte:02x}"))
93                .collect(),
94            build_profile,
95            os: os.into(),
96            arch: arch.into(),
97        }
98    }
99}
100
101#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
102#[serde(rename_all = "camelCase", deny_unknown_fields)]
103pub struct CompileMeasurements {
104    pub first_ms: f64,
105    pub repeated: BenchmarkStatistics,
106}
107
108#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
109#[serde(rename_all = "camelCase", deny_unknown_fields)]
110pub struct DispatchMeasurements {
111    pub first_ms: f64,
112    pub repeated: BenchmarkStatistics,
113    pub batch_wall_ms: f64,
114    pub throughput_per_second: f64,
115}
116
117impl PortableBenchmarkReceipt {
118    /// Validate the closed cross-host receipt contract before persistence.
119    pub fn validate(&self) -> Result<(), String> {
120        if self.schema_version != PORTABLE_BENCHMARK_SCHEMA_VERSION {
121            return Err("portable benchmark schema version is not supported".to_string());
122        }
123        if self.source.is_empty() || self.entry.is_empty() || self.artifact_bytes == 0 {
124            return Err("portable benchmark identity fields must not be empty".to_string());
125        }
126        if self.iterations == 0 || self.iterations > PORTABLE_MAX_DISPATCH_ITERATIONS {
127            return Err(
128                "portable benchmark dispatch iteration count is outside limits".to_string(),
129            );
130        }
131        if self.workers == 0 || self.workers > PORTABLE_MAX_WORKERS {
132            return Err("portable benchmark worker count is outside limits".to_string());
133        }
134        if self.workers > self.iterations {
135            return Err(
136                "portable benchmark workers must not exceed dispatch iterations".to_string(),
137            );
138        }
139        if self.compile.repeated.iterations == 0
140            || self.compile.repeated.iterations > PORTABLE_MAX_COMPILE_ITERATIONS
141        {
142            return Err("portable benchmark compile iteration count is outside limits".to_string());
143        }
144        if self.dispatch.repeated.iterations != self.iterations {
145            return Err(
146                "portable benchmark dispatch statistics do not match iterations".to_string(),
147            );
148        }
149        if let Some(decode) = self.decode {
150            if decode.iterations != self.compile.repeated.iterations {
151                return Err(
152                    "portable benchmark decode statistics do not match compilation samples"
153                        .to_string(),
154                );
155            }
156        }
157        match self.target {
158            BenchmarkTarget::Native
159                if self.initialization_ms.is_some() || self.decode.is_none() =>
160            {
161                return Err("native portable benchmarks require decode samples and no adapter initialization".to_string());
162            }
163            BenchmarkTarget::Browser
164                if self.initialization_ms.is_none() || self.decode.is_some() =>
165            {
166                return Err("browser portable benchmarks require adapter initialization and include decode in dispatch".to_string());
167            }
168            BenchmarkTarget::Native | BenchmarkTarget::Browser => {}
169        }
170        if !is_digest(&self.artifact_digest)
171            || !is_digest(&self.terminal_digest)
172            || !is_digest(&self.provenance.semantic_abi_fingerprint)
173            || !is_digest(&self.provenance.opcode_abi_fingerprint)
174        {
175            return Err("portable benchmark digests must be lowercase 32-byte hex".to_string());
176        }
177        if self.provenance.harn_version.is_empty()
178            || self.provenance.kernel_version.is_empty()
179            || self.provenance.os.is_empty()
180            || self.provenance.arch.is_empty()
181            || self.provenance.artifact_format_version == 0
182        {
183            return Err("portable benchmark provenance is incomplete".to_string());
184        }
185        for value in [
186            self.initialization_ms.unwrap_or(0.0),
187            self.compile.first_ms,
188            self.dispatch.first_ms,
189            self.dispatch.batch_wall_ms,
190            self.dispatch.throughput_per_second,
191        ] {
192            if !value.is_finite() || value < 0.0 {
193                return Err(
194                    "portable benchmark measurements must be finite and non-negative".to_string(),
195                );
196            }
197        }
198        for statistics in [
199            &self.compile.repeated,
200            &self.dispatch.repeated,
201            self.decode.as_ref().unwrap_or(&self.compile.repeated),
202        ] {
203            if !valid_statistics(statistics) {
204                return Err(
205                    "portable benchmark statistics must be finite and non-negative".to_string(),
206                );
207            }
208        }
209        if self.dispatch.batch_wall_ms == 0.0 || self.dispatch.throughput_per_second == 0.0 {
210            return Err("portable benchmark batch measurements must be positive".to_string());
211        }
212        Ok(())
213    }
214}
215
216fn is_digest(value: &str) -> bool {
217    value.len() == 64
218        && value
219            .bytes()
220            .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
221}
222
223fn valid_statistics(statistics: &BenchmarkStatistics) -> bool {
224    statistics.iterations > 0
225        && [
226            statistics.min_ms,
227            statistics.mean_ms,
228            statistics.p50_ms,
229            statistics.p95_ms,
230            statistics.max_ms,
231            statistics.stddev_ms,
232            statistics.total_ms,
233        ]
234        .into_iter()
235        .all(|value| value.is_finite() && value >= 0.0)
236}
237
238/// Digest the canonical tagged-JSON representation used in benchmark receipts.
239///
240/// Native, browser, and future adapters call this projection instead of
241/// choosing their own JSON normalization or hash input.
242pub fn benchmark_terminal_digest(value: &DataValue) -> String {
243    blake3::hash(value.to_json().to_string().as_bytes())
244        .to_hex()
245        .to_string()
246}
247
248/// Aggregate wall-time statistics used by portable benchmark receipts.
249///
250/// Percentiles use R-7 linear interpolation, the default used by common
251/// statistical tools. Inputs must be finite, non-negative millisecond values.
252#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
253#[non_exhaustive]
254pub struct BenchmarkStatistics {
255    pub iterations: usize,
256    pub min_ms: f64,
257    pub mean_ms: f64,
258    pub p50_ms: f64,
259    pub p95_ms: f64,
260    pub max_ms: f64,
261    pub stddev_ms: f64,
262    pub total_ms: f64,
263}
264
265impl BenchmarkStatistics {
266    /// Aggregate a finite, non-empty set of elapsed-time samples.
267    ///
268    /// The population standard deviation is reported because a benchmark
269    /// receipt describes every observed iteration rather than estimating a
270    /// larger population from a sample.
271    pub fn from_samples(
272        samples: impl IntoIterator<Item = f64>,
273    ) -> Result<Self, BenchmarkStatisticsError> {
274        let mut sorted = samples.into_iter().collect::<Vec<_>>();
275        if sorted.is_empty() {
276            return Err(BenchmarkStatisticsError::Empty);
277        }
278        for (index, sample) in sorted.iter_mut().enumerate() {
279            if !sample.is_finite() {
280                return Err(BenchmarkStatisticsError::NonFinite { index });
281            }
282            if *sample < 0.0 {
283                return Err(BenchmarkStatisticsError::Negative { index });
284            }
285            // Equal elapsed times must have one byte-level representation on
286            // every host; JavaScript and native clocks can otherwise surface
287            // signed zero differently in receipts.
288            if *sample == 0.0 {
289                *sample = 0.0;
290            }
291        }
292
293        sorted.sort_by(f64::total_cmp);
294        let iterations = sorted.len();
295        let total_ms = compensated_sum(sorted.iter().copied());
296        if !total_ms.is_finite() {
297            return Err(BenchmarkStatisticsError::AggregateOverflow);
298        }
299        let mean_ms = total_ms / iterations as f64;
300        let variance = compensated_sum(sorted.iter().map(|sample| {
301            let delta = sample - mean_ms;
302            delta * delta
303        })) / iterations as f64;
304        if !variance.is_finite() {
305            return Err(BenchmarkStatisticsError::AggregateOverflow);
306        }
307
308        Ok(Self {
309            iterations,
310            min_ms: sorted[0],
311            mean_ms,
312            p50_ms: percentile_r7(&sorted, 0.50),
313            p95_ms: percentile_r7(&sorted, 0.95),
314            max_ms: sorted[iterations - 1],
315            stddev_ms: variance.sqrt(),
316            total_ms,
317        })
318    }
319}
320
321/// Deterministic reason benchmark samples could not be aggregated.
322#[derive(Debug, Clone, Copy, PartialEq, Eq)]
323pub enum BenchmarkStatisticsError {
324    Empty,
325    NonFinite { index: usize },
326    Negative { index: usize },
327    AggregateOverflow,
328}
329
330impl BenchmarkStatisticsError {
331    pub const fn code(self) -> &'static str {
332        match self {
333            Self::Empty => "benchmark_samples_empty",
334            Self::NonFinite { .. } => "benchmark_sample_non_finite",
335            Self::Negative { .. } => "benchmark_sample_negative",
336            Self::AggregateOverflow => "benchmark_aggregate_overflow",
337        }
338    }
339}
340
341impl fmt::Display for BenchmarkStatisticsError {
342    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
343        match self {
344            Self::Empty => formatter.write_str("benchmark samples must not be empty"),
345            Self::NonFinite { index } => {
346                write!(formatter, "benchmark sample {index} must be finite")
347            }
348            Self::Negative { index } => {
349                write!(formatter, "benchmark sample {index} must not be negative")
350            }
351            Self::AggregateOverflow => {
352                formatter.write_str("benchmark sample aggregate exceeds finite range")
353            }
354        }
355    }
356}
357
358impl std::error::Error for BenchmarkStatisticsError {}
359
360fn percentile_r7(sorted: &[f64], probability: f64) -> f64 {
361    if sorted.len() == 1 {
362        return sorted[0];
363    }
364    let rank = probability * (sorted.len() - 1) as f64;
365    let lower = rank.floor() as usize;
366    let upper = rank.ceil() as usize;
367    if lower == upper {
368        sorted[lower]
369    } else {
370        let weight = rank - lower as f64;
371        sorted[lower] * (1.0 - weight) + sorted[upper] * weight
372    }
373}
374
375fn compensated_sum(values: impl IntoIterator<Item = f64>) -> f64 {
376    let mut sum = 0.0;
377    let mut compensation = 0.0;
378    for value in values {
379        let corrected = value - compensation;
380        let next = sum + corrected;
381        compensation = (next - sum) - corrected;
382        sum = next;
383    }
384    sum
385}
386
387#[cfg(test)]
388mod tests {
389    use super::*;
390
391    #[test]
392    fn aggregates_unsorted_samples_with_r7_percentiles() {
393        let stats = BenchmarkStatistics::from_samples([30.0, 10.0, 40.0, 20.0]).unwrap();
394
395        assert_eq!(stats.iterations, 4);
396        assert_eq!(stats.min_ms, 10.0);
397        assert_eq!(stats.mean_ms, 25.0);
398        assert_eq!(stats.p50_ms, 25.0);
399        assert_eq!(stats.p95_ms, 38.5);
400        assert_eq!(stats.max_ms, 40.0);
401        assert_eq!(stats.stddev_ms, 125.0_f64.sqrt());
402        assert_eq!(stats.total_ms, 100.0);
403    }
404
405    #[test]
406    fn single_sample_has_zero_variance_and_exact_percentiles() {
407        let stats = BenchmarkStatistics::from_samples([1.25]).unwrap();
408
409        assert_eq!(stats.p50_ms, 1.25);
410        assert_eq!(stats.p95_ms, 1.25);
411        assert_eq!(stats.stddev_ms, 0.0);
412    }
413
414    #[test]
415    fn normalizes_signed_zero_for_cross_host_receipts() {
416        let stats = BenchmarkStatistics::from_samples([-0.0, 0.0]).unwrap();
417
418        assert_eq!(stats.min_ms.to_bits(), 0.0_f64.to_bits());
419        assert_eq!(stats.max_ms.to_bits(), 0.0_f64.to_bits());
420    }
421
422    #[test]
423    fn rejects_empty_or_invalid_elapsed_times() {
424        assert_eq!(
425            BenchmarkStatistics::from_samples([]).unwrap_err(),
426            BenchmarkStatisticsError::Empty
427        );
428        assert_eq!(
429            BenchmarkStatistics::from_samples([1.0, f64::NAN]).unwrap_err(),
430            BenchmarkStatisticsError::NonFinite { index: 1 }
431        );
432        assert_eq!(
433            BenchmarkStatistics::from_samples([1.0, f64::INFINITY]).unwrap_err(),
434            BenchmarkStatisticsError::NonFinite { index: 1 }
435        );
436        assert_eq!(
437            BenchmarkStatistics::from_samples([1.0, -0.1]).unwrap_err(),
438            BenchmarkStatisticsError::Negative { index: 1 }
439        );
440        assert_eq!(
441            BenchmarkStatistics::from_samples([f64::MAX, f64::MAX]).unwrap_err(),
442            BenchmarkStatisticsError::AggregateOverflow
443        );
444    }
445
446    #[test]
447    fn serialized_field_names_match_the_receipt_contract() {
448        let stats = BenchmarkStatistics::from_samples([10.0, 20.0]).unwrap();
449        let value = serde_json::to_value(stats).unwrap();
450
451        assert_eq!(value["iterations"], 2);
452        assert_eq!(value["p50_ms"], 15.0);
453        assert_eq!(value["p95_ms"], 19.5);
454    }
455
456    #[test]
457    fn terminal_digest_uses_canonical_tagged_json() {
458        let left = DataValue::Record(std::collections::BTreeMap::from([
459            ("z".to_string(), DataValue::Float(f64::NAN)),
460            ("a".to_string(), DataValue::Int(i64::MAX)),
461        ]));
462        let right = DataValue::from_json(left.to_json()).unwrap();
463
464        assert_eq!(
465            benchmark_terminal_digest(&left),
466            benchmark_terminal_digest(&right)
467        );
468    }
469}
harn_kernel/benchmark.rs

harn_kernel/
benchmark.rs
