harn_kernel/
benchmark.rs

1//! Deterministic aggregation for benchmark receipts.
2//!
3//! Hosts own clocks and sample capture. Keeping aggregation here gives native,
4//! browser, and future edge adapters one definition of the receipt statistics
5//! without giving the portable kernel clock authority.
6
7use std::fmt;
8
9use serde::{Deserialize, Serialize};
10
11use crate::{opcode_abi_fingerprint, semantic_abi_fingerprint_hex, DataValue, ARTIFACT_VERSION};
12
13mod schema;
14pub use schema::portable_benchmark_json_schema;
15
16pub const PORTABLE_BENCHMARK_SCHEMA_VERSION: &str = "harn.portable_kernel.benchmark.v1";
17pub const PORTABLE_MAX_DISPATCH_ITERATIONS: usize = 1_000_000;
18pub const PORTABLE_MAX_COMPILE_ITERATIONS: usize = 100_000;
19pub const PORTABLE_MAX_WORKERS: usize = 256;
20
21/// Closed, shared receipt emitted by native and browser benchmark adapters.
22/// Hosts own clocks and sample capture; the kernel owns field names,
23/// aggregation, provenance, bounds, and serialization.
24#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
25#[serde(rename_all = "camelCase", deny_unknown_fields)]
26pub struct PortableBenchmarkReceipt {
27    pub schema_version: String,
28    pub target: BenchmarkTarget,
29    pub source: String,
30    pub entry: String,
31    pub entry_kind: BenchmarkEntryKind,
32    pub artifact_bytes: usize,
33    pub artifact_digest: String,
34    pub iterations: usize,
35    pub workers: usize,
36    pub provenance: BenchmarkProvenance,
37    pub initialization_ms: Option<f64>,
38    pub compile: CompileMeasurements,
39    pub decode: Option<BenchmarkStatistics>,
40    pub dispatch: DispatchMeasurements,
41    pub terminal_digest: String,
42}
43
44#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
45#[serde(rename_all = "snake_case")]
46pub enum BenchmarkTarget {
47    Native,
48    Browser,
49}
50
51#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
52#[serde(rename_all = "snake_case")]
53pub enum BenchmarkEntryKind {
54    Function,
55    Pipeline,
56}
57
58#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
59#[serde(rename_all = "snake_case")]
60pub enum BenchmarkBuildProfile {
61    Debug,
62    Release,
63}
64
65#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
66#[serde(rename_all = "camelCase", deny_unknown_fields)]
67pub struct BenchmarkProvenance {
68    pub harn_version: String,
69    pub kernel_version: String,
70    pub artifact_format_version: u16,
71    pub semantic_abi_fingerprint: String,
72    pub opcode_abi_fingerprint: String,
73    pub build_profile: BenchmarkBuildProfile,
74    pub os: String,
75    pub arch: String,
76}
77
78impl BenchmarkProvenance {
79    pub fn current(
80        harn_version: impl Into<String>,
81        build_profile: BenchmarkBuildProfile,
82        os: impl Into<String>,
83        arch: impl Into<String>,
84    ) -> Self {
85        Self {
86            harn_version: harn_version.into(),
87            kernel_version: crate::KERNEL_VERSION.to_string(),
88            artifact_format_version: ARTIFACT_VERSION,
89            semantic_abi_fingerprint: semantic_abi_fingerprint_hex(),
90            opcode_abi_fingerprint: opcode_abi_fingerprint()
91                .iter()
92                .map(|byte| format!("{byte:02x}"))
93                .collect(),
94            build_profile,
95            os: os.into(),
96            arch: arch.into(),
97        }
98    }
99}
100
101#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
102#[serde(rename_all = "camelCase", deny_unknown_fields)]
103pub struct CompileMeasurements {
104    pub first_ms: f64,
105    pub repeated: BenchmarkStatistics,
106}
107
108#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
109#[serde(rename_all = "camelCase", deny_unknown_fields)]
110pub struct DispatchMeasurements {
111    pub first_ms: f64,
112    pub repeated: BenchmarkStatistics,
113    pub batch_wall_ms: f64,
114    pub throughput_per_second: f64,
115}
116
117impl PortableBenchmarkReceipt {
118    /// Validate the closed cross-host receipt contract before persistence.
119    pub fn validate(&self) -> Result<(), String> {
120        if self.schema_version != PORTABLE_BENCHMARK_SCHEMA_VERSION {
121            return Err("portable benchmark schema version is not supported".to_string());
122        }
123        if self.source.is_empty() || self.entry.is_empty() || self.artifact_bytes == 0 {
124            return Err("portable benchmark identity fields must not be empty".to_string());
125        }
126        if self.iterations == 0 || self.iterations > PORTABLE_MAX_DISPATCH_ITERATIONS {
127            return Err(
128                "portable benchmark dispatch iteration count is outside limits".to_string(),
129            );
130        }
131        if self.workers == 0 || self.workers > PORTABLE_MAX_WORKERS {
132            return Err("portable benchmark worker count is outside limits".to_string());
133        }
134        if self.workers > self.iterations {
135            return Err(
136                "portable benchmark workers must not exceed dispatch iterations".to_string(),
137            );
138        }
139        if self.compile.repeated.iterations == 0
140            || self.compile.repeated.iterations > PORTABLE_MAX_COMPILE_ITERATIONS
141        {
142            return Err("portable benchmark compile iteration count is outside limits".to_string());
143        }
144        if self.dispatch.repeated.iterations != self.iterations {
145            return Err(
146                "portable benchmark dispatch statistics do not match iterations".to_string(),
147            );
148        }
149        if let Some(decode) = self.decode {
150            if decode.iterations != self.compile.repeated.iterations {
151                return Err(
152                    "portable benchmark decode statistics do not match compilation samples"
153                        .to_string(),
154                );
155            }
156        }
157        match self.target {
158            BenchmarkTarget::Native
159                if self.initialization_ms.is_some() || self.decode.is_none() =>
160            {
161                return Err("native portable benchmarks require decode samples and no adapter initialization".to_string());
162            }
163            BenchmarkTarget::Browser
164                if self.initialization_ms.is_none() || self.decode.is_some() =>
165            {
166                return Err("browser portable benchmarks require adapter initialization and include decode in dispatch".to_string());
167            }
168            BenchmarkTarget::Native | BenchmarkTarget::Browser => {}
169        }
170        if !is_digest(&self.artifact_digest)
171            || !is_digest(&self.terminal_digest)
172            || !is_digest(&self.provenance.semantic_abi_fingerprint)
173            || !is_digest(&self.provenance.opcode_abi_fingerprint)
174        {
175            return Err("portable benchmark digests must be lowercase 32-byte hex".to_string());
176        }
177        if self.provenance.harn_version.is_empty()
178            || self.provenance.kernel_version.is_empty()
179            || self.provenance.os.is_empty()
180            || self.provenance.arch.is_empty()
181            || self.provenance.artifact_format_version == 0
182        {
183            return Err("portable benchmark provenance is incomplete".to_string());
184        }
185        for value in [
186            self.initialization_ms.unwrap_or(0.0),
187            self.compile.first_ms,
188            self.dispatch.first_ms,
189            self.dispatch.batch_wall_ms,
190            self.dispatch.throughput_per_second,
191        ] {
192            if !value.is_finite() || value < 0.0 {
193                return Err(
194                    "portable benchmark measurements must be finite and non-negative".to_string(),
195                );
196            }
197        }
198        for statistics in [
199            &self.compile.repeated,
200            &self.dispatch.repeated,
201            self.decode.as_ref().unwrap_or(&self.compile.repeated),
202        ] {
203            if !valid_statistics(statistics) {
204                return Err(
205                    "portable benchmark statistics must be finite and non-negative".to_string(),
206                );
207            }
208        }
209        if self.dispatch.batch_wall_ms == 0.0 || self.dispatch.throughput_per_second == 0.0 {
210            return Err("portable benchmark batch measurements must be positive".to_string());
211        }
212        Ok(())
213    }
214}
215
216fn is_digest(value: &str) -> bool {
217    value.len() == 64
218        && value
219            .bytes()
220            .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
221}
222
223fn valid_statistics(statistics: &BenchmarkStatistics) -> bool {
224    statistics.iterations > 0
225        && [
226            statistics.min_ms,
227            statistics.mean_ms,
228            statistics.p50_ms,
229            statistics.p95_ms,
230            statistics.max_ms,
231            statistics.stddev_ms,
232            statistics.total_ms,
233        ]
234        .into_iter()
235        .all(|value| value.is_finite() && value >= 0.0)
236}
237
238/// Digest the canonical tagged-JSON representation used in benchmark receipts.
239///
240/// Native, browser, and future adapters call this projection instead of
241/// choosing their own JSON normalization or hash input.
242pub fn benchmark_terminal_digest(value: &DataValue) -> String {
243    blake3::hash(value.to_json().to_string().as_bytes())
244        .to_hex()
245        .to_string()
246}
247
248/// Aggregate wall-time statistics used by portable benchmark receipts.
249///
250/// Percentiles use R-7 linear interpolation, the default used by common
251/// statistical tools. Inputs must be finite, non-negative millisecond values.
252#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
253#[non_exhaustive]
254pub struct BenchmarkStatistics {
255    pub iterations: usize,
256    pub min_ms: f64,
257    pub mean_ms: f64,
258    pub p50_ms: f64,
259    pub p95_ms: f64,
260    pub max_ms: f64,
261    pub stddev_ms: f64,
262    pub total_ms: f64,
263}
264
265impl BenchmarkStatistics {
266    /// Aggregate a finite, non-empty set of elapsed-time samples.
267    ///
268    /// The population standard deviation is reported because a benchmark
269    /// receipt describes every observed iteration rather than estimating a
270    /// larger population from a sample.
271    pub fn from_samples(
272        samples: impl IntoIterator<Item = f64>,
273    ) -> Result<Self, BenchmarkStatisticsError> {
274        let mut sorted = samples.into_iter().collect::<Vec<_>>();
275        if sorted.is_empty() {
276            return Err(BenchmarkStatisticsError::Empty);
277        }
278        for (index, sample) in sorted.iter_mut().enumerate() {
279            if !sample.is_finite() {
280                return Err(BenchmarkStatisticsError::NonFinite { index });
281            }
282            if *sample < 0.0 {
283                return Err(BenchmarkStatisticsError::Negative { index });
284            }
285            // Equal elapsed times must have one byte-level representation on
286            // every host; JavaScript and native clocks can otherwise surface
287            // signed zero differently in receipts.
288            if *sample == 0.0 {
289                *sample = 0.0;
290            }
291        }
292
293        sorted.sort_by(f64::total_cmp);
294        let iterations = sorted.len();
295        let total_ms = compensated_sum(sorted.iter().copied());
296        if !total_ms.is_finite() {
297            return Err(BenchmarkStatisticsError::AggregateOverflow);
298        }
299        let mean_ms = total_ms / iterations as f64;
300        let variance = compensated_sum(sorted.iter().map(|sample| {
301            let delta = sample - mean_ms;
302            delta * delta
303        })) / iterations as f64;
304        if !variance.is_finite() {
305            return Err(BenchmarkStatisticsError::AggregateOverflow);
306        }
307
308        Ok(Self {
309            iterations,
310            min_ms: sorted[0],
311            mean_ms,
312            p50_ms: percentile_r7(&sorted, 0.50),
313            p95_ms: percentile_r7(&sorted, 0.95),
314            max_ms: sorted[iterations - 1],
315            stddev_ms: variance.sqrt(),
316            total_ms,
317        })
318    }
319}
320
321/// Deterministic reason benchmark samples could not be aggregated.
322#[derive(Debug, Clone, Copy, PartialEq, Eq)]
323#[non_exhaustive]
324pub enum BenchmarkStatisticsError {
325    Empty,
326    NonFinite { index: usize },
327    Negative { index: usize },
328    AggregateOverflow,
329}
330
331impl BenchmarkStatisticsError {
332    pub const fn code(self) -> &'static str {
333        match self {
334            Self::Empty => "benchmark_samples_empty",
335            Self::NonFinite { .. } => "benchmark_sample_non_finite",
336            Self::Negative { .. } => "benchmark_sample_negative",
337            Self::AggregateOverflow => "benchmark_aggregate_overflow",
338        }
339    }
340}
341
342impl fmt::Display for BenchmarkStatisticsError {
343    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
344        match self {
345            Self::Empty => formatter.write_str("benchmark samples must not be empty"),
346            Self::NonFinite { index } => {
347                write!(formatter, "benchmark sample {index} must be finite")
348            }
349            Self::Negative { index } => {
350                write!(formatter, "benchmark sample {index} must not be negative")
351            }
352            Self::AggregateOverflow => {
353                formatter.write_str("benchmark sample aggregate exceeds finite range")
354            }
355        }
356    }
357}
358
359impl std::error::Error for BenchmarkStatisticsError {}
360
361fn percentile_r7(sorted: &[f64], probability: f64) -> f64 {
362    if sorted.len() == 1 {
363        return sorted[0];
364    }
365    let rank = probability * (sorted.len() - 1) as f64;
366    let lower = rank.floor() as usize;
367    let upper = rank.ceil() as usize;
368    if lower == upper {
369        sorted[lower]
370    } else {
371        let weight = rank - lower as f64;
372        sorted[lower] * (1.0 - weight) + sorted[upper] * weight
373    }
374}
375
376fn compensated_sum(values: impl IntoIterator<Item = f64>) -> f64 {
377    let mut sum = 0.0;
378    let mut compensation = 0.0;
379    for value in values {
380        let corrected = value - compensation;
381        let next = sum + corrected;
382        compensation = (next - sum) - corrected;
383        sum = next;
384    }
385    sum
386}
387
388#[cfg(test)]
389mod tests {
390    use super::*;
391
392    #[test]
393    fn aggregates_unsorted_samples_with_r7_percentiles() {
394        let stats = BenchmarkStatistics::from_samples([30.0, 10.0, 40.0, 20.0]).unwrap();
395
396        assert_eq!(stats.iterations, 4);
397        assert_eq!(stats.min_ms, 10.0);
398        assert_eq!(stats.mean_ms, 25.0);
399        assert_eq!(stats.p50_ms, 25.0);
400        assert_eq!(stats.p95_ms, 38.5);
401        assert_eq!(stats.max_ms, 40.0);
402        assert_eq!(stats.stddev_ms, 125.0_f64.sqrt());
403        assert_eq!(stats.total_ms, 100.0);
404    }
405
406    #[test]
407    fn single_sample_has_zero_variance_and_exact_percentiles() {
408        let stats = BenchmarkStatistics::from_samples([1.25]).unwrap();
409
410        assert_eq!(stats.p50_ms, 1.25);
411        assert_eq!(stats.p95_ms, 1.25);
412        assert_eq!(stats.stddev_ms, 0.0);
413    }
414
415    #[test]
416    fn normalizes_signed_zero_for_cross_host_receipts() {
417        let stats = BenchmarkStatistics::from_samples([-0.0, 0.0]).unwrap();
418
419        assert_eq!(stats.min_ms.to_bits(), 0.0_f64.to_bits());
420        assert_eq!(stats.max_ms.to_bits(), 0.0_f64.to_bits());
421    }
422
423    #[test]
424    fn rejects_empty_or_invalid_elapsed_times() {
425        assert_eq!(
426            BenchmarkStatistics::from_samples([]).unwrap_err(),
427            BenchmarkStatisticsError::Empty
428        );
429        assert_eq!(
430            BenchmarkStatistics::from_samples([1.0, f64::NAN]).unwrap_err(),
431            BenchmarkStatisticsError::NonFinite { index: 1 }
432        );
433        assert_eq!(
434            BenchmarkStatistics::from_samples([1.0, f64::INFINITY]).unwrap_err(),
435            BenchmarkStatisticsError::NonFinite { index: 1 }
436        );
437        assert_eq!(
438            BenchmarkStatistics::from_samples([1.0, -0.1]).unwrap_err(),
439            BenchmarkStatisticsError::Negative { index: 1 }
440        );
441        assert_eq!(
442            BenchmarkStatistics::from_samples([f64::MAX, f64::MAX]).unwrap_err(),
443            BenchmarkStatisticsError::AggregateOverflow
444        );
445    }
446
447    #[test]
448    fn serialized_field_names_match_the_receipt_contract() {
449        let stats = BenchmarkStatistics::from_samples([10.0, 20.0]).unwrap();
450        let value = serde_json::to_value(stats).unwrap();
451
452        assert_eq!(value["iterations"], 2);
453        assert_eq!(value["p50_ms"], 15.0);
454        assert_eq!(value["p95_ms"], 19.5);
455    }
456
457    #[test]
458    fn terminal_digest_uses_canonical_tagged_json() {
459        let left = DataValue::Record(std::collections::BTreeMap::from([
460            ("z".to_string(), DataValue::Float(f64::NAN)),
461            ("a".to_string(), DataValue::Int(i64::MAX)),
462        ]));
463        let right = DataValue::from_json(left.to_json()).unwrap();
464
465        assert_eq!(
466            benchmark_terminal_digest(&left),
467            benchmark_terminal_digest(&right)
468        );
469    }
470}
harn_kernel/benchmark.rs

harn_kernel/
benchmark.rs
