use std::collections::BTreeMap;
use std::fmt;
use hdrhistogram::Histogram;
use serde::{Deserialize, Serialize};
use crate::bench::BenchError;
use crate::bench::fingerprint::EnvironmentFingerprint;
use crate::bench::pack::BenchPack;
use crate::bench::posture::PostureRecord;
use crate::bench::relative::RelativeIndex;
pub const RESULT_FILE_STEM: &str = "bench-result";
pub const SUBMITTABLE_REPETITIONS: usize = 3;
pub const SUBMITTABLE_BASELINES: usize = 1;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum SubmissionRequirement {
Repetitions,
Baseline,
ErrorShare,
}
impl SubmissionRequirement {
pub const ALL: &[SubmissionRequirement] = &[
SubmissionRequirement::Repetitions,
SubmissionRequirement::Baseline,
SubmissionRequirement::ErrorShare,
];
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
SubmissionRequirement::Repetitions => "repetitions",
SubmissionRequirement::Baseline => "baseline",
SubmissionRequirement::ErrorShare => "error_share",
}
}
#[must_use]
pub const fn statement(self) -> &'static str {
match self {
SubmissionRequirement::Repetitions => {
"at least 3 repetitions, because one repetition measures a moment rather than a system"
}
SubmissionRequirement::Baseline => {
"at least one same-machine baseline, because an absolute number without an anchor describes the machine as much as the system"
}
SubmissionRequirement::ErrorShare => {
"every repetition, phase and operation, on the target and on every baseline, at or below the pack's failed-arrival ceiling, because percentiles taken over failed arrivals measure the failure rather than the system"
}
}
}
pub fn parse(token: &str) -> Result<Self, BenchError> {
Self::ALL
.iter()
.copied()
.find(|requirement| requirement.as_str() == token)
.ok_or_else(|| BenchError::UnknownToken {
vocabulary: "submission requirement",
token: token.to_owned(),
accepted: Self::ALL
.iter()
.map(|requirement| requirement.as_str())
.collect::<Vec<_>>()
.join(", "),
})
}
}
impl fmt::Display for SubmissionRequirement {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
impl Serialize for SubmissionRequirement {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
serializer.serialize_str(self.as_str())
}
}
impl<'de> Deserialize<'de> for SubmissionRequirement {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let token = String::deserialize(deserializer)?;
SubmissionRequirement::parse(&token).map_err(serde::de::Error::custom)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RecipeReference {
pub repository: String,
pub git_ref: String,
pub file: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct BaselineResources {
pub server_cpus: String,
pub server_memory: String,
pub database_cpus: String,
pub database_memory: String,
pub database_shm_size: String,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct BaselineRecord {
pub cdr: String,
pub display_name: String,
pub images: BTreeMap<String, String>,
pub recipe: RecipeReference,
pub resources: BaselineResources,
pub base_url: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub sut_version: Option<String>,
pub started_at: String,
pub finished_at: String,
pub seed_phases: Vec<SeedPhaseRecord>,
pub repetitions: Vec<RepetitionRecord>,
pub cross: BTreeMap<String, CrossPhase>,
pub posture: PostureRecord,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum LoopRegime {
ClosedLoop,
OpenLoop,
}
impl LoopRegime {
pub const ALL: &[LoopRegime] = &[LoopRegime::ClosedLoop, LoopRegime::OpenLoop];
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
LoopRegime::ClosedLoop => "closed-loop",
LoopRegime::OpenLoop => "open-loop",
}
}
}
impl fmt::Display for LoopRegime {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum ErrorClass {
Http2xx,
Http3xx,
Http4xx,
Http5xx,
HttpOther,
Transport,
Timeout,
}
impl ErrorClass {
pub const ALL: &[ErrorClass] = &[
ErrorClass::Http2xx,
ErrorClass::Http3xx,
ErrorClass::Http4xx,
ErrorClass::Http5xx,
ErrorClass::HttpOther,
ErrorClass::Timeout,
ErrorClass::Transport,
];
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
ErrorClass::Http2xx => "http_2xx",
ErrorClass::Http3xx => "http_3xx",
ErrorClass::Http4xx => "http_4xx",
ErrorClass::Http5xx => "http_5xx",
ErrorClass::HttpOther => "http_other",
ErrorClass::Transport => "transport",
ErrorClass::Timeout => "timeout",
}
}
pub fn parse(token: &str) -> Result<Self, BenchError> {
Self::ALL
.iter()
.copied()
.find(|class| class.as_str() == token)
.ok_or_else(|| BenchError::UnknownToken {
vocabulary: "error class",
token: token.to_owned(),
accepted: Self::ALL
.iter()
.map(|class| class.as_str())
.collect::<Vec<_>>()
.join(", "),
})
}
#[must_use]
pub fn of_status(status: reqwest::StatusCode) -> Self {
if status.is_success() {
ErrorClass::Http2xx
} else if status.is_redirection() {
ErrorClass::Http3xx
} else if status.is_client_error() {
ErrorClass::Http4xx
} else if status.is_server_error() {
ErrorClass::Http5xx
} else {
ErrorClass::HttpOther
}
}
}
impl fmt::Display for ErrorClass {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct OperationStats {
pub count: u64,
pub errors: u64,
pub errors_by_class: BTreeMap<String, u64>,
pub throughput_ops_s: f64,
pub p50_us: u64,
pub p75_us: u64,
pub p90_us: u64,
pub p99_us: u64,
pub p999_us: u64,
pub max_us: u64,
pub hdr_v2_base64: String,
}
impl OperationStats {
pub fn from_histogram(
histogram: &Histogram<u64>,
errors_by_class: BTreeMap<String, u64>,
measured_span_s: f64,
) -> Result<Self, BenchError> {
let count = histogram.len();
let errors = errors_by_class
.values()
.fold(0_u64, |a, b| a.saturating_add(*b));
#[expect(
clippy::as_conversions,
clippy::cast_precision_loss,
reason = "arrival counts are far below 2^52, so the throughput divisor loses nothing"
)]
let throughput_ops_s = if measured_span_s > 0.0 {
count as f64 / measured_span_s
} else {
0.0
};
Ok(Self {
count,
errors,
errors_by_class,
throughput_ops_s,
p50_us: histogram.value_at_quantile(0.50),
p75_us: histogram.value_at_quantile(0.75),
p90_us: histogram.value_at_quantile(0.90),
p99_us: histogram.value_at_quantile(0.99),
p999_us: histogram.value_at_quantile(0.999),
max_us: histogram.max(),
hdr_v2_base64: crate::perf::encode_hdr_v2(histogram).map_err(BenchError::Histogram)?,
})
}
pub fn decode_histogram(&self) -> Result<Histogram<u64>, BenchError> {
crate::perf::decode_hdr_v2(&self.hdr_v2_base64).map_err(BenchError::Histogram)
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SeedPhaseRecord {
pub name: String,
pub regime: LoopRegime,
pub ehrs: u64,
pub compositions_per_ehr: u64,
pub workers: u64,
pub elapsed_s: f64,
pub bulk_load_writes_per_s: f64,
pub whole_loop_ms_per_composition: f64,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SweepPhaseRecord {
pub name: String,
pub regime: LoopRegime,
pub workers: u64,
pub compositions: u64,
pub requests_per_composition: u64,
pub requests: u64,
pub elapsed_s: f64,
pub whole_loop_us_per_request: f64,
pub operations: BTreeMap<String, OperationStats>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ScaleRecord {
pub factor: f64,
pub declared_workers: bool,
pub reference_configuration: bool,
}
impl ScaleRecord {
#[must_use]
pub fn new(factor: f64, declared_workers: bool) -> Self {
let reference_scale = (factor - 1.0).abs() < f64::EPSILON;
Self {
factor,
declared_workers,
reference_configuration: reference_scale && declared_workers,
}
}
}
impl Default for ScaleRecord {
fn default() -> Self {
Self {
factor: 1.0,
declared_workers: true,
reference_configuration: true,
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MeasuredPhaseRecord {
pub regime: LoopRegime,
pub rate_per_s: f64,
pub warmup_s: u64,
pub duration_s: u64,
pub planned_measured_arrivals: u64,
pub dispatched_measured_arrivals: u64,
pub warmup_arrivals: u64,
pub offered_load_sustained_per_s: f64,
pub generator_bound: bool,
pub operations: BTreeMap<String, OperationStats>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RepetitionRecord {
pub repetition: u32,
pub phases: BTreeMap<String, MeasuredPhaseRecord>,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub sweeps: BTreeMap<String, SweepPhaseRecord>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CrossStat {
pub median: f64,
pub iqr: f64,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CrossOperation {
pub repetitions: u32,
pub p50_us: CrossStat,
pub p75_us: CrossStat,
pub p90_us: CrossStat,
pub p99_us: CrossStat,
pub p999_us: CrossStat,
pub throughput_ops_s: CrossStat,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CrossPhase {
pub regime: LoopRegime,
pub operations: BTreeMap<String, CrossOperation>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct PackRecord {
pub id: String,
pub version: String,
pub description: String,
pub max_failed_share: f64,
pub seed: u64,
pub fixtures: BTreeMap<String, String>,
}
impl PackRecord {
#[must_use]
pub fn of(pack: &BenchPack) -> Self {
Self {
id: pack.id.as_str().to_owned(),
version: pack.version.clone(),
description: pack.description.clone(),
max_failed_share: pack.max_failed_share,
seed: pack.seed,
fixtures: pack.fixture_pins(),
}
}
}
#[must_use]
pub fn failed_share(count: u64, errors: u64) -> f64 {
if count == 0 {
return 1.0;
}
#[expect(
clippy::as_conversions,
clippy::cast_precision_loss,
reason = "arrival counts are far below 2^52, so neither side of the ratio loses a digit"
)]
let share = errors.min(count) as f64 / count as f64;
share
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub enum MeasuredSide {
Target,
Baseline(String),
}
impl fmt::Display for MeasuredSide {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
MeasuredSide::Target => f.write_str("the target"),
MeasuredSide::Baseline(cdr) => write!(f, "the {cdr} baseline"),
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct FailedSharePhase {
pub side: MeasuredSide,
pub repetition: u32,
pub phase: String,
pub regime: LoopRegime,
pub count: u64,
pub errors: u64,
pub share: f64,
pub worst_operation: Option<String>,
pub worst_share: f64,
}
impl FailedSharePhase {
#[must_use]
pub fn breaches(&self, ceiling: f64) -> bool {
self.worst_share > ceiling
}
#[must_use]
pub fn sentence(&self, ceiling: f64) -> String {
let where_it_went = match &self.worst_operation {
Some(operation) => format!(
"lost {:.3} of the arrivals it recorded for `{operation}`, above the pack ceiling of {ceiling:.2}",
self.worst_share
),
None => "recorded no operation at all, so it measured nothing".to_owned(),
};
format!(
"on {}, repetition {} of phase `{}` ({}) {where_it_went}",
self.side, self.repetition, self.phase, self.regime
)
}
}
fn side_readings(side: &MeasuredSide, repetitions: &[RepetitionRecord]) -> Vec<FailedSharePhase> {
let mut readings = Vec::new();
for repetition in repetitions {
for (phase, measured) in &repetition.phases {
readings.push(one_reading(
side,
repetition.repetition,
phase,
measured.regime,
&measured.operations,
));
}
for (phase, sweep) in &repetition.sweeps {
readings.push(one_reading(
side,
repetition.repetition,
phase,
sweep.regime,
&sweep.operations,
));
}
}
readings
}
fn one_reading(
side: &MeasuredSide,
repetition: u32,
phase: &str,
regime: LoopRegime,
operations: &BTreeMap<String, OperationStats>,
) -> FailedSharePhase {
let mut count = 0_u64;
let mut errors = 0_u64;
let mut worst_operation: Option<String> = None;
let mut worst_share = 1.0_f64;
for (operation, stats) in operations {
count = count.saturating_add(stats.count);
errors = errors.saturating_add(stats.errors);
let share = failed_share(stats.count, stats.errors);
if worst_operation.is_none() || share > worst_share {
worst_operation = Some(operation.clone());
worst_share = share;
}
}
FailedSharePhase {
side: side.clone(),
repetition,
phase: phase.to_owned(),
regime,
count,
errors,
share: failed_share(count, errors),
worst_operation,
worst_share,
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TargetRecord {
pub base_url: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub sut_version: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Methodology {
pub statement: String,
pub open_loop: bool,
pub coordinated_omission_free: bool,
pub seed_once_measure_n: bool,
pub repetitions: u32,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct BenchResult {
pub schema_version: String,
pub boundary_statement: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub label: Option<String>,
pub pack: PackRecord,
pub target: TargetRecord,
pub environment: EnvironmentFingerprint,
pub started_at: String,
pub finished_at: String,
pub scale: ScaleRecord,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub version_at_time: Option<String>,
pub seed_phases: Vec<SeedPhaseRecord>,
pub repetitions: Vec<RepetitionRecord>,
pub cross: BTreeMap<String, CrossPhase>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub baselines: Vec<BaselineRecord>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub relative: Vec<RelativeIndex>,
pub methodology: Methodology,
pub submittable: bool,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub submittable_unmet: Vec<SubmissionRequirement>,
pub posture: PostureRecord,
}
impl BenchResult {
#[must_use]
pub fn failed_share_readings(&self) -> Vec<FailedSharePhase> {
let mut readings = side_readings(&MeasuredSide::Target, &self.repetitions);
for baseline in &self.baselines {
readings.extend(side_readings(
&MeasuredSide::Baseline(baseline.cdr.clone()),
&baseline.repetitions,
));
}
readings
}
#[must_use]
pub fn failed_share_breaches(&self) -> Vec<FailedSharePhase> {
let ceiling = self.pack.max_failed_share;
self.failed_share_readings()
.into_iter()
.filter(|reading| reading.breaches(ceiling))
.collect()
}
#[must_use]
pub fn worst_failed_share(&self) -> f64 {
self.failed_share_readings()
.iter()
.map(|reading| reading.worst_share)
.fold(0.0_f64, f64::max)
}
#[must_use]
pub fn unmet_requirements(&self) -> Vec<SubmissionRequirement> {
SubmissionRequirement::ALL
.iter()
.copied()
.filter(|requirement| match requirement {
SubmissionRequirement::Repetitions => {
self.repetitions.len() < SUBMITTABLE_REPETITIONS
}
SubmissionRequirement::Baseline => self.baselines.len() < SUBMITTABLE_BASELINES,
SubmissionRequirement::ErrorShare => !self.failed_share_breaches().is_empty(),
})
.collect()
}
#[must_use]
pub fn is_submittable(&self) -> bool {
self.unmet_requirements().is_empty()
}
pub fn attach_baselines(&mut self, baselines: Vec<BaselineRecord>) {
self.relative = baselines
.iter()
.map(|baseline| crate::bench::relative::derive(&self.cross, baseline))
.collect();
self.baselines = baselines;
self.settle_submittability();
}
pub fn settle_submittability(&mut self) {
self.submittable_unmet = self.unmet_requirements();
self.submittable = self.submittable_unmet.is_empty();
}
pub fn to_document(&self) -> Result<String, BenchError> {
let mut text =
serde_json::to_string_pretty(self).map_err(|source| BenchError::Serialize {
context: "bench result",
source,
})?;
text.push('\n');
Ok(text)
}
#[must_use]
pub fn file_name(&self) -> String {
match self.label.as_deref().map(slug) {
Some(slug) if !slug.is_empty() => format!("{RESULT_FILE_STEM}-{slug}.json"),
_ => format!("{RESULT_FILE_STEM}.json"),
}
}
}
fn slug(label: &str) -> String {
label
.chars()
.map(|c| {
if c.is_ascii_alphanumeric() {
c.to_ascii_lowercase()
} else {
'-'
}
})
.collect::<String>()
.trim_matches('-')
.to_owned()
}
#[cfg(test)]
#[expect(
clippy::panic_in_result_fn,
reason = "three Result-returning tests in the Book ch11 shape, each asserting; \
clippy offers no allow-in-tests knob for this lint"
)]
mod tests {
use super::*;
use crate::bench::posture::{
Assurance, Bracket, CanaryOutcome, CanaryReading, MINIMAL, PostureDisclosure, PostureItem,
};
fn declared_only_posture() -> PostureRecord {
let reading = |bracket: Bracket| CanaryReading {
bracket,
outcome: CanaryOutcome::NotObservable,
observed: "(not observable)".to_owned(),
evidence: "a record built for this test observes nothing".to_owned(),
};
PostureRecord {
profile: MINIMAL.name.to_owned(),
summary: MINIMAL.summary.to_owned(),
items: PostureItem::ALL
.iter()
.copied()
.map(|item| PostureDisclosure {
item,
declared: MINIMAL.declared(item).unwrap_or("none").to_owned(),
assurance: Assurance::DeclaredOnly,
readings: vec![reading(Bracket::Before), reading(Bracket::After)],
})
.collect(),
comparability: Vec::new(),
}
}
#[test]
fn statuses_class_by_their_wire_range() {
assert_eq!(
ErrorClass::of_status(reqwest::StatusCode::CREATED),
ErrorClass::Http2xx
);
assert_eq!(
ErrorClass::of_status(reqwest::StatusCode::FOUND),
ErrorClass::Http3xx
);
assert_eq!(
ErrorClass::of_status(reqwest::StatusCode::NOT_FOUND),
ErrorClass::Http4xx
);
assert_eq!(
ErrorClass::of_status(reqwest::StatusCode::INTERNAL_SERVER_ERROR),
ErrorClass::Http5xx
);
assert_eq!(
ErrorClass::of_status(reqwest::StatusCode::CONTINUE),
ErrorClass::HttpOther
);
}
#[test]
fn every_error_class_token_round_trips() -> Result<(), BenchError> {
for class in ErrorClass::ALL {
assert_eq!(ErrorClass::parse(class.as_str())?, *class);
}
assert!(ErrorClass::parse("http_6xx").is_err());
Ok(())
}
#[test]
fn the_encoded_histogram_reproduces_the_recorded_percentiles()
-> Result<(), Box<dyn std::error::Error>> {
let mut histogram = Histogram::<u64>::new_with_bounds(1, 600_000_000, 3)?;
for value in 1..=1000_u64 {
histogram.record(value * 100)?;
}
let stats = OperationStats::from_histogram(&histogram, BTreeMap::new(), 10.0)?;
let decoded = stats.decode_histogram()?;
assert_eq!(decoded.value_at_quantile(0.50), stats.p50_us);
assert_eq!(decoded.value_at_quantile(0.99), stats.p99_us);
assert_eq!(decoded.value_at_quantile(0.999), stats.p999_us);
assert_eq!(decoded.max(), stats.max_us);
assert_eq!(stats.count, 1000);
assert!((stats.throughput_ops_s - 100.0).abs() < 1e-9);
Ok(())
}
#[test]
fn errors_sum_from_their_classes() -> Result<(), Box<dyn std::error::Error>> {
let mut histogram = Histogram::<u64>::new_with_bounds(1, 600_000_000, 3)?;
histogram.record(10)?;
let mut classes = BTreeMap::new();
let _replaced = classes.insert(ErrorClass::Http5xx.as_str().to_owned(), 3_u64);
let _replaced = classes.insert(ErrorClass::Timeout.as_str().to_owned(), 1_u64);
let stats = OperationStats::from_histogram(&histogram, classes, 1.0)?;
assert_eq!(stats.errors, 4);
Ok(())
}
#[test]
fn only_the_pinned_configuration_is_the_reference_one() {
assert!(ScaleRecord::new(1.0, true).reference_configuration);
assert!(!ScaleRecord::new(0.1, true).reference_configuration);
assert!(!ScaleRecord::new(1.0, false).reference_configuration);
assert!(!ScaleRecord::new(0.1, false).reference_configuration);
assert_eq!(ScaleRecord::default(), ScaleRecord::new(1.0, true));
}
#[test]
fn the_file_name_follows_the_label() {
assert_eq!(slug("EHRbase 2.x / run A"), "ehrbase-2-x---run-a");
assert_eq!(slug("---"), "");
}
#[test]
fn every_submission_requirement_token_round_trips() -> Result<(), Box<dyn std::error::Error>> {
for requirement in SubmissionRequirement::ALL {
assert_eq!(
SubmissionRequirement::parse(requirement.as_str())?,
*requirement
);
let text = serde_json::to_string(requirement)?;
let back: SubmissionRequirement = serde_json::from_str(&text)?;
assert_eq!(back, *requirement);
assert!(!requirement.statement().is_empty());
}
assert!(SubmissionRequirement::parse("vibes").is_err());
Ok(())
}
#[test]
fn submittability_names_the_requirement_it_misses() {
let mut result = minimal_result(3);
result.settle_submittability();
assert!(!result.submittable);
assert_eq!(
result.submittable_unmet,
vec![SubmissionRequirement::Baseline]
);
result.attach_baselines(vec![empty_baseline()]);
assert!(result.submittable, "{:?}", result.submittable_unmet);
assert!(result.submittable_unmet.is_empty());
}
#[test]
fn a_thin_record_names_every_requirement_it_misses() {
let mut result = minimal_result(1);
result.settle_submittability();
assert_eq!(
result.submittable_unmet,
vec![
SubmissionRequirement::Repetitions,
SubmissionRequirement::Baseline
]
);
assert!(!result.is_submittable());
}
#[test]
fn attaching_a_baseline_derives_its_relative_index() {
let mut result = minimal_result(3);
result.attach_baselines(vec![empty_baseline()]);
assert_eq!(result.relative.len(), 1);
assert_eq!(
result.relative.first().map(|index| index.baseline.as_str()),
Some("ehrbase")
);
}
#[test]
fn a_baseline_free_record_omits_the_new_keys() -> Result<(), Box<dyn std::error::Error>> {
let mut result = minimal_result(3);
result.settle_submittability();
let text = result.to_document()?;
assert!(!text.contains("\"baselines\""), "{text}");
assert!(!text.contains("\"relative\""), "{text}");
assert!(text.contains("\"submittable_unmet\""), "{text}");
let back: BenchResult = serde_json::from_str(&text)?;
assert_eq!(back, result);
Ok(())
}
const CEILING: f64 = crate::bench::pack::DEFAULT_MAX_FAILED_SHARE;
fn stats(count: u64, errors: u64) -> OperationStats {
let mut errors_by_class = BTreeMap::new();
if errors > 0 {
let _replaced = errors_by_class.insert(ErrorClass::Http5xx.as_str().to_owned(), errors);
}
OperationStats {
count,
errors,
errors_by_class,
throughput_ops_s: 1.0,
p50_us: 1,
p75_us: 1,
p90_us: 1,
p99_us: 1,
p999_us: 1,
max_us: 1,
hdr_v2_base64: String::new(),
}
}
fn measured(operations: BTreeMap<String, OperationStats>) -> MeasuredPhaseRecord {
MeasuredPhaseRecord {
regime: LoopRegime::OpenLoop,
rate_per_s: 1.0,
warmup_s: 0,
duration_s: 1,
planned_measured_arrivals: 0,
dispatched_measured_arrivals: 0,
warmup_arrivals: 0,
offered_load_sustained_per_s: 0.0,
generator_bound: false,
operations,
}
}
fn phases_of(count: u64, errors: u64) -> BTreeMap<String, MeasuredPhaseRecord> {
let mut operations = BTreeMap::new();
let _replaced = operations.insert("get_ehr".to_owned(), stats(count, errors));
let mut phases = BTreeMap::new();
let _replaced = phases.insert("mixed".to_owned(), measured(operations));
phases
}
fn record_of(count: u64, errors: u64) -> BenchResult {
let mut result = minimal_result(3);
for repetition in &mut result.repetitions {
repetition.phases = phases_of(count, errors);
}
result.attach_baselines(vec![empty_baseline()]);
result
}
#[test]
fn a_zero_arrival_operation_is_fully_failed() {
assert!((failed_share(0, 0) - 1.0).abs() < f64::EPSILON);
assert!((failed_share(0, 7) - 1.0).abs() < f64::EPSILON);
assert!((failed_share(100, 1) - 0.01).abs() < f64::EPSILON);
assert!(failed_share(100, 0).abs() < f64::EPSILON);
assert!((failed_share(444, 444) - 1.0).abs() < f64::EPSILON);
}
#[test]
fn the_target_passes_at_the_ceiling_and_refuses_above_it() {
let at = record_of(100, 1);
assert!(at.submittable, "{:?}", at.submittable_unmet);
assert!(at.failed_share_breaches().is_empty());
assert!((at.worst_failed_share() - CEILING).abs() < f64::EPSILON);
let above = record_of(100, 2);
assert!(!above.submittable);
assert_eq!(
above.submittable_unmet,
vec![SubmissionRequirement::ErrorShare]
);
let breaches = above.failed_share_breaches();
assert_eq!(breaches.len(), 3, "one per repetition");
let Some(first) = breaches.first() else {
panic!("the breach list lost its entries");
};
assert_eq!(first.side, MeasuredSide::Target);
let sentence = first.sentence(CEILING);
assert!(sentence.contains("the target"), "{sentence}");
assert!(sentence.contains("`mixed`"), "{sentence}");
assert!(sentence.contains("`get_ehr`"), "{sentence}");
assert!(sentence.contains("0.01"), "{sentence}");
}
#[test]
fn a_baseline_above_the_ceiling_refuses_the_record() {
let mut result = minimal_result(3);
for repetition in &mut result.repetitions {
repetition.phases = phases_of(100, 0);
}
let mut baseline = empty_baseline();
baseline.repetitions = vec![RepetitionRecord {
repetition: 1,
phases: phases_of(100, 1),
sweeps: BTreeMap::new(),
}];
result.attach_baselines(vec![baseline.clone()]);
assert!(result.submittable, "{:?}", result.submittable_unmet);
baseline.repetitions = vec![RepetitionRecord {
repetition: 1,
phases: phases_of(100, 2),
sweeps: BTreeMap::new(),
}];
result.attach_baselines(vec![baseline]);
assert_eq!(
result.submittable_unmet,
vec![SubmissionRequirement::ErrorShare]
);
let breaches = result.failed_share_breaches();
let Some(breach) = breaches.first() else {
panic!("the baseline breach was not recorded");
};
assert_eq!(breach.side, MeasuredSide::Baseline("ehrbase".to_owned()));
assert!(
breach.sentence(CEILING).contains("the ehrbase baseline"),
"{}",
breach.sentence(CEILING)
);
}
#[test]
fn a_wholly_failed_operation_refuses_the_record() {
let result = record_of(444, 444);
assert!(!result.submittable);
assert!(
result
.submittable_unmet
.contains(&SubmissionRequirement::ErrorShare)
);
assert!((result.worst_failed_share() - 1.0).abs() < f64::EPSILON);
}
fn minimal_result(repetitions: usize) -> BenchResult {
BenchResult {
schema_version: "0".to_owned(),
boundary_statement: crate::bench::BOUNDARY_STATEMENT.to_owned(),
label: None,
pack: PackRecord {
id: "smoke".to_owned(),
version: "1.0.0".to_owned(),
description: "a pack".to_owned(),
max_failed_share: CEILING,
seed: 1,
fixtures: BTreeMap::new(),
},
target: TargetRecord {
base_url: "http://sut.invalid/openehr/v1".to_owned(),
sut_version: None,
},
environment: EnvironmentFingerprint::detect(),
started_at: "2026-08-29T00:00:00Z".to_owned(),
finished_at: "2026-08-29T00:01:00Z".to_owned(),
scale: ScaleRecord::default(),
version_at_time: None,
seed_phases: Vec::new(),
repetitions: (1..=repetitions)
.map(|index| RepetitionRecord {
repetition: u32::try_from(index).unwrap_or(1),
phases: BTreeMap::new(),
sweeps: BTreeMap::new(),
})
.collect(),
cross: BTreeMap::new(),
baselines: Vec::new(),
relative: Vec::new(),
methodology: Methodology {
statement: crate::bench::METHODOLOGY.to_owned(),
open_loop: true,
coordinated_omission_free: true,
seed_once_measure_n: true,
repetitions: u32::try_from(repetitions).unwrap_or(1),
},
submittable: false,
submittable_unmet: Vec::new(),
posture: declared_only_posture(),
}
}
fn empty_baseline() -> BaselineRecord {
BaselineRecord {
cdr: "ehrbase".to_owned(),
display_name: "EHRbase".to_owned(),
images: BTreeMap::new(),
recipe: RecipeReference {
repository: "https://example.invalid/cdr".to_owned(),
git_ref: "v1.0.0".to_owned(),
file: "docker-compose.yml".to_owned(),
},
resources: crate::bench::baselines::pinned_resources(),
base_url: "http://127.0.0.1:18091/ehrbase/rest/openehr/v1".to_owned(),
sut_version: None,
started_at: "2026-08-29T00:02:00Z".to_owned(),
finished_at: "2026-08-29T00:03:00Z".to_owned(),
seed_phases: Vec::new(),
repetitions: Vec::new(),
cross: BTreeMap::new(),
posture: declared_only_posture(),
}
}
}