use std::collections::HashMap;
use serde_json::Value;
use super::ids;
use super::metrics::{FidelityRecord, MetricValue, fidelity_from_json, is_finite_num};
use super::{EsrNamcoreEnvelope, PerformanceEntry, QualityContract};
pub const MANDATORY_FIDELITY_PHASES: [&str; 3] =
["golden_vectors", "reference_oracle_f64", "quick_parity"];
pub const REGRESSION_GATE_PHASE: &str = "regression_gate";
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PhaseRecord {
pub phase_id: String,
pub status: String,
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq)]
pub struct LatencyRecord {
pub label: String,
pub median_latency_us: f64,
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq)]
pub struct VerifyReport {
pub phases: Vec<PhaseRecord>,
pub fidelity: Vec<FidelityRecord>,
pub latency: Vec<LatencyRecord>,
}
pub fn parse_verify_report(input: &str) -> Result<VerifyReport, VerifyError> {
let mut report = VerifyReport {
phases: Vec::new(),
fidelity: Vec::new(),
latency: Vec::new(),
};
for (line_no, raw_line) in input.lines().enumerate() {
let line = raw_line.trim();
if line.is_empty() {
continue;
}
let value: Value =
serde_json::from_str(line).map_err(|source| VerifyError::MalformedLine {
line: line_no + 1,
source,
})?;
if value.get("phase_id").is_some() {
report.phases.push(phase_from_json(&value, line_no + 1)?);
} else if value.get("median_latency_us").is_some() {
report.latency.push(latency_from_json(&value, line_no + 1)?);
} else if let Some(record) = fidelity_from_json(&value) {
report.fidelity.push(record);
}
}
Ok(report)
}
pub fn parse_verify_report_file(
path: impl AsRef<std::path::Path>,
) -> Result<VerifyReport, VerifyError> {
let input = std::fs::read_to_string(path)?;
parse_verify_report(&input)
}
#[derive(Debug, thiserror::Error)]
pub enum VerifyError {
#[error("malformed report line {line}: {source}")]
MalformedLine {
line: usize,
source: serde_json::Error,
},
#[error("phase record on line {line} must have string `phase_id` and `status`")]
InvalidPhaseRecord {
line: usize,
},
#[error(
"latency record on line {line} must have string `label` and finite numeric `median_latency_us`"
)]
InvalidLatencyRecord {
line: usize,
},
#[error("cannot read verify report: {0}")]
Io(#[from] std::io::Error),
}
fn phase_from_json(value: &Value, line: usize) -> Result<PhaseRecord, VerifyError> {
let phase_id = value
.get("phase_id")
.and_then(Value::as_str)
.ok_or(VerifyError::InvalidPhaseRecord { line })?;
let status = value
.get("status")
.and_then(Value::as_str)
.ok_or(VerifyError::InvalidPhaseRecord { line })?;
Ok(PhaseRecord {
phase_id: phase_id.to_string(),
status: status.to_string(),
})
}
fn latency_from_json(value: &Value, line: usize) -> Result<LatencyRecord, VerifyError> {
let label = value
.get("label")
.and_then(Value::as_str)
.ok_or(VerifyError::InvalidLatencyRecord { line })?;
let median_latency_us = value
.get("median_latency_us")
.and_then(Value::as_f64)
.filter(|v| v.is_finite())
.ok_or(VerifyError::InvalidLatencyRecord { line })?;
Ok(LatencyRecord {
label: label.to_string(),
median_latency_us,
})
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FidelityVerdict {
Ok,
Fail {
violations: u32,
},
}
impl FidelityVerdict {
pub fn is_ok(&self) -> bool {
matches!(self, Self::Ok)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PerformanceVerdict {
Ok,
NotVerified,
Fail {
violations: u32,
},
}
impl PerformanceVerdict {
pub fn is_ok(&self) -> bool {
matches!(self, Self::Ok)
}
pub fn is_not_verified(&self) -> bool {
matches!(self, Self::NotVerified)
}
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq)]
pub struct VerifyOutcome {
pub fidelity: FidelityVerdict,
pub performance: PerformanceVerdict,
pub review_required: u32,
pub fidelity_checks: Vec<FidelityCheck>,
pub perf_checks: Vec<PerfCheck>,
}
impl VerifyOutcome {
pub fn exit_code(&self) -> i32 {
if self.fidelity.is_ok() && self.performance.is_ok() && self.review_required == 0 {
0
} else {
1
}
}
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq)]
pub struct FidelityCheck {
pub id: String,
pub label: String,
pub outcome: FidelityOutcome,
}
#[derive(Debug, Clone, PartialEq)]
pub enum FidelityOutcome {
OptionalSkipped,
MissingLabel,
Measured(Vec<MetricCheck>),
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq)]
pub struct MetricCheck {
pub metric: Metric,
pub outcome: MetricOutcome,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Metric {
EsrNamcore,
EsrF64,
SnrDb,
Mrstft,
}
#[derive(Debug, Clone, PartialEq)]
pub enum MetricOutcome {
Ok,
SafetyCeiling {
current: f64,
limit: f64,
baseline: f64,
},
NoiseEnvelope {
current: f64,
limit: f64,
baseline: f64,
},
Envelope {
current: f64,
limit: f64,
baseline: f64,
},
Malformed(MalformedReason),
Missing,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum MalformedReason {
Missing,
NonFinite(String),
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq)]
pub struct PerfCheck {
pub id: String,
pub label: String,
pub result: PerfResult,
}
#[derive(Debug, Clone, PartialEq)]
pub enum PerfResult {
Ok {
median_us: f64,
},
Regressed {
median_us: f64,
limit_us: f64,
baseline_us: f64,
},
MissingLabel,
MissingLatency,
}
pub fn verify_contract(contract: &QualityContract, report: &VerifyReport) -> VerifyOutcome {
let mut fidelity_violations: u32 = 0;
let mut perf_violations: u32 = 0;
let mut review_required: u32 = 0;
for phase in &report.phases {
if phase.status == "FAIL" && phase.phase_id != REGRESSION_GATE_PHASE {
fidelity_violations += 1;
}
}
for phase_id in MANDATORY_FIDELITY_PHASES {
let status = phase_status(report, phase_id);
if status != "PASS" && status != "FAIL" {
fidelity_violations += 1;
}
}
let mut fidelity_checks = Vec::new();
if !contract.fidelity.is_empty() {
let mut report_by_label: HashMap<&str, &FidelityRecord> = HashMap::new();
for record in &report.fidelity {
report_by_label
.entry(record.label.as_str())
.or_insert(record);
}
for record in &report.fidelity {
if let Some(canonical) = ids::resolve_fidelity_alias(&record.label) {
report_by_label.entry(canonical).or_insert(record);
}
}
let envelopes = &contract.envelopes;
for entry in &contract.fidelity {
let id = entry.id.clone();
let label = entry.label.clone();
let Some(record) = report_by_label.get(entry.label.as_str()).copied() else {
if entry.optional {
fidelity_checks.push(FidelityCheck {
id,
label,
outcome: FidelityOutcome::OptionalSkipped,
});
} else {
fidelity_violations += 1;
fidelity_checks.push(FidelityCheck {
id,
label,
outcome: FidelityOutcome::MissingLabel,
});
}
continue;
};
let mut metrics = Vec::new();
let mut namcore_ok = true;
let esr_baseline = entry.esr_namcore;
if esr_baseline.is_finite() {
match &record.esr {
MetricValue::Na => {
fidelity_violations += 1;
namcore_ok = false;
metrics.push(MetricCheck {
metric: Metric::EsrNamcore,
outcome: MetricOutcome::Malformed(MalformedReason::Missing),
});
}
MetricValue::Raw(raw) if !is_finite_num(raw) => {
fidelity_violations += 1;
namcore_ok = false;
metrics.push(MetricCheck {
metric: Metric::EsrNamcore,
outcome: MetricOutcome::Malformed(MalformedReason::NonFinite(
raw.clone(),
)),
});
}
MetricValue::Raw(raw) => {
let current = parse_finite(raw);
let (noise_limit, safety_limit) =
esr_limits(esr_baseline, &envelopes.esr_namcore);
if current > safety_limit {
fidelity_violations += 1;
namcore_ok = false;
metrics.push(MetricCheck {
metric: Metric::EsrNamcore,
outcome: MetricOutcome::SafetyCeiling {
current,
limit: safety_limit,
baseline: esr_baseline,
},
});
} else if current > noise_limit {
fidelity_violations += 1;
namcore_ok = false;
metrics.push(MetricCheck {
metric: Metric::EsrNamcore,
outcome: MetricOutcome::NoiseEnvelope {
current,
limit: noise_limit,
baseline: esr_baseline,
},
});
} else {
metrics.push(MetricCheck {
metric: Metric::EsrNamcore,
outcome: MetricOutcome::Ok,
});
}
}
}
}
if let Some(f64_baseline) = entry.esr_f64.filter(|v| v.is_finite()) {
match record.esr_f64.as_raw() {
None => {
fidelity_violations += 1;
metrics.push(MetricCheck {
metric: Metric::EsrF64,
outcome: MetricOutcome::Missing,
});
}
Some(raw) => match raw.parse::<f64>() {
Err(_) => {
fidelity_violations += 1;
metrics.push(MetricCheck {
metric: Metric::EsrF64,
outcome: MetricOutcome::Malformed(MalformedReason::NonFinite(
raw.to_string(),
)),
});
}
Ok(current) => {
let (noise_limit, safety_limit) =
esr_limits(f64_baseline, &envelopes.esr_namcore);
if current > safety_limit {
fidelity_violations += 1;
if namcore_ok {
review_required += 1;
}
metrics.push(MetricCheck {
metric: Metric::EsrF64,
outcome: MetricOutcome::SafetyCeiling {
current,
limit: safety_limit,
baseline: f64_baseline,
},
});
} else if current > noise_limit {
fidelity_violations += 1;
if namcore_ok {
review_required += 1;
}
metrics.push(MetricCheck {
metric: Metric::EsrF64,
outcome: MetricOutcome::NoiseEnvelope {
current,
limit: noise_limit,
baseline: f64_baseline,
},
});
} else {
if !namcore_ok {
review_required += 1;
}
metrics.push(MetricCheck {
metric: Metric::EsrF64,
outcome: MetricOutcome::Ok,
});
}
if esr_baseline > 0.0
&& let Some(cur_n_raw) = record.esr.as_finite()
{
let cur_n = parse_finite(cur_n_raw);
let rn = cur_n / esr_baseline;
let rf = current / f64_baseline;
if (rn < 0.85 && rf > 1.15) || (rn > 1.15 && rf < 0.85) {
review_required += 1;
}
}
}
},
}
}
if let Some(snr_baseline) = entry.snr_db.filter(|v| v.is_finite()) {
match &record.snr_db {
MetricValue::Na => {
fidelity_violations += 1;
metrics.push(MetricCheck {
metric: Metric::SnrDb,
outcome: MetricOutcome::Malformed(MalformedReason::Missing),
});
}
MetricValue::Raw(raw) if !is_finite_num(raw) => {
fidelity_violations += 1;
metrics.push(MetricCheck {
metric: Metric::SnrDb,
outcome: MetricOutcome::Malformed(MalformedReason::NonFinite(
raw.clone(),
)),
});
}
MetricValue::Raw(raw) => {
let current = parse_finite(raw);
let limit = snr_baseline - envelopes.snr_db_drop;
if current < limit {
fidelity_violations += 1;
metrics.push(MetricCheck {
metric: Metric::SnrDb,
outcome: MetricOutcome::Envelope {
current,
limit,
baseline: snr_baseline,
},
});
} else {
metrics.push(MetricCheck {
metric: Metric::SnrDb,
outcome: MetricOutcome::Ok,
});
}
}
}
}
if entry.mrstft.is_finite() {
match &record.mrstft {
MetricValue::Na => {
fidelity_violations += 1;
metrics.push(MetricCheck {
metric: Metric::Mrstft,
outcome: MetricOutcome::Malformed(MalformedReason::Missing),
});
}
MetricValue::Raw(raw) if !is_finite_num(raw) => {
fidelity_violations += 1;
metrics.push(MetricCheck {
metric: Metric::Mrstft,
outcome: MetricOutcome::Malformed(MalformedReason::NonFinite(
raw.clone(),
)),
});
}
MetricValue::Raw(raw) => {
let current = parse_finite(raw);
let limit = entry.mrstft * envelopes.mrstft_mult;
if current > limit {
fidelity_violations += 1;
metrics.push(MetricCheck {
metric: Metric::Mrstft,
outcome: MetricOutcome::Envelope {
current,
limit,
baseline: entry.mrstft,
},
});
} else {
metrics.push(MetricCheck {
metric: Metric::Mrstft,
outcome: MetricOutcome::Ok,
});
}
}
}
}
fidelity_checks.push(FidelityCheck {
id,
label,
outcome: FidelityOutcome::Measured(metrics),
});
}
}
let mut perf_checks = Vec::new();
let performance = if contract.performance.is_empty() {
PerformanceVerdict::Ok
} else if phase_status(report, REGRESSION_GATE_PHASE) != "PASS" {
PerformanceVerdict::NotVerified
} else {
let envelopes = &contract.envelopes;
for entry in &contract.performance {
let id = entry.id.clone();
let label = entry.label.clone();
let Some(record) = match_latency(entry, &report.latency) else {
perf_violations += 1;
perf_checks.push(PerfCheck {
id,
label,
result: PerfResult::MissingLabel,
});
continue;
};
let baseline_us = entry.median_latency_us;
let limit_us = (baseline_us * envelopes.latency_mult)
.max(baseline_us + envelopes.latency_floor_us);
if record.median_latency_us > limit_us {
perf_violations += 1;
perf_checks.push(PerfCheck {
id,
label,
result: PerfResult::Regressed {
median_us: record.median_latency_us,
limit_us,
baseline_us,
},
});
} else {
perf_checks.push(PerfCheck {
id,
label,
result: PerfResult::Ok {
median_us: record.median_latency_us,
},
});
}
}
if perf_violations > 0 {
PerformanceVerdict::Fail {
violations: perf_violations,
}
} else {
PerformanceVerdict::Ok
}
};
VerifyOutcome {
fidelity: if fidelity_violations > 0 {
FidelityVerdict::Fail {
violations: fidelity_violations,
}
} else {
FidelityVerdict::Ok
},
performance,
review_required,
fidelity_checks,
perf_checks,
}
}
fn phase_status<'a>(report: &'a VerifyReport, phase_id: &str) -> &'a str {
report
.phases
.iter()
.rev()
.find(|p| p.phase_id == phase_id)
.map(|p| p.status.as_str())
.unwrap_or("NOT_RUN")
}
fn esr_limits(baseline: f64, env: &EsrNamcoreEnvelope) -> (f64, f64) {
let noise = (baseline * env.noise_mult).max(baseline + env.noise_floor_abs);
let safety = (baseline * env.safety_mult).max(env.safety_floor_abs);
(round_printf_2e(noise), round_printf_2e(safety))
}
fn round_printf_2e(v: f64) -> f64 {
if v == 0.0 || !v.is_finite() {
return v;
}
format!("{v:.2e}").parse::<f64>().unwrap_or(v)
}
fn parse_finite(raw: &str) -> f64 {
raw.parse::<f64>()
.expect("is_finite_num accepted a non-parseable literal")
}
fn match_latency<'a>(
entry: &PerformanceEntry,
report: &'a [LatencyRecord],
) -> Option<&'a LatencyRecord> {
report.iter().find(|record| {
record.label == entry.label
|| record.label == entry.id
|| normalize_bench_label(&record.label) == normalize_bench_label(&entry.label)
})
}
fn normalize_bench_label(label: &str) -> String {
label
.replace('×', "x")
.replace('→', "->")
.replace(" ", " ")
.to_lowercase()
}
#[cfg(test)]
#[path = "verify_test.rs"]
mod verify_test;