#![expect(
clippy::disallowed_types,
reason = "dev/verification tooling over JSON artifacts (the catalogue, results, wire \
exchanges) — not the application (#1694)"
)]
#![allow(
clippy::print_stdout,
clippy::print_stderr,
reason = "this IS the CLI: stdout carries the run report and stderr the \
diagnostics; only library crates are restricted to `tracing`"
)]
use std::path::PathBuf;
use std::process::ExitCode;
use clap::{Parser, Subcommand};
use veredictum::artifacts::load_root;
use veredictum::load::compile_schema;
use veredictum::party::{Results, Statement};
use veredictum::render::{render_certificate, render_report, render_statement};
use veredictum::schema::{emit_all, render, results_schema, statement_schema};
use veredictum::validate::{Context, render_coverage_report, validate};
use veredictum::verdict::compute;
#[derive(Parser)]
#[command(
name = "veredictum",
about = "The independent conformance instrument for openEHR clinical data repositories",
version
)]
struct Cli {
#[command(subcommand)]
command: Command,
}
#[derive(Subcommand)]
enum Command {
EmitSchemas {
#[arg(long)]
out: PathBuf,
},
Run {
#[arg(long)]
root: PathBuf,
#[arg(long)]
ixit: PathBuf,
#[arg(long)]
out: PathBuf,
#[arg(long, default_value = "ferroehr")]
sut_name: String,
#[arg(long, default_value = "dev")]
sut_version: String,
#[arg(long)]
filter: Option<String>,
#[arg(long)]
statement: Option<PathBuf>,
},
Validate {
#[arg(long)]
root: PathBuf,
#[arg(long)]
specs: Option<PathBuf>,
#[arg(long)]
write_report: bool,
},
Perf {
#[arg(long)]
root: PathBuf,
#[arg(long)]
ixit: PathBuf,
#[arg(long)]
results: PathBuf,
#[arg(long)]
class: String,
#[arg(long, default_value_t = 16)]
seed_workers: usize,
#[arg(long, default_value_t = 1)]
hours: u64,
},
Stress {
#[arg(long)]
root: PathBuf,
#[arg(long)]
ixit: PathBuf,
#[arg(long)]
out: PathBuf,
#[arg(long, default_value = "POC")]
corpus_class: String,
#[arg(long, default_value_t = 16)]
seed_workers: usize,
#[arg(long, default_value_t = 120)]
step_secs: u64,
#[arg(long, default_value_t = 3)]
bisections: u32,
#[arg(long, default_value_t = 4096.0)]
max_rate: f64,
},
AqlProbe {
#[arg(long)]
root: PathBuf,
#[arg(long)]
ixit: PathBuf,
#[arg(long)]
out: PathBuf,
#[arg(long, default_value = "POC")]
corpus_class: String,
#[arg(long, default_value_t = 16)]
seed_workers: usize,
#[arg(long, default_value_t = 20)]
requests: u32,
},
StressCompare {
#[arg(long)]
left: PathBuf,
#[arg(long)]
left_label: String,
#[arg(long)]
right: PathBuf,
#[arg(long)]
right_label: String,
#[arg(long)]
out: PathBuf,
},
PerfAssets {
#[arg(long)]
root: PathBuf,
#[arg(long)]
results: PathBuf,
#[arg(long)]
out: PathBuf,
#[arg(long)]
summary: Option<PathBuf>,
#[arg(long)]
stress: Option<PathBuf>,
},
ConformanceAssets {
#[arg(long)]
root: PathBuf,
#[arg(long)]
results: PathBuf,
#[arg(long)]
verdicts: PathBuf,
#[arg(long)]
out: PathBuf,
#[arg(long, default_value = "")]
suffix: String,
},
Verdicts {
#[arg(long)]
statement: PathBuf,
#[arg(long)]
results: PathBuf,
#[arg(long)]
root: PathBuf,
#[arg(long)]
out: PathBuf,
},
}
fn load_party_json<T: serde::de::DeserializeOwned>(
path: &std::path::Path,
schema: &serde_json::Value,
schema_name: &str,
) -> Result<T, String> {
let text = std::fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?;
let value: serde_json::Value =
serde_json::from_str(&text).map_err(|e| format!("{}: JSON: {e}", path.display()))?;
let validator = compile_schema(schema, schema_name).map_err(|e| e.to_string())?;
let violations: Vec<String> = validator
.iter_errors(&value)
.map(|e| format!("{}: {e}", e.instance_path()))
.collect();
if !violations.is_empty() {
return Err(format!(
"{}: schema: {}",
path.display(),
violations.join("; ")
));
}
serde_json::from_value(value).map_err(|e| format!("{}: model: {e}", path.display()))
}
fn main() -> ExitCode {
match Cli::parse().command {
Command::EmitSchemas { out } => emit_schemas_command(&out),
Command::Run {
root,
ixit,
out,
sut_name,
sut_version,
filter,
statement,
} => run_command(
&root,
&ixit,
&out,
&sut_name,
&sut_version,
filter.as_deref(),
statement.as_deref(),
),
Command::Validate {
root,
specs,
write_report,
} => validate_command(&root, specs.as_deref(), write_report),
Command::Perf {
root,
ixit,
results,
class,
seed_workers,
hours,
} => perf_command(&root, &ixit, &results, &class, seed_workers, hours),
Command::Stress {
root,
ixit,
out,
corpus_class,
seed_workers,
step_secs,
bisections,
max_rate,
} => stress_command(
&root,
&ixit,
&out,
&corpus_class,
seed_workers,
step_secs,
bisections,
max_rate,
),
Command::AqlProbe {
root,
ixit,
out,
corpus_class,
seed_workers,
requests,
} => probe_command(&root, &ixit, &out, &corpus_class, seed_workers, requests),
Command::StressCompare {
left,
left_label,
right,
right_label,
out,
} => stress_compare_command(&left, &left_label, &right, &right_label, &out),
Command::PerfAssets {
root,
results,
out,
summary,
stress,
} => perf_assets_command(&root, &results, &out, summary.as_deref(), stress.as_deref()),
Command::ConformanceAssets {
root,
results,
verdicts,
out,
suffix,
} => conformance_assets_command(&root, &results, &verdicts, &out, &suffix),
Command::Verdicts {
statement,
results,
root,
out,
} => run_verdicts(&statement, &results, &root, &out),
}
}
fn emit_schemas_command(out: &std::path::Path) -> ExitCode {
if let Err(e) = std::fs::create_dir_all(out) {
eprintln!("cannot create {}: {e}", out.display());
return ExitCode::from(2);
}
for (name, schema) in emit_all() {
let path = out.join(name);
if let Err(e) = std::fs::write(&path, render(&schema)) {
eprintln!("cannot write {}: {e}", path.display());
return ExitCode::from(2);
}
println!("wrote {}", path.display());
}
ExitCode::SUCCESS
}
fn validate_command(
root: &std::path::Path,
specs: Option<&std::path::Path>,
write_report: bool,
) -> ExitCode {
let loaded = match load_root(root) {
Ok(loaded) => loaded,
Err(e) => {
eprintln!("runner defect: {e}");
return ExitCode::from(2);
}
};
let findings = validate(&Context {
set: &loaded.set,
load_errors: &loaded.errors,
spec_root: specs,
});
for finding in &findings {
println!("{finding}");
}
if write_report
&& let Some(specs) = specs
&& let Some(docs) = specs.parent().and_then(std::path::Path::parent)
{
let report_path = docs.join("conformance/coverage-report.md");
let body = render_coverage_report(&loaded.set, Some(specs));
match report_path
.parent()
.map_or(Ok(()), std::fs::create_dir_all)
.and_then(|()| std::fs::write(&report_path, body))
{
Ok(()) => println!("wrote {}", report_path.display()),
Err(e) => eprintln!("warning: cannot write {}: {e}", report_path.display()),
}
}
println!(
"{} case(s), {} binding(s), {} party statement(s), {} finding(s)",
loaded.set.cases.len(),
loaded.set.bindings.len(),
loaded.set.parties.len(),
findings.len()
);
if findings.is_empty() {
ExitCode::SUCCESS
} else {
ExitCode::from(1)
}
}
#[expect(clippy::too_many_lines, reason = "the one-shot orchestration seam")]
fn run_verdicts(
statement_path: &std::path::Path,
results_path: &std::path::Path,
root: &std::path::Path,
out: &std::path::Path,
) -> ExitCode {
let statement: Statement =
match load_party_json(statement_path, &statement_schema(), "statement.schema.json") {
Ok(s) => s,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
let results: Results =
match load_party_json(results_path, &results_schema(), "results.schema.json") {
Ok(r) => r,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
if let Err(errors) = results.check_invariants() {
for e in &errors {
eprintln!("{e}");
}
return ExitCode::from(2);
}
let loaded = match load_root(root) {
Ok(loaded) => loaded,
Err(e) => {
eprintln!("runner defect: {e}");
return ExitCode::from(2);
}
};
if !loaded.errors.is_empty() {
for e in &loaded.errors {
eprintln!("{e}");
}
return ExitCode::from(2);
}
let Some((_, matrix)) = &loaded.set.matrix else {
eprintln!("artifact tree carries no capability matrix");
return ExitCode::from(2);
};
let Some((_, register)) = &loaded.set.register else {
eprintln!("artifact tree carries no ambiguity register");
return ExitCode::from(2);
};
let cases: Vec<_> = loaded.set.cases.iter().map(|(_, c)| c.clone()).collect();
let perf_cases: Vec<_> = loaded
.set
.performance
.iter()
.map(|(_, c)| c.clone())
.collect();
let report = compute(&statement, &results, &cases, &perf_cases, matrix, register);
let served_extensions = match &loaded.set.wire_surface {
Some((_, wire_surface)) => wire_surface.served_extensions.as_slice(),
None => &[],
};
if let Err(e) = std::fs::create_dir_all(out) {
eprintln!("cannot create {}: {e}", out.display());
return ExitCode::from(2);
}
let artifacts: [(&str, String); 4] = [
(
"verdicts.json",
match serde_json::to_string_pretty(&report) {
Ok(mut json) => {
json.push('\n');
json
}
Err(e) => {
eprintln!("cannot serialize verdicts: {e}");
return ExitCode::from(2);
}
},
),
(
"CONFORMANCE_REPORT.md",
match render_report(&results, &report, &statement) {
Ok(markdown) => markdown,
Err(e) => {
eprintln!("cannot render the report: {e}");
return ExitCode::from(2);
}
},
),
(
"CONFORMANCE_STATEMENT.md",
render_statement(&statement, &report, served_extensions),
),
(
"CONFORMANCE_CERTIFICATE.md",
render_certificate(&statement, &results, &report, matrix),
),
];
let mut artifacts: Vec<(String, String)> = artifacts
.into_iter()
.map(|(name, body)| (name.to_owned(), body))
.collect();
for named in veredictum::badges::badges(
&report,
matrix,
veredictum::badges::CaseCounts::of(&results),
) {
match serde_json::to_string_pretty(&named.badge) {
Ok(mut json) => {
json.push('\n');
artifacts.push((named.file, json));
}
Err(e) => {
eprintln!("cannot serialize the {} badge: {e}", named.file);
return ExitCode::from(2);
}
}
}
for (name, body) in &artifacts {
let path = out.join(name);
if let Err(e) = std::fs::write(&path, body) {
eprintln!("cannot write {}: {e}", path.display());
return ExitCode::from(2);
}
println!("wrote {}", path.display());
}
for finding in &report.review {
println!("static-review: {}", finding.message);
}
println!(
"{} capability verdict(s), {} of {} cases driven, {} review finding(s)",
report.capabilities.len(),
report.coverage.driven,
report.coverage.selected,
report.review.len(),
);
if report.review.is_empty() {
ExitCode::SUCCESS
} else {
ExitCode::from(1)
}
}
fn conformance_assets_command(
root: &std::path::Path,
results_path: &std::path::Path,
verdicts_path: &std::path::Path,
out: &std::path::Path,
suffix: &str,
) -> ExitCode {
#[derive(serde::Deserialize)]
struct VerdictSlice {
capabilities: Vec<(String, veredictum::verdict::Evidence)>,
}
let loaded = match load_root(root) {
Ok(loaded) => loaded,
Err(e) => {
eprintln!("runner defect: {e}");
return ExitCode::from(2);
}
};
let Some((_, matrix)) = &loaded.set.matrix else {
eprintln!("artifact set has no capability matrix");
return ExitCode::from(2);
};
let results: Results =
match load_party_json(results_path, &results_schema(), "results.schema.json") {
Ok(results) => results,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
let verdicts: VerdictSlice = match std::fs::read_to_string(verdicts_path)
.map_err(|e| format!("cannot read {}: {e}", verdicts_path.display()))
.and_then(|text| serde_json::from_str(&text).map_err(|e| format!("verdicts: {e}")))
{
Ok(verdicts) => verdicts,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
if let Err(e) = std::fs::create_dir_all(out) {
eprintln!("cannot create {}: {e}", out.display());
return ExitCode::from(2);
}
let sut_label = format!("{} {}", results.sut.name, results.sut.version);
let chapters = match veredictum::conf_assets::chapter_counts(&results) {
Ok(chapters) => chapters,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
let assets = [
(
format!("conformance-heat-grid{suffix}.svg"),
veredictum::conf_assets::heat_grid_svg(&sut_label, matrix, &verdicts.capabilities),
),
(
format!("conformance-chapter-bars{suffix}.svg"),
veredictum::conf_assets::chapter_bars_svg(&sut_label, &chapters),
),
];
for (name, body) in &assets {
let path = out.join(name);
if let Err(e) = std::fs::write(&path, body) {
eprintln!("cannot write {}: {e}", path.display());
return ExitCode::from(2);
}
println!("wrote {}", path.display());
}
ExitCode::SUCCESS
}
#[expect(clippy::too_many_lines, reason = "one-shot orchestration seam")]
fn perf_assets_command(
root: &std::path::Path,
results_path: &std::path::Path,
out: &std::path::Path,
summary: Option<&std::path::Path>,
stress: Option<&std::path::Path>,
) -> ExitCode {
let loaded = match load_root(root) {
Ok(loaded) => loaded,
Err(e) => {
eprintln!("runner defect: {e}");
return ExitCode::from(2);
}
};
if !loaded.errors.is_empty() {
for e in &loaded.errors {
eprintln!("{e}");
}
return ExitCode::from(2);
}
let results: Results =
match load_party_json(results_path, &results_schema(), "results.schema.json") {
Ok(results) => results,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
if let Err(e) = std::fs::create_dir_all(out) {
eprintln!("cannot create {}: {e}", out.display());
return ExitCode::from(2);
}
let perf_cases: Vec<_> = loaded
.set
.performance
.iter()
.map(|(_, c)| c.clone())
.collect();
let mut files: Vec<(String, String)> = vec![(
"perf-class-ladder.svg".to_owned(),
veredictum::perf_assets::class_ladder_svg(&perf_cases, &results.measurements),
)];
if let Some(stress_path) = stress {
let report: veredictum::stress::StressReport = match std::fs::read_to_string(stress_path)
.map_err(|e| format!("cannot read {}: {e}", stress_path.display()))
.and_then(|text| {
serde_json::from_str(&text).map_err(|e| format!("{}: {e}", stress_path.display()))
}) {
Ok(report) => report,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
match veredictum::perf_assets::stress_curve_svg(&report) {
Ok(svg) => files.push(("perf-stress-curve.svg".to_owned(), svg)),
Err(e) => {
eprintln!("stress curve: {e}");
return ExitCode::from(2);
}
}
}
for measurement in &results.measurements {
match veredictum::perf_assets::latency_percentiles_svg(measurement) {
Ok(svg) => files.push((
format!("perf-latency-class-{}.svg", measurement.class.token()),
svg,
)),
Err(e) => {
eprintln!("{}: {e}", measurement.case);
return ExitCode::from(2);
}
}
if let Some(svg) = veredictum::perf_assets::resources_timeseries_svg(measurement) {
files.push((
format!("perf-resources-class-{}.svg", measurement.class.token()),
svg,
));
}
}
if let Some(svg) = veredictum::perf_assets::disk_growth_svg(&results.measurements) {
files.push(("perf-disk-growth.svg".to_owned(), svg));
}
for (name, body) in &files {
let path = out.join(name);
if let Err(e) = std::fs::write(&path, body) {
eprintln!("cannot write {}: {e}", path.display());
return ExitCode::from(2);
}
println!("wrote {}", path.display());
}
if let Some(summary_path) = summary {
let body =
match veredictum::perf_assets::summary_markdown(&perf_cases, &results.measurements) {
Ok(body) => body,
Err(e) => {
eprintln!("summary: {e}");
return ExitCode::from(2);
}
};
if let Some(parent) = summary_path.parent()
&& let Err(e) = std::fs::create_dir_all(parent)
{
eprintln!("cannot create {}: {e}", parent.display());
return ExitCode::from(2);
}
if let Err(e) = std::fs::write(summary_path, body) {
eprintln!("cannot write {}: {e}", summary_path.display());
return ExitCode::from(2);
}
println!("wrote {}", summary_path.display());
}
ExitCode::SUCCESS
}
fn scale_opt_xml(loaded: &veredictum::artifacts::Loaded) -> Result<String, String> {
let corpus_dir = loaded
.set
.corpus_dir
.as_deref()
.ok_or_else(|| "artifact set has no corpus directory".to_owned())?;
let key =
veredictum::ids::CorpusKey::parse("cnf.opt.blood_pressure").map_err(|e| e.to_string())?;
let source = loaded
.set
.corpus
.as_ref()
.and_then(|(_, m)| m.get(&key))
.and_then(|entry| entry.source.clone())
.ok_or_else(|| "corpus manifest has no cnf.opt.blood_pressure fixture".to_owned())?;
std::fs::read_to_string(corpus_dir.join(&source))
.map_err(|e| format!("cannot read OPT fixture {source}: {e}"))
}
fn journey_context(
loaded: &veredictum::artifacts::Loaded,
) -> Result<
(
veredictum::perf::JourneyCatalogue,
veredictum::perf_run::pack::JourneyPack,
),
String,
> {
let catalogue = loaded
.set
.journeys
.as_ref()
.map(|(_, catalogue)| catalogue.clone())
.ok_or_else(|| "artifact set has no vocab/journey_catalogue.yaml".to_owned())?;
let corpus_dir = loaded
.set
.corpus_dir
.as_deref()
.ok_or_else(|| "artifact set has no corpus directory".to_owned())?;
let manifest = loaded
.set
.corpus
.as_ref()
.map(|(_, manifest)| manifest)
.ok_or_else(|| "artifact set has no corpus manifest".to_owned())?;
let pack = veredictum::perf_run::pack::JourneyPack::load(corpus_dir, manifest, &catalogue)?;
Ok((catalogue, pack))
}
#[derive(Debug, Clone, Copy)]
enum SeedStage {
BeforeScale,
AfterScale,
AfterWard,
}
fn seed_corpus(
client: &veredictum::perf_run::client::PerfClient,
corpus_key: &str,
opt_xml: &str,
journey_pack: &veredictum::perf_run::pack::JourneyPack,
seed_workers: usize,
progress: &(dyn Fn(String) + Sync),
stage: &mut dyn FnMut(SeedStage),
) -> Result<veredictum::perf_run::corpus::SeededCorpus, String> {
use veredictum::perf_run::corpus;
let (ehrs, versions) = corpus::scale_shape(corpus_key)?;
stage(SeedStage::BeforeScale);
let mut seeded = corpus::seed_scale_ladder(
client,
corpus_key,
opt_xml,
ehrs,
versions,
seed_workers,
progress,
)
.map_err(|e| format!("seeding failed: {e}"))?;
stage(SeedStage::AfterScale);
corpus::seed_ward(client, &mut seeded, journey_pack, seed_workers, progress)
.map_err(|e| format!("ward seeding failed: {e}"))?;
stage(SeedStage::AfterWard);
Ok(seeded)
}
#[expect(
clippy::too_many_arguments,
clippy::too_many_lines,
reason = "one-shot orchestration seam"
)]
fn stress_command(
root: &std::path::Path,
ixit_path: &std::path::Path,
out: &std::path::Path,
corpus_class: &str,
seed_workers: usize,
step_secs: u64,
bisections: u32,
max_rate: f64,
) -> ExitCode {
use veredictum::perf::PerfClass;
use veredictum::perf_run;
let class = match PerfClass::parse(corpus_class) {
Ok(class) => class,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
let loaded = match load_root(root) {
Ok(loaded) => loaded,
Err(e) => {
eprintln!("runner defect: {e}");
return ExitCode::from(2);
}
};
if !loaded.errors.is_empty() {
for e in &loaded.errors {
eprintln!("{e}");
}
return ExitCode::from(2);
}
let mut ixit: veredictum::ixit::Ixit = match std::fs::read_to_string(ixit_path)
.map_err(|e| format!("cannot read {}: {e}", ixit_path.display()))
.and_then(|text| serde_json::from_str(&text).map_err(|e| format!("ixit: {e}")))
{
Ok(ixit) => ixit,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
ixit.rebase_paths(ixit_path.parent().unwrap_or(std::path::Path::new(".")));
let (principals, environment) = match perf_run::window::measured_run_context(&ixit) {
Ok(context) => context,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
let client = principals.primary().clone();
let Some((_, case)) = loaded
.set
.performance
.iter()
.find(|(_, c)| c.class == class)
else {
eprintln!("no performance case of class {corpus_class} in the catalogue");
return ExitCode::from(2);
};
let opt_xml = match scale_opt_xml(&loaded) {
Ok(xml) => xml,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
let (catalogue, journey_pack) = match journey_context(&loaded) {
Ok(context) => context,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
let progress = |message: String| eprintln!("[stress] {message}");
let corpus = match seed_corpus(
&client,
case.corpus.as_str(),
&opt_xml,
&journey_pack,
seed_workers,
&progress,
&mut |_| {},
) {
Ok(corpus) => corpus,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
let options = veredictum::stress::StressOptions {
step_hold_s: step_secs.max(10),
bisections,
max_rate,
..veredictum::stress::StressOptions::default()
};
let workload = perf_run::schedule::JourneyWorkload {
catalogue: &catalogue,
shares: &case.workload.journeys,
pack: &journey_pack,
curve: veredictum::perf::ArrivalCurve::Uniform,
principals: &principals,
};
let report = match veredictum::stress::run_stress(
&principals,
&corpus,
&workload,
environment,
ixit.containers.as_ref(),
&options,
&progress,
) {
Ok(report) => report,
Err(e) => {
eprintln!("stress run failed: {e}");
return ExitCode::from(2);
}
};
if perf_run::rate_limited_observed() {
eprintln!("{}", perf_run::rate_limited_refusal("stress"));
return ExitCode::from(2);
}
match serde_json::to_string_pretty(&report) {
Ok(mut text) => {
text.push('\n');
if let Some(parent) = out.parent()
&& let Err(e) = std::fs::create_dir_all(parent)
{
eprintln!("cannot create {}: {e}", parent.display());
return ExitCode::from(2);
}
if let Err(e) = std::fs::write(out, text) {
eprintln!("cannot write {}: {e}", out.display());
return ExitCode::from(2);
}
}
Err(e) => {
eprintln!("serialize: {e}");
return ExitCode::from(2);
}
}
println!("{}", report.remark);
println!(
"wrote {} ({} steps, max sustainable {:.1}/s)",
out.display(),
report.steps.len(),
report.max_sustainable_throughput_per_s
);
ExitCode::SUCCESS
}
fn stress_compare_command(
left: &std::path::Path,
left_label: &str,
right: &std::path::Path,
right_label: &str,
out: &std::path::Path,
) -> ExitCode {
let read = |path: &std::path::Path| -> Result<veredictum::stress::StressReport, String> {
std::fs::read_to_string(path)
.map_err(|e| format!("cannot read {}: {e}", path.display()))
.and_then(|text| {
serde_json::from_str(&text).map_err(|e| format!("{}: {e}", path.display()))
})
};
let (left_report, right_report) = match (read(left), read(right)) {
(Ok(a), Ok(b)) => (a, b),
(Err(e), _) | (_, Err(e)) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
let svg = match veredictum::perf_assets::stress_compare_svg(
(left_label, &left_report),
(right_label, &right_report),
) {
Ok(svg) => svg,
Err(e) => {
eprintln!("stress compare: {e}");
return ExitCode::from(2);
}
};
if let Some(parent) = out.parent()
&& let Err(e) = std::fs::create_dir_all(parent)
{
eprintln!("cannot create {}: {e}", parent.display());
return ExitCode::from(2);
}
if let Err(e) = std::fs::write(out, svg) {
eprintln!("cannot write {}: {e}", out.display());
return ExitCode::from(2);
}
println!("wrote {}", out.display());
ExitCode::SUCCESS
}
#[expect(clippy::too_many_lines, reason = "one-shot orchestration seam")]
fn probe_command(
root: &std::path::Path,
ixit_path: &std::path::Path,
out: &std::path::Path,
corpus_class: &str,
seed_workers: usize,
requests: u32,
) -> ExitCode {
use veredictum::perf::PerfClass;
use veredictum::perf_run;
let class = match PerfClass::parse(corpus_class) {
Ok(class) => class,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
let loaded = match load_root(root) {
Ok(loaded) => loaded,
Err(e) => {
eprintln!("runner defect: {e}");
return ExitCode::from(2);
}
};
if !loaded.errors.is_empty() {
for e in &loaded.errors {
eprintln!("{e}");
}
return ExitCode::from(2);
}
let mut ixit: veredictum::ixit::Ixit = match std::fs::read_to_string(ixit_path)
.map_err(|e| format!("cannot read {}: {e}", ixit_path.display()))
.and_then(|text| serde_json::from_str(&text).map_err(|e| format!("ixit: {e}")))
{
Ok(ixit) => ixit,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
ixit.rebase_paths(ixit_path.parent().unwrap_or(std::path::Path::new(".")));
let (principals, environment) = match perf_run::window::measured_run_context(&ixit) {
Ok(context) => context,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
let client = principals.primary().clone();
let Some((_, case)) = loaded
.set
.performance
.iter()
.find(|(_, c)| c.class == class)
else {
eprintln!("no performance case of class {corpus_class} in the catalogue");
return ExitCode::from(2);
};
let opt_xml = match scale_opt_xml(&loaded) {
Ok(xml) => xml,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
let (_, journey_pack) = match journey_context(&loaded) {
Ok(context) => context,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
let progress = |message: String| eprintln!("[probe] {message}");
let corpus = match seed_corpus(
&client,
case.corpus.as_str(),
&opt_xml,
&journey_pack,
seed_workers,
&progress,
&mut |_| {},
) {
Ok(corpus) => corpus,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
let options = veredictum::probe::ProbeOptions { requests };
let report = match veredictum::probe::run_probe(
&client,
&corpus,
environment,
ixit.containers.as_ref(),
&options,
&progress,
) {
Ok(report) => report,
Err(e) => {
eprintln!("probe run failed: {e}");
return ExitCode::from(2);
}
};
match serde_json::to_string_pretty(&report) {
Ok(mut text) => {
text.push('\n');
if let Some(parent) = out.parent()
&& let Err(e) = std::fs::create_dir_all(parent)
{
eprintln!("cannot create {}: {e}", parent.display());
return ExitCode::from(2);
}
if let Err(e) = std::fs::write(out, text) {
eprintln!("cannot write {}: {e}", out.display());
return ExitCode::from(2);
}
}
Err(e) => {
eprintln!("serialize: {e}");
return ExitCode::from(2);
}
}
println!("wrote {} ({} probes)", out.display(), report.probes.len());
ExitCode::SUCCESS
}
#[expect(clippy::too_many_lines, reason = "one-shot orchestration seam")]
fn perf_command(
root: &std::path::Path,
ixit_path: &std::path::Path,
results_path: &std::path::Path,
class_token: &str,
seed_workers: usize,
hours: u64,
) -> ExitCode {
use veredictum::perf::PerfClass;
use veredictum::perf_run;
if ![1, 2, 4, 6, 8, 12].contains(&hours) {
eprintln!("--hours must be one of 1 | 2 | 4 | 6 | 8 | 12 (got {hours})");
return ExitCode::from(2);
}
let class = match PerfClass::parse(class_token) {
Ok(class) => class,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
let loaded = match load_root(root) {
Ok(loaded) => loaded,
Err(e) => {
eprintln!("runner defect: {e}");
return ExitCode::from(2);
}
};
if !loaded.errors.is_empty() {
for e in &loaded.errors {
eprintln!("{e}");
}
return ExitCode::from(2);
}
let mut ixit: veredictum::ixit::Ixit = match std::fs::read_to_string(ixit_path)
.map_err(|e| format!("cannot read {}: {e}", ixit_path.display()))
.and_then(|text| serde_json::from_str(&text).map_err(|e| format!("ixit: {e}")))
{
Ok(ixit) => ixit,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
ixit.rebase_paths(ixit_path.parent().unwrap_or(std::path::Path::new(".")));
let (principals, environment) = match perf_run::window::measured_run_context(&ixit) {
Ok(context) => context,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
let client = principals.primary().clone();
let selected: Vec<_> = loaded
.set
.performance
.iter()
.filter(|(_, c)| c.class == class)
.collect();
if selected.is_empty() {
eprintln!("no performance case of class {class_token} in the catalogue");
return ExitCode::from(2);
}
let opt_xml = match scale_opt_xml(&loaded) {
Ok(xml) => xml,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
let (catalogue, journey_pack) = match journey_context(&loaded) {
Ok(context) => context,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
let progress = |message: String| eprintln!("[perf] {message}");
let containers = ixit.containers.clone();
if containers.is_none() {
progress("resources: not sampled (ixit declares no `containers` block)".to_owned());
}
let mut earned_all = true;
for (path, case) in selected {
println!(
"case {} (class {class_token}) from {}",
case.id,
path.display()
);
let mut disk = veredictum::perf::DiskAnchors {
before_scale_seed_bytes: None,
after_scale_seed_bytes: None,
after_ward_seed_bytes: None,
after_window_bytes: None,
seed_compositions: perf_run::corpus::scale_shape(case.corpus.as_str())
.ok()
.and_then(|(ehrs, versions)| u64::try_from(ehrs.saturating_mul(versions)).ok()),
};
let probe_volume = |label: &str| -> Option<u64> {
let db = &containers.as_ref()?.db;
match perf_run::resources::db_volume_bytes(db) {
Ok(bytes) => {
progress(format!("disk anchor {label}: {bytes} bytes"));
Some(bytes)
}
Err(e) => {
progress(format!("disk anchor {label} unavailable: {e}"));
None
}
}
};
let corpus = match seed_corpus(
&client,
case.corpus.as_str(),
&opt_xml,
&journey_pack,
seed_workers,
&progress,
&mut |milestone| match milestone {
SeedStage::BeforeScale => {
disk.before_scale_seed_bytes = probe_volume("before scale seed");
}
SeedStage::AfterScale => {
disk.after_scale_seed_bytes = probe_volume("after scale seed");
}
SeedStage::AfterWard => {
disk.after_ward_seed_bytes = probe_volume("after preflight + ward seed");
}
},
) {
Ok(corpus) => corpus,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
if let Some(c) = &containers {
progress(
"settling maintenance before the measured window (vacuumdb --analyze)".to_owned(),
);
if let Err(e) = perf_run::resources::settle_maintenance(&c.db) {
progress(format!("maintenance not settled: {e}"));
}
}
let warmup_s = case.workload.warmup.0;
let duration_s = case.workload.duration.0.max(hours.saturating_mul(3600));
let sampler = containers
.as_ref()
.map(|c| perf_run::resources::ResourceSampler::start(c, warmup_s, duration_s));
let mut measurement = match perf_run::window::drive_case(
case,
&principals,
&corpus,
&journey_pack,
&catalogue,
environment,
warmup_s,
duration_s,
&progress,
) {
Ok(measurement) => measurement,
Err(e) => {
eprintln!("measured run failed: {e}");
return ExitCode::from(2);
}
};
if let Some(sampler) = sampler {
let (series, notes) = sampler.stop();
for note in notes {
progress(note);
}
disk.after_window_bytes = probe_volume("after measured window");
let sampled_any = series.iter().any(|s| !s.samples.is_empty());
let anchored_any = disk.before_scale_seed_bytes.is_some()
|| disk.after_scale_seed_bytes.is_some()
|| disk.after_ward_seed_bytes.is_some()
|| disk.after_window_bytes.is_some();
if sampled_any || anchored_any {
measurement.resources = Some(veredictum::perf::ResourcesRecord {
sample_interval_s: perf_run::resources::SAMPLE_INTERVAL.as_secs(),
containers: series,
disk: Some(disk),
});
} else {
progress(
"resources: not sampled (container runtime unreachable for the whole run)"
.to_owned(),
);
}
}
for op in &measurement.operations {
println!(
" {}: {} requests, {} errors, p50 {:.1}ms p90 {:.1}ms p99 {:.1}ms",
op.operation,
op.requests,
op.errors,
op.latency_ms_p50,
op.latency_ms_p90,
op.latency_ms_p99
);
}
println!(" {}", veredictum::perf::verdict_evidence(&measurement));
if measurement.verdict != veredictum::perf::ClassVerdict::Earned {
earned_all = false;
}
if perf_run::rate_limited_observed() {
eprintln!("{}", perf_run::rate_limited_refusal("perf"));
return ExitCode::from(2);
}
let mut results: Results =
match load_party_json(results_path, &results_schema(), "results.schema.json") {
Ok(results) => results,
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
};
results.measurements.retain(|m| m.case != measurement.case);
results.measurements.retain(|m| {
let known = loaded.set.performance.iter().any(|(_, c)| c.id == m.case);
if !known {
println!(" pruned orphaned measurement for retired case {}", m.case);
}
known
});
results.measurements.push(measurement);
results
.measurements
.sort_by(|a, b| a.case.as_str().cmp(b.case.as_str()));
match serde_json::to_string_pretty(&results) {
Ok(mut text) => {
text.push('\n');
if let Err(e) = std::fs::write(results_path, text) {
eprintln!("cannot write {}: {e}", results_path.display());
return ExitCode::from(2);
}
println!(" measurement merged into {}", results_path.display());
}
Err(e) => {
eprintln!("serialize: {e}");
return ExitCode::from(2);
}
}
}
if earned_all {
ExitCode::SUCCESS
} else {
ExitCode::from(1)
}
}
#[expect(clippy::too_many_lines, reason = "the one-shot orchestration seam")]
fn run_command(
root: &std::path::Path,
ixit_path: &std::path::Path,
out: &std::path::Path,
sut_name: &str,
sut_version: &str,
filter: Option<&str>,
statement_path: Option<&std::path::Path>,
) -> ExitCode {
let loaded = match load_root(root) {
Ok(loaded) => loaded,
Err(e) => {
eprintln!("runner defect: {e}");
return ExitCode::from(2);
}
};
if !loaded.errors.is_empty() {
for e in &loaded.errors {
eprintln!("{e}");
}
return ExitCode::from(2);
}
let ixit_text = match std::fs::read_to_string(ixit_path) {
Ok(text) => text,
Err(e) => {
eprintln!("cannot read {}: {e}", ixit_path.display());
return ExitCode::from(2);
}
};
let mut ixit: veredictum::ixit::Ixit = match serde_json::from_str(&ixit_text) {
Ok(ixit) => ixit,
Err(e) => {
eprintln!("ixit: {e}");
return ExitCode::from(2);
}
};
ixit.rebase_paths(ixit_path.parent().unwrap_or(std::path::Path::new(".")));
let mut set = loaded.set;
if let Some(needle) = filter {
set.cases.retain(|(_, c)| c.id.as_str().contains(needle));
}
let statement: Option<Statement> = match statement_path {
None => None,
Some(path) => match std::fs::read_to_string(path)
.map_err(|e| format!("cannot read {}: {e}", path.display()))
.and_then(|text| serde_json::from_str(&text).map_err(|e| format!("statement: {e}")))
{
Ok(statement) => Some(statement),
Err(e) => {
eprintln!("{e}");
return ExitCode::from(2);
}
},
};
let report = match veredictum::run::execute(&set, &ixit, statement.as_ref()) {
Ok(report) => report,
Err(e) => {
eprintln!("execution defect: {e}");
return ExitCode::from(2);
}
};
let outcomes: Vec<veredictum::party::OutcomeRecord> = report
.records
.iter()
.map(veredictum::party::OutcomeRecord::from)
.collect();
let (passed, failed, errored, na) = outcomes.iter().fold((0, 0, 0, 0), |acc, o| {
use veredictum::party::OutcomeStatus;
match o.status {
OutcomeStatus::Passed => (acc.0 + 1, acc.1, acc.2, acc.3),
OutcomeStatus::Failed => (acc.0, acc.1 + 1, acc.2, acc.3),
OutcomeStatus::Errored => (acc.0, acc.1, acc.2 + 1, acc.3),
_ => (acc.0, acc.1, acc.2, acc.3 + 1),
}
});
let ixit_digest = {
use std::hash::{Hash, Hasher};
let mut hasher = std::collections::hash_map::DefaultHasher::new();
ixit_text.hash(&mut hasher);
format!("{:016x}", hasher.finish())
};
let carried_measurements: Vec<veredictum::perf::Measurement> = {
let prior_path = out.join("results.json");
let prior = match std::fs::read_to_string(&prior_path) {
Ok(text) => match serde_json::from_str::<Results>(&text) {
Ok(prior) => Some(prior),
Err(e) => {
eprintln!(
"runner defect: {} exists but does not parse as results.json ({e}) — \
its measurement records cannot be carried forward",
prior_path.display()
);
return ExitCode::from(2);
}
},
Err(e) if e.kind() == std::io::ErrorKind::NotFound => None,
Err(e) => {
eprintln!(
"runner defect: {} is unreadable ({e})",
prior_path.display()
);
return ExitCode::from(2);
}
};
prior
.filter(|prior| prior.sut.name == sut_name)
.map(|prior| {
if prior.sut.version != sut_version && !prior.measurements.is_empty() {
eprintln!(
"warning: carrying {} measurement record(s) taken at SUT version {} into a run at {sut_version} — re-measure or attest the surface unchanged",
prior.measurements.len(),
prior.sut.version
);
}
prior.measurements
})
.unwrap_or_default()
};
let results = Results {
sut: veredictum::party::Sut {
name: sut_name.to_owned(),
version: sut_version.to_owned(),
},
runner: veredictum::party::Runner {
name: "veredictum".to_owned(),
version: env!("CARGO_PKG_VERSION").to_owned(),
verification_pack_status: veredictum::party::VerificationPackStatus::Passed,
},
schedule_release: "cnf-2.0-w2".to_owned(),
tech_profile: veredictum::party::TechProfile {
its: veredictum::vocab::ItsName::ItsRest,
formats: statement
.as_ref()
.and_then(|s| {
s.tech_profiles
.iter()
.find(|p| p.its == veredictum::vocab::ItsName::ItsRest)
})
.map_or_else(
|| veredictum::vocab::FormatName::ALL.to_vec(),
|p| p.formats.clone(),
),
},
ixit_digest,
restapi_specs_version: report.restapi_specs_version.clone(),
outcomes,
measurements: carried_measurements,
ambiguity_dispositions: Vec::new(),
};
if let Err(errors) = results.check_invariants() {
for e in errors {
eprintln!("results invariant: {e}");
}
return ExitCode::from(2);
}
if let Err(e) = std::fs::create_dir_all(out) {
eprintln!("cannot create {}: {e}", out.display());
return ExitCode::from(2);
}
let results_path = out.join("results.json");
match serde_json::to_string_pretty(&results) {
Ok(mut text) => {
text.push('\n');
if let Err(e) = std::fs::write(&results_path, text) {
eprintln!("cannot write {}: {e}", results_path.display());
return ExitCode::from(2);
}
}
Err(e) => {
eprintln!("serialize: {e}");
return ExitCode::from(2);
}
}
let exceptions_path = out.join("run-exceptions.json");
if let Ok(mut text) = serde_json::to_string_pretty(
&report
.exceptions
.iter()
.map(|(case, e)| serde_json::json!({ "case": case.to_string(), "exception": e }))
.collect::<Vec<_>>(),
) {
text.push('\n');
let _write = std::fs::write(&exceptions_path, text);
}
println!(
"{} case-records: {passed} passed / {failed} failed / {errored} errored / {na} n-a; interpreter coverage {:.1}% ({} exceptions); wrote {}",
report.records.len(),
report.interpreter_coverage() * 100.0,
report.exceptions.len(),
results_path.display()
);
if failed == 0 && errored == 0 {
ExitCode::SUCCESS
} else {
ExitCode::from(1)
}
}