#![allow(
clippy::disallowed_types,
reason = "dev/verification tooling over JSON artifacts (the catalogue, results, wire \
exchanges), whose shapes belong to the artifacts and the SUT; the carriers \
here are cfg(test)-only, so #[expect] would be unfulfilled in the non-test build"
)]
pub mod assertions;
pub mod bodies;
pub mod content_synth;
pub mod driver;
pub mod headers;
pub mod opt_synth;
pub mod outcome;
pub mod player;
pub mod recipes;
pub mod resolve;
pub mod resultset;
pub mod signature;
pub mod state;
pub mod transport;
pub mod versioned;
use crate::ids::CaseId;
use crate::model::case::{CaseCore, FlowStep};
use crate::vocab::{FormatName, OutcomeKind};
use assertions::AssertionOutcome;
use outcome::{Observation, StepJudgement};
use state::VarStore;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RowOutcome {
Passed,
Failed {
step: u32,
reason: String,
},
Errored {
step: u32,
reason: String,
},
NotApplicable {
citation: String,
},
Skipped {
citation: String,
},
}
#[derive(Debug, Clone)]
pub struct CaseRecord {
pub case: CaseId,
pub format: Option<FormatName>,
pub rows: Vec<RowOutcome>,
pub rows_driven: usize,
pub rows_total: usize,
pub advisories: Vec<String>,
}
impl CaseRecord {
#[must_use]
pub fn passed(&self) -> bool {
!self.rows.is_empty()
&& self
.rows
.iter()
.all(|r| matches!(r, RowOutcome::Passed | RowOutcome::NotApplicable { .. }))
}
}
#[derive(Debug)]
pub struct StepObservation {
pub observation: Observation,
pub assertion_failures: Vec<AssertionOutcome>,
pub advisories: Vec<String>,
}
impl StepObservation {
#[must_use]
pub fn transport(message: String) -> Self {
Self {
observation: Observation::Transport(message),
assertion_failures: Vec::new(),
advisories: Vec::new(),
}
}
#[must_use]
pub fn labelled_advisories(&self, row: usize, step: u32) -> Vec<String> {
self.advisories
.iter()
.map(|advisory| format!("row {row} step {step}: {advisory}"))
.collect()
}
}
#[derive(Debug, Default)]
pub struct PostconditionOutcomes {
pub failures: Vec<AssertionOutcome>,
pub advisories: Vec<String>,
}
pub trait StepDriver {
fn perform(
&mut self,
case: &CaseCore,
step: &FlowStep,
expected: OutcomeKind,
row: usize,
vars: &mut VarStore,
) -> Result<StepObservation, String>;
fn provision(
&mut self,
case: &CaseCore,
row: usize,
vars: &mut VarStore,
) -> Result<Provisioned, String>;
fn postconditions(
&mut self,
case: &CaseCore,
row: usize,
vars: &mut VarStore,
) -> Result<PostconditionOutcomes, String>;
fn aggregates(&mut self, case: &CaseCore, all_rows: &[VarStore])
-> Result<Vec<String>, String>;
}
#[derive(Debug, Clone, PartialEq)]
pub enum Provisioned {
Ready,
RowNotApplicable {
citation: String,
},
RowErrored {
reason: String,
},
}
fn expected_kind(case: &CaseCore, step: &FlowStep, row: usize) -> Option<OutcomeKind> {
if let Some(matrix) = case.parameters.as_ref().and_then(|p| p.matrix.as_ref())
&& let Some(col) = matrix.columns.iter().position(|c| c == "expected")
&& let Some(crate::model::case::MatrixCell::Literal(serde_json::Value::String(s))) =
matrix.rows.get(row).and_then(|cells| cells.get(col))
&& let Some(kind) = OutcomeKind::from_token(s)
{
return Some(kind);
}
match step.expect {
crate::model::case::ExpectSpec::Kind(kind) => Some(kind),
crate::model::case::ExpectSpec::FixtureExpected => case
.parameters
.as_ref()
.and_then(|p| p.fixture_set.as_ref())
.and_then(|fixtures| fixtures.get(row))
.map(|f| f.expected),
}
}
#[must_use]
pub fn row_count(case: &CaseCore) -> usize {
case.parameters
.as_ref()
.map_or(1, |p| {
p.matrix
.as_ref()
.map(|m| m.rows.len())
.or_else(|| p.fixture_set.as_ref().map(Vec::len))
.unwrap_or(1)
})
.max(1)
}
fn judged_first(failures: &[AssertionOutcome]) -> Option<&AssertionOutcome> {
failures
.iter()
.find(|failure| matches!(**failure, AssertionOutcome::Mismatch(_)))
.or_else(|| failures.first())
}
fn row_from_assertion(step: u32, failure: &AssertionOutcome) -> RowOutcome {
match failure {
AssertionOutcome::Mismatch(reason) => RowOutcome::Failed {
step,
reason: reason.clone(),
},
AssertionOutcome::Unjudgeable(reason) => RowOutcome::Errored {
step,
reason: reason.clone(),
},
}
}
pub fn run_case<D: StepDriver>(
case: &CaseCore,
format: Option<FormatName>,
driver: &mut D,
) -> Result<CaseRecord, String> {
let total = row_count(case);
let reset_per_row = case
.parameters
.as_ref()
.is_none_or(|p| matches!(p.iteration, crate::vocab::Iteration::ResetPerRow));
let mut rows = Vec::with_capacity(total);
let mut row_states: Vec<VarStore> = Vec::with_capacity(total);
let mut advisories: Vec<String> = Vec::new();
let mut vars = VarStore::default();
for row in 0..total {
if reset_per_row || row == 0 {
vars = VarStore::default();
match driver.provision(case, row, &mut vars)? {
Provisioned::Ready => {}
Provisioned::RowNotApplicable { citation } => {
row_states.push(vars.clone());
rows.push(RowOutcome::NotApplicable { citation });
continue;
}
Provisioned::RowErrored { reason } => {
row_states.push(vars.clone());
rows.push(RowOutcome::Errored { step: 0, reason });
continue;
}
}
}
let mut row_outcome = RowOutcome::Passed;
'steps: for step in &case.flow {
let Some(expected) = expected_kind(case, step, row) else {
row_outcome = RowOutcome::Errored {
step: step.step,
reason: "no expected kind resolvable for this row".to_owned(),
};
break 'steps;
};
let observed = driver.perform(case, step, expected, row, &mut vars)?;
advisories.extend(observed.labelled_advisories(row, step.step));
match outcome::judge(expected, &observed.observation) {
StepJudgement::Continue => {
if let Some(failure) = judged_first(&observed.assertion_failures) {
row_outcome = row_from_assertion(step.step, failure);
break 'steps; }
}
StepJudgement::Failed { expected, observed } => {
row_outcome = RowOutcome::Failed {
step: step.step,
reason: format!(
"expected `{}`, observed `{}`",
expected.token(),
observed.token()
),
};
break 'steps; }
StepJudgement::Errored(reason) => {
row_outcome = RowOutcome::Errored {
step: step.step,
reason,
};
break 'steps;
}
}
}
if matches!(row_outcome, RowOutcome::Passed) {
let postconditions = driver.postconditions(case, row, &mut vars)?;
advisories.extend(
postconditions
.advisories
.iter()
.map(|line| format!("row {row} postconditions: {line}")),
);
if let Some(failure) = judged_first(&postconditions.failures) {
row_outcome = row_from_assertion(0, failure);
}
}
row_states.push(vars.clone());
rows.push(row_outcome);
}
if rows.iter().all(|r| matches!(r, RowOutcome::Passed))
&& let Some(failure) = driver.aggregates(case, &row_states)?.into_iter().next()
&& let Some(last) = rows.last_mut()
{
{
*last = RowOutcome::Failed {
step: 0,
reason: format!("aggregate: {failure}"),
};
}
}
Ok(CaseRecord {
case: case.id.clone(),
format,
rows_driven: rows.len(),
rows_total: total,
rows,
advisories,
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::vocab::OutcomeKind;
struct Scripted {
provisioned: usize,
script: Vec<Observation>,
aggregate_failure: Option<String>,
cursor: usize,
}
impl StepDriver for Scripted {
fn perform(
&mut self,
_case: &CaseCore,
_step: &FlowStep,
_expected: OutcomeKind,
_row: usize,
_vars: &mut VarStore,
) -> Result<StepObservation, String> {
let observation = self.script.get(self.cursor).cloned().unwrap();
self.cursor += 1;
Ok(StepObservation {
observation,
assertion_failures: Vec::new(),
advisories: Vec::new(),
})
}
fn provision(
&mut self,
_c: &CaseCore,
_r: usize,
_v: &mut VarStore,
) -> Result<Provisioned, String> {
self.provisioned += 1;
Ok(Provisioned::Ready)
}
fn postconditions(
&mut self,
_c: &CaseCore,
_r: usize,
_v: &mut VarStore,
) -> Result<PostconditionOutcomes, String> {
Ok(PostconditionOutcomes::default())
}
fn aggregates(&mut self, _c: &CaseCore, _rows: &[VarStore]) -> Result<Vec<String>, String> {
Ok(self.aggregate_failure.clone().into_iter().collect())
}
}
fn two_row_case() -> CaseCore {
serde_json::from_value(serde_json::json!({
"id": "I_EHR_SERVICE.create_ehr-law_test",
"kind": "functional",
"component": "EHR",
"sm_operation": "I_EHR_SERVICE.create_ehr",
"capabilities": ["EhrOperations"],
"profiles": ["CORE"],
"test_purpose": "laws",
"description": "laws",
"spec_refs": ["CNF platform_test_schedule master06 §create_ehr data sets"],
"requires": { "server": "empty" },
"parameters": { "iteration": "reset_per_row",
"matrix": { "columns": ["ehr_id"], "rows": [["absent"], ["provided"]] } },
"flow": [
{ "step": 1, "call": "create_ehr", "expect": "created" },
{ "step": 2, "call": "create_ehr", "expect": "already_exists" }
]
}))
.unwrap()
}
#[test]
fn law_a_reprovisions_per_row_and_law_b_aborts() {
let case = two_row_case();
let mut driver = Scripted {
provisioned: 0,
script: vec![
Observation::Kind(OutcomeKind::Created),
Observation::Kind(OutcomeKind::Created), Observation::Kind(OutcomeKind::Created),
Observation::Kind(OutcomeKind::AlreadyExists),
],
aggregate_failure: None,
cursor: 0,
};
let record = run_case(&case, None, &mut driver).unwrap();
assert_eq!(driver.provisioned, 2); assert!(matches!(record.rows[0], RowOutcome::Failed { step: 2, .. }));
assert!(matches!(record.rows[1], RowOutcome::Passed));
assert!(!record.passed());
}
#[test]
fn law_c_errored_is_not_failed_and_law_e_runs_last() {
let case = two_row_case();
let mut driver = Scripted {
provisioned: 0,
script: vec![
Observation::Kind(OutcomeKind::Created),
Observation::Kind(OutcomeKind::AlreadyExists),
Observation::Kind(OutcomeKind::Created),
Observation::Kind(OutcomeKind::AlreadyExists),
],
aggregate_failure: Some("ehr_id values are not pairwise distinct".to_owned()),
cursor: 0,
};
let record = run_case(&case, None, &mut driver).unwrap();
assert!(matches!(record.rows[1], RowOutcome::Failed { step: 0, .. }));
let mut errored = Scripted {
provisioned: 0,
script: vec![
Observation::Transport("connection refused".into()),
Observation::Kind(OutcomeKind::Created),
Observation::Kind(OutcomeKind::AlreadyExists),
],
aggregate_failure: None,
cursor: 0,
};
let record = run_case(&case, None, &mut errored).unwrap();
assert!(matches!(
record.rows[0],
RowOutcome::Errored { step: 1, .. }
));
assert!(matches!(record.rows[1], RowOutcome::Passed));
}
struct Provisioning {
grounds: Vec<Provisioned>,
assertion_failures: Vec<Vec<String>>,
postconditions: Vec<String>,
performed: usize,
cursor: usize,
}
impl Provisioning {
fn ready(assertion_failures: Vec<Vec<String>>) -> Self {
Self {
grounds: vec![Provisioned::Ready; assertion_failures.len().max(1)],
assertion_failures,
postconditions: Vec::new(),
performed: 0,
cursor: 0,
}
}
}
impl StepDriver for Provisioning {
fn perform(
&mut self,
_case: &CaseCore,
_step: &FlowStep,
expected: OutcomeKind,
_row: usize,
_vars: &mut VarStore,
) -> Result<StepObservation, String> {
let assertion_failures = self
.assertion_failures
.get(self.performed)
.cloned()
.unwrap_or_default()
.into_iter()
.map(AssertionOutcome::Mismatch)
.collect();
self.performed += 1;
Ok(StepObservation {
observation: Observation::Kind(expected),
assertion_failures,
advisories: Vec::new(),
})
}
fn provision(
&mut self,
_c: &CaseCore,
_r: usize,
_v: &mut VarStore,
) -> Result<Provisioned, String> {
let outcome = self
.grounds
.get(self.cursor)
.cloned()
.unwrap_or(Provisioned::Ready);
self.cursor += 1;
Ok(outcome)
}
fn postconditions(
&mut self,
_c: &CaseCore,
_r: usize,
_v: &mut VarStore,
) -> Result<PostconditionOutcomes, String> {
Ok(PostconditionOutcomes {
failures: self
.postconditions
.iter()
.cloned()
.map(AssertionOutcome::Mismatch)
.collect(),
advisories: Vec::new(),
})
}
fn aggregates(&mut self, _c: &CaseCore, _rows: &[VarStore]) -> Result<Vec<String>, String> {
Ok(Vec::new())
}
}
#[test]
fn an_unprovisionable_row_is_excused_or_inconclusive_and_never_driven() {
let case = two_row_case();
let mut driver = Provisioning {
grounds: vec![
Provisioned::RowNotApplicable {
citation: "AMB-99: no such ground on this profile".to_owned(),
},
Provisioned::RowErrored {
reason: "template upload answered 500".to_owned(),
},
],
assertion_failures: Vec::new(),
postconditions: Vec::new(),
performed: 0,
cursor: 0,
};
let record = run_case(&case, None, &mut driver).unwrap();
assert_eq!(
record.rows[0],
RowOutcome::NotApplicable {
citation: "AMB-99: no such ground on this profile".to_owned()
}
);
assert_eq!(
record.rows[1],
RowOutcome::Errored {
step: 0,
reason: "template upload answered 500".to_owned()
}
);
assert_eq!(
driver.performed, 0,
"neither row's steps are driven once provisioning did not succeed"
);
assert!(
!record.passed(),
"an inconclusive row is not a passing case"
);
}
#[test]
fn a_wholly_excused_case_rolls_up_as_passed() {
let case = two_row_case();
let mut driver = Provisioning {
grounds: vec![
Provisioned::RowNotApplicable {
citation: "AMB-99".to_owned(),
};
2
],
assertion_failures: Vec::new(),
postconditions: Vec::new(),
performed: 0,
cursor: 0,
};
let record = run_case(&case, None, &mut driver).unwrap();
assert!(record.passed());
assert_eq!(record.rows_driven, 2);
assert_eq!(record.rows_total, 2);
}
#[test]
fn a_failed_step_assertion_fails_the_row_at_its_own_step() {
let case = two_row_case();
let mut driver = Provisioning::ready(vec![
vec!["body/uid did not match".to_owned()],
Vec::new(),
Vec::new(),
]);
let record = run_case(&case, None, &mut driver).unwrap();
assert_eq!(
record.rows[0],
RowOutcome::Failed {
step: 1,
reason: "body/uid did not match".to_owned()
}
);
assert!(matches!(record.rows[1], RowOutcome::Passed));
assert_eq!(driver.performed, 3, "row 0 aborted after its first step");
let mut clean = Provisioning::ready(vec![Vec::new(); 4]);
let record = run_case(&case, None, &mut clean).unwrap();
assert!(record.passed());
assert_eq!(clean.performed, 4, "both steps of both rows drove");
}
#[test]
fn row_postconditions_fail_the_row_at_step_zero() {
let case = two_row_case();
let mut driver = Provisioning::ready(vec![Vec::new(); 4]);
driver.postconditions = vec!["the read-back is not equivalent".to_owned()];
let record = run_case(&case, None, &mut driver).unwrap();
for row in &record.rows {
assert_eq!(
row,
&RowOutcome::Failed {
step: 0,
reason: "the read-back is not equivalent".to_owned()
}
);
}
}
#[test]
fn the_reserved_expected_column_overrides_the_flow_expectation() {
let case: CaseCore = serde_json::from_value(serde_json::json!({
"id": "I_EHR_SERVICE.create_ehr-per_row_expectation",
"kind": "functional", "component": "EHR",
"sm_operation": "I_EHR_SERVICE.create_ehr",
"capabilities": ["EhrOperations"], "profiles": ["CORE"],
"test_purpose": "per-row expectation", "description": "per-row expectation",
"spec_refs": ["CNF platform_test_schedule master06 §create_ehr data sets"],
"parameters": { "iteration": "reset_per_row", "matrix": {
"columns": ["ehr_id", "expected"],
"rows": [["absent", "created"], ["provided", "already_exists"]]
} },
"flow": [{ "step": 1, "call": "create_ehr", "expect": "created" }]
}))
.unwrap();
assert_eq!(
expected_kind(&case, &case.flow[0], 0),
Some(OutcomeKind::Created)
);
assert_eq!(
expected_kind(&case, &case.flow[0], 1),
Some(OutcomeKind::AlreadyExists),
"row 1's own column, not the flow's `created`"
);
assert_eq!(
expected_kind(&case, &case.flow[0], 9),
Some(OutcomeKind::Created),
"a row past the matrix inherits the flow expectation"
);
}
#[test]
fn a_fixture_row_carries_its_own_expectation_and_a_missing_one_errors() {
let case: CaseCore = serde_json::from_value(serde_json::json!({
"id": "I_EHR_COMPOSITION.create_composition-fixtures",
"kind": "functional", "component": "EHR_COMPOSITION",
"sm_operation": "I_EHR_COMPOSITION.create_composition",
"capabilities": ["CompositionOperations"], "profiles": ["CORE"],
"test_purpose": "fixture expectations", "description": "fixture expectations",
"spec_refs": ["CNF platform_test_schedule master06 §create_ehr data sets"],
"parameters": { "iteration": "reset_per_row", "fixture_set": [
{ "data_set": "cnf.valid.one", "expected": "created" },
{ "data_set": "cnf.invalid.one", "expected": "validation_failed",
"defect": "empty 1..* list", "spec_ref": "RM data_structures §ITEM_LIST" }
] },
"flow": [{ "step": 1, "call": "create_composition", "expect": "${fixture.expected}" }]
}))
.unwrap();
assert_eq!(row_count(&case), 2, "the fixture set is the row axis");
assert_eq!(
expected_kind(&case, &case.flow[0], 0),
Some(OutcomeKind::Created)
);
assert_eq!(
expected_kind(&case, &case.flow[0], 1),
Some(OutcomeKind::ValidationFailed)
);
assert_eq!(
expected_kind(&case, &case.flow[0], 2),
None,
"a row the fixture set does not carry resolves to no expectation"
);
let mut truncated = case.clone();
if let Some(parameters) = &mut truncated.parameters
&& let Some(fixtures) = &mut parameters.fixture_set
{
fixtures.clear();
}
let mut driver = Provisioning::ready(Vec::new());
let record = run_case(&truncated, None, &mut driver).unwrap();
assert_eq!(record.rows_total, 1, "an empty axis still drives one row");
assert_eq!(
record.rows[0],
RowOutcome::Errored {
step: 1,
reason: "no expected kind resolvable for this row".to_owned()
}
);
assert_eq!(driver.performed, 0);
}
#[test]
fn the_row_axis_is_never_empty_and_an_empty_record_never_passes() {
let case: CaseCore = serde_json::from_value(serde_json::json!({
"id": "I_EHR_SERVICE.create_ehr-single",
"kind": "functional", "component": "EHR",
"sm_operation": "I_EHR_SERVICE.create_ehr",
"capabilities": ["EhrOperations"], "profiles": ["CORE"],
"test_purpose": "single row", "description": "single row",
"spec_refs": ["CNF platform_test_schedule master06 §create_ehr data sets"],
"flow": [{ "step": 1, "call": "create_ehr", "expect": "created" }]
}))
.unwrap();
assert_eq!(row_count(&case), 1);
let empty = CaseRecord {
case: case.id.clone(),
format: None,
rows: Vec::new(),
rows_driven: 0,
rows_total: 1,
advisories: Vec::new(),
};
assert!(
!empty.passed(),
"a case that produced no row proves nothing"
);
}
#[test]
fn a_driver_internal_failure_is_a_transport_observation() {
let observed = StepObservation::transport("connection refused".to_owned());
assert_eq!(
observed.observation,
Observation::Transport("connection refused".to_owned())
);
assert!(observed.assertion_failures.is_empty());
}
}