#![allow(
clippy::disallowed_types,
reason = "dev/verification tooling over JSON artifacts (the catalogue, results, wire \
exchanges), whose shapes belong to the artifacts and the SUT; the carriers \
here are cfg(test)-only, so #[expect] would be unfulfilled in the non-test build"
)]
use serde::Serialize;
use crate::ids::{CapabilityName, CaseId, OptionTag};
use crate::model::capability::CapabilityMatrix;
use crate::model::case::CaseCore;
use crate::model::register::AmbiguityRegister;
use crate::party::{OutcomeStatus, Results, Statement};
use crate::perf::{ClassVerdict, PerfClass, PerformanceCase, class_verdict};
use crate::vocab::{CaseStatus, Disposition, Family, FormatName, Tier};
#[derive(Debug, Clone, Serialize)]
pub struct ReviewFinding {
pub message: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, serde::Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Evidence {
Passed,
Failed,
Inconclusive,
NotEvidenced,
NotClaimed,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, serde::Deserialize)]
pub struct CapabilityTally {
pub passed: usize,
pub failed: usize,
pub inconclusive: usize,
pub unevidenced: usize,
}
impl CapabilityTally {
#[must_use]
pub fn selected(&self) -> usize {
self.passed + self.failed + self.inconclusive + self.unevidenced
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum ProfileVerdict {
Pass,
Fail,
NotClaimed,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum SecBasicVerdict {
Pass,
Fail,
}
#[derive(Debug, Clone, Serialize)]
pub struct PerformanceVerdict {
pub case: CaseId,
pub class: PerfClass,
pub claimed: bool,
pub verdict: ClassVerdict,
pub violations: Vec<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub struct Coverage {
pub selected: usize,
pub driven: usize,
#[serde(default)]
pub passed: usize,
#[serde(default)]
pub failed: usize,
#[serde(default)]
pub inconclusive: usize,
}
#[derive(Debug, Clone, Serialize)]
pub struct VerdictReport {
pub review: Vec<ReviewFinding>,
pub capabilities: Vec<(CapabilityName, Evidence)>,
pub capability_tallies: Vec<(CapabilityName, CapabilityTally)>,
pub profiles: Vec<(Tier, ProfileVerdict)>,
pub security: Option<SecBasicVerdict>,
pub performance: Vec<PerformanceVerdict>,
pub coverage: Coverage,
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum Effective {
Passed,
Failed,
Errored,
NotApplicable,
ExcusedByRegister,
Skipped,
NotDriven,
Deselected,
}
impl Effective {
fn driven(&self) -> bool {
matches!(
self,
Effective::Passed | Effective::Failed | Effective::Errored
)
}
}
struct Selected<'a> {
case: &'a CaseCore,
gating: bool,
effective: Effective,
}
#[must_use]
pub fn compute(
statement: &Statement,
results: &Results,
cases: &[CaseCore],
perf_cases: &[PerformanceCase],
matrix: &CapabilityMatrix,
register: &AmbiguityRegister,
) -> VerdictReport {
let mut review = static_review(statement, cases, matrix, register);
if let Some(claimed) = statement
.tech_profiles
.iter()
.find(|p| p.its == results.tech_profile.its)
{
let mut recorded = results.tech_profile.formats.clone();
let mut declared = claimed.formats.clone();
recorded.sort_unstable_by_key(|f| format!("{f:?}"));
declared.sort_unstable_by_key(|f| format!("{f:?}"));
if recorded != declared {
review.push(ReviewFinding {
message: format!(
"results.tech_profile.formats {:?} diverges from the statement's {:?} \
tech-profile claim {:?} — the run's gating selection does not match the \
certified claim; re-run against the current statement",
results.tech_profile.formats, results.tech_profile.its, claimed.formats
),
});
}
}
if let (Some(served), Some(declared)) = (
results.restapi_specs_version.as_deref(),
statement.spec_versions.its_rest.as_deref(),
) {
let same = match (
semver::Version::parse(served),
semver::Version::parse(declared),
) {
(Ok(observed), Ok(claimed)) => observed == claimed,
_ => served == declared,
};
if !same {
review.push(ReviewFinding {
message: format!(
"the SUT's System OPTIONS manifest advertises restapi_specs_version \
{served} but the statement declares spec_versions.its_rest {declared} — \
the declaration and the deployment disagree; fix whichever is wrong \
(the manifest member is optional and is never the source of truth)"
),
});
}
}
let performance = measured_verdicts(statement, results, perf_cases, &mut review);
let selected = select(statement, results, cases, register);
let capabilities: Vec<(CapabilityName, Evidence)> = matrix
.entries()
.iter()
.map(|(name, _)| {
(
name.clone(),
capability_evidence(name, &statement.claims.capabilities, &selected),
)
})
.collect();
let capability_tallies: Vec<(CapabilityName, CapabilityTally)> = matrix
.entries()
.iter()
.map(|(name, _)| (name.clone(), capability_tally(name, &selected)))
.collect();
let profiles = platform_profiles(statement, matrix, &capabilities);
let security = security_verdict(statement, matrix, &capabilities);
let coverage = Coverage {
selected: selected.len(),
driven: selected.iter().filter(|s| s.effective.driven()).count(),
passed: selected
.iter()
.filter(|s| s.effective == Effective::Passed)
.count(),
failed: selected
.iter()
.filter(|s| s.effective == Effective::Failed)
.count(),
inconclusive: selected
.iter()
.filter(|s| s.effective == Effective::Errored)
.count(),
};
VerdictReport {
review,
capabilities,
capability_tallies,
profiles,
security,
performance,
coverage,
}
}
fn measured_verdicts(
statement: &Statement,
results: &Results,
perf_cases: &[PerformanceCase],
review: &mut Vec<ReviewFinding>,
) -> Vec<PerformanceVerdict> {
let claimed_class: Option<PerfClass> = match &statement.performance {
None => None,
Some(claim) => match PerfClass::parse(&claim.class) {
Ok(class) => Some(class),
Err(e) => {
review.push(ReviewFinding {
message: format!("performance claim: {e}"),
});
None
}
},
};
let mut verdicts = Vec::new();
for measurement in &results.measurements {
let Some(case) = perf_cases.iter().find(|c| c.id == measurement.case) else {
review.push(ReviewFinding {
message: format!(
"measurement references performance case {} not in the catalogue",
measurement.case
),
});
continue;
};
match class_verdict(
case,
measurement.offered_load_sustained,
&measurement.operations,
) {
Ok((verdict, violations)) => {
if verdict != measurement.verdict {
review.push(ReviewFinding {
message: format!(
"measurement {}: stored verdict does not re-derive from the \
embedded histograms (stored {:?}, derived {verdict:?})",
measurement.case, measurement.verdict
),
});
}
verdicts.push(PerformanceVerdict {
case: measurement.case.clone(),
class: case.class,
claimed: claimed_class == Some(case.class),
verdict,
violations,
});
}
Err(e) => review.push(ReviewFinding {
message: format!("measurement {}: {e}", measurement.case),
}),
}
}
if let Some(class) = claimed_class
&& !verdicts.iter().any(|v| v.class == class)
{
review.push(ReviewFinding {
message: format!(
"performance class {} is claimed but the results carry no measurement for it",
class.token()
),
});
}
verdicts
}
fn static_review(
statement: &Statement,
cases: &[CaseCore],
matrix: &CapabilityMatrix,
register: &AmbiguityRegister,
) -> Vec<ReviewFinding> {
let mut findings = Vec::new();
let claimed = &statement.claims.capabilities;
for cap in claimed {
if matrix.get(cap).is_none() {
findings.push(ReviewFinding {
message: format!("claimed capability {cap} is not in the capability matrix"),
});
}
}
let claims_profile = |t: Tier| statement.claims.profiles.contains(&t);
if claims_profile(Tier::Core) {
require_tier_caps(&[Tier::Core], "CORE", claimed, matrix, &mut findings);
}
if claims_profile(Tier::Standard) {
if !claims_profile(Tier::Core) {
findings.push(ReviewFinding {
message: "STANDARD is claimed but CORE is not (STANDARD requires CORE)".to_owned(),
});
}
require_tier_caps(
&[Tier::Core, Tier::Standard],
"STANDARD",
claimed,
matrix,
&mut findings,
);
}
if claims_profile(Tier::SecBasic) {
require_tier_caps(
&[Tier::SecBasic],
"SEC-BASIC",
claimed,
matrix,
&mut findings,
);
}
for tier in &statement.claims.profiles {
if *tier == Tier::Options {
continue; }
let represented = claimed
.iter()
.filter_map(|c| matrix.get(c))
.any(|e| e.tier == *tier);
if !represented {
findings.push(ReviewFinding {
message: format!(
"profile {tier:?} is claimed but no claimed capability carries that tier"
),
});
}
}
for (id, entry) in register.entries() {
if entry.disposition != Disposition::OptionSelect {
continue;
}
let sibling_intersects = cases.iter().any(|c| {
c.status == CaseStatus::Active
&& c.option.as_ref().is_some_and(|t| entry.options.contains(t))
&& intersects(&c.capabilities, claimed)
});
if sibling_intersects && !entry.options.iter().any(|o| statement.options.contains(o)) {
findings.push(ReviewFinding {
message: format!(
"option_select {id} governs claimed cases but the ICS declares none of its options {:?}",
entry.options.iter().map(OptionTag::as_str).collect::<Vec<_>>()
),
});
}
}
if let Some(perf) = &statement.performance
&& perf.environment_ref.trim().is_empty()
{
findings.push(ReviewFinding {
message: format!(
"performance class {:?} is claimed without an environment reference",
perf.class
),
});
}
findings
}
fn require_tier_caps(
tiers: &[Tier],
label: &str,
claimed: &[CapabilityName],
matrix: &CapabilityMatrix,
findings: &mut Vec<ReviewFinding>,
) {
for (name, entry) in matrix.entries() {
if entry.required && tiers.contains(&entry.tier) && !claimed.contains(name) {
findings.push(ReviewFinding {
message: format!(
"{label} is claimed but required capability {name} is not claimed"
),
});
}
}
}
fn select<'a>(
statement: &Statement,
results: &Results,
cases: &'a [CaseCore],
register: &AmbiguityRegister,
) -> Vec<Selected<'a>> {
let mut selected = Vec::new();
for case in cases {
if case.status != CaseStatus::Active {
continue;
}
if !intersects(&case.capabilities, &statement.claims.capabilities) {
continue; }
if !applies_satisfied(case, &statement.spec_versions) {
continue; }
let gating = !is_report_only(case, register);
let effective = effective_outcome(case, statement, results);
selected.push(Selected {
case,
gating,
effective,
});
}
selected
}
fn applies_satisfied(case: &CaseCore, versions: &crate::party::SpecVersions) -> bool {
case.applies.satisfied_by(versions)
}
fn is_report_only(case: &CaseCore, register: &AmbiguityRegister) -> bool {
case.ambiguities.iter().any(|id| {
register
.get(id)
.is_some_and(|e| e.disposition == Disposition::ReportOnly)
})
}
fn effective_outcome(case: &CaseCore, statement: &Statement, results: &Results) -> Effective {
if let Some(tag) = &case.option
&& !statement.options.contains(tag)
{
return Effective::Deselected;
}
rollup_results(&case.id, &results.outcomes, &results.tech_profile.formats)
}
fn rollup_results(
case: &CaseId,
outcomes: &[crate::party::OutcomeRecord],
formats: &[FormatName],
) -> Effective {
let relevant: Vec<&crate::party::OutcomeRecord> = outcomes
.iter()
.filter(|o| &o.case == case && format_in_profile(o.format, formats))
.collect();
if relevant.is_empty() {
return Effective::NotDriven;
}
let has = |s: OutcomeStatus| relevant.iter().any(|o| o.status == s);
let all_na = relevant
.iter()
.all(|o| o.status == OutcomeStatus::NotApplicable);
if all_na
&& relevant
.iter()
.all(|o| o.citation.as_deref().is_some_and(|c| c.contains("AMB-")))
{
return Effective::ExcusedByRegister;
}
if has(OutcomeStatus::Failed) {
Effective::Failed
} else if has(OutcomeStatus::Errored) {
Effective::Errored
} else if has(OutcomeStatus::Passed) {
Effective::Passed
} else if relevant
.iter()
.all(|o| o.status == OutcomeStatus::NotApplicable)
{
Effective::NotApplicable
} else if relevant.iter().all(|o| o.status == OutcomeStatus::Skipped) {
Effective::Skipped
} else {
Effective::NotApplicable
}
}
fn format_in_profile(format: Option<FormatName>, formats: &[FormatName]) -> bool {
format.is_none_or(|f| formats.is_empty() || formats.contains(&f))
}
fn capability_evidence(
cap: &CapabilityName,
claimed: &[CapabilityName],
selected: &[Selected<'_>],
) -> Evidence {
if !claimed.contains(cap) {
return Evidence::NotClaimed;
}
let relevant: Vec<&Selected<'_>> = selected
.iter()
.filter(|s| s.gating && s.case.capabilities.contains(cap))
.collect();
if relevant.iter().any(|s| s.effective == Effective::Failed) {
return Evidence::Failed;
}
if relevant.iter().any(|s| s.effective == Effective::Errored) {
return Evidence::Inconclusive;
}
if relevant.iter().any(|s| s.effective == Effective::Passed) {
return Evidence::Passed;
}
Evidence::NotEvidenced
}
fn capability_tally(cap: &CapabilityName, selected: &[Selected<'_>]) -> CapabilityTally {
let mut tally = CapabilityTally::default();
for case in selected
.iter()
.filter(|s| s.gating && s.case.capabilities.contains(cap))
{
match case.effective {
Effective::Passed => tally.passed += 1,
Effective::Failed => tally.failed += 1,
Effective::Errored => tally.inconclusive += 1,
Effective::NotApplicable
| Effective::ExcusedByRegister
| Effective::Skipped
| Effective::NotDriven
| Effective::Deselected => tally.unevidenced += 1,
}
}
tally
}
fn evidence_of(caps: &[(CapabilityName, Evidence)], name: &CapabilityName) -> Option<Evidence> {
caps.iter().find(|(n, _)| n == name).map(|(_, e)| *e)
}
#[must_use]
pub fn tier_members(tier: Tier, matrix: &CapabilityMatrix) -> Vec<CapabilityName> {
let cumulative: &[Tier] = match tier {
Tier::Core => &[Tier::Core],
Tier::Standard => &[Tier::Core, Tier::Standard],
_ => &[],
};
matrix
.entries()
.iter()
.filter(|(_, e)| match tier {
Tier::Core | Tier::Standard => e.required && cumulative.contains(&e.tier),
Tier::Options => e.family == Family::Platform && !e.required && e.tier == Tier::Options,
Tier::SecBasic => e.required && e.family == Family::Security,
Tier::EnterpriseD | Tier::EnterpriseM | Tier::EnterpriseX => false,
})
.map(|(name, _)| name.clone())
.collect()
}
fn required_all_passed(
tiers: &[Tier],
matrix: &CapabilityMatrix,
caps: &[(CapabilityName, Evidence)],
) -> bool {
matrix
.entries()
.iter()
.filter(|(_, e)| e.required && tiers.contains(&e.tier))
.all(|(name, _)| {
matches!(evidence_of(caps, name), Some(Evidence::Passed))
})
}
fn platform_profiles(
statement: &Statement,
matrix: &CapabilityMatrix,
caps: &[(CapabilityName, Evidence)],
) -> Vec<(Tier, ProfileVerdict)> {
let claimed = |t: Tier| statement.claims.profiles.contains(&t);
let mut out = Vec::new();
for tier in [Tier::Core, Tier::Standard, Tier::Options] {
let verdict = if claimed(tier) {
match tier {
Tier::Core => bool_verdict(required_all_passed(&[Tier::Core], matrix, caps)),
Tier::Standard => bool_verdict(required_all_passed(
&[Tier::Core, Tier::Standard],
matrix,
caps,
)),
Tier::Options => bool_verdict(matrix.entries().iter().any(|(name, e)| {
e.family == Family::Platform
&& !e.required
&& evidence_of(caps, name) == Some(Evidence::Passed)
})),
_ => ProfileVerdict::NotClaimed,
}
} else {
ProfileVerdict::NotClaimed
};
out.push((tier, verdict));
}
out
}
fn security_verdict(
statement: &Statement,
matrix: &CapabilityMatrix,
caps: &[(CapabilityName, Evidence)],
) -> Option<SecBasicVerdict> {
if !statement.claims.profiles.contains(&Tier::SecBasic) {
return None;
}
let pass = matrix
.entries()
.iter()
.filter(|(_, e)| e.required && e.family == Family::Security)
.all(|(name, _)| evidence_of(caps, name) == Some(Evidence::Passed));
Some(if pass {
SecBasicVerdict::Pass
} else {
SecBasicVerdict::Fail
})
}
fn bool_verdict(pass: bool) -> ProfileVerdict {
if pass {
ProfileVerdict::Pass
} else {
ProfileVerdict::Fail
}
}
fn intersects(a: &[CapabilityName], b: &[CapabilityName]) -> bool {
a.iter().any(|x| b.contains(x))
}
#[cfg(test)]
mod tests {
use super::*;
fn matrix() -> CapabilityMatrix {
serde_json::from_value(serde_json::json!({
"EhrOperations": { "family": "Platform", "tier": "CORE", "required": true },
"AqlBasic": { "family": "Platform", "tier": "STANDARD", "required": true },
"SimplifiedFormats": { "family": "Platform", "tier": "OPTIONS", "required": false },
"AuthenticatedAccess": { "family": "Security", "tier": "SEC-BASIC", "required": true }
}))
.unwrap()
}
fn register() -> AmbiguityRegister {
serde_json::from_value(serde_json::json!({
"AMB-5": { "ambiguity": "a", "source": "s", "handling": "h",
"disposition": "report_only" },
"AMB-39": { "ambiguity": "a", "source": "s", "handling": "h",
"disposition": "option_select",
"options": ["sf-deprecated-types-supported", "sf-deprecated-types-unsupported"] }
}))
.unwrap()
}
fn case(json: serde_json::Value) -> CaseCore {
serde_json::from_value(json).unwrap()
}
fn functional_case(id: &str, caps: &[&str], profiles: &[&str]) -> CaseCore {
case(serde_json::json!({
"id": id, "kind": "functional", "component": "EHR",
"sm_operation": "I_EHR_SERVICE.create_ehr",
"capabilities": caps, "profiles": profiles,
"test_purpose": "t", "description": "d",
"spec_refs": ["CNF platform_test_schedule master06 §x"],
"flow": [ { "step": 1, "call": "create_ehr", "expect": "created" } ]
}))
}
fn statement(caps: &[&str], profiles: &[&str], options: &[&str]) -> Statement {
serde_json::from_value(serde_json::json!({
"product": { "name": "p", "version": "1", "vendor": "v", "identifier": "i" },
"schedule_release": "CNF-2.0",
"spec_versions": { "rm": "1.2.0", "its_rest": "1.1.0" },
"claims": { "capabilities": caps, "profiles": profiles },
"tech_profiles": [ { "its": "its-rest", "formats": ["canonical-json"] } ],
"options": options
}))
.unwrap()
}
fn results(outcomes: serde_json::Value) -> Results {
let mut value = serde_json::json!({
"sut": { "name": "s", "version": "1" },
"runner": { "name": "veredictum", "version": "0", "verification_pack_status": "passed" },
"schedule_release": "CNF-2.0",
"tech_profile": { "its": "its-rest", "formats": ["canonical-json"] },
"ixit_digest": "d"
});
if let serde_json::Value::Object(map) = &mut value {
map.insert("outcomes".to_owned(), outcomes);
}
serde_json::from_value(value).unwrap()
}
#[test]
fn manifest_advertised_its_rest_divergence_is_a_review_finding() {
let cases = vec![functional_case(
"I_EHR_SERVICE.create_ehr-main",
&["EhrOperations"],
&["CORE"],
)];
let st = statement(&["EhrOperations"], &["CORE"], &[]);
let outcomes = serde_json::json!([
{ "case": "I_EHR_SERVICE.create_ehr-main", "format": "canonical-json",
"status": "passed", "rows_driven": 1, "rows_total": 1 }
]);
let mut diverging = results(outcomes.clone());
diverging.restapi_specs_version = Some("1.0.3".to_owned());
let report = compute(&st, &diverging, &cases, &[], &matrix(), ®ister());
assert_eq!(
report
.review
.iter()
.filter(|f| f.message.contains("restapi_specs_version"))
.count(),
1,
"review: {:?}",
report.review
);
let mut agreeing = results(outcomes.clone());
agreeing.restapi_specs_version = Some("1.1.0".to_owned());
let report = compute(&st, &agreeing, &cases, &[], &matrix(), ®ister());
assert!(
report
.review
.iter()
.all(|f| !f.message.contains("restapi_specs_version")),
"review: {:?}",
report.review
);
let report = compute(&st, &results(outcomes), &cases, &[], &matrix(), ®ister());
assert!(
report
.review
.iter()
.all(|f| !f.message.contains("restapi_specs_version"))
);
}
#[test]
fn passing_core_claim_passes() {
let cases = vec![functional_case(
"I_EHR_SERVICE.create_ehr-main",
&["EhrOperations"],
&["CORE"],
)];
let st = statement(&["EhrOperations"], &["CORE"], &[]);
let rs = results(serde_json::json!([
{ "case": "I_EHR_SERVICE.create_ehr-main", "format": "canonical-json",
"status": "passed", "rows_driven": 1, "rows_total": 1 }
]));
let report = compute(&st, &rs, &cases, &[], &matrix(), ®ister());
assert!(report.review.is_empty(), "{:?}", report.review);
assert_eq!(
evidence_of(
&report.capabilities,
&CapabilityName::parse("EhrOperations").unwrap()
),
Some(Evidence::Passed)
);
assert_eq!(
report
.profiles
.iter()
.find(|(t, _)| *t == Tier::Core)
.map(|(_, v)| *v),
Some(ProfileVerdict::Pass)
);
assert_eq!(report.coverage.selected, 1);
assert_eq!(report.coverage.driven, 1);
}
#[test]
fn failing_case_marks_capability_failed() {
let cases = vec![functional_case(
"I_EHR_SERVICE.create_ehr-main",
&["EhrOperations"],
&["CORE"],
)];
let st = statement(&["EhrOperations"], &["CORE"], &[]);
let rs = results(serde_json::json!([
{ "case": "I_EHR_SERVICE.create_ehr-main", "format": "canonical-json",
"status": "failed", "rows_driven": 1, "rows_total": 1, "failing_step": 1 }
]));
let report = compute(&st, &rs, &cases, &[], &matrix(), ®ister());
assert_eq!(
evidence_of(
&report.capabilities,
&CapabilityName::parse("EhrOperations").unwrap()
),
Some(Evidence::Failed)
);
assert_eq!(
report
.profiles
.iter()
.find(|(t, _)| *t == Tier::Core)
.map(|(_, v)| *v),
Some(ProfileVerdict::Fail)
);
}
#[test]
fn diverging_recorded_tech_profile_is_a_review_finding() {
let cases = vec![functional_case(
"I_EHR_SERVICE.create_ehr-main",
&["EhrOperations"],
&["CORE"],
)];
let mut st = statement(&["EhrOperations"], &["CORE"], &[]);
st.tech_profiles[0].formats = vec![FormatName::CanonicalJson, FormatName::CanonicalXml];
let rs = results(serde_json::json!([
{ "case": "I_EHR_SERVICE.create_ehr-main", "format": "canonical-json",
"status": "passed", "rows_driven": 1, "rows_total": 1 }
]));
let report = compute(&st, &rs, &cases, &[], &matrix(), ®ister());
assert!(
report
.review
.iter()
.any(|f| f.message.contains("diverges from the statement's")),
"expected the tech-profile divergence review finding, got {:?}",
report.review
);
st.tech_profiles[0].formats = vec![FormatName::CanonicalJson];
let report = compute(&st, &rs, &cases, &[], &matrix(), ®ister());
assert!(
!report
.review
.iter()
.any(|f| f.message.contains("diverges from the statement's")),
"{:?}",
report.review
);
}
#[test]
fn option_deselection_yields_no_gating_evidence() {
let mut c = functional_case(
"SF-FLAT-deprecated_unsupported",
&["SimplifiedFormats"],
&["OPTIONS"],
);
c.option = Some(OptionTag::parse("sf-deprecated-types-unsupported").unwrap());
c.ambiguities = vec![crate::ids::AmbiguityId::parse("AMB-39").unwrap()];
let cases = vec![c];
let st = statement(
&["SimplifiedFormats"],
&["OPTIONS"],
&["sf-deprecated-types-supported"],
);
let rs = results(serde_json::json!([
{ "case": "SF-FLAT-deprecated_unsupported", "format": "canonical-json",
"status": "passed", "rows_driven": 1, "rows_total": 1 }
]));
let report = compute(&st, &rs, &cases, &[], &matrix(), ®ister());
assert_eq!(
evidence_of(
&report.capabilities,
&CapabilityName::parse("SimplifiedFormats").unwrap()
),
Some(Evidence::NotEvidenced)
);
assert!(report.review.is_empty(), "{:?}", report.review);
}
#[test]
fn report_only_case_never_gates() {
let mut c = functional_case(
"I_EHR_COMPOSITION.persistent-uniqueness",
&["EhrOperations"],
&["CORE"],
);
c.ambiguities = vec![crate::ids::AmbiguityId::parse("AMB-5").unwrap()];
let cases = vec![c];
let st = statement(&["EhrOperations"], &["CORE"], &[]);
let rs = results(serde_json::json!([
{ "case": "I_EHR_COMPOSITION.persistent-uniqueness", "format": "canonical-json",
"status": "failed", "rows_driven": 1, "rows_total": 1, "failing_step": 1 }
]));
let report = compute(&st, &rs, &cases, &[], &matrix(), ®ister());
assert_eq!(
evidence_of(
&report.capabilities,
&CapabilityName::parse("EhrOperations").unwrap()
),
Some(Evidence::NotEvidenced)
);
}
#[test]
fn no_cases_coverage_prints() {
let cases: Vec<CaseCore> = Vec::new();
let st = statement(&["AqlBasic"], &[], &[]);
let rs = results(serde_json::json!([]));
let report = compute(&st, &rs, &cases, &[], &matrix(), ®ister());
assert_eq!(
evidence_of(
&report.capabilities,
&CapabilityName::parse("AqlBasic").unwrap()
),
Some(Evidence::NotEvidenced)
);
assert_eq!(report.coverage.selected, 0);
assert_eq!(report.coverage.driven, 0);
}
#[test]
fn security_verdict_is_optional_and_computed() {
let cases = vec![functional_case(
"I_EHR_SERVICE.auth",
&["AuthenticatedAccess"],
&["SEC-BASIC"],
)];
let unclaimed = statement(&["EhrOperations"], &["CORE"], &[]);
let rs = results(serde_json::json!([]));
assert!(
compute(&unclaimed, &rs, &cases, &[], &matrix(), ®ister())
.security
.is_none()
);
let claimed = statement(&["AuthenticatedAccess"], &["SEC-BASIC"], &[]);
let rs = results(serde_json::json!([
{ "case": "I_EHR_SERVICE.auth", "format": "canonical-json",
"status": "passed", "rows_driven": 1, "rows_total": 1 }
]));
assert_eq!(
compute(&claimed, &rs, &cases, &[], &matrix(), ®ister()).security,
Some(SecBasicVerdict::Pass)
);
}
#[test]
fn an_unclaimed_capability_is_not_claimed_never_not_evidenced() {
let cases = vec![
functional_case(
"I_EHR_SERVICE.create_ehr-main",
&["EhrOperations"],
&["CORE"],
),
functional_case("SF-FLAT-main", &["SimplifiedFormats"], &["OPTIONS"]),
];
let st = statement(&["EhrOperations", "AqlBasic"], &["CORE"], &[]);
let rs = results(serde_json::json!([
{ "case": "I_EHR_SERVICE.create_ehr-main", "format": "canonical-json",
"status": "passed", "rows_driven": 1, "rows_total": 1 },
{ "case": "SF-FLAT-main", "format": "canonical-json",
"status": "failed", "rows_driven": 1, "rows_total": 1, "failing_step": 1 }
]));
let report = compute(&st, &rs, &cases, &[], &matrix(), ®ister());
assert_eq!(
evidence_of(
&report.capabilities,
&CapabilityName::parse("SimplifiedFormats").unwrap()
),
Some(Evidence::NotClaimed),
"an unclaimed capability's failing rows are out of scope, and the token says so"
);
assert_eq!(
evidence_of(
&report.capabilities,
&CapabilityName::parse("AqlBasic").unwrap()
),
Some(Evidence::NotEvidenced),
"claimed-but-unevidenced keeps its own token"
);
assert_eq!(
evidence_of(
&report.capabilities,
&CapabilityName::parse("EhrOperations").unwrap()
),
Some(Evidence::Passed)
);
let claims_options = statement(&["EhrOperations"], &["CORE", "OPTIONS"], &[]);
let report = compute(&claims_options, &rs, &cases, &[], &matrix(), ®ister());
assert_eq!(
report
.profiles
.iter()
.find(|(t, _)| *t == Tier::Options)
.map(|(_, v)| *v),
Some(ProfileVerdict::Fail)
);
}
#[test]
fn standard_requires_core_static_review() {
let cases: Vec<CaseCore> = Vec::new();
let st = statement(&["AqlBasic"], &["STANDARD"], &[]);
let report = compute(
&st,
&results(serde_json::json!([])),
&cases,
&[],
&matrix(),
®ister(),
);
assert!(
report
.review
.iter()
.any(|f| f.message.contains("STANDARD requires CORE")),
"{:?}",
report.review
);
}
fn messages(report: &VerdictReport) -> Vec<&str> {
report.review.iter().map(|f| f.message.as_str()).collect()
}
#[test]
fn an_unknown_claimed_capability_is_a_review_finding() {
let st = statement(&["NoSuchCapability"], &[], &[]);
let report = compute(
&st,
&results(serde_json::json!([])),
&[],
&[],
&matrix(),
®ister(),
);
assert!(
messages(&report)
.contains(&"claimed capability NoSuchCapability is not in the capability matrix"),
"{:?}",
report.review
);
}
#[test]
fn a_profile_tier_no_claimed_capability_carries_is_a_review_finding() {
let st = statement(&["EhrOperations"], &["CORE", "SEC-BASIC"], &[]);
let report = compute(
&st,
&results(serde_json::json!([])),
&[],
&[],
&matrix(),
®ister(),
);
assert!(
messages(&report).iter().any(|m| m.contains(
"profile SecBasic is claimed but no claimed capability carries that tier"
)),
"{:?}",
report.review
);
}
#[test]
fn an_option_select_over_claimed_cases_needs_a_declared_option() {
let mut governed = functional_case(
"SF-deprecated_types-supported",
&["SimplifiedFormats"],
&["OPTIONS"],
);
governed.option = Some(OptionTag::parse("sf-deprecated-types-supported").unwrap());
let st = statement(&["SimplifiedFormats"], &["OPTIONS"], &[]);
let report = compute(
&st,
&results(serde_json::json!([])),
&[governed],
&[],
&matrix(),
®ister(),
);
assert!(
messages(&report)
.iter()
.any(|m| m.starts_with("option_select AMB-39 governs claimed cases")),
"{:?}",
report.review
);
let declaring = statement(
&["SimplifiedFormats"],
&["OPTIONS"],
&["sf-deprecated-types-supported"],
);
let mut governed = functional_case(
"SF-deprecated_types-supported",
&["SimplifiedFormats"],
&["OPTIONS"],
);
governed.option = Some(OptionTag::parse("sf-deprecated-types-supported").unwrap());
let report = compute(
&declaring,
&results(serde_json::json!([])),
&[governed],
&[],
&matrix(),
®ister(),
);
assert!(
messages(&report)
.iter()
.all(|m| !m.starts_with("option_select")),
"{:?}",
report.review
);
}
fn statement_with_performance(class: &str, environment_ref: &str) -> Statement {
let mut st = statement(&["EhrOperations"], &[], &[]);
st.performance = Some(crate::party::Performance {
class: class.to_owned(),
environment_ref: environment_ref.to_owned(),
});
st
}
#[test]
fn a_malformed_performance_claim_is_reviewed_on_both_of_its_parts() {
let report = compute(
&statement_with_performance("XL", " "),
&results(serde_json::json!([])),
&[],
&[],
&matrix(),
®ister(),
);
assert!(
messages(&report).contains(&"performance claim: unknown performance class \"XL\""),
"{:?}",
report.review
);
assert!(
messages(&report)
.iter()
.any(|m| m.contains("is claimed without an environment reference")),
"{:?}",
report.review
);
}
#[test]
fn a_claimed_class_with_no_measurement_is_a_review_finding() {
let report = compute(
&statement_with_performance("POC", "env-1"),
&results(serde_json::json!([])),
&[],
&[],
&matrix(),
®ister(),
);
assert_eq!(
messages(&report),
["performance class POC is claimed but the results carry no measurement for it"]
);
}
fn measurement(stored_verdict: &str, operation: &str) -> crate::perf::Measurement {
serde_json::from_value(serde_json::json!({
"case": "PERF-hospital_sim-class_POC",
"class": "POC",
"verdict": stored_verdict,
"offered_load_sustained": 2.0,
"duration_s": 3600,
"warmup_s": 300,
"environment": { "hardware_class": "laptop", "cores": 8, "memory_gb": 16,
"storage_class": "nvme", "topology": "single-node" },
"operations": [
{ "operation": operation, "requests": 10, "errors": 0,
"latency_ms_p50": 1.0, "latency_ms_p90": 2.0, "latency_ms_p99": 3.0,
"hdr_v2_base64": "" }
],
"violations": []
}))
.unwrap()
}
fn perf_case(threshold: &serde_json::Value) -> PerformanceCase {
serde_json::from_value(serde_json::json!({
"id": "PERF-hospital_sim-class_POC",
"kind": "performance", "component": "PERFORMANCE",
"description": "d", "test_purpose": "t", "spec_refs": [],
"class": "POC", "corpus": "cnf.scale.10k",
"workload": {
"arrival_rate": "2/s", "warmup": "PT5M", "duration": "PT1H",
"journeys": { "chart_review": "100%" }
},
"thresholds": [threshold]
}))
.unwrap()
}
fn results_with(measurement: crate::perf::Measurement) -> Results {
let mut rs = results(serde_json::json!([]));
rs.measurements = vec![measurement];
rs
}
#[test]
fn a_measurement_without_its_catalogue_case_is_a_review_finding() {
let report = compute(
&statement(&["EhrOperations"], &[], &[]),
&results_with(measurement("earned", "ehr_create")),
&[],
&[],
&matrix(),
®ister(),
);
assert_eq!(
messages(&report),
[
"measurement references performance case PERF-hospital_sim-class_POC not in the catalogue"
]
);
assert!(report.performance.is_empty());
}
#[test]
fn a_stored_verdict_that_does_not_re_derive_is_flagged() {
let case = perf_case(&serde_json::json!({ "metric": "offered_load_sustained", "min": 2 }));
let report = compute(
&statement(&["EhrOperations"], &[], &[]),
&results_with(measurement("not-earned", "ehr_create")),
&[],
std::slice::from_ref(&case),
&matrix(),
®ister(),
);
assert!(
messages(&report).iter().any(|m| m.contains(
"stored verdict does not re-derive from the embedded histograms (stored NotEarned, derived Earned)"
)),
"{:?}",
report.review
);
assert_eq!(report.performance.len(), 1);
let report = compute(
&statement(&["EhrOperations"], &[], &[]),
&results_with(measurement("earned", "ehr_create")),
&[],
std::slice::from_ref(&case),
&matrix(),
®ister(),
);
assert!(report.review.is_empty(), "{:?}", report.review);
}
#[test]
fn a_threshold_over_an_unmeasured_operation_fails_the_re_derivation() {
let case = perf_case(
&serde_json::json!({ "metric": "latency_p99", "operation": "aql_query", "max": 1000 }),
);
let report = compute(
&statement(&["EhrOperations"], &[], &[]),
&results_with(measurement("earned", "ehr_create")),
&[],
&[case],
&matrix(),
®ister(),
);
assert_eq!(
messages(&report),
[
"measurement PERF-hospital_sim-class_POC: threshold references unmeasured operation aql_query"
]
);
assert!(report.performance.is_empty());
}
#[test]
fn a_non_semver_manifest_version_is_compared_literally() {
let st = statement(&["EhrOperations"], &[], &[]);
let mut same = results(serde_json::json!([]));
same.restapi_specs_version = Some("1.1.0-rc.candidate".to_owned());
let mut statement_same = st.clone();
statement_same.spec_versions.its_rest = Some("1.1.0-rc.candidate".to_owned());
let report = compute(&statement_same, &same, &[], &[], &matrix(), ®ister());
assert!(
messages(&report)
.iter()
.all(|m| !m.contains("restapi_specs_version")),
"{:?}",
report.review
);
let mut different = results(serde_json::json!([]));
different.restapi_specs_version = Some("release-candidate".to_owned());
let report = compute(&st, &different, &[], &[], &matrix(), ®ister());
assert!(
messages(&report)
.iter()
.any(|m| m.contains("restapi_specs_version")),
"{:?}",
report.review
);
}
fn one_case_report(status: &str, citation: Option<&str>) -> VerdictReport {
let case = functional_case(
"I_EHR_SERVICE.create_ehr-main",
&["EhrOperations"],
&["CORE"],
);
let mut record = serde_json::json!({
"case": "I_EHR_SERVICE.create_ehr-main", "format": "canonical-json",
"status": status, "rows_driven": 1, "rows_total": 1
});
if let (Some(citation), Some(map)) = (citation, record.as_object_mut()) {
map.insert("citation".to_owned(), serde_json::json!(citation));
}
compute(
&statement(&["EhrOperations"], &["CORE"], &[]),
&results(serde_json::json!([record])),
std::slice::from_ref(&case),
&[],
&matrix(),
®ister(),
)
}
fn evidence_for(report: &VerdictReport, name: &str) -> Evidence {
let wanted = CapabilityName::parse(name).unwrap();
evidence_of(&report.capabilities, &wanted).unwrap()
}
fn tally_for(report: &VerdictReport, name: &str) -> CapabilityTally {
let wanted = CapabilityName::parse(name).unwrap();
report
.capability_tallies
.iter()
.find(|(n, _)| *n == wanted)
.map(|(_, t)| *t)
.unwrap()
}
#[test]
fn an_errored_row_makes_its_capability_inconclusive() {
let report = one_case_report("errored", None);
assert_eq!(
evidence_for(&report, "EhrOperations"),
Evidence::Inconclusive
);
assert_eq!(tally_for(&report, "EhrOperations").inconclusive, 1);
assert_eq!(report.coverage.inconclusive, 1);
}
#[test]
fn a_row_excused_by_the_register_evidences_nothing() {
let report = one_case_report("not_applicable", Some("AMB-32 — no released wire"));
assert_eq!(
evidence_for(&report, "EhrOperations"),
Evidence::NotEvidenced
);
assert_eq!(tally_for(&report, "EhrOperations").unevidenced, 1);
}
#[test]
fn a_not_applicable_or_skipped_row_evidences_nothing() {
for (status, citation) in [
("not_applicable", "ITS-REST master05 §Directory — no route"),
("skipped", "operator selection"),
] {
let report = one_case_report(status, Some(citation));
assert_eq!(
evidence_for(&report, "EhrOperations"),
Evidence::NotEvidenced,
"{status}"
);
assert_eq!(
tally_for(&report, "EhrOperations").unevidenced,
1,
"{status}"
);
}
}
#[test]
fn sec_basic_fails_without_passing_security_evidence() {
let st = statement(&["AuthenticatedAccess"], &["SEC-BASIC"], &[]);
let report = compute(
&st,
&results(serde_json::json!([])),
&[],
&[],
&matrix(),
®ister(),
);
assert_eq!(report.security, Some(SecBasicVerdict::Fail));
}
#[test]
fn the_enterprise_tiers_have_no_member_set() {
let matrix = matrix();
for tier in [Tier::EnterpriseD, Tier::EnterpriseM, Tier::EnterpriseX] {
assert!(tier_members(tier, &matrix).is_empty(), "{tier:?}");
}
assert_eq!(
tier_members(Tier::SecBasic, &matrix),
vec![CapabilityName::parse("AuthenticatedAccess").unwrap()]
);
}
}