use crate::meta::component::ComponentSpec;
use crate::meta::harness::meta_mutation_probe;
use crate::meta::registry::COMPONENT_REGISTRY;
use std::path::Path;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum MetaGauntletFinding {
UnroutableAdversary {
adversary_id: String,
claimed_target: String,
},
EmptyFailureSignature {
adversary_id: String,
component: String,
},
UncoveredMutationClass {
component: String,
class: String,
},
UndetectedBreakage {
component: String,
adversary_id: String,
message: String,
},
}
#[derive(Debug, Clone, Default)]
pub struct MetaGauntletReport {
pub findings: Vec<MetaGauntletFinding>,
pub components_checked: usize,
pub adversaries_checked: usize,
}
impl MetaGauntletReport {
#[must_use]
#[inline]
pub fn is_clean(&self) -> bool {
self.findings.is_empty()
}
}
#[must_use]
#[inline]
pub fn run_meta_gauntlet() -> MetaGauntletReport {
let mut report = MetaGauntletReport::default();
for component in COMPONENT_REGISTRY {
report.components_checked += 1;
check_component(component, &mut report);
}
report
}
fn check_component(component: &ComponentSpec, report: &mut MetaGauntletReport) {
for adversary in component.adversaries {
report.adversaries_checked += 1;
if adversary.targets != component.name {
report
.findings
.push(MetaGauntletFinding::UnroutableAdversary {
adversary_id: adversary.id.to_string(),
claimed_target: adversary.targets.to_string(),
});
}
if adversary.expected_failure_signature.is_empty() {
report
.findings
.push(MetaGauntletFinding::EmptyFailureSignature {
adversary_id: adversary.id.to_string(),
component: component.name.to_string(),
});
}
}
for class in component.meta_mutation_sensitivity {
let covered = component
.adversaries
.iter()
.any(|adv| adv.covers_classes.contains(class));
if !covered {
report
.findings
.push(MetaGauntletFinding::UncoveredMutationClass {
component: component.name.to_string(),
class: format!("{class:?}"),
});
}
}
let crate_root = Path::new(env!("CARGO_MANIFEST_DIR"));
let probe = meta_mutation_probe(
component,
crate_root,
component.test_suite_filter,
component.meta_mutation_sensitivity,
);
let killed: std::collections::HashSet<&str> = probe
.adversaries_killed
.iter()
.map(String::as_str)
.collect();
for adversary in component.adversaries {
if !killed.contains(adversary.id) {
report
.findings
.push(MetaGauntletFinding::UndetectedBreakage {
component: component.name.to_string(),
adversary_id: adversary.id.to_string(),
message: format!(
"Fix: adversary `{}` for component `{}` was not detected by the meta-test suite. \
Add or strengthen a meta-test that produces the expected signature: {:?}",
adversary.id, component.name, adversary.expected_failure_signature
),
});
}
}
}
#[cfg(test)]
mod tests {
use super::{run_meta_gauntlet, MetaGauntletReport};
#[test]
fn gauntlet_runs_without_panic_on_current_registry() {
let report = run_meta_gauntlet();
assert!(report.components_checked >= 1);
assert!(report.adversaries_checked >= 1);
let _ = report.is_clean();
}
#[test]
fn gauntlet_report_is_default_constructible() {
let report = MetaGauntletReport::default();
assert!(report.is_clean());
assert_eq!(report.components_checked, 0);
assert_eq!(report.adversaries_checked, 0);
}
}