trellis_testing/conformance/
runner.rs

1use std::collections::{BTreeMap, BTreeSet};
2use std::fmt;
3
4use super::{ConformanceLevel, ConformanceReport, ConformanceSuite};
5
6/// Result returned by one executable conformance check.
7#[derive(Clone, Debug, Eq, PartialEq)]
8pub enum ConformanceCheckResult {
9    /// The check passed.
10    Passed,
11    /// The check was skipped because the app does not support the hook.
12    Unsupported(String),
13    /// The check ran and failed with scenario/trace/invariant detail.
14    Failed(String),
15}
16
17impl ConformanceCheckResult {
18    /// Creates a passed result.
19    pub const fn passed() -> Self {
20        Self::Passed
21    }
22
23    /// Creates an unsupported result with a reason.
24    pub fn unsupported(reason: impl Into<String>) -> Self {
25        Self::Unsupported(reason.into())
26    }
27
28    /// Creates a failed result with diagnostic detail.
29    pub fn failed(detail: impl Into<String>) -> Self {
30        Self::Failed(detail.into())
31    }
32}
33
34/// Executed conformance check summary.
35#[derive(Clone, Debug, Eq, PartialEq)]
36pub struct ConformanceCheckReport {
37    /// Conformance level exercised by this check.
38    pub level: ConformanceLevel,
39    /// Stable invariant or scenario name.
40    pub invariant: String,
41    /// Result returned by the check.
42    pub result: ConformanceCheckResult,
43}
44
45/// Failure returned by an executable conformance suite.
46#[derive(Clone, Debug, Eq, PartialEq)]
47pub struct ConformanceFailure {
48    /// Conformance level that failed.
49    pub level: ConformanceLevel,
50    /// Stable invariant or scenario name that failed.
51    pub invariant: String,
52    /// Scenario, trace, or assertion detail from the failing check.
53    pub detail: String,
54}
55
56impl fmt::Display for ConformanceFailure {
57    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
58        write!(
59            f,
60            "{:?} conformance failed for {}: {}",
61            self.level, self.invariant, self.detail
62        )
63    }
64}
65
66impl std::error::Error for ConformanceFailure {}
67
68/// Creates an empty executable conformance runner.
69pub fn conformance() -> ConformanceRunner {
70    ConformanceSuite::new().runner()
71}
72
73/// Executable opt-in conformance suite for application graphs.
74pub struct ConformanceRunner {
75    suite: ConformanceSuite,
76    checks: Vec<ConformanceCheck>,
77    unsupported: BTreeMap<ConformanceLevel, Vec<String>>,
78}
79
80impl ConformanceRunner {
81    /// Creates a runner from a required-level suite.
82    pub fn new(suite: ConformanceSuite) -> Self {
83        Self {
84            suite,
85            checks: Vec::new(),
86            unsupported: BTreeMap::new(),
87        }
88    }
89
90    /// Requires a level even if no check has been registered yet.
91    pub fn require(mut self, level: ConformanceLevel) -> Self {
92        self.suite = self.suite.require(level);
93        self
94    }
95
96    /// Registers an executable conformance check.
97    pub fn check(
98        mut self,
99        level: ConformanceLevel,
100        invariant: impl Into<String>,
101        run: impl FnMut() -> ConformanceCheckResult + 'static,
102    ) -> Self {
103        self.suite = self.suite.require(level);
104        self.checks.push(ConformanceCheck {
105            level,
106            invariant: invariant.into(),
107            run: Box::new(run),
108        });
109        self
110    }
111
112    /// Marks a level unsupported with an explicit reason.
113    pub fn unsupported(mut self, level: ConformanceLevel, reason: impl Into<String>) -> Self {
114        self.suite = self.suite.require(level);
115        self.unsupported
116            .entry(level)
117            .or_default()
118            .push(reason.into());
119        self
120    }
121
122    /// Runs all checks and returns supported/unsupported level reporting.
123    pub fn run(mut self) -> Result<ConformanceReport, ConformanceFailure> {
124        let mut report = ConformanceReport::new();
125        let mut seen = BTreeSet::new();
126        let mut unsupported = self.unsupported;
127
128        for check in &mut self.checks {
129            seen.insert(check.level);
130            let result = (check.run)();
131            match &result {
132                ConformanceCheckResult::Passed => {}
133                ConformanceCheckResult::Unsupported(reason) => {
134                    unsupported
135                        .entry(check.level)
136                        .or_default()
137                        .push(format!("{}: {reason}", check.invariant));
138                }
139                ConformanceCheckResult::Failed(detail) => {
140                    return Err(ConformanceFailure {
141                        level: check.level,
142                        invariant: check.invariant.clone(),
143                        detail: detail.clone(),
144                    });
145                }
146            }
147            report = report.record_check(ConformanceCheckReport {
148                level: check.level,
149                invariant: check.invariant.clone(),
150                result,
151            });
152        }
153
154        for level in self.suite.required_levels() {
155            if let Some(reasons) = unsupported.remove(level) {
156                for reason in reasons {
157                    report = report.unsupported_with_reason(*level, reason);
158                }
159            } else if seen.contains(level) {
160                report = report.support(*level);
161            } else {
162                report = report.unsupported_with_reason(
163                    *level,
164                    "no conformance check registered for required level",
165                );
166            }
167        }
168        Ok(report)
169    }
170}
171
172struct ConformanceCheck {
173    level: ConformanceLevel,
174    invariant: String,
175    run: Box<dyn FnMut() -> ConformanceCheckResult>,
176}
trellis_testing/conformance/runner.rs

trellis_testing/conformance/
runner.rs
