Skip to main content

treeboot_spec/
suite.rs

1use std::any::Any;
2use std::sync::Arc;
3use std::time::{Duration, Instant};
4
5use thiserror::Error;
6
7use crate::cases::support::{CaseContext, ExecutionFailure, install_case_panic_hook, with_context};
8use crate::{
9    CaseMetadata, CaseOutcome, CaseRequirement, CaseResult, CommandTemplate, LocalProcessRunner,
10    Runner, SuiteReport,
11};
12
13/// Selects the compatibility guarantees exercised by a suite run.
14#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
15#[non_exhaustive]
16pub enum ConformanceProfile {
17    /// Runs functional behavior and exact specification identity cases.
18    #[default]
19    Full,
20    /// Runs portable behavior while allowing specification-version drift.
21    Functional,
22}
23
24impl ConformanceProfile {
25    /// Returns the stable lowercase profile name used by the CLI and reports.
26    pub const fn as_str(self) -> &'static str {
27        match self {
28            Self::Full => "full",
29            Self::Functional => "functional",
30        }
31    }
32
33    fn selects(self, requirement: CaseRequirement) -> bool {
34        self == Self::Full || requirement == CaseRequirement::Functional
35    }
36}
37
38/// A synchronous progress event emitted while a suite runs.
39#[derive(Debug)]
40#[non_exhaustive]
41pub enum SuiteEvent<'a> {
42    /// The final case selection is known and execution is about to begin.
43    SuiteStarted {
44        /// Number of selected cases, including cases that will skip.
45        selected_cases: usize,
46    },
47    /// A selected case is about to run or report its static skip.
48    CaseStarted {
49        /// One-based position in the selected case order.
50        index: usize,
51        /// Total number of selected cases.
52        total: usize,
53        /// Stable metadata for the selected case.
54        case: CaseMetadata,
55    },
56    /// A selected case has produced its final result.
57    CaseFinished {
58        /// One-based position in the selected case order.
59        index: usize,
60        /// Total number of selected cases.
61        total: usize,
62        /// Final case result. The reference remains valid for this callback only.
63        result: &'a CaseResult,
64    },
65}
66
67/// Options controlling one suite execution.
68#[derive(Debug, Clone, PartialEq, Eq)]
69#[non_exhaustive]
70pub struct RunOptions {
71    /// Compatibility profile to execute.
72    pub profile: ConformanceProfile,
73    /// Optional substring matched against stable case identifiers.
74    pub filter: Option<String>,
75    /// Default timeout applied to each candidate invocation.
76    pub invocation_timeout: Duration,
77}
78
79impl RunOptions {
80    /// Returns options for the full profile with no filter and a 30-second timeout.
81    pub const fn new() -> Self {
82        Self {
83            profile: ConformanceProfile::Full,
84            filter: None,
85            invocation_timeout: Duration::from_secs(30),
86        }
87    }
88
89    /// Selects the compatibility profile to execute.
90    #[must_use]
91    pub const fn with_profile(mut self, profile: ConformanceProfile) -> Self {
92        self.profile = profile;
93        self
94    }
95
96    /// Restricts execution to case identifiers containing `filter`.
97    #[must_use]
98    pub fn with_filter(mut self, filter: impl Into<String>) -> Self {
99        self.filter = Some(filter.into());
100        self
101    }
102
103    /// Sets the default timeout applied to each candidate invocation.
104    #[must_use]
105    pub const fn with_invocation_timeout(mut self, timeout: Duration) -> Self {
106        self.invocation_timeout = timeout;
107        self
108    }
109}
110
111impl Default for RunOptions {
112    fn default() -> Self {
113        Self::new()
114    }
115}
116
117/// The case registry for one specification version.
118#[derive(Debug, Clone, Copy, Default)]
119pub struct Suite;
120
121impl Suite {
122    /// Returns the registry for the specification embedded in this crate.
123    pub const fn current() -> Self {
124        Self
125    }
126
127    /// Returns ordered metadata for every conformance case.
128    pub fn cases(self) -> impl Iterator<Item = CaseMetadata> {
129        crate::cases::definitions().map(|definition| definition.metadata)
130    }
131
132    /// Runs the suite using the default local-process runner.
133    ///
134    /// The command is resolved before any case changes its working directory.
135    ///
136    /// # Errors
137    ///
138    /// Returns [`SuiteError`] when a path-like candidate command cannot be
139    /// resolved before execution.
140    pub fn run(
141        self,
142        command: &CommandTemplate,
143        options: RunOptions,
144    ) -> Result<SuiteReport, SuiteError> {
145        self.run_observed(command, options, |_| {})
146    }
147
148    /// Runs the suite through the local-process runner and synchronously reports progress.
149    ///
150    /// The observer runs on the suite thread. It must not retain borrowed event data.
151    ///
152    /// # Errors
153    ///
154    /// Returns [`SuiteError`] when a path-like candidate command cannot be
155    /// resolved before execution.
156    pub fn run_observed<F>(
157        self,
158        command: &CommandTemplate,
159        options: RunOptions,
160        observer: F,
161    ) -> Result<SuiteReport, SuiteError>
162    where
163        F: FnMut(SuiteEvent<'_>),
164    {
165        let command = command
166            .resolve()
167            .map_err(|source| SuiteError::ResolveCandidate { source })?;
168        Ok(self.run_with_observer(
169            Arc::new(LocalProcessRunner::new(command)),
170            options,
171            observer,
172        ))
173    }
174
175    /// Runs the same registry through a custom execution adapter.
176    ///
177    /// The adapter must make the candidate see fixture files and honor each
178    /// invocation's native arguments, working directory, environment changes,
179    /// input mode, and timeout.
180    pub fn run_with(self, runner: Arc<dyn Runner>, options: RunOptions) -> SuiteReport {
181        self.run_with_observer(runner, options, |_| {})
182    }
183
184    /// Runs the registry through a custom adapter and synchronously reports progress.
185    ///
186    /// Selection by case identifier and profile completes before the first event.
187    /// The observer runs on the suite thread and must not retain borrowed event data.
188    pub fn run_with_observer<F>(
189        self,
190        runner: Arc<dyn Runner>,
191        options: RunOptions,
192        mut observer: F,
193    ) -> SuiteReport
194    where
195        F: FnMut(SuiteEvent<'_>),
196    {
197        install_case_panic_hook();
198        let matching = crate::cases::definitions()
199            .filter(|definition| {
200                options
201                    .filter
202                    .as_ref()
203                    .is_none_or(|filter| definition.metadata.id().contains(filter))
204            })
205            .collect::<Vec<_>>();
206        let omitted_exact_case_count = matching
207            .iter()
208            .filter(|definition| !options.profile.selects(definition.metadata.requirement()))
209            .count();
210        let selected = matching
211            .into_iter()
212            .filter(|definition| options.profile.selects(definition.metadata.requirement()))
213            .collect::<Vec<_>>();
214        let total = selected.len();
215        observer(SuiteEvent::SuiteStarted {
216            selected_cases: total,
217        });
218        let mut results = Vec::new();
219        for (offset, definition) in selected.into_iter().enumerate() {
220            let index = offset + 1;
221            observer(SuiteEvent::CaseStarted {
222                index,
223                total,
224                case: definition.metadata,
225            });
226
227            if let Some(reason) = definition.skip_reason {
228                let result = CaseResult {
229                    case: definition.metadata,
230                    duration_ms: 0,
231                    outcome: CaseOutcome::Skipped {
232                        reason: reason.to_owned(),
233                    },
234                };
235                observer(SuiteEvent::CaseFinished {
236                    index,
237                    total,
238                    result: &result,
239                });
240                results.push(result);
241                continue;
242            }
243
244            let context = Arc::new(CaseContext::new(
245                Arc::clone(&runner),
246                options.invocation_timeout,
247            ));
248            let started = Instant::now();
249            let Some(run) = definition.run else {
250                let result = CaseResult {
251                    case: definition.metadata,
252                    duration_ms: started.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
253                    outcome: malformed_definition_outcome(definition.metadata),
254                };
255                observer(SuiteEvent::CaseFinished {
256                    index,
257                    total,
258                    result: &result,
259                });
260                results.push(result);
261                continue;
262            };
263            let execution = std::panic::catch_unwind(std::panic::AssertUnwindSafe({
264                let context = Arc::clone(&context);
265                move || with_context(context, run)
266            }));
267            let recorded = context.take_failure();
268            let outcome = match (execution, recorded) {
269                (Ok(()), None) => CaseOutcome::Passed,
270                (_, Some(ExecutionFailure::Skipped(reason))) => CaseOutcome::Skipped { reason },
271                (_, Some(ExecutionFailure::Runner(details))) => CaseOutcome::Error { details },
272                (_, Some(ExecutionFailure::Fixture(details))) => CaseOutcome::Error {
273                    details: format!("fixture setup failed: {details}"),
274                },
275                (_, Some(ExecutionFailure::TimedOut(details))) => CaseOutcome::TimedOut { details },
276                (Err(payload), None) if !context.candidate_invoked() => CaseOutcome::Error {
277                    details: format!("fixture setup failed: {}", panic_message(payload)),
278                },
279                (Err(payload), None) => CaseOutcome::Failed {
280                    details: panic_message(payload),
281                },
282            };
283            let result = CaseResult {
284                case: definition.metadata,
285                duration_ms: started.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
286                outcome,
287            };
288            observer(SuiteEvent::CaseFinished {
289                index,
290                total,
291                result: &result,
292            });
293            results.push(result);
294        }
295
296        SuiteReport {
297            crate_version: env!("CARGO_PKG_VERSION"),
298            specification_version: crate::SPEC_VERSION,
299            host_platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
300            candidate: runner.command().report(),
301            cases: results,
302            profile: options.profile,
303            omitted_exact_case_count,
304        }
305    }
306}
307
308fn malformed_definition_outcome(metadata: CaseMetadata) -> CaseOutcome {
309    CaseOutcome::Error {
310        details: format!(
311            "malformed conformance registry: case '{}' has neither a function nor a skip reason",
312            metadata.id()
313        ),
314    }
315}
316
317fn panic_message(payload: Box<dyn Any + Send>) -> String {
318    if let Some(message) = payload.downcast_ref::<String>() {
319        return message.clone();
320    }
321    if let Some(message) = payload.downcast_ref::<&str>() {
322        return (*message).to_owned();
323    }
324    "conformance assertion panicked without a message".to_owned()
325}
326
327/// Suite setup failure that occurs before any case executes.
328#[derive(Debug, Error)]
329#[non_exhaustive]
330pub enum SuiteError {
331    /// A path-like candidate could not be resolved before fixture execution.
332    #[error("failed to resolve candidate command: {source}")]
333    ResolveCandidate {
334        /// Candidate path resolution failure.
335        #[source]
336        source: std::io::Error,
337    },
338}
339
340#[cfg(test)]
341mod tests {
342    use super::*;
343
344    struct PassingRunner {
345        command: CommandTemplate,
346    }
347
348    impl Runner for PassingRunner {
349        fn command(&self) -> &CommandTemplate {
350            &self.command
351        }
352
353        fn run(
354            &self,
355            _invocation: &crate::Invocation,
356        ) -> Result<crate::InvocationResult, crate::RunnerError> {
357            Ok(crate::InvocationResult::new(
358                crate::Termination::Exited { code: 0 },
359                Vec::new(),
360                Vec::new(),
361                Duration::from_millis(1),
362            ))
363        }
364    }
365
366    #[test]
367    fn malformed_definition_should_be_an_infrastructure_error() {
368        let metadata = CaseMetadata::closure("malformed.case", &["#test"]);
369
370        let outcome = malformed_definition_outcome(metadata);
371
372        assert!(matches!(outcome, CaseOutcome::Error { .. }));
373    }
374
375    #[test]
376    fn fixture_failure_after_candidate_should_be_an_infrastructure_error() {
377        let report = Suite::current().run_with(
378            Arc::new(PassingRunner {
379                command: CommandTemplate::new("test-candidate"),
380            }),
381            RunOptions {
382                filter: Some("test.fixture-failure.after-candidate".to_owned()),
383                ..RunOptions::default()
384            },
385        );
386
387        assert_eq!(report.cases.len(), 1);
388        assert!(matches!(
389            &report.cases[0].outcome,
390            CaseOutcome::Error { details } if details.starts_with("fixture setup failed:")
391        ));
392    }
393}