Skip to main content

treeboot_spec/
suite.rs

1use std::any::Any;
2use std::num::NonZeroUsize;
3use std::sync::atomic::{AtomicUsize, Ordering};
4use std::sync::{Arc, mpsc};
5use std::thread;
6use std::time::{Duration, Instant};
7
8use thiserror::Error;
9
10use crate::case::CaseDefinition;
11use crate::cases::support::{CaseContext, ExecutionFailure, install_case_panic_hook, with_context};
12use crate::{
13    CaseMetadata, CaseOutcome, CaseRequirement, CaseResult, CommandTemplate, LocalProcessRunner,
14    Runner, SuiteReport,
15};
16
17/// Selects the compatibility guarantees exercised by a suite run.
18#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
19#[non_exhaustive]
20pub enum ConformanceProfile {
21    /// Runs functional behavior and exact specification identity cases.
22    #[default]
23    Full,
24    /// Runs portable behavior while allowing specification-version drift.
25    Functional,
26}
27
28impl ConformanceProfile {
29    /// Returns the stable lowercase profile name used by the CLI and reports.
30    pub const fn as_str(self) -> &'static str {
31        match self {
32            Self::Full => "full",
33            Self::Functional => "functional",
34        }
35    }
36
37    fn selects(self, requirement: CaseRequirement) -> bool {
38        self == Self::Full || requirement == CaseRequirement::Functional
39    }
40}
41
42/// A synchronous progress event emitted while a suite runs.
43#[derive(Debug)]
44#[non_exhaustive]
45pub enum SuiteEvent<'a> {
46    /// The final case selection is known and execution is about to begin.
47    SuiteStarted {
48        /// Number of selected cases, including cases that will skip.
49        selected_cases: usize,
50    },
51    /// A selected case is ready to run or report its static skip.
52    ///
53    /// Events from different cases may interleave when concurrency is greater
54    /// than one.
55    CaseStarted {
56        /// One-based position in the selected case order.
57        index: usize,
58        /// Total number of selected cases.
59        total: usize,
60        /// Stable metadata for the selected case.
61        case: CaseMetadata,
62    },
63    /// A selected case has produced its final result.
64    CaseFinished {
65        /// One-based position in the selected case order.
66        index: usize,
67        /// Total number of selected cases.
68        total: usize,
69        /// Final case result. The reference remains valid for this callback only.
70        result: &'a CaseResult,
71    },
72}
73
74/// Options controlling one suite execution.
75#[derive(Debug, Clone, PartialEq, Eq)]
76#[non_exhaustive]
77pub struct RunOptions {
78    /// Compatibility profile to execute.
79    pub profile: ConformanceProfile,
80    /// Optional substring matched against stable case identifiers.
81    pub filter: Option<String>,
82    /// Default timeout applied to each candidate invocation.
83    pub invocation_timeout: Duration,
84    /// Maximum number of conformance cases executed at once.
85    ///
86    /// The default is one so existing callers retain serial execution.
87    pub concurrency: NonZeroUsize,
88}
89
90impl RunOptions {
91    /// Returns options for the full profile with no filter, a 30-second timeout,
92    /// and serial execution.
93    pub const fn new() -> Self {
94        Self {
95            profile: ConformanceProfile::Full,
96            filter: None,
97            invocation_timeout: Duration::from_secs(30),
98            concurrency: NonZeroUsize::MIN,
99        }
100    }
101
102    /// Selects the compatibility profile to execute.
103    #[must_use]
104    pub const fn with_profile(mut self, profile: ConformanceProfile) -> Self {
105        self.profile = profile;
106        self
107    }
108
109    /// Restricts execution to case identifiers containing `filter`.
110    #[must_use]
111    pub fn with_filter(mut self, filter: impl Into<String>) -> Self {
112        self.filter = Some(filter.into());
113        self
114    }
115
116    /// Sets the default timeout applied to each candidate invocation.
117    #[must_use]
118    pub const fn with_invocation_timeout(mut self, timeout: Duration) -> Self {
119        self.invocation_timeout = timeout;
120        self
121    }
122
123    /// Sets the maximum number of conformance cases executed at once.
124    #[must_use]
125    pub const fn with_concurrency(mut self, concurrency: NonZeroUsize) -> Self {
126        self.concurrency = concurrency;
127        self
128    }
129}
130
131impl Default for RunOptions {
132    fn default() -> Self {
133        Self::new()
134    }
135}
136
137/// The case registry for one specification version.
138#[derive(Debug, Clone, Copy, Default)]
139pub struct Suite;
140
141impl Suite {
142    /// Returns the registry for the specification embedded in this crate.
143    pub const fn current() -> Self {
144        Self
145    }
146
147    /// Returns ordered metadata for every conformance case.
148    pub fn cases(self) -> impl Iterator<Item = CaseMetadata> {
149        crate::cases::definitions().map(|definition| definition.metadata)
150    }
151
152    /// Runs the suite using the default local-process runner.
153    ///
154    /// The command is resolved before any case changes its working directory.
155    ///
156    /// # Errors
157    ///
158    /// Returns [`SuiteError`] when a path-like candidate command cannot be
159    /// resolved before execution.
160    pub fn run(
161        self,
162        command: &CommandTemplate,
163        options: RunOptions,
164    ) -> Result<SuiteReport, SuiteError> {
165        self.run_observed(command, options, |_| {})
166    }
167
168    /// Runs the suite through the local-process runner and synchronously reports progress.
169    ///
170    /// The observer runs on the suite thread. It must not retain borrowed event data.
171    ///
172    /// # Errors
173    ///
174    /// Returns [`SuiteError`] when a path-like candidate command cannot be
175    /// resolved before execution.
176    pub fn run_observed<F>(
177        self,
178        command: &CommandTemplate,
179        options: RunOptions,
180        observer: F,
181    ) -> Result<SuiteReport, SuiteError>
182    where
183        F: FnMut(SuiteEvent<'_>),
184    {
185        let command = command
186            .resolve()
187            .map_err(|source| SuiteError::ResolveCandidate { source })?;
188        Ok(self.run_with_observer(
189            Arc::new(LocalProcessRunner::new(command)),
190            options,
191            observer,
192        ))
193    }
194
195    /// Runs the same registry through a custom execution adapter.
196    ///
197    /// The adapter must make the candidate see fixture files and honor each
198    /// invocation's native arguments, working directory, environment changes,
199    /// input mode, and timeout.
200    pub fn run_with(self, runner: Arc<dyn Runner>, options: RunOptions) -> SuiteReport {
201        self.run_with_observer(runner, options, |_| {})
202    }
203
204    /// Runs the registry through a custom adapter and synchronously reports progress.
205    ///
206    /// Selection by case identifier and profile completes before the first event.
207    /// The observer runs on the suite thread and must not retain borrowed event data.
208    pub fn run_with_observer<F>(
209        self,
210        runner: Arc<dyn Runner>,
211        options: RunOptions,
212        mut observer: F,
213    ) -> SuiteReport
214    where
215        F: FnMut(SuiteEvent<'_>),
216    {
217        install_case_panic_hook();
218        let matching = crate::cases::definitions()
219            .filter(|definition| {
220                options
221                    .filter
222                    .as_ref()
223                    .is_none_or(|filter| definition.metadata.id().contains(filter))
224            })
225            .collect::<Vec<_>>();
226        let omitted_exact_case_count = matching
227            .iter()
228            .filter(|definition| !options.profile.selects(definition.metadata.requirement()))
229            .count();
230        let selected = matching
231            .into_iter()
232            .filter(|definition| options.profile.selects(definition.metadata.requirement()))
233            .collect::<Vec<_>>();
234        let total = selected.len();
235        observer(SuiteEvent::SuiteStarted {
236            selected_cases: total,
237        });
238        let results = if options.concurrency.get() == 1 || total < 2 {
239            run_serial(
240                &selected,
241                &runner,
242                options.invocation_timeout,
243                &mut observer,
244            )
245        } else {
246            run_concurrent(
247                &selected,
248                &runner,
249                options.invocation_timeout,
250                options.concurrency,
251                &mut observer,
252            )
253        };
254
255        SuiteReport {
256            crate_version: env!("CARGO_PKG_VERSION"),
257            specification_version: crate::SPEC_VERSION,
258            host_platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
259            candidate: runner.command().report(),
260            cases: results,
261            profile: options.profile,
262            omitted_exact_case_count,
263        }
264    }
265}
266
267fn run_serial<F>(
268    selected: &[&CaseDefinition],
269    runner: &Arc<dyn Runner>,
270    invocation_timeout: Duration,
271    observer: &mut F,
272) -> Vec<CaseResult>
273where
274    F: FnMut(SuiteEvent<'_>),
275{
276    let total = selected.len();
277    selected
278        .iter()
279        .enumerate()
280        .map(|(offset, definition)| {
281            let index = offset + 1;
282            observer(SuiteEvent::CaseStarted {
283                index,
284                total,
285                case: definition.metadata,
286            });
287            let result = execute_case(definition, runner, invocation_timeout);
288            observer(SuiteEvent::CaseFinished {
289                index,
290                total,
291                result: &result,
292            });
293            result
294        })
295        .collect()
296}
297
298enum WorkerEvent {
299    Started { offset: usize, case: CaseMetadata },
300    Finished { offset: usize, result: CaseResult },
301}
302
303fn run_concurrent<F>(
304    selected: &[&CaseDefinition],
305    runner: &Arc<dyn Runner>,
306    invocation_timeout: Duration,
307    concurrency: NonZeroUsize,
308    observer: &mut F,
309) -> Vec<CaseResult>
310where
311    F: FnMut(SuiteEvent<'_>),
312{
313    let total = selected.len();
314    let worker_count = concurrency.get().min(total);
315    let next = AtomicUsize::new(0);
316    let (sender, receiver) = mpsc::channel();
317    let mut results = Vec::with_capacity(total);
318
319    thread::scope(|scope| {
320        for _ in 0..worker_count {
321            let sender = sender.clone();
322            let runner = Arc::clone(runner);
323            let next = &next;
324            scope.spawn(move || {
325                loop {
326                    let offset = next.fetch_add(1, Ordering::Relaxed);
327                    let Some(definition) = selected.get(offset).copied() else {
328                        break;
329                    };
330                    if sender
331                        .send(WorkerEvent::Started {
332                            offset,
333                            case: definition.metadata,
334                        })
335                        .is_err()
336                    {
337                        break;
338                    }
339                    let result = execute_case(definition, &runner, invocation_timeout);
340                    if sender
341                        .send(WorkerEvent::Finished { offset, result })
342                        .is_err()
343                    {
344                        break;
345                    }
346                }
347            });
348        }
349        drop(sender);
350
351        for event in receiver {
352            match event {
353                WorkerEvent::Started { offset, case } => observer(SuiteEvent::CaseStarted {
354                    index: offset + 1,
355                    total,
356                    case,
357                }),
358                WorkerEvent::Finished { offset, result } => {
359                    observer(SuiteEvent::CaseFinished {
360                        index: offset + 1,
361                        total,
362                        result: &result,
363                    });
364                    results.push((offset, result));
365                }
366            }
367        }
368    });
369
370    results.sort_unstable_by_key(|(offset, _)| *offset);
371    results.into_iter().map(|(_, result)| result).collect()
372}
373
374fn execute_case(
375    definition: &CaseDefinition,
376    runner: &Arc<dyn Runner>,
377    invocation_timeout: Duration,
378) -> CaseResult {
379    if let Some(reason) = definition.skip_reason {
380        return CaseResult {
381            case: definition.metadata,
382            duration_ms: 0,
383            outcome: CaseOutcome::Skipped {
384                reason: reason.to_owned(),
385            },
386        };
387    }
388
389    let context = Arc::new(CaseContext::new(Arc::clone(runner), invocation_timeout));
390    let started = Instant::now();
391    let Some(run) = definition.run else {
392        return CaseResult {
393            case: definition.metadata,
394            duration_ms: started.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
395            outcome: malformed_definition_outcome(definition.metadata),
396        };
397    };
398    let execution = std::panic::catch_unwind(std::panic::AssertUnwindSafe({
399        let context = Arc::clone(&context);
400        move || with_context(context, run)
401    }));
402    let recorded = context.take_failure();
403    let outcome = match (execution, recorded) {
404        (Ok(()), None) => CaseOutcome::Passed,
405        (_, Some(ExecutionFailure::Skipped(reason))) => CaseOutcome::Skipped { reason },
406        (_, Some(ExecutionFailure::Runner(details))) => CaseOutcome::Error { details },
407        (_, Some(ExecutionFailure::Fixture(details))) => CaseOutcome::Error {
408            details: format!("fixture setup failed: {details}"),
409        },
410        (_, Some(ExecutionFailure::TimedOut(details))) => CaseOutcome::TimedOut { details },
411        (Err(payload), None) if !context.candidate_invoked() => CaseOutcome::Error {
412            details: format!("fixture setup failed: {}", panic_message(payload)),
413        },
414        (Err(payload), None) => CaseOutcome::Failed {
415            details: panic_message(payload),
416        },
417    };
418    CaseResult {
419        case: definition.metadata,
420        duration_ms: started.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
421        outcome,
422    }
423}
424
425fn malformed_definition_outcome(metadata: CaseMetadata) -> CaseOutcome {
426    CaseOutcome::Error {
427        details: format!(
428            "malformed conformance registry: case '{}' has neither a function nor a skip reason",
429            metadata.id()
430        ),
431    }
432}
433
434fn panic_message(payload: Box<dyn Any + Send>) -> String {
435    if let Some(message) = payload.downcast_ref::<String>() {
436        return message.clone();
437    }
438    if let Some(message) = payload.downcast_ref::<&str>() {
439        return (*message).to_owned();
440    }
441    "conformance assertion panicked without a message".to_owned()
442}
443
444/// Suite setup failure that occurs before any case executes.
445#[derive(Debug, Error)]
446#[non_exhaustive]
447pub enum SuiteError {
448    /// A path-like candidate could not be resolved before fixture execution.
449    #[error("failed to resolve candidate command: {source}")]
450    ResolveCandidate {
451        /// Candidate path resolution failure.
452        #[source]
453        source: std::io::Error,
454    },
455}
456
457#[cfg(test)]
458mod tests {
459    use super::*;
460
461    struct PassingRunner {
462        command: CommandTemplate,
463    }
464
465    impl Runner for PassingRunner {
466        fn command(&self) -> &CommandTemplate {
467            &self.command
468        }
469
470        fn run(
471            &self,
472            _invocation: &crate::Invocation,
473        ) -> Result<crate::InvocationResult, crate::RunnerError> {
474            Ok(crate::InvocationResult::new(
475                crate::Termination::Exited { code: 0 },
476                Vec::new(),
477                Vec::new(),
478                Duration::from_millis(1),
479            ))
480        }
481    }
482
483    #[test]
484    fn malformed_definition_should_be_an_infrastructure_error() {
485        let metadata = CaseMetadata::closure("malformed.case", &["#test"]);
486
487        let outcome = malformed_definition_outcome(metadata);
488
489        assert!(matches!(outcome, CaseOutcome::Error { .. }));
490    }
491
492    #[test]
493    fn fixture_failure_after_candidate_should_be_an_infrastructure_error() {
494        let report = Suite::current().run_with(
495            Arc::new(PassingRunner {
496                command: CommandTemplate::new("test-candidate"),
497            }),
498            RunOptions {
499                filter: Some("test.fixture-failure.after-candidate".to_owned()),
500                ..RunOptions::default()
501            },
502        );
503
504        assert_eq!(report.cases.len(), 1);
505        assert!(matches!(
506            &report.cases[0].outcome,
507            CaseOutcome::Error { details } if details.starts_with("fixture setup failed:")
508        ));
509    }
510}