1use std::any::Any;
2use std::sync::Arc;
3use std::time::{Duration, Instant};
4
5use thiserror::Error;
6
7use crate::cases::support::{CaseContext, ExecutionFailure, install_case_panic_hook, with_context};
8use crate::{
9 CaseMetadata, CaseOutcome, CaseRequirement, CaseResult, CommandTemplate, LocalProcessRunner,
10 Runner, SuiteReport,
11};
12
13#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
15#[non_exhaustive]
16pub enum ConformanceProfile {
17 #[default]
19 Full,
20 Functional,
22}
23
24impl ConformanceProfile {
25 pub const fn as_str(self) -> &'static str {
27 match self {
28 Self::Full => "full",
29 Self::Functional => "functional",
30 }
31 }
32
33 fn selects(self, requirement: CaseRequirement) -> bool {
34 self == Self::Full || requirement == CaseRequirement::Functional
35 }
36}
37
38#[derive(Debug)]
40#[non_exhaustive]
41pub enum SuiteEvent<'a> {
42 SuiteStarted {
44 selected_cases: usize,
46 },
47 CaseStarted {
49 index: usize,
51 total: usize,
53 case: CaseMetadata,
55 },
56 CaseFinished {
58 index: usize,
60 total: usize,
62 result: &'a CaseResult,
64 },
65}
66
67#[derive(Debug, Clone, PartialEq, Eq)]
69#[non_exhaustive]
70pub struct RunOptions {
71 pub profile: ConformanceProfile,
73 pub filter: Option<String>,
75 pub invocation_timeout: Duration,
77}
78
79impl RunOptions {
80 pub const fn new() -> Self {
82 Self {
83 profile: ConformanceProfile::Full,
84 filter: None,
85 invocation_timeout: Duration::from_secs(30),
86 }
87 }
88
89 #[must_use]
91 pub const fn with_profile(mut self, profile: ConformanceProfile) -> Self {
92 self.profile = profile;
93 self
94 }
95
96 #[must_use]
98 pub fn with_filter(mut self, filter: impl Into<String>) -> Self {
99 self.filter = Some(filter.into());
100 self
101 }
102
103 #[must_use]
105 pub const fn with_invocation_timeout(mut self, timeout: Duration) -> Self {
106 self.invocation_timeout = timeout;
107 self
108 }
109}
110
111impl Default for RunOptions {
112 fn default() -> Self {
113 Self::new()
114 }
115}
116
117#[derive(Debug, Clone, Copy, Default)]
119pub struct Suite;
120
121impl Suite {
122 pub const fn current() -> Self {
124 Self
125 }
126
127 pub fn cases(self) -> impl Iterator<Item = CaseMetadata> {
129 crate::cases::definitions().map(|definition| definition.metadata)
130 }
131
132 pub fn run(
141 self,
142 command: &CommandTemplate,
143 options: RunOptions,
144 ) -> Result<SuiteReport, SuiteError> {
145 self.run_observed(command, options, |_| {})
146 }
147
148 pub fn run_observed<F>(
157 self,
158 command: &CommandTemplate,
159 options: RunOptions,
160 observer: F,
161 ) -> Result<SuiteReport, SuiteError>
162 where
163 F: FnMut(SuiteEvent<'_>),
164 {
165 let command = command
166 .resolve()
167 .map_err(|source| SuiteError::ResolveCandidate { source })?;
168 Ok(self.run_with_observer(
169 Arc::new(LocalProcessRunner::new(command)),
170 options,
171 observer,
172 ))
173 }
174
175 pub fn run_with(self, runner: Arc<dyn Runner>, options: RunOptions) -> SuiteReport {
181 self.run_with_observer(runner, options, |_| {})
182 }
183
184 pub fn run_with_observer<F>(
189 self,
190 runner: Arc<dyn Runner>,
191 options: RunOptions,
192 mut observer: F,
193 ) -> SuiteReport
194 where
195 F: FnMut(SuiteEvent<'_>),
196 {
197 install_case_panic_hook();
198 let matching = crate::cases::definitions()
199 .filter(|definition| {
200 options
201 .filter
202 .as_ref()
203 .is_none_or(|filter| definition.metadata.id().contains(filter))
204 })
205 .collect::<Vec<_>>();
206 let omitted_exact_case_count = matching
207 .iter()
208 .filter(|definition| !options.profile.selects(definition.metadata.requirement()))
209 .count();
210 let selected = matching
211 .into_iter()
212 .filter(|definition| options.profile.selects(definition.metadata.requirement()))
213 .collect::<Vec<_>>();
214 let total = selected.len();
215 observer(SuiteEvent::SuiteStarted {
216 selected_cases: total,
217 });
218 let mut results = Vec::new();
219 for (offset, definition) in selected.into_iter().enumerate() {
220 let index = offset + 1;
221 observer(SuiteEvent::CaseStarted {
222 index,
223 total,
224 case: definition.metadata,
225 });
226
227 if let Some(reason) = definition.skip_reason {
228 let result = CaseResult {
229 case: definition.metadata,
230 duration_ms: 0,
231 outcome: CaseOutcome::Skipped {
232 reason: reason.to_owned(),
233 },
234 };
235 observer(SuiteEvent::CaseFinished {
236 index,
237 total,
238 result: &result,
239 });
240 results.push(result);
241 continue;
242 }
243
244 let context = Arc::new(CaseContext::new(
245 Arc::clone(&runner),
246 options.invocation_timeout,
247 ));
248 let started = Instant::now();
249 let Some(run) = definition.run else {
250 let result = CaseResult {
251 case: definition.metadata,
252 duration_ms: started.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
253 outcome: malformed_definition_outcome(definition.metadata),
254 };
255 observer(SuiteEvent::CaseFinished {
256 index,
257 total,
258 result: &result,
259 });
260 results.push(result);
261 continue;
262 };
263 let execution = std::panic::catch_unwind(std::panic::AssertUnwindSafe({
264 let context = Arc::clone(&context);
265 move || with_context(context, run)
266 }));
267 let recorded = context.take_failure();
268 let outcome = match (execution, recorded) {
269 (Ok(()), None) => CaseOutcome::Passed,
270 (_, Some(ExecutionFailure::Skipped(reason))) => CaseOutcome::Skipped { reason },
271 (_, Some(ExecutionFailure::Runner(details))) => CaseOutcome::Error { details },
272 (_, Some(ExecutionFailure::Fixture(details))) => CaseOutcome::Error {
273 details: format!("fixture setup failed: {details}"),
274 },
275 (_, Some(ExecutionFailure::TimedOut(details))) => CaseOutcome::TimedOut { details },
276 (Err(payload), None) if !context.candidate_invoked() => CaseOutcome::Error {
277 details: format!("fixture setup failed: {}", panic_message(payload)),
278 },
279 (Err(payload), None) => CaseOutcome::Failed {
280 details: panic_message(payload),
281 },
282 };
283 let result = CaseResult {
284 case: definition.metadata,
285 duration_ms: started.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
286 outcome,
287 };
288 observer(SuiteEvent::CaseFinished {
289 index,
290 total,
291 result: &result,
292 });
293 results.push(result);
294 }
295
296 SuiteReport {
297 crate_version: env!("CARGO_PKG_VERSION"),
298 specification_version: crate::SPEC_VERSION,
299 host_platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
300 candidate: runner.command().report(),
301 cases: results,
302 profile: options.profile,
303 omitted_exact_case_count,
304 }
305 }
306}
307
308fn malformed_definition_outcome(metadata: CaseMetadata) -> CaseOutcome {
309 CaseOutcome::Error {
310 details: format!(
311 "malformed conformance registry: case '{}' has neither a function nor a skip reason",
312 metadata.id()
313 ),
314 }
315}
316
317fn panic_message(payload: Box<dyn Any + Send>) -> String {
318 if let Some(message) = payload.downcast_ref::<String>() {
319 return message.clone();
320 }
321 if let Some(message) = payload.downcast_ref::<&str>() {
322 return (*message).to_owned();
323 }
324 "conformance assertion panicked without a message".to_owned()
325}
326
327#[derive(Debug, Error)]
329#[non_exhaustive]
330pub enum SuiteError {
331 #[error("failed to resolve candidate command: {source}")]
333 ResolveCandidate {
334 #[source]
336 source: std::io::Error,
337 },
338}
339
340#[cfg(test)]
341mod tests {
342 use super::*;
343
344 struct PassingRunner {
345 command: CommandTemplate,
346 }
347
348 impl Runner for PassingRunner {
349 fn command(&self) -> &CommandTemplate {
350 &self.command
351 }
352
353 fn run(
354 &self,
355 _invocation: &crate::Invocation,
356 ) -> Result<crate::InvocationResult, crate::RunnerError> {
357 Ok(crate::InvocationResult::new(
358 crate::Termination::Exited { code: 0 },
359 Vec::new(),
360 Vec::new(),
361 Duration::from_millis(1),
362 ))
363 }
364 }
365
366 #[test]
367 fn malformed_definition_should_be_an_infrastructure_error() {
368 let metadata = CaseMetadata::closure("malformed.case", &["#test"]);
369
370 let outcome = malformed_definition_outcome(metadata);
371
372 assert!(matches!(outcome, CaseOutcome::Error { .. }));
373 }
374
375 #[test]
376 fn fixture_failure_after_candidate_should_be_an_infrastructure_error() {
377 let report = Suite::current().run_with(
378 Arc::new(PassingRunner {
379 command: CommandTemplate::new("test-candidate"),
380 }),
381 RunOptions {
382 filter: Some("test.fixture-failure.after-candidate".to_owned()),
383 ..RunOptions::default()
384 },
385 );
386
387 assert_eq!(report.cases.len(), 1);
388 assert!(matches!(
389 &report.cases[0].outcome,
390 CaseOutcome::Error { details } if details.starts_with("fixture setup failed:")
391 ));
392 }
393}