1use std::any::Any;
2use std::num::NonZeroUsize;
3use std::sync::atomic::{AtomicUsize, Ordering};
4use std::sync::{Arc, mpsc};
5use std::thread;
6use std::time::{Duration, Instant};
7
8use thiserror::Error;
9
10use crate::case::CaseDefinition;
11use crate::cases::support::{CaseContext, ExecutionFailure, install_case_panic_hook, with_context};
12use crate::{
13 CaseMetadata, CaseOutcome, CaseRequirement, CaseResult, CommandTemplate, LocalProcessRunner,
14 Runner, SuiteReport,
15};
16
17#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
19#[non_exhaustive]
20pub enum ConformanceProfile {
21 #[default]
23 Full,
24 Functional,
26}
27
28impl ConformanceProfile {
29 pub const fn as_str(self) -> &'static str {
31 match self {
32 Self::Full => "full",
33 Self::Functional => "functional",
34 }
35 }
36
37 fn selects(self, requirement: CaseRequirement) -> bool {
38 self == Self::Full || requirement == CaseRequirement::Functional
39 }
40}
41
42#[derive(Debug)]
44#[non_exhaustive]
45pub enum SuiteEvent<'a> {
46 SuiteStarted {
48 selected_cases: usize,
50 },
51 CaseStarted {
56 index: usize,
58 total: usize,
60 case: CaseMetadata,
62 },
63 CaseFinished {
65 index: usize,
67 total: usize,
69 result: &'a CaseResult,
71 },
72}
73
74#[derive(Debug, Clone, PartialEq, Eq)]
76#[non_exhaustive]
77pub struct RunOptions {
78 pub profile: ConformanceProfile,
80 pub filter: Option<String>,
82 pub invocation_timeout: Duration,
84 pub concurrency: NonZeroUsize,
88}
89
90impl RunOptions {
91 pub const fn new() -> Self {
94 Self {
95 profile: ConformanceProfile::Full,
96 filter: None,
97 invocation_timeout: Duration::from_secs(30),
98 concurrency: NonZeroUsize::MIN,
99 }
100 }
101
102 #[must_use]
104 pub const fn with_profile(mut self, profile: ConformanceProfile) -> Self {
105 self.profile = profile;
106 self
107 }
108
109 #[must_use]
111 pub fn with_filter(mut self, filter: impl Into<String>) -> Self {
112 self.filter = Some(filter.into());
113 self
114 }
115
116 #[must_use]
118 pub const fn with_invocation_timeout(mut self, timeout: Duration) -> Self {
119 self.invocation_timeout = timeout;
120 self
121 }
122
123 #[must_use]
125 pub const fn with_concurrency(mut self, concurrency: NonZeroUsize) -> Self {
126 self.concurrency = concurrency;
127 self
128 }
129}
130
131impl Default for RunOptions {
132 fn default() -> Self {
133 Self::new()
134 }
135}
136
137#[derive(Debug, Clone, Copy, Default)]
139pub struct Suite;
140
141impl Suite {
142 pub const fn current() -> Self {
144 Self
145 }
146
147 pub fn cases(self) -> impl Iterator<Item = CaseMetadata> {
149 crate::cases::definitions().map(|definition| definition.metadata)
150 }
151
152 pub fn run(
161 self,
162 command: &CommandTemplate,
163 options: RunOptions,
164 ) -> Result<SuiteReport, SuiteError> {
165 self.run_observed(command, options, |_| {})
166 }
167
168 pub fn run_observed<F>(
177 self,
178 command: &CommandTemplate,
179 options: RunOptions,
180 observer: F,
181 ) -> Result<SuiteReport, SuiteError>
182 where
183 F: FnMut(SuiteEvent<'_>),
184 {
185 let command = command
186 .resolve()
187 .map_err(|source| SuiteError::ResolveCandidate { source })?;
188 Ok(self.run_with_observer(
189 Arc::new(LocalProcessRunner::new(command)),
190 options,
191 observer,
192 ))
193 }
194
195 pub fn run_with(self, runner: Arc<dyn Runner>, options: RunOptions) -> SuiteReport {
201 self.run_with_observer(runner, options, |_| {})
202 }
203
204 pub fn run_with_observer<F>(
209 self,
210 runner: Arc<dyn Runner>,
211 options: RunOptions,
212 mut observer: F,
213 ) -> SuiteReport
214 where
215 F: FnMut(SuiteEvent<'_>),
216 {
217 install_case_panic_hook();
218 let matching = crate::cases::definitions()
219 .filter(|definition| {
220 options
221 .filter
222 .as_ref()
223 .is_none_or(|filter| definition.metadata.id().contains(filter))
224 })
225 .collect::<Vec<_>>();
226 let omitted_exact_case_count = matching
227 .iter()
228 .filter(|definition| !options.profile.selects(definition.metadata.requirement()))
229 .count();
230 let selected = matching
231 .into_iter()
232 .filter(|definition| options.profile.selects(definition.metadata.requirement()))
233 .collect::<Vec<_>>();
234 let total = selected.len();
235 observer(SuiteEvent::SuiteStarted {
236 selected_cases: total,
237 });
238 let results = if options.concurrency.get() == 1 || total < 2 {
239 run_serial(
240 &selected,
241 &runner,
242 options.invocation_timeout,
243 &mut observer,
244 )
245 } else {
246 run_concurrent(
247 &selected,
248 &runner,
249 options.invocation_timeout,
250 options.concurrency,
251 &mut observer,
252 )
253 };
254
255 SuiteReport {
256 crate_version: env!("CARGO_PKG_VERSION"),
257 specification_version: crate::SPEC_VERSION,
258 host_platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
259 candidate: runner.command().report(),
260 cases: results,
261 profile: options.profile,
262 omitted_exact_case_count,
263 }
264 }
265}
266
267fn run_serial<F>(
268 selected: &[&CaseDefinition],
269 runner: &Arc<dyn Runner>,
270 invocation_timeout: Duration,
271 observer: &mut F,
272) -> Vec<CaseResult>
273where
274 F: FnMut(SuiteEvent<'_>),
275{
276 let total = selected.len();
277 selected
278 .iter()
279 .enumerate()
280 .map(|(offset, definition)| {
281 let index = offset + 1;
282 observer(SuiteEvent::CaseStarted {
283 index,
284 total,
285 case: definition.metadata,
286 });
287 let result = execute_case(definition, runner, invocation_timeout);
288 observer(SuiteEvent::CaseFinished {
289 index,
290 total,
291 result: &result,
292 });
293 result
294 })
295 .collect()
296}
297
298enum WorkerEvent {
299 Started { offset: usize, case: CaseMetadata },
300 Finished { offset: usize, result: CaseResult },
301}
302
303fn run_concurrent<F>(
304 selected: &[&CaseDefinition],
305 runner: &Arc<dyn Runner>,
306 invocation_timeout: Duration,
307 concurrency: NonZeroUsize,
308 observer: &mut F,
309) -> Vec<CaseResult>
310where
311 F: FnMut(SuiteEvent<'_>),
312{
313 let total = selected.len();
314 let worker_count = concurrency.get().min(total);
315 let next = AtomicUsize::new(0);
316 let (sender, receiver) = mpsc::channel();
317 let mut results = Vec::with_capacity(total);
318
319 thread::scope(|scope| {
320 for _ in 0..worker_count {
321 let sender = sender.clone();
322 let runner = Arc::clone(runner);
323 let next = &next;
324 scope.spawn(move || {
325 loop {
326 let offset = next.fetch_add(1, Ordering::Relaxed);
327 let Some(definition) = selected.get(offset).copied() else {
328 break;
329 };
330 if sender
331 .send(WorkerEvent::Started {
332 offset,
333 case: definition.metadata,
334 })
335 .is_err()
336 {
337 break;
338 }
339 let result = execute_case(definition, &runner, invocation_timeout);
340 if sender
341 .send(WorkerEvent::Finished { offset, result })
342 .is_err()
343 {
344 break;
345 }
346 }
347 });
348 }
349 drop(sender);
350
351 for event in receiver {
352 match event {
353 WorkerEvent::Started { offset, case } => observer(SuiteEvent::CaseStarted {
354 index: offset + 1,
355 total,
356 case,
357 }),
358 WorkerEvent::Finished { offset, result } => {
359 observer(SuiteEvent::CaseFinished {
360 index: offset + 1,
361 total,
362 result: &result,
363 });
364 results.push((offset, result));
365 }
366 }
367 }
368 });
369
370 results.sort_unstable_by_key(|(offset, _)| *offset);
371 results.into_iter().map(|(_, result)| result).collect()
372}
373
374fn execute_case(
375 definition: &CaseDefinition,
376 runner: &Arc<dyn Runner>,
377 invocation_timeout: Duration,
378) -> CaseResult {
379 if let Some(reason) = definition.skip_reason {
380 return CaseResult {
381 case: definition.metadata,
382 duration_ms: 0,
383 outcome: CaseOutcome::Skipped {
384 reason: reason.to_owned(),
385 },
386 };
387 }
388
389 let context = Arc::new(CaseContext::new(Arc::clone(runner), invocation_timeout));
390 let started = Instant::now();
391 let Some(run) = definition.run else {
392 return CaseResult {
393 case: definition.metadata,
394 duration_ms: started.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
395 outcome: malformed_definition_outcome(definition.metadata),
396 };
397 };
398 let execution = std::panic::catch_unwind(std::panic::AssertUnwindSafe({
399 let context = Arc::clone(&context);
400 move || with_context(context, run)
401 }));
402 let recorded = context.take_failure();
403 let outcome = match (execution, recorded) {
404 (Ok(()), None) => CaseOutcome::Passed,
405 (_, Some(ExecutionFailure::Skipped(reason))) => CaseOutcome::Skipped { reason },
406 (_, Some(ExecutionFailure::Runner(details))) => CaseOutcome::Error { details },
407 (_, Some(ExecutionFailure::Fixture(details))) => CaseOutcome::Error {
408 details: format!("fixture setup failed: {details}"),
409 },
410 (_, Some(ExecutionFailure::TimedOut(details))) => CaseOutcome::TimedOut { details },
411 (Err(payload), None) if !context.candidate_invoked() => CaseOutcome::Error {
412 details: format!("fixture setup failed: {}", panic_message(payload)),
413 },
414 (Err(payload), None) => CaseOutcome::Failed {
415 details: panic_message(payload),
416 },
417 };
418 CaseResult {
419 case: definition.metadata,
420 duration_ms: started.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
421 outcome,
422 }
423}
424
425fn malformed_definition_outcome(metadata: CaseMetadata) -> CaseOutcome {
426 CaseOutcome::Error {
427 details: format!(
428 "malformed conformance registry: case '{}' has neither a function nor a skip reason",
429 metadata.id()
430 ),
431 }
432}
433
434fn panic_message(payload: Box<dyn Any + Send>) -> String {
435 if let Some(message) = payload.downcast_ref::<String>() {
436 return message.clone();
437 }
438 if let Some(message) = payload.downcast_ref::<&str>() {
439 return (*message).to_owned();
440 }
441 "conformance assertion panicked without a message".to_owned()
442}
443
444#[derive(Debug, Error)]
446#[non_exhaustive]
447pub enum SuiteError {
448 #[error("failed to resolve candidate command: {source}")]
450 ResolveCandidate {
451 #[source]
453 source: std::io::Error,
454 },
455}
456
457#[cfg(test)]
458mod tests {
459 use super::*;
460
461 struct PassingRunner {
462 command: CommandTemplate,
463 }
464
465 impl Runner for PassingRunner {
466 fn command(&self) -> &CommandTemplate {
467 &self.command
468 }
469
470 fn run(
471 &self,
472 _invocation: &crate::Invocation,
473 ) -> Result<crate::InvocationResult, crate::RunnerError> {
474 Ok(crate::InvocationResult::new(
475 crate::Termination::Exited { code: 0 },
476 Vec::new(),
477 Vec::new(),
478 Duration::from_millis(1),
479 ))
480 }
481 }
482
483 #[test]
484 fn malformed_definition_should_be_an_infrastructure_error() {
485 let metadata = CaseMetadata::closure("malformed.case", &["#test"]);
486
487 let outcome = malformed_definition_outcome(metadata);
488
489 assert!(matches!(outcome, CaseOutcome::Error { .. }));
490 }
491
492 #[test]
493 fn fixture_failure_after_candidate_should_be_an_infrastructure_error() {
494 let report = Suite::current().run_with(
495 Arc::new(PassingRunner {
496 command: CommandTemplate::new("test-candidate"),
497 }),
498 RunOptions {
499 filter: Some("test.fixture-failure.after-candidate".to_owned()),
500 ..RunOptions::default()
501 },
502 );
503
504 assert_eq!(report.cases.len(), 1);
505 assert!(matches!(
506 &report.cases[0].outcome,
507 CaseOutcome::Error { details } if details.starts_with("fixture setup failed:")
508 ));
509 }
510}