use std::any::Any;
use std::sync::Arc;
use std::time::{Duration, Instant};
use thiserror::Error;
use crate::cases::support::{CaseContext, ExecutionFailure, install_case_panic_hook, with_context};
use crate::{
CaseMetadata, CaseOutcome, CaseRequirement, CaseResult, CommandTemplate, LocalProcessRunner,
Runner, SuiteReport,
};
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
#[non_exhaustive]
pub enum ConformanceProfile {
#[default]
Full,
Functional,
}
impl ConformanceProfile {
pub const fn as_str(self) -> &'static str {
match self {
Self::Full => "full",
Self::Functional => "functional",
}
}
fn selects(self, requirement: CaseRequirement) -> bool {
self == Self::Full || requirement == CaseRequirement::Functional
}
}
#[derive(Debug)]
#[non_exhaustive]
pub enum SuiteEvent<'a> {
SuiteStarted {
selected_cases: usize,
},
CaseStarted {
index: usize,
total: usize,
case: CaseMetadata,
},
CaseFinished {
index: usize,
total: usize,
result: &'a CaseResult,
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct RunOptions {
pub profile: ConformanceProfile,
pub filter: Option<String>,
pub invocation_timeout: Duration,
}
impl RunOptions {
pub const fn new() -> Self {
Self {
profile: ConformanceProfile::Full,
filter: None,
invocation_timeout: Duration::from_secs(30),
}
}
#[must_use]
pub const fn with_profile(mut self, profile: ConformanceProfile) -> Self {
self.profile = profile;
self
}
#[must_use]
pub fn with_filter(mut self, filter: impl Into<String>) -> Self {
self.filter = Some(filter.into());
self
}
#[must_use]
pub const fn with_invocation_timeout(mut self, timeout: Duration) -> Self {
self.invocation_timeout = timeout;
self
}
}
impl Default for RunOptions {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone, Copy, Default)]
pub struct Suite;
impl Suite {
pub const fn current() -> Self {
Self
}
pub fn cases(self) -> impl Iterator<Item = CaseMetadata> {
crate::cases::definitions().map(|definition| definition.metadata)
}
pub fn run(
self,
command: &CommandTemplate,
options: RunOptions,
) -> Result<SuiteReport, SuiteError> {
self.run_observed(command, options, |_| {})
}
pub fn run_observed<F>(
self,
command: &CommandTemplate,
options: RunOptions,
observer: F,
) -> Result<SuiteReport, SuiteError>
where
F: FnMut(SuiteEvent<'_>),
{
let command = command
.resolve()
.map_err(|source| SuiteError::ResolveCandidate { source })?;
Ok(self.run_with_observer(
Arc::new(LocalProcessRunner::new(command)),
options,
observer,
))
}
pub fn run_with(self, runner: Arc<dyn Runner>, options: RunOptions) -> SuiteReport {
self.run_with_observer(runner, options, |_| {})
}
pub fn run_with_observer<F>(
self,
runner: Arc<dyn Runner>,
options: RunOptions,
mut observer: F,
) -> SuiteReport
where
F: FnMut(SuiteEvent<'_>),
{
install_case_panic_hook();
let matching = crate::cases::definitions()
.filter(|definition| {
options
.filter
.as_ref()
.is_none_or(|filter| definition.metadata.id().contains(filter))
})
.collect::<Vec<_>>();
let omitted_exact_case_count = matching
.iter()
.filter(|definition| !options.profile.selects(definition.metadata.requirement()))
.count();
let selected = matching
.into_iter()
.filter(|definition| options.profile.selects(definition.metadata.requirement()))
.collect::<Vec<_>>();
let total = selected.len();
observer(SuiteEvent::SuiteStarted {
selected_cases: total,
});
let mut results = Vec::new();
for (offset, definition) in selected.into_iter().enumerate() {
let index = offset + 1;
observer(SuiteEvent::CaseStarted {
index,
total,
case: definition.metadata,
});
if let Some(reason) = definition.skip_reason {
let result = CaseResult {
case: definition.metadata,
duration_ms: 0,
outcome: CaseOutcome::Skipped {
reason: reason.to_owned(),
},
};
observer(SuiteEvent::CaseFinished {
index,
total,
result: &result,
});
results.push(result);
continue;
}
let context = Arc::new(CaseContext::new(
Arc::clone(&runner),
options.invocation_timeout,
));
let started = Instant::now();
let Some(run) = definition.run else {
let result = CaseResult {
case: definition.metadata,
duration_ms: started.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
outcome: malformed_definition_outcome(definition.metadata),
};
observer(SuiteEvent::CaseFinished {
index,
total,
result: &result,
});
results.push(result);
continue;
};
let execution = std::panic::catch_unwind(std::panic::AssertUnwindSafe({
let context = Arc::clone(&context);
move || with_context(context, run)
}));
let recorded = context.take_failure();
let outcome = match (execution, recorded) {
(Ok(()), None) => CaseOutcome::Passed,
(_, Some(ExecutionFailure::Skipped(reason))) => CaseOutcome::Skipped { reason },
(_, Some(ExecutionFailure::Runner(details))) => CaseOutcome::Error { details },
(_, Some(ExecutionFailure::Fixture(details))) => CaseOutcome::Error {
details: format!("fixture setup failed: {details}"),
},
(_, Some(ExecutionFailure::TimedOut(details))) => CaseOutcome::TimedOut { details },
(Err(payload), None) if !context.candidate_invoked() => CaseOutcome::Error {
details: format!("fixture setup failed: {}", panic_message(payload)),
},
(Err(payload), None) => CaseOutcome::Failed {
details: panic_message(payload),
},
};
let result = CaseResult {
case: definition.metadata,
duration_ms: started.elapsed().as_millis().try_into().unwrap_or(u64::MAX),
outcome,
};
observer(SuiteEvent::CaseFinished {
index,
total,
result: &result,
});
results.push(result);
}
SuiteReport {
crate_version: env!("CARGO_PKG_VERSION"),
specification_version: crate::SPEC_VERSION,
host_platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
candidate: runner.command().report(),
cases: results,
profile: options.profile,
omitted_exact_case_count,
}
}
}
fn malformed_definition_outcome(metadata: CaseMetadata) -> CaseOutcome {
CaseOutcome::Error {
details: format!(
"malformed conformance registry: case '{}' has neither a function nor a skip reason",
metadata.id()
),
}
}
fn panic_message(payload: Box<dyn Any + Send>) -> String {
if let Some(message) = payload.downcast_ref::<String>() {
return message.clone();
}
if let Some(message) = payload.downcast_ref::<&str>() {
return (*message).to_owned();
}
"conformance assertion panicked without a message".to_owned()
}
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum SuiteError {
#[error("failed to resolve candidate command: {source}")]
ResolveCandidate {
#[source]
source: std::io::Error,
},
}
#[cfg(test)]
mod tests {
use super::*;
struct PassingRunner {
command: CommandTemplate,
}
impl Runner for PassingRunner {
fn command(&self) -> &CommandTemplate {
&self.command
}
fn run(
&self,
_invocation: &crate::Invocation,
) -> Result<crate::InvocationResult, crate::RunnerError> {
Ok(crate::InvocationResult::new(
crate::Termination::Exited { code: 0 },
Vec::new(),
Vec::new(),
Duration::from_millis(1),
))
}
}
#[test]
fn malformed_definition_should_be_an_infrastructure_error() {
let metadata = CaseMetadata::closure("malformed.case", &["#test"]);
let outcome = malformed_definition_outcome(metadata);
assert!(matches!(outcome, CaseOutcome::Error { .. }));
}
#[test]
fn fixture_failure_after_candidate_should_be_an_infrastructure_error() {
let report = Suite::current().run_with(
Arc::new(PassingRunner {
command: CommandTemplate::new("test-candidate"),
}),
RunOptions {
filter: Some("test.fixture-failure.after-candidate".to_owned()),
..RunOptions::default()
},
);
assert_eq!(report.cases.len(), 1);
assert!(matches!(
&report.cases[0].outcome,
CaseOutcome::Error { details } if details.starts_with("fixture setup failed:")
));
}
}