use std::fmt;
use std::future::Future;
use std::path::PathBuf;
use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::time::{Duration, Instant};
use tokio::sync::Mutex;
use crate::fixture::FixturePool;
use crate::reporter::EventBus;
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct TestId {
pub file: String,
pub suite: Option<String>,
pub name: String,
pub line: Option<usize>,
}
impl TestId {
#[must_use]
pub fn full_name(&self) -> String {
match &self.suite {
Some(s) => format!("{} > {} > {}", self.file, s, self.name),
None => format!("{} > {}", self.file, self.name),
}
}
#[must_use]
pub fn file_location(&self) -> String {
match self.line {
Some(line) => format!("{}:{}", self.file, line),
None => self.file.clone(),
}
}
}
impl fmt::Display for TestId {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.full_name())
}
}
pub type TestFn =
Arc<dyn Fn(FixturePool) -> Pin<Box<dyn Future<Output = Result<(), TestFailure>> + Send>> + Send + Sync>;
#[derive(Clone)]
pub struct TestCase {
pub id: TestId,
pub test_fn: TestFn,
pub fixture_requests: Vec<String>,
pub annotations: Vec<TestAnnotation>,
pub timeout: Option<Duration>,
pub retries: Option<u32>,
pub expected_status: ExpectedStatus,
pub use_options: Option<serde_json::Value>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum SuiteMode {
#[default]
Parallel,
Serial,
}
#[derive(Clone)]
pub struct TestSuite {
pub name: String,
pub file: String,
pub tests: Vec<TestCase>,
pub hooks: Hooks,
pub annotations: Vec<TestAnnotation>,
pub mode: SuiteMode,
}
#[derive(Clone)]
pub struct Hooks {
pub before_all: Vec<SuiteHookFn>,
pub after_all: Vec<SuiteHookFn>,
pub before_each: Vec<HookFn>,
pub after_each: Vec<HookFn>,
}
impl Default for Hooks {
fn default() -> Self {
Self {
before_all: Vec::new(),
after_all: Vec::new(),
before_each: Vec::new(),
after_each: Vec::new(),
}
}
}
pub type SuiteHookFn =
Arc<dyn Fn(FixturePool) -> Pin<Box<dyn Future<Output = Result<(), TestFailure>> + Send>> + Send + Sync>;
pub type HookFn = Arc<
dyn Fn(FixturePool, Arc<TestInfo>) -> Pin<Box<dyn Future<Output = Result<(), TestFailure>> + Send>> + Send + Sync,
>;
#[derive(Clone, Default)]
pub struct TestHooks {
pub global_setup_fns: Vec<SuiteHookFn>,
pub global_teardown_fns: Vec<SuiteHookFn>,
}
impl std::fmt::Debug for TestHooks {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TestHooks")
.field("global_setup_fns", &format!("[{} fn(s)]", self.global_setup_fns.len()))
.field(
"global_teardown_fns",
&format!("[{} fn(s)]", self.global_teardown_fns.len()),
)
.finish()
}
}
#[derive(Clone)]
pub struct TestPlan {
pub suites: Vec<TestSuite>,
pub total_tests: usize,
pub shard: Option<ShardInfo>,
}
#[derive(Debug, Clone)]
pub struct ShardInfo {
pub current: u32,
pub total: u32,
}
pub struct SuiteDef {
pub id: String,
pub name: String,
pub file: String,
pub mode: SuiteMode,
}
pub struct HookDef {
pub suite_id: String,
pub kind: HookKind,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum HookPhase {
Before,
After,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum HookScope {
Suite,
Scenario,
Step,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum HookOwner {
Root,
Suite(String),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct HookRegistration {
pub phase: HookPhase,
pub scope: HookScope,
pub owner: HookOwner,
pub tags: Option<String>,
pub requested_fixtures: Vec<String>,
}
pub enum HookKind {
BeforeAll(SuiteHookFn),
AfterAll(SuiteHookFn),
BeforeEach(HookFn),
AfterEach(HookFn),
}
pub struct TestPlanBuilder {
tests: Vec<TestCase>,
suites: Vec<SuiteDef>,
hooks: Vec<HookDef>,
}
impl Default for TestPlanBuilder {
fn default() -> Self {
Self::new()
}
}
impl TestPlanBuilder {
pub fn new() -> Self {
Self {
tests: Vec::new(),
suites: Vec::new(),
hooks: Vec::new(),
}
}
pub fn add_test(&mut self, test: TestCase) {
self.tests.push(test);
}
pub fn add_suite(&mut self, suite: SuiteDef) {
self.suites.push(suite);
}
pub fn add_hook(&mut self, hook: HookDef) {
self.hooks.push(hook);
}
pub fn build(self) -> TestPlan {
use rustc_hash::FxHashMap;
let suite_meta: FxHashMap<String, (String, String, SuiteMode)> = self
.suites
.into_iter()
.map(|s| (s.id, (s.name, s.file, s.mode)))
.collect();
let mut grouped: FxHashMap<String, Vec<TestCase>> = FxHashMap::default();
for tc in self.tests {
let key = tc.id.suite.clone().unwrap_or_default();
grouped.entry(key).or_default().push(tc);
}
let mut hook_map: FxHashMap<String, Hooks> = FxHashMap::default();
for h in self.hooks {
let hooks = hook_map.entry(h.suite_id).or_default();
match h.kind {
HookKind::BeforeAll(f) => hooks.before_all.push(f),
HookKind::AfterAll(f) => hooks.after_all.push(f),
HookKind::BeforeEach(f) => hooks.before_each.push(f),
HookKind::AfterEach(f) => hooks.after_each.push(f),
}
}
let mut plan_suites: Vec<TestSuite> = Vec::new();
let mut total = 0usize;
for (suite_key, tests) in grouped {
total += tests.len();
let (name, file, mode) = if suite_key.is_empty() {
("tests".to_string(), String::new(), SuiteMode::Parallel)
} else if let Some((n, f, m)) = suite_meta.get(&suite_key) {
(n.clone(), f.clone(), *m)
} else {
(suite_key.clone(), String::new(), SuiteMode::Parallel)
};
let hooks = hook_map.remove(&suite_key).unwrap_or_default();
plan_suites.push(TestSuite {
name,
file,
tests,
hooks,
annotations: Vec::new(),
mode,
});
}
TestPlan {
suites: plan_suites,
total_tests: total,
shard: None,
}
}
}
#[derive(Clone)]
pub struct TestInfo {
pub test_id: TestId,
pub title_path: Vec<String>,
pub retry: u32,
pub worker_index: u32,
pub parallel_index: u32,
pub repeat_each_index: u32,
pub output_dir: PathBuf,
pub snapshot_dir: PathBuf,
pub snapshot_path_template: Option<String>,
pub update_snapshots: crate::config::UpdateSnapshotsMode,
pub ignore_snapshots: bool,
pub attachments: Arc<Mutex<Vec<Attachment>>>,
pub steps: Arc<Mutex<Vec<TestStep>>>,
pub soft_errors: Arc<Mutex<Vec<TestFailure>>>,
pub errors: Arc<Mutex<Vec<TestFailure>>>,
pub snapshot_suffix: Arc<Mutex<String>>,
pub column: Option<u32>,
pub project: Option<crate::config::ProjectConfig>,
pub config_snapshot: Option<Arc<crate::config::TestConfig>>,
pub timeout: Duration,
pub tags: Vec<String>,
pub start_time: Instant,
pub event_bus: Option<EventBus>,
pub annotations: Arc<Mutex<Vec<TestAnnotation>>>,
}
impl TestInfo {
pub fn new_anonymous() -> Self {
Self {
test_id: TestId {
file: String::new(),
suite: None,
name: "anonymous".into(),
line: None,
},
title_path: Vec::new(),
retry: 0,
worker_index: 0,
parallel_index: 0,
repeat_each_index: 0,
output_dir: PathBuf::new(),
snapshot_dir: PathBuf::new(),
snapshot_path_template: None,
update_snapshots: crate::config::UpdateSnapshotsMode::default(),
ignore_snapshots: false,
attachments: Arc::new(Mutex::new(Vec::new())),
steps: Arc::new(Mutex::new(Vec::new())),
soft_errors: Arc::new(Mutex::new(Vec::new())),
errors: Arc::new(Mutex::new(Vec::new())),
snapshot_suffix: Arc::new(Mutex::new(String::new())),
column: None,
project: None,
config_snapshot: None,
timeout: Duration::from_secs(30),
tags: Vec::new(),
start_time: Instant::now(),
event_bus: None,
annotations: Arc::new(Mutex::new(Vec::new())),
}
}
pub async fn annotate(&self, type_name: impl Into<String>, description: impl Into<String>) {
let mut annotations = self.annotations.lock().await;
annotations.push(TestAnnotation::Info {
type_name: type_name.into(),
description: description.into(),
});
}
pub async fn get_annotations(&self) -> Vec<TestAnnotation> {
let annotations = self.annotations.lock().await;
annotations.clone()
}
pub async fn attach(&self, name: String, content_type: String, body: AttachmentBody) {
let mut attachments = self.attachments.lock().await;
attachments.push(Attachment {
name,
content_type,
body,
});
}
pub async fn add_soft_error(&self, error: TestFailure) {
let mut errors = self.soft_errors.lock().await;
errors.push(error);
}
pub async fn has_soft_errors(&self) -> bool {
let errors = self.soft_errors.lock().await;
!errors.is_empty()
}
pub async fn drain_soft_errors(&self) -> Vec<TestFailure> {
let mut errors = self.soft_errors.lock().await;
errors.drain(..).collect()
}
pub async fn push_step(&self, step: TestStep) {
let mut steps = self.steps.lock().await;
steps.push(step);
}
pub fn elapsed(&self) -> Duration {
self.start_time.elapsed()
}
pub async fn begin_step(&self, title: impl Into<String>, category: StepCategory) -> StepHandle {
let title = title.into();
let step_id = format!("{}@{}", category, STEP_ID_COUNTER.fetch_add(1, Ordering::Relaxed));
if let Some(bus) = &self.event_bus {
bus.emit(crate::reporter::ReporterEvent::StepStarted(Box::new(
crate::reporter::StepStartedEvent {
test_id: self.test_id.clone(),
step_id: step_id.clone(),
parent_step_id: None,
title: title.clone(),
category: category.clone(),
},
)));
}
StepHandle {
step_id,
test_id: self.test_id.clone(),
title,
category,
parent_step_id: None,
start: Instant::now(),
metadata: None,
event_bus: self.event_bus.clone(),
steps: Arc::clone(&self.steps),
}
}
pub async fn begin_child_step(
&self,
title: impl Into<String>,
category: StepCategory,
parent_step_id: &str,
) -> StepHandle {
let title = title.into();
let step_id = format!("{}@{}", category, STEP_ID_COUNTER.fetch_add(1, Ordering::Relaxed));
if let Some(bus) = &self.event_bus {
bus.emit(crate::reporter::ReporterEvent::StepStarted(Box::new(
crate::reporter::StepStartedEvent {
test_id: self.test_id.clone(),
step_id: step_id.clone(),
parent_step_id: Some(parent_step_id.to_string()),
title: title.clone(),
category: category.clone(),
},
)));
}
StepHandle {
step_id,
test_id: self.test_id.clone(),
title,
category,
parent_step_id: Some(parent_step_id.to_string()),
start: Instant::now(),
metadata: None,
event_bus: self.event_bus.clone(),
steps: Arc::clone(&self.steps),
}
}
pub async fn record_step(
&self,
title: impl Into<String>,
category: StepCategory,
status: StepStatus,
duration: Duration,
error: Option<String>,
metadata: Option<serde_json::Value>,
) {
let title = title.into();
let step_id = format!("{}@{}", category, STEP_ID_COUNTER.fetch_add(1, Ordering::Relaxed));
if let Some(bus) = &self.event_bus {
bus.emit(crate::reporter::ReporterEvent::StepStarted(Box::new(
crate::reporter::StepStartedEvent {
test_id: self.test_id.clone(),
step_id: step_id.clone(),
parent_step_id: None,
title: title.clone(),
category: category.clone(),
},
)));
bus.emit(crate::reporter::ReporterEvent::StepFinished(Box::new(
crate::reporter::StepFinishedEvent {
test_id: self.test_id.clone(),
step_id: step_id.clone(),
title: title.clone(),
category: category.clone(),
duration,
error: error.clone(),
metadata: metadata.clone(),
},
)));
}
self.steps.lock().await.push(TestStep {
step_id,
title,
category,
duration,
status,
error,
location: None,
parent_step_id: None,
metadata,
steps: Vec::new(),
});
}
}
static STEP_ID_COUNTER: AtomicU64 = AtomicU64::new(0);
pub struct StepHandle {
pub step_id: String,
pub test_id: TestId,
pub title: String,
pub category: StepCategory,
pub parent_step_id: Option<String>,
pub start: Instant,
pub metadata: Option<serde_json::Value>,
event_bus: Option<EventBus>,
steps: Arc<Mutex<Vec<TestStep>>>,
}
impl StepHandle {
pub async fn end(self, error: Option<String>) {
let duration = self.start.elapsed();
let status = if error.is_some() {
StepStatus::Failed
} else {
StepStatus::Passed
};
if let Some(bus) = &self.event_bus {
bus.emit(crate::reporter::ReporterEvent::StepFinished(Box::new(
crate::reporter::StepFinishedEvent {
test_id: self.test_id.clone(),
step_id: self.step_id.clone(),
title: self.title.clone(),
category: self.category.clone(),
duration,
error: error.clone(),
metadata: self.metadata.clone(),
},
)));
}
let step = TestStep {
step_id: self.step_id,
title: self.title,
category: self.category,
duration,
status,
error,
location: None,
parent_step_id: self.parent_step_id,
metadata: self.metadata.clone(),
steps: Vec::new(),
};
self.steps.lock().await.push(step);
}
pub async fn skip(self, reason: Option<String>) {
self.finish_with_status(StepStatus::Skipped, reason).await;
}
pub async fn pending(self, reason: Option<String>) {
self.finish_with_status(StepStatus::Pending, reason).await;
}
async fn finish_with_status(self, status: StepStatus, error: Option<String>) {
let duration = self.start.elapsed();
if let Some(bus) = &self.event_bus {
bus.emit(crate::reporter::ReporterEvent::StepFinished(Box::new(
crate::reporter::StepFinishedEvent {
test_id: self.test_id.clone(),
step_id: self.step_id.clone(),
title: self.title.clone(),
category: self.category.clone(),
duration,
error: error.clone(),
metadata: self.metadata.clone(),
},
)));
}
let step = TestStep {
step_id: self.step_id,
title: self.title,
category: self.category,
duration,
status,
error,
location: None,
parent_step_id: self.parent_step_id,
metadata: self.metadata,
steps: Vec::new(),
};
self.steps.lock().await.push(step);
}
}
#[derive(Debug, Clone)]
pub struct TestStep {
pub step_id: String,
pub title: String,
pub category: StepCategory,
pub duration: Duration,
pub status: StepStatus,
pub error: Option<String>,
pub location: Option<String>,
pub parent_step_id: Option<String>,
pub metadata: Option<serde_json::Value>,
pub steps: Vec<TestStep>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StepStatus {
Passed,
Failed,
Skipped,
Pending,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum StepCategory {
TestStep,
Expect,
Fixture,
Hook,
PwApi,
}
impl StepCategory {
pub fn is_visible(&self) -> bool {
matches!(self, Self::TestStep | Self::Hook)
}
}
impl fmt::Display for StepCategory {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::TestStep => write!(f, "test.step"),
Self::Expect => write!(f, "expect"),
Self::Fixture => write!(f, "fixture"),
Self::Hook => write!(f, "hook"),
Self::PwApi => write!(f, "pw:api"),
}
}
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum TestAnnotation {
Skip {
reason: Option<String>,
condition: Option<String>,
},
Slow {
reason: Option<String>,
condition: Option<String>,
},
Fixme {
reason: Option<String>,
condition: Option<String>,
},
Fail {
reason: Option<String>,
condition: Option<String>,
},
Only,
Tag(String),
Info {
type_name: String,
description: String,
},
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub enum ExpectedStatus {
#[default]
Pass,
Fail,
}
pub struct TestModifiers {
pub skipped: AtomicBool,
pub skip_reason: std::sync::Mutex<Option<String>>,
pub expected_failure: AtomicBool,
pub slow: AtomicBool,
pub timeout_override: std::sync::Mutex<Option<u64>>,
}
impl Default for TestModifiers {
fn default() -> Self {
Self {
skipped: AtomicBool::new(false),
skip_reason: std::sync::Mutex::new(None),
expected_failure: AtomicBool::new(false),
slow: AtomicBool::new(false),
timeout_override: std::sync::Mutex::new(None),
}
}
}
impl std::fmt::Debug for TestModifiers {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TestModifiers")
.field("skipped", &self.skipped.load(Ordering::Relaxed))
.field("expected_failure", &self.expected_failure.load(Ordering::Relaxed))
.field("slow", &self.slow.load(Ordering::Relaxed))
.finish_non_exhaustive()
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum TestStatus {
Passed,
Failed,
TimedOut,
Skipped,
Flaky,
Interrupted,
}
impl fmt::Display for TestStatus {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Passed => write!(f, "passed"),
Self::Failed => write!(f, "failed"),
Self::TimedOut => write!(f, "timed out"),
Self::Skipped => write!(f, "skipped"),
Self::Flaky => write!(f, "flaky"),
Self::Interrupted => write!(f, "interrupted"),
}
}
}
#[derive(Debug, Clone)]
pub struct TestOutcome {
pub test_id: TestId,
pub status: TestStatus,
pub duration: Duration,
pub attempt: u32,
pub max_attempts: u32,
pub error: Option<TestFailure>,
pub attachments: Vec<Attachment>,
pub steps: Vec<TestStep>,
pub stdout: String,
pub stderr: String,
pub annotations: Vec<TestAnnotation>,
pub metadata: serde_json::Value,
}
#[derive(Debug, Clone)]
pub struct TestFailure {
pub message: String,
pub stack: Option<String>,
pub diff: Option<String>,
pub screenshot: Option<Vec<u8>>,
}
impl fmt::Display for TestFailure {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.message)?;
if let Some(diff) = &self.diff {
write!(f, "\n{diff}")?;
}
Ok(())
}
}
impl std::error::Error for TestFailure {}
impl TestFailure {
#[must_use]
pub fn wrap(prefix: impl std::fmt::Display, err: ferridriver::FerriError) -> Self {
Self {
message: format!("{prefix}: {}", err.display_named()),
stack: None,
diff: None,
screenshot: None,
}
}
}
impl From<String> for TestFailure {
fn from(message: String) -> Self {
Self {
message,
stack: None,
diff: None,
screenshot: None,
}
}
}
impl From<&str> for TestFailure {
fn from(message: &str) -> Self {
Self::from(message.to_string())
}
}
impl From<ferridriver::FerriError> for TestFailure {
fn from(err: ferridriver::FerriError) -> Self {
Self {
message: err.display_named(),
stack: None,
diff: None,
screenshot: None,
}
}
}
#[derive(Debug, Clone)]
pub struct Attachment {
pub name: String,
pub content_type: String,
pub body: AttachmentBody,
}
#[derive(Debug, Clone)]
pub enum AttachmentBody {
Bytes(Vec<u8>),
Path(PathBuf),
}
#[derive(Clone)]
pub struct TestFixtures {
pub browser: Arc<ferridriver::Browser>,
pub page: Arc<ferridriver::Page>,
pub context: Arc<ferridriver::context::ContextRef>,
pub request: Arc<ferridriver::http_client::HttpClient>,
pub test_info: Arc<TestInfo>,
pub modifiers: Arc<TestModifiers>,
pub browser_config: crate::config::BrowserConfig,
pub bdd_args: Option<Vec<serde_json::Value>>,
pub bdd_data_table: Option<Vec<Vec<String>>>,
pub bdd_doc_string: Option<String>,
}
#[cfg(test)]
mod tests {
use super::*;
use ferridriver::FerriError;
#[test]
fn testfailure_from_timeout_keeps_class_prefix() {
let tf = TestFailure::from(FerriError::timeout("navigating", 30_000));
assert_eq!(tf.message, "TimeoutError: Timeout 30000ms exceeded while navigating");
}
#[test]
fn testfailure_from_target_closed_keeps_class_prefix() {
let tf = TestFailure::from(FerriError::target_closed(Some("crashed".into())));
assert_eq!(
tf.message,
"TargetClosedError: Target page, context or browser has been closed: crashed"
);
}
#[test]
fn testfailure_from_backend_has_no_prefix() {
let tf = TestFailure::from(FerriError::backend("launch failed"));
assert_eq!(tf.message, "backend error: launch failed");
}
#[test]
fn testfailure_wrap_preserves_timeout_class_after_prefix() {
let tf = TestFailure::wrap("fixture 'browser' failed", FerriError::timeout("launch", 30_000));
assert_eq!(
tf.message,
"fixture 'browser' failed: TimeoutError: Timeout 30000ms exceeded while launch"
);
}
#[test]
fn testfailure_wrap_unnamed_keeps_message_only() {
let tf = TestFailure::wrap("fixture 'page' failed", FerriError::backend("oops"));
assert_eq!(tf.message, "fixture 'page' failed: backend error: oops");
}
}