use super::faulty_transport::{FaultAction, FaultyTransport};
use crate::transport::{BoxFuture, Transport};
use proptest::collection::vec;
use proptest::prelude::{Just, prop_oneof};
use proptest::strategy::{BoxedStrategy, Strategy};
use proptest::test_runner::{Config, TestCaseError, TestRunner};
use serde_json::{Map, Value};
use std::collections::HashMap;
use std::future::Future;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
fn runtime() -> Result<tokio::runtime::Runtime, String> {
tokio::runtime::Builder::new_current_thread()
.enable_time()
.start_paused(true)
.build()
.map_err(|e| format!("cannot build a runtime for the axioms: {e}"))
}
#[derive(Default)]
pub struct LivenessScenario {
pub outputs: Vec<BoxFuture<'static, ()>>,
pub settlements: Vec<BoxFuture<'static, ()>>,
}
pub type Respond = dyn Fn(&str, &Map<String, Value>) -> Result<Option<Value>, String> + Send + Sync;
pub struct LivenessSpec<Setup> {
pub setup: Setup,
pub timeout: Duration,
pub retry_budget: usize,
pub schedules: Option<BoxedStrategy<Vec<FaultAction>>>,
pub respond: Option<Arc<Respond>>,
pub cases: u32,
}
pub fn fault_actions() -> impl Strategy<Value = FaultAction> {
prop_oneof![
Just(FaultAction::Deliver),
Just(FaultAction::DropReply),
(0u64..=5).prop_map(|ms| FaultAction::Delay(Duration::from_millis(ms))),
Just(FaultAction::DuplicateReply),
Just(FaultAction::RejectEmit),
]
}
pub fn fault_schedules() -> impl Strategy<Value = Vec<FaultAction>> {
vec(fault_actions(), 1..=8)
}
fn described(schedule: &[FaultAction]) -> String {
schedule
.iter()
.map(|action| match action {
FaultAction::Deliver => "deliver".to_owned(),
FaultAction::DropReply => "drop-reply".to_owned(),
FaultAction::Delay(after) => format!("delay({})", after.as_millis()),
FaultAction::DuplicateReply => "duplicate-reply".to_owned(),
FaultAction::RejectEmit => "reject-emit".to_owned(),
})
.collect::<Vec<_>>()
.join(",")
}
pub fn assert_liveness_axioms<Setup>(spec: LivenessSpec<Setup>) -> Result<(), String>
where
Setup: Fn(FaultyTransport) -> LivenessScenario,
{
let runtime = runtime()?;
let schedules = spec
.schedules
.clone()
.unwrap_or_else(|| fault_schedules().boxed());
let mut runner = TestRunner::new(Config {
cases: spec.cases,
failure_persistence: None,
..Config::default()
});
runner
.run(&schedules, |schedule| {
let delays: Duration = schedule
.iter()
.map(|action| match action {
FaultAction::Delay(after) => *after,
_ => Duration::ZERO,
})
.sum();
let bound = (spec.timeout * (1 + spec.retry_budget as u32) + delays) * 4;
runtime
.block_on(async {
let transport = match &spec.respond {
Some(respond) => {
let respond = respond.clone();
FaultyTransport::answering(schedule.clone(), move |operation, payload| {
respond(operation, payload)
})
}
None => FaultyTransport::new(schedule.clone()),
};
let scenario = (spec.setup)(transport.clone());
let watch = |paths: Vec<BoxFuture<'static, ()>>| -> Vec<Arc<AtomicBool>> {
paths
.into_iter()
.map(|path| {
let done = Arc::new(AtomicBool::new(false));
let seen = done.clone();
tokio::spawn(async move {
path.await;
seen.store(true, Ordering::SeqCst);
});
done
})
.collect()
};
let notified = watch(scenario.outputs);
let settled = watch(scenario.settlements);
tokio::time::sleep(bound).await;
let under = format!("under schedule ⟨{}⟩", described(&schedule));
if let Some(i) = settled.iter().position(|s| !s.load(Ordering::SeqCst)) {
return Err(format!(
"L2: settlement #{i} did not settle within {}ms {under}",
bound.as_millis()
));
}
let mut issued: HashMap<String, (usize, bool)> = HashMap::new();
for entry in transport.request_log() {
let faulted = matches!(
entry.action,
FaultAction::DropReply | FaultAction::RejectEmit
);
let issue = issued.entry(entry.retry_key).or_insert((0, false));
*issue = (issue.0 + 1, faulted);
}
let silent = !notified.iter().any(|n| n.load(Ordering::SeqCst));
for (key, (count, last_faulted)) in &issued {
if *count > 1 + spec.retry_budget {
return Err(format!(
"L2: request ⟨{key}⟩ was sent {count} times, past the retry budget (1 + {}) {under}",
spec.retry_budget
));
}
if *last_faulted
&& *count <= spec.retry_budget
&& !notified.is_empty()
&& silent
{
return Err(format!(
"L2: faulted request ⟨{key}⟩ was neither sent again nor surfaced {under}: its failure was swallowed"
));
}
}
if let Some(i) = notified.iter().position(|n| !n.load(Ordering::SeqCst)) {
return Err(format!(
"L1: output #{i} said nothing within {}ms {under}: silently pending",
bound.as_millis()
));
}
transport.close().await;
Ok(())
})
.map_err(TestCaseError::fail)
})
.map_err(|failure| failure.to_string())
}
pub trait DeliverySubject {
fn write(&self, event_id: &str);
fn transition(&self) -> BoxFuture<'_, ()>;
fn delivered(&self) -> BoxFuture<'_, Vec<String>>;
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DeliveryOp {
Write,
Transition,
}
pub fn delivery_ops(longest: usize) -> impl Strategy<Value = Vec<DeliveryOp>> {
vec(
prop_oneof![Just(DeliveryOp::Write), Just(DeliveryOp::Transition)],
1..=longest,
)
.prop_map(|mut ops| {
if !ops.contains(&DeliveryOp::Write) {
ops.push(DeliveryOp::Write);
}
ops
})
}
pub fn assert_exactly_once_delivery<S, Building>(
setup: impl Fn() -> Building,
ops: Option<BoxedStrategy<Vec<DeliveryOp>>>,
cases: u32,
) -> Result<(), String>
where
S: DeliverySubject,
Building: Future<Output = S>,
{
assert_exactly_once_delivery_on(&runtime()?, setup, ops, cases)
}
pub fn assert_exactly_once_delivery_on<S, Building>(
runtime: &tokio::runtime::Runtime,
setup: impl Fn() -> Building,
ops: Option<BoxedStrategy<Vec<DeliveryOp>>>,
cases: u32,
) -> Result<(), String>
where
S: DeliverySubject,
Building: Future<Output = S>,
{
let ops = ops.unwrap_or_else(|| delivery_ops(12).boxed());
let mut runner = TestRunner::new(Config {
cases,
failure_persistence: None,
..Config::default()
});
runner
.run(&ops, |ops| {
runtime
.block_on(async {
let subject = setup().await;
let mut written = Vec::new();
for (i, op) in ops.iter().enumerate() {
match op {
DeliveryOp::Write => {
let id = format!("e{i}");
subject.write(&id);
written.push(id);
}
DeliveryOp::Transition => subject.transition().await,
}
}
let delivered = subject.delivered().await;
let under = format!(
"under ⟨{}⟩",
ops.iter()
.map(|op| match op {
DeliveryOp::Write => "write",
DeliveryOp::Transition => "transition",
})
.collect::<Vec<_>>()
.join(",")
);
for id in &written {
match delivered.iter().filter(|d| *d == id).count() {
1 => {}
0 => {
return Err(format!(
"L3: event ⟨{id}⟩ was delivered 0 times {under}: lost across a transition"
));
}
n => {
return Err(format!(
"L3: event ⟨{id}⟩ was delivered {n} times {under}"
));
}
}
}
Ok(())
})
.map_err(TestCaseError::fail)
})
.map_err(|failure| failure.to_string())
}