use happenstance_core::{
AppendError, Event, EventStore, Query, ReadOptions, SequencePosition, SequencedEvent, collect,
};
use crate::Fixture;
pub trait ConcurrentFixture: Fixture<Store: Send> {}
impl<F> ConcurrentFixture for F
where
F: Fixture,
F::Store: Send,
{
}
pub const CONTENDERS: usize = 64;
#[derive(Debug, Clone, PartialEq, Eq)]
enum Attempt {
Committed(SequencePosition),
Rejected,
Busy,
Failed(String),
}
impl Attempt {
fn of<E: core::fmt::Display>(result: Result<SequencePosition, AppendError<E>>) -> Self {
match result {
Ok(position) => Self::Committed(position),
Err(AppendError::ConditionViolated(_)) => Self::Rejected,
Err(AppendError::Busy(_)) => Self::Busy,
Err(other) => Self::Failed(format!("{other}")),
}
}
const fn position(&self) -> Option<SequencePosition> {
match self {
Self::Committed(position) => Some(*position),
_ => None,
}
}
const fn is_busy(&self) -> bool {
matches!(self, Self::Busy)
}
const fn is_unconditional_outcome(&self) -> bool {
matches!(self, Self::Committed(_) | Self::Busy)
}
}
fn race<S, T, B>(stores: Vec<S>, body: B) -> Vec<T>
where
S: Send,
T: Send,
B: Fn(usize, S) -> T + Sync,
{
let gate = std::sync::Barrier::new(stores.len());
std::thread::scope(|scope| {
let running: Vec<_> = stores
.into_iter()
.enumerate()
.map(|(index, store)| {
let body = &body;
let gate = &gate;
scope.spawn(move || {
gate.wait();
body(index, store)
})
})
.collect();
running
.into_iter()
.map(|thread| match thread.join() {
Ok(value) => value,
Err(payload) => std::panic::resume_unwind(payload),
})
.collect()
})
}
pub mod rules {
#![allow(clippy::missing_panics_doc)]
use core::sync::atomic::{AtomicBool, Ordering};
use happenstance_core::{
Event, EventStore, Query, ReadOptions, SequencePosition, SequencedEvent,
};
use super::{Attempt, CONTENDERS, append_ok, connect_many, race, read_ok, sorted_positions};
use crate::fixtures::{
condition, condition_after, event, query_of, query_of_types, tagged_event,
};
use crate::{Fixture, RuleOutcome};
pub async fn exactly_one_of_n_contenders_commits<F>(open: impl AsyncFn() -> F) -> RuleOutcome
where
F: Fixture,
F::Store: EventStore + Send,
{
let fixture = open().await;
let boundary = {
let setup = fixture.connect().await;
append_ok(&setup, &[event("CourseDefined")]).await
};
let query = query_of(&["StudentSubscribed"], &[("course", "c1")]);
let guard = condition_after(query.clone(), boundary.get());
let subscribe = tagged_event("StudentSubscribed", &[("course", "c1")]);
let stores = connect_many(&fixture, CONTENDERS).await;
let attempts = race(stores, |_, store| {
crate::block_on(async {
Attempt::of(
store
.append(core::slice::from_ref(&subscribe), Some(&guard))
.await,
)
})
});
let failures: Vec<&Attempt> = attempts
.iter()
.filter(|attempt| matches!(attempt, Attempt::Failed(_)))
.collect();
assert!(
failures.is_empty(),
"a contender must commit, be told `ConditionViolated`, or be refused \
as `Busy` (ES-43); anything else is a store failure under \
contention rather than a signal a caller can act on. Got \
{failures:?}"
);
let committed: Vec<SequencePosition> =
attempts.iter().filter_map(Attempt::position).collect();
assert!(
committed.len() <= 1,
"at most one of {CONTENDERS} handlers that decided from the same \
snapshot may commit — more than one means the consistency boundary \
is not enforced under contention. Attempts: {attempts:?}"
);
assert!(
!committed.is_empty(),
"no contender committed, out of {CONTENDERS}. Neither answer the \
rest were given is possible without a winner: `ConditionViolated` \
needs a matching event to have landed, and a store that answers \
`Busy` to every contender has made no progress. Either the store \
rejected a decision nothing had invalidated, or it wrote a batch \
and told its caller otherwise. Attempts: {attempts:?}"
);
let observer = fixture.connect().await;
let landed = read_ok(&observer, &query, ReadOptions::new()).await;
assert_eq!(
sorted_positions(&landed),
committed,
"the store must hold exactly the one batch it acknowledged, at the \
position it told the winner"
);
RuleOutcome::Ran
}
pub async fn k_disjoint_boundaries_never_conflict<F>(open: impl AsyncFn() -> F) -> RuleOutcome
where
F: Fixture,
F::Store: EventStore + Send,
{
const BOUNDARIES: usize = 4;
const PER_BOUNDARY: usize = 3;
let fixture = open().await;
let courses: Vec<String> = (0..BOUNDARIES).map(|index| format!("c{index}")).collect();
let boundary = {
let setup = fixture.connect().await;
let defined: Vec<_> = courses
.iter()
.map(|course| tagged_event("CourseDefined", &[("course", course.as_str())]))
.collect();
append_ok(&setup, &defined).await
};
let queries: Vec<Query> = courses
.iter()
.map(|course| query_of(&["StudentSubscribed"], &[("course", course.as_str())]))
.collect();
let guards: Vec<_> = queries
.iter()
.map(|query| condition_after(query.clone(), boundary.get()))
.collect();
let subscribes: Vec<_> = courses
.iter()
.map(|course| tagged_event("StudentSubscribed", &[("course", course.as_str())]))
.collect();
let contenders = BOUNDARIES * PER_BOUNDARY;
let stores = connect_many(&fixture, contenders).await;
let attempts = race(stores, |index, store| {
let seat = index % BOUNDARIES;
crate::block_on(async {
Attempt::of(
store
.append(
core::slice::from_ref(&subscribes[seat]),
Some(&guards[seat]),
)
.await,
)
})
});
let failures: Vec<&Attempt> = attempts
.iter()
.filter(|attempt| matches!(attempt, Attempt::Failed(_)))
.collect();
assert!(
failures.is_empty(),
"a contender must commit, be told `ConditionViolated`, or be refused \
as `Busy` (ES-43). Got {failures:?}"
);
assert!(
attempts.iter().any(|attempt| attempt.position().is_some()),
"no contender committed on any of {BOUNDARIES} boundaries, so the \
store made no progress at all. All attempts: {attempts:?}"
);
let observer = fixture.connect().await;
for (seat, query) in queries.iter().enumerate() {
let (won, rejected) = attempts
.iter()
.enumerate()
.filter(|(index, _)| index % BOUNDARIES == seat)
.fold(
(0_usize, 0_usize),
|(won, rejected), (_, attempt)| match attempt {
Attempt::Committed(_) => (won + 1, rejected),
Attempt::Rejected => (won, rejected + 1),
Attempt::Busy | Attempt::Failed(_) => (won, rejected),
},
);
assert!(
won > 0 || rejected == 0,
"boundary {seat} of {BOUNDARIES} elected no winner out of \
{PER_BOUNDARY} contenders, and told {rejected} of them \
`ConditionViolated`. Nothing on it had committed, and nothing \
else could have invalidated them: their conditions name a \
query no other boundary writes to. A store that answers \
`ConditionViolated` here is reporting a conflict between \
commands that share nothing — a global version check, or a \
serialisation failure mapped onto the DCB retry signal — or it \
wrote this boundary's batch and told its caller otherwise. All \
attempts: {attempts:?}"
);
assert!(
won <= 1,
"boundary {seat} of {BOUNDARIES} must elect at most one winner \
out of {PER_BOUNDARY} contenders, and elected {won}. All of \
them decided from the same snapshot and all of them named the \
same query, so the condition did not hold under contention. \
All attempts: {attempts:?}"
);
let landed = read_ok(&observer, query, ReadOptions::new()).await;
assert_eq!(
landed.len(),
won,
"and boundary {seat} must hold exactly the batch it \
acknowledged, and nothing when every contender on it was \
refused as `Busy`"
);
}
RuleOutcome::Ran
}
pub async fn positions_are_unique_under_concurrent_appends<F>(
open: impl AsyncFn() -> F,
) -> RuleOutcome
where
F: Fixture,
F::Store: EventStore + Send,
{
const BATCH: usize = 2;
let fixture = open().await;
let batch = [event("Appended"), event("Appended")];
let stores = connect_many(&fixture, CONTENDERS).await;
let attempts = race(stores, |_, store| {
crate::block_on(async { Attempt::of(store.append(&batch, None).await) })
});
let unexpected: Vec<&Attempt> = attempts
.iter()
.filter(|attempt| !attempt.is_unconditional_outcome())
.collect();
assert!(
unexpected.is_empty(),
"an unconditional append has nothing to be rejected by, so every \
contender must commit or be refused as `Busy` (ES-43). Got \
{unexpected:?}"
);
let committed: Vec<SequencePosition> =
attempts.iter().filter_map(Attempt::position).collect();
assert!(
!committed.is_empty(),
"no contender committed, out of {CONTENDERS}: every one was refused \
as `Busy`, and a store that makes no progress under contention has \
not shown it can assign a position at all. Attempts: {attempts:?}"
);
let observer = fixture.connect().await;
let all = read_ok(&observer, &Query::all(), ReadOptions::new()).await;
assert_eq!(
all.len(),
committed.len() * BATCH,
"the store must hold every event of every committed batch and \
nothing more: {} of {CONTENDERS} contenders committed a batch of \
{BATCH}, and a `Busy` refusal wrote nothing (ES-43)",
committed.len()
);
let mut positions = sorted_positions(&all);
let total = positions.len();
positions.dedup();
assert_eq!(
positions.len(),
total,
"no two events anywhere in the store may share a position"
);
RuleOutcome::Ran
}
pub async fn append_returns_the_callers_own_last_position<F>(
open: impl AsyncFn() -> F,
) -> RuleOutcome
where
F: Fixture,
F::Store: EventStore + Send,
{
const BATCH: usize = 2;
let fixture = open().await;
let writers: Vec<String> = (0..CONTENDERS).map(|index| format!("w{index}")).collect();
let batches: Vec<Vec<_>> = writers
.iter()
.map(|writer| {
(0..BATCH)
.map(|_| tagged_event("Written", &[("writer", writer.as_str())]))
.collect()
})
.collect();
let stores = connect_many(&fixture, CONTENDERS).await;
let attempts = race(stores, |index, store| {
crate::block_on(async { Attempt::of(store.append(&batches[index], None).await) })
});
assert!(
attempts.iter().any(|attempt| attempt.position().is_some()),
"no contender committed, out of {CONTENDERS}, so there is no \
returned position to check. Attempts: {attempts:?}"
);
let observer = fixture.connect().await;
for (index, writer) in writers.iter().enumerate() {
let returned = match &attempts[index] {
Attempt::Committed(returned) => *returned,
Attempt::Busy => continue,
other => panic!(
"contender {index} did not commit an unconditional append \
and was not refused as `Busy`: {other:?}. All attempts: \
{attempts:?}"
),
};
let mine = read_ok(
&observer,
&query_of(&["Written"], &[("writer", writer.as_str())]),
ReadOptions::new(),
)
.await;
assert_eq!(
mine.len(),
BATCH,
"contender {index}'s batch must be in the store whole"
);
let last = sorted_positions(&mine).pop();
assert_eq!(
last,
Some(returned),
"`append` must return the position of the caller's own last \
event. Contender {index} was told {returned:?} and its own \
events are at {:?} — a store that answers with its head hands \
the caller a checkpoint that skips somebody else's batch",
sorted_positions(&mine)
);
}
RuleOutcome::Ran
}
pub async fn a_busy_append_left_nothing_behind<F>(open: impl AsyncFn() -> F) -> RuleOutcome
where
F: Fixture,
F::Store: EventStore + Send,
{
const BATCH: usize = 2;
let fixture = open().await;
let writers: Vec<String> = (0..CONTENDERS).map(|index| format!("b{index}")).collect();
let queries: Vec<Query> = writers
.iter()
.map(|writer| query_of(&["Written"], &[("writer", writer.as_str())]))
.collect();
let guards: Vec<_> = queries
.iter()
.map(|query| condition(query.clone()))
.collect();
let batches: Vec<Vec<_>> = writers
.iter()
.map(|writer| {
(0..BATCH)
.map(|_| tagged_event("Written", &[("writer", writer.as_str())]))
.collect()
})
.collect();
let stores = connect_many(&fixture, CONTENDERS).await;
let attempts = race(stores, |index, store| {
crate::block_on(async {
Attempt::of(store.append(&batches[index], Some(&guards[index])).await)
})
});
assert!(
attempts.iter().any(|attempt| attempt.position().is_some()),
"no contender committed, out of {CONTENDERS}, so the anchor below \
has nothing to read and a store whose reads return nothing would \
pass. Attempts: {attempts:?}"
);
let observer = fixture.connect().await;
for (index, query) in queries.iter().enumerate() {
let mine = read_ok(&observer, query, ReadOptions::new()).await;
match &attempts[index] {
Attempt::Busy => assert!(
mine.is_empty(),
"contender {index} was refused as `Busy`, and {} of its \
events are in the store at {:?}. `Busy` promises that the \
batch took no effect (ES-43), and a caller that believes it \
runs the command again. An outcome the store cannot vouch \
for must be reported as `Store`",
mine.len(),
sorted_positions(&mine)
),
Attempt::Committed(_) => assert_eq!(
mine.len(),
BATCH,
"contender {index} committed, so its batch must be in the \
store whole"
),
Attempt::Rejected | Attempt::Failed(_) => {}
}
}
RuleOutcome::Ran
}
pub async fn a_concurrent_reader_never_sees_a_partial_batch<F>(
open: impl AsyncFn() -> F,
) -> RuleOutcome
where
F: Fixture,
F::Store: EventStore + Send,
{
const WRITERS: usize = 4;
const ROUNDS: usize = 4;
const BATCH: usize = 3;
let fixture = open().await;
let types: Vec<String> = (0..WRITERS)
.flat_map(|writer| (0..ROUNDS).map(move |round| format!("Batch{writer}x{round}")))
.collect();
let batches: Vec<Vec<_>> = types
.iter()
.map(|name| (0..BATCH).map(|_| event(name)).collect())
.collect();
let mut stores = connect_many(&fixture, WRITERS + 1).await;
let Some(reading) = stores.pop() else {
panic!("connect_many returned nothing for {WRITERS} writers and a reader");
};
let (attempts, sightings) =
observe_while_writing(stores, reading, &batches, &types, ROUNDS, BATCH);
let failures: Vec<&Attempt> = attempts
.iter()
.filter(|attempt| !attempt.is_unconditional_outcome())
.collect();
assert!(
failures.is_empty(),
"an unconditional append has nothing to be rejected by, so each one \
must commit or be refused as `Busy` (ES-43). Got {failures:?}"
);
let committed = attempts
.iter()
.filter(|attempt| attempt.position().is_some())
.count();
assert!(
committed > 0,
"no batch committed, out of {}: every append was refused as `Busy`, \
so there was never a batch in flight to be caught part-written",
attempts.len()
);
let mut distinct = sightings.unreadable.clone();
distinct.sort_unstable();
distinct.dedup();
assert!(
sightings.unreadable.is_empty(),
"the store could not be read while an append was in flight, on {} \
of the reader's passes. That is a defect in the read path under \
contention rather than a partial batch, and this run says nothing \
about atomicity either way. Distinct errors: {distinct:?}",
sightings.unreadable.len()
);
assert!(
sightings.partial.is_empty(),
"a reader observed {} batch(es) part-written: {:?}. Either every \
event of a batch is visible or none is (ES-18), and a reader that \
can see half of one can build a decision model from a command that \
never completed",
sightings.partial.len(),
sightings.partial
);
let observer = fixture.connect().await;
let all = read_ok(&observer, &Query::all(), ReadOptions::new()).await;
assert_eq!(
all.len(),
committed * BATCH,
"every event of every acknowledged batch must still be there once \
the writers have finished, and nothing more: {committed} of {} \
batches committed, and a `Busy` refusal wrote nothing (ES-43)",
attempts.len()
);
for (name, attempt) in types.iter().zip(&attempts) {
if attempt.is_busy() {
continue;
}
let found = read_ok(
&observer,
&query_of_types(&[name.as_str()]),
ReadOptions::new(),
)
.await;
assert_eq!(
found.len(),
BATCH,
"batch `{name}` was acknowledged and must be present whole"
);
}
RuleOutcome::Ran
}
fn observe_while_writing<S: EventStore + Send>(
writers: Vec<S>,
reading: S,
batches: &[Vec<Event>],
types: &[String],
rounds: usize,
batch: usize,
) -> (Vec<Attempt>, Sightings) {
let done = AtomicBool::new(false);
drop(incomplete_batches(&reading, types, batch));
std::thread::scope(|scope| {
let flag = &done;
let names = types;
let reader = scope.spawn(move || {
let mut partial = Sightings::default();
while !flag.load(Ordering::Acquire) {
partial.absorb(incomplete_batches(&reading, names, batch));
}
partial.absorb(incomplete_batches(&reading, names, batch));
partial
});
let writing: Vec<_> = writers
.into_iter()
.enumerate()
.map(|(writer, store)| {
scope.spawn(move || {
crate::block_on(async {
let mut attempts = Vec::with_capacity(rounds);
for round in 0..rounds {
let index = writer * rounds + round;
attempts
.push(Attempt::of(store.append(&batches[index], None).await));
}
attempts
})
})
})
.collect();
let joined: Vec<std::thread::Result<Vec<Attempt>>> = writing
.into_iter()
.map(std::thread::ScopedJoinHandle::join)
.collect();
done.store(true, Ordering::Release);
let observed = reader.join();
let mut committed = Vec::new();
for writer in joined {
match writer {
Ok(attempts) => committed.extend(attempts),
Err(payload) => std::panic::resume_unwind(payload),
}
}
match observed {
Ok(partial) => (committed, partial),
Err(payload) => std::panic::resume_unwind(payload),
}
})
}
#[derive(Debug, Default)]
struct Sightings {
partial: Vec<String>,
unreadable: Vec<String>,
}
impl Sightings {
fn absorb(&mut self, pass: Self) {
self.partial.extend(pass.partial);
self.unreadable.extend(pass.unreadable);
}
}
fn incomplete_batches<S: EventStore>(store: &S, types: &[String], batch: usize) -> Sightings {
let seen: Vec<SequencedEvent> = match crate::block_on(happenstance_core::collect(
store.read(&Query::all(), ReadOptions::new()),
)) {
Ok(events) => events,
Err(err) => {
return Sightings {
partial: Vec::new(),
unreadable: std::vec![format!("{err}")],
};
}
};
Sightings {
partial: types
.iter()
.filter(|name| {
let count = seen
.iter()
.filter(|event| event.event_type().as_str() == name.as_str())
.count();
count != 0 && count != batch
})
.cloned()
.collect(),
unreadable: Vec::new(),
}
}
}
async fn connect_many<F: Fixture>(fixture: &F, count: usize) -> Vec<F::Store> {
let mut stores = Vec::with_capacity(count);
for _ in 0..count {
stores.push(fixture.connect().await);
}
stores
}
async fn append_ok<S: EventStore>(store: &S, events: &[Event]) -> SequencePosition {
match store.append(events, None).await {
Ok(position) => position,
Err(err) => panic!("unconditional append should succeed, got {err:?}"),
}
}
async fn read_ok<S: EventStore>(
store: &S,
query: &Query,
options: ReadOptions,
) -> Vec<SequencedEvent> {
match collect(store.read(query, options)).await {
Ok(events) => events,
Err(err) => panic!("read should succeed, got {err:?}"),
}
}
fn sorted_positions(events: &[SequencedEvent]) -> Vec<SequencePosition> {
let mut positions: Vec<SequencePosition> = events.iter().map(|event| event.position).collect();
positions.sort_unstable();
positions
}
#[macro_export]
macro_rules! for_each_concurrency_rule {
($($callback:tt)+) => {
$($callback)+! {
exactly_one_of_n_contenders_commits,
k_disjoint_boundaries_never_conflict,
positions_are_unique_under_concurrent_appends,
append_returns_the_callers_own_last_position,
a_busy_append_left_nothing_behind,
a_concurrent_reader_never_sees_a_partial_batch,
}
};
}
#[macro_export]
macro_rules! emit_concurrency_tokio {
($($name:ident),* $(,)?) => {
$(
#[tokio::test(flavor = "multi_thread")]
async fn $name() {
$crate::__private::concurrency_rules::$name(__conformance_fixture)
.await
.report(::core::stringify!($name));
}
)*
};
}
#[macro_export]
macro_rules! emit_concurrency_blocking {
($($name:ident),* $(,)?) => {
$(
#[test]
fn $name() {
$crate::__private::block_on($crate::__private::concurrency_rules::$name(__conformance_fixture))
.report(::core::stringify!($name));
}
)*
};
}
#[macro_export]
macro_rules! event_store_concurrency_conformance {
(mod_name = $mod_name:ident, emit = $emit:path, fixture = $fixture:expr) => {
mod $mod_name {
#![allow(clippy::unwrap_used, unused_imports)]
use super::*;
async fn __conformance_fixture() -> impl $crate::__private::ConcurrentFixture {
$fixture
}
$crate::for_each_concurrency_rule!($emit);
}
};
(mod_name = $mod_name:ident, fixture = $fixture:expr) => {
$crate::event_store_concurrency_conformance!(
mod_name = $mod_name,
emit = $crate::emit_concurrency_tokio,
fixture = $fixture
);
};
($fixture:expr) => {
$crate::event_store_concurrency_conformance!(
mod_name = dcb_concurrency_conformance,
emit = $crate::emit_concurrency_tokio,
fixture = $fixture
);
};
}