use std::{any::Any, fmt, panic::resume_unwind, pin::pin, thread, time::Duration};
use futures::{FutureExt, Stream, StreamExt, stream};
use tokio::{runtime, time::timeout};
use super::harness::{expect_next, expect_no_more};
use super::helpers::unique_subject;
use crate::{
AckError, Connected, ConnectedBroker, HeaderMap, IncomingMessage, OutgoingMessage, Publisher,
RawMessage, Subscribe, Subscriber, SubscriptionSource,
testing::{Backlog, Coordinator, InProcess, TestableBroker, TestableRegistration},
};
const DELIVERY_TIMEOUT: Duration = Duration::from_secs(2);
const QUIET: Duration = Duration::from_millis(100);
const SERVER_TIMEOUT: Duration = Duration::from_secs(30);
const SERVER_QUIET: Duration = Duration::from_millis(500);
const REDELIVERY_DELAY: Duration = Duration::from_secs(2);
const SHARED: u32 = 10;
pub(crate) async fn run<B, F>(factory: F)
where
B: InProcess,
B::Connected: Subscribe,
F: Fn() -> B,
{
own_publishes_are_logged(connect_in_process(factory()).await).await;
one_name_two_subscriptions(connect_in_process(factory()).await).await;
counts_balance(factory());
}
async fn connect_in_process<B: InProcess>(broker: B) -> B::Connected {
broker
.connect_in_process()
.await
.expect("broker must connect in process before a suite scenario")
}
fn registration_of<C: TestableBroker + 'static>(
connected: &C,
) -> Option<&'static TestableRegistration> {
inventory::iter::<TestableRegistration>
.into_iter()
.find(|registration| registration.resolve(connected as &dyn Any).is_some())
}
async fn publish_own<C>(
registration: &TestableRegistration,
connected: &C,
msg: OutgoingMessage<'_>,
) where
C: TestableBroker + 'static,
{
let what = format!("{:?}", msg.name());
(registration.live())(connected, msg)
.await
.unwrap_or_else(|err| {
panic!("a publish to {what} through the broker's own default publisher failed: {err}")
});
}
async fn publish_as_service<C>(
connected: &C,
registration: Option<&TestableRegistration>,
msg: OutgoingMessage<'_>,
) -> bool
where
C: TestableBroker + 'static,
{
if let Some(registration) = registration {
(registration.live())(connected, msg).await.is_ok()
} else {
connected.inject(msg);
true
}
}
async fn settle_ack<M: IncomingMessage>(msg: M) {
match msg.ack().await {
Ok(()) | Err(AckError::Unsupported) => {}
Err(other) => panic!("ack must succeed or be unsupported, got: {other:?}"),
}
}
fn quiescent(coordinator: &Coordinator) -> bool {
coordinator.drive().now_or_never().is_some()
}
async fn own_publishes_are_logged<C: TestableBroker + Subscribe>(connected: C) {
let name = unique_subject("conformance.ownpublish");
let subscriber = Subscribe::subscribe(&connected, &name)
.await
.expect("subscribe failed");
let registration = registration_of(&connected);
let mut headers = HeaderMap::new();
headers.insert("x-conformance", "own");
let first = OutgoingMessage::new(name.as_str(), b"first".as_slice()).with_headers(headers);
let Some(registration) = registration else {
drop(subscriber);
connected.shutdown().await.expect("shutdown failed");
return;
};
if publish_as_service(&connected, Some(registration), first).await {
connected.inject(OutgoingMessage::new(name.as_str(), b"injected".as_slice()));
publish_own(
registration,
&connected,
OutgoingMessage::new(name.as_str(), b"second".as_slice()),
)
.await;
let log = connected.published(&name);
let payloads: Vec<&[u8]> = log.iter().map(RawMessage::payload).collect();
assert_eq!(
payloads,
[b"first".as_slice(), b"injected", b"second"],
"the publish log must record what the broker's own publisher sent as well as what was \
injected, in publish order: a handler's reply is published that way and TestApp's \
`published` assertions read the log",
);
assert_eq!(
log[0].headers().get("x-conformance"),
Some(b"own".as_slice()),
"the publish log must keep the headers the broker's own publisher sent",
);
} else {
assert_eq!(
connected.published(&name),
Vec::new(),
"the publish log recorded a publish the broker's own publisher refused",
);
}
drop(subscriber);
connected.shutdown().await.expect("shutdown failed");
}
struct Received {
position: usize,
message: u32,
}
async fn one_name_two_subscriptions<C: TestableBroker + Subscribe>(connected: C) {
let coordinator = Coordinator::new(usize::MAX);
connected.install_coordinator(coordinator.clone());
let name = unique_subject("conformance.shared");
let other = unique_subject("conformance.elsewhere");
let mut subscribers = vec![(
name.as_str(),
Subscribe::subscribe(&connected, &name)
.await
.expect("subscribe failed"),
)];
let second = Subscribe::subscribe(&connected, &name).await.ok();
subscribers.extend(second.map(|second| (name.as_str(), second)));
let third = Subscribe::subscribe(&connected, &other).await.ok();
subscribers.extend(third.map(|third| (other.as_str(), third)));
let names: Vec<&str> = subscribers.iter().map(|(name, _)| *name).collect();
let registration = registration_of(&connected);
let mut routed = Vec::new();
for message in 0..SHARED {
let answer = connected.routes(&name, &names);
let taken = publish_as_service(
&connected,
registration,
OutgoingMessage::new(name.as_str(), message.to_be_bytes().as_slice()),
)
.await;
routed.push(if taken { answer } else { Vec::new() });
}
let owed: usize = routed.iter().map(Vec::len).sum();
let received = {
let mut merged = stream::select_all(subscribers.iter_mut().enumerate().map(
|(position, (_, subscriber))| {
Box::pin(subscriber.stream().map(move |item| (position, item)))
},
));
let mut received = Vec::new();
while received.len() < owed {
match timeout(DELIVERY_TIMEOUT, merged.next()).await {
Ok(Some(delivery)) => received.push(read_shared(delivery).await),
Ok(None) | Err(_) => break,
}
}
while let Ok(Some(delivery)) = timeout(QUIET, merged.next()).await {
received.push(read_shared(delivery).await);
}
received
};
for (message, answer) in (0..SHARED).zip(&routed) {
let mut expected = answer.clone();
expected.sort_unstable();
let mut delivered: Vec<usize> = received
.iter()
.filter(|delivery| delivery.message == message)
.map(|delivery| delivery.position)
.collect();
delivered.sort_unstable();
assert_eq!(
delivered, expected,
"message {message} published to {name:?} with subscriptions {names:?}: `routes` \
answers {answer:?}, so exactly those subscriptions must receive it, once each; a live \
TestApp waits on that answer",
);
}
assert!(
quiescent(&coordinator),
"one_name_two_subscriptions: every delivery was settled, yet the coordinator still counts \
one in flight; each delivery is counted once by `enqueued` and released once by \
`consumed`",
);
drop(subscribers);
connected.shutdown().await.expect("shutdown failed");
}
async fn read_shared<M, E>((position, item): (usize, Result<M, E>)) -> Received
where
M: IncomingMessage,
E: fmt::Debug,
{
let msg = item
.unwrap_or_else(|err| panic!("one_name_two_subscriptions: stream yielded error: {err:?}"));
let message = msg.payload().try_into().map(u32::from_be_bytes).expect(
"one_name_two_subscriptions: a delivery carries a payload it was not published with",
);
settle_ack(msg).await;
Received { position, message }
}
fn counts_balance<B>(broker: B)
where
B: InProcess,
B::Connected: Subscribe,
{
thread::scope(|scope| {
let joined = thread::Builder::new()
.name("conformance-paused-clock".to_owned())
.spawn_scoped(scope, move || {
runtime::Builder::new_current_thread()
.enable_all()
.start_paused(true)
.build()
.expect("a paused current-thread runtime must build")
.block_on(counts(broker));
})
.expect("the paused-clock thread must start")
.join();
if let Err(panic) = joined {
resume_unwind(panic);
}
});
}
async fn counts<B>(broker: B)
where
B: InProcess,
B::Connected: Subscribe,
{
let connected = connect_in_process(broker).await;
let coordinator = Coordinator::new(usize::MAX);
connected.install_coordinator(coordinator.clone());
let name = unique_subject("conformance.counted");
let mut subscriber = Subscribe::subscribe(&connected, &name)
.await
.expect("subscribe failed");
let registration = registration_of(&connected);
{
let mut stream = pin!(subscriber.stream());
let inject = |payload: &'static [u8]| {
connected.inject(OutgoingMessage::new(name.as_str(), payload));
};
assert!(
stream.next().now_or_never().is_none(),
"counts: a subscription to a fresh name delivered before anything was published",
);
assert!(
quiescent(&coordinator),
"counts: nothing is published yet, and the coordinator already counts a delivery in \
flight",
);
inject(b"injected");
assert!(
!quiescent(&coordinator),
"counts: a message injected into a subscription must be counted in flight \
(`Coordinator::enqueued`) before `inject` returns; TestApp would stop waiting before \
the handler ran",
);
let msg = expect_next(&mut stream, "counts injected").await;
assert!(
!quiescent(&coordinator),
"counts: a delivery in the handler's hands must stay counted until it is settled",
);
settle_ack(msg).await;
assert!(
quiescent(&coordinator),
"counts: an acked delivery must be released exactly once (`Coordinator::consumed`, \
from its `Drop`)",
);
if let Some(registration) = registration {
let reaches = !connected.routes(&name, &[name.as_str()]).is_empty();
let taken = publish_as_service(
&connected,
Some(registration),
OutgoingMessage::new(name.as_str(), b"own".as_slice()),
)
.await;
if taken && reaches {
assert!(
!quiescent(&coordinator),
"counts: a message the broker's own publisher sent must be counted in flight \
by the time the publish returns; a handler's publish is followed by its \
settlement, and TestApp would stop waiting in between",
);
settle_ack(expect_next(&mut stream, "counts own publish").await).await;
}
assert!(
quiescent(&coordinator),
"counts: a delivery of the broker's own publish must be released exactly once, \
and a publish that reaches no subscription must not be counted",
);
}
let unread = unique_subject("conformance.unread");
let reaches = !connected.routes(&unread, &[name.as_str()]).is_empty();
let taken = publish_as_service(
&connected,
registration_of(&connected),
OutgoingMessage::new(unread.as_str(), b"unread".as_slice()),
)
.await;
if taken && reaches {
assert!(
!quiescent(&coordinator),
"counts: `routes` says the subscription receives a publish to {unread:?}, so it \
must be counted in flight",
);
settle_ack(expect_next(&mut stream, "counts routed elsewhere").await).await;
}
assert!(
quiescent(&coordinator),
"counts: a publish no subscription receives must not be counted in flight; TestApp \
would wait for a handler that never runs",
);
settlement_counts(&coordinator, &mut stream, inject).await;
}
drop(subscriber);
connected.shutdown().await.expect("shutdown failed");
}
async fn settlement_counts<M, E, S>(
coordinator: &Coordinator,
stream: &mut S,
inject: impl Fn(&'static [u8]),
) where
M: IncomingMessage,
E: fmt::Debug,
S: Stream<Item = Result<M, E>> + Unpin,
{
let coordinator = coordinator.clone();
{
inject(b"dropped");
let msg = expect_next(stream, "counts nack(false)").await;
match msg.nack(false).await {
Ok(()) | Err(AckError::Unsupported) => {}
Err(other) => panic!("nack must succeed or be unsupported, got: {other:?}"),
}
assert!(
quiescent(&coordinator),
"counts: a delivery nacked without requeue must be released exactly once",
);
expect_no_more(stream, "counts nack(false)").await;
inject(b"requeued");
let msg = expect_next(stream, "counts nack(true)").await;
match msg.nack(true).await {
Ok(()) => {
assert!(
!quiescent(&coordinator),
"counts: a requeued message must be counted in flight again before `nack` \
returns; TestApp would stop waiting before the redelivery was handled",
);
settle_ack(expect_next(stream, "counts requeued").await).await;
assert!(
quiescent(&coordinator),
"counts: an acked redelivery must be released exactly once",
);
}
Err(AckError::Unsupported) => assert!(
quiescent(&coordinator),
"counts: a delivery whose requeue is unsupported must still be released",
),
Err(other) => panic!("nack must succeed or be unsupported, got: {other:?}"),
}
inject(b"delayed");
let msg = expect_next(stream, "counts nack_after").await;
if msg.supports_nack_after() {
delayed_redelivery(msg, &coordinator, stream).await;
} else {
settle_ack(msg).await;
}
inject(b"unsettled");
drop(expect_next(stream, "counts unsettled drop").await);
if !quiescent(&coordinator) {
settle_ack(expect_next(stream, "counts unsettled redelivery").await).await;
assert!(
quiescent(&coordinator),
"counts: an unsettled delivery the transport took back is counted once more than \
it redelivers",
);
}
}
}
async fn delayed_redelivery<M, E, S>(msg: M, coordinator: &Coordinator, stream: &mut S)
where
M: IncomingMessage,
E: fmt::Debug,
S: Stream<Item = Result<M, E>> + Unpin,
{
match msg.nack_after(REDELIVERY_DELAY).await {
Ok(()) => {}
Err(AckError::Unsupported) => {
assert!(
quiescent(coordinator),
"counts: a delivery whose delayed nack is unsupported must still be released",
);
return;
}
Err(other) => panic!("nack_after must succeed or be unsupported, got: {other:?}"),
}
assert!(
quiescent(coordinator),
"counts: after `nack_after` the delivery must be released and the redelivery must wait \
for its delay; TestApp returns from the publish there and delivers it on `advance`",
);
let half = REDELIVERY_DELAY / 2;
tokio::time::advance(half).await;
coordinator.fire_due_timers().await;
assert!(
quiescent(coordinator) && stream.next().now_or_never().is_none(),
"counts: a delayed nack of {REDELIVERY_DELAY:?} came back after {half:?}",
);
tokio::time::advance(REDELIVERY_DELAY.saturating_sub(half)).await;
coordinator.fire_due_timers().await;
assert!(
!quiescent(coordinator),
"counts: a delayed redelivery must be counted in flight once its delay ran out and \
TestApp::advance fired the due timers; schedule it with \
`Coordinator::schedule_redelivery`",
);
settle_ack(expect_next(stream, "counts delayed redelivery").await).await;
assert!(
quiescent(coordinator),
"counts: an acked delayed redelivery must be released exactly once",
);
}
#[derive(Debug, Clone, Copy)]
enum Transport {
Server,
InProcess,
}
impl Transport {
async fn connect<B: InProcess>(self, broker: B) -> B::Connected {
match self {
Self::Server => broker
.connect()
.await
.unwrap_or_else(|err| panic!("the broker must connect to its server: {err}")),
Self::InProcess => connect_in_process(broker).await,
}
}
const fn delivery_timeout(self) -> Duration {
match self {
Self::Server => SERVER_TIMEOUT,
Self::InProcess => DELIVERY_TIMEOUT,
}
}
const fn quiet(self) -> Duration {
match self {
Self::Server => SERVER_QUIET,
Self::InProcess => QUIET,
}
}
}
impl fmt::Display for Transport {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(match self {
Self::Server => "against the server",
Self::InProcess => "in process",
})
}
}
async fn next_on<S, M, E>(stream: &mut S, transport: Transport, label: &str) -> M
where
S: Stream<Item = Result<M, E>> + Unpin,
M: IncomingMessage,
E: fmt::Debug,
{
timeout(transport.delivery_timeout(), stream.next())
.await
.unwrap_or_else(|_| panic!("{label} {transport}: no delivery arrived"))
.unwrap_or_else(|| panic!("{label} {transport}: the stream ended"))
.unwrap_or_else(|err| panic!("{label} {transport}: the stream yielded an error: {err:?}"))
}
pub async fn backlog_matches_server<B, MkBroker, Pub, MkPub>(
make_broker: MkBroker,
make_publisher: MkPub,
) where
B: InProcess,
B::Connected: Subscribe,
MkBroker: Fn() -> B,
Pub: Publisher,
MkPub: Fn(&Connected<B>) -> Pub,
{
let declared = {
let connected = connect_in_process(make_broker()).await;
let declared = connected.backlog();
connected.shutdown().await.expect("shutdown failed");
declared
};
let mut server_refused = None;
for transport in [Transport::Server, Transport::InProcess] {
let connected = transport.connect(make_broker()).await;
let subject = unique_subject("conformance.backlog");
let publisher = make_publisher(&connected);
let before = publisher
.publish(
OutgoingMessage::new(subject.as_str(), b"before-subscribe".as_slice()),
None,
)
.await
.map_err(|err| err.to_string());
let refused_before = match (transport, before, server_refused.as_deref()) {
(Transport::Server, Err(reason), _) => {
server_refused = Some(reason);
true
}
(Transport::InProcess, Ok(()), Some(reason)) => panic!(
"backlog_matches_server: the server refused a publish to a name no subscription \
had opened ({reason}), and the in-process transport accepted it; a test passes \
where production fails"
),
(Transport::InProcess, Err(_), Some(_)) => true,
(Transport::InProcess, Err(reason), None) => panic!(
"backlog_matches_server in process: a publish to {subject:?} was refused \
({reason}), and the server accepted it; a test fails where production succeeds"
),
(_, Ok(()), _) => false,
};
let publish = async |payload: &[u8]| {
publisher
.publish(OutgoingMessage::new(subject.as_str(), payload), None)
.await
.unwrap_or_else(|err| {
panic!(
"backlog_matches_server {transport}: a publish to {subject:?} failed: {err}"
)
});
};
let mut subscriber = Subscribe::subscribe(&connected, &subject)
.await
.unwrap_or_else(|err| {
panic!(
"backlog_matches_server {transport}: subscribing to {subject:?} failed: {err}"
)
});
publish(b"after-subscribe").await;
{
let mut stream = pin!(subscriber.stream());
let first = next_on(&mut stream, transport, "backlog_matches_server").await;
let expected: &[u8] = match declared {
Backlog::Delivered if !refused_before => b"before-subscribe",
_ => b"after-subscribe",
};
assert_eq!(
first.payload(),
expected,
"backlog_matches_server {transport}: {}, so a subscription opened after a publish \
must first receive {:?}",
if refused_before {
"the publish before the subscription opened was refused".to_owned()
} else {
format!("the in-process transport declares `Backlog::{declared:?}`")
},
String::from_utf8_lossy(expected),
);
settle_ack(first).await;
if expected == b"before-subscribe" {
let second = next_on(&mut stream, transport, "backlog_matches_server").await;
assert_eq!(
second.payload(),
b"after-subscribe",
"backlog_matches_server {transport}: the message published after the \
subscription opened comes next",
);
settle_ack(second).await;
}
}
drop(subscriber);
connected.shutdown().await.expect("shutdown failed");
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub enum Refusal<Src> {
PayloadOver {
name: String,
limit: usize,
},
Publish {
name: String,
},
Subscription {
source: Src,
},
Conflicting {
open: Src,
refused: Src,
},
}
pub async fn refuses_like_the_server<B, MkBroker, Src, Pub, MkPub>(
make_broker: MkBroker,
make_publisher: MkPub,
refusals: impl IntoIterator<Item = Refusal<Src>> + Send,
) where
B: InProcess,
B::Connected: Subscribe,
MkBroker: Fn() -> B + Send,
Src: SubscriptionSource<Connected<B>> + Clone + Send + Sync,
Pub: Publisher,
MkPub: Fn(&Connected<B>) -> Pub + Send + Sync,
{
let refusals: Vec<Refusal<Src>> = refusals.into_iter().collect();
assert!(
!refusals.is_empty(),
"pass at least one probe the server refuses; an empty list passes whatever the in-process \
transport accepts",
);
for refusal in &refusals {
let mut refused = [false; 2];
for (leg, transport) in [Transport::Server, Transport::InProcess]
.into_iter()
.enumerate()
{
let connected = transport.connect(make_broker()).await;
refused[leg] = probe(&connected, refusal, transport, &make_publisher).await;
connected.shutdown().await.expect("shutdown failed");
}
let what = describe::<Connected<B>, Src>(refusal);
match refused {
[true, true] => {}
[true, false] => panic!(
"refuses_like_the_server: {what}: the server refused it and the in-process \
transport accepted it; a test passes where production fails"
),
[false, true] => panic!(
"refuses_like_the_server: {what}: the in-process transport refused it and the \
server accepted it; a test fails where production succeeds"
),
[false, false] => panic!(
"refuses_like_the_server: {what}: the server accepted it, so it proves nothing; \
pass a probe the server refuses"
),
}
}
admits_like_the_server(make_broker).await;
}
async fn admits_like_the_server<B, MkBroker>(make_broker: MkBroker)
where
B: InProcess,
B::Connected: Subscribe,
MkBroker: Fn() -> B + Send,
{
let mut admitted = [[false; 3]; 2];
for (leg, transport) in [Transport::Server, Transport::InProcess]
.into_iter()
.enumerate()
{
let connected = transport.connect(make_broker()).await;
let name = unique_subject("conformance.admitted");
let other = unique_subject("conformance.admitted.elsewhere");
let mut opened = Vec::new();
for (slot, subject) in [&name, &name, &other].into_iter().enumerate() {
if let Ok(subscriber) = Subscribe::subscribe(&connected, subject).await {
admitted[leg][slot] = true;
opened.push(subscriber);
}
}
drop(opened);
connected.shutdown().await.expect("shutdown failed");
}
let [server, in_process] = admitted;
assert_eq!(
in_process, server,
"refuses_like_the_server: a subscription to a name, a second one to it and one to another \
name are admitted as {in_process:?} in process and as {server:?} by the server; the \
in-process transport must admit what the server admits, or a test passes or fails where \
production does the opposite",
);
}
fn describe<C, Src>(refusal: &Refusal<Src>) -> String
where
C: ConnectedBroker,
Src: SubscriptionSource<C>,
{
match refusal {
Refusal::PayloadOver { name, limit } => format!(
"a payload of {} bytes to {name:?}, one over the limit",
limit + 1
),
Refusal::Publish { name } => format!("a publish to {name:?}"),
Refusal::Subscription { source } => format!("the subscription {:?}", source.name()),
Refusal::Conflicting { open, refused } => format!(
"the subscription {:?} beside {:?}",
refused.name(),
open.name()
),
}
}
async fn probe<C, Src, Pub, MkPub>(
connected: &C,
refusal: &Refusal<Src>,
transport: Transport,
make_publisher: &MkPub,
) -> bool
where
C: Subscribe,
Src: SubscriptionSource<C> + Clone + Sync,
Pub: Publisher,
MkPub: Fn(&C) -> Pub + Sync,
{
match refusal {
Refusal::PayloadOver { name, limit } => {
let mut subscriber = Subscribe::subscribe(connected, name).await.unwrap_or_else(
|err| {
panic!(
"refuses_like_the_server {transport}: subscribing to {name:?} failed: {err}"
)
},
);
let publisher = make_publisher(connected);
let at_limit = vec![b'x'; *limit];
if let Err(err) = publisher
.publish(
OutgoingMessage::new(name.as_str(), at_limit.as_slice()),
None,
)
.await
{
panic!(
"refuses_like_the_server {transport}: a payload of exactly {limit} bytes to \
{name:?} was refused: {err}"
);
}
let over = vec![b'x'; limit + 1];
let refused = publisher
.publish(OutgoingMessage::new(name.as_str(), over.as_slice()), None)
.await
.is_err();
let mut stream = pin!(subscriber.stream());
let delivered = next_on(&mut stream, transport, "refuses_like_the_server").await;
assert_eq!(
delivered.payload().len(),
*limit,
"refuses_like_the_server {transport}: the payload of exactly the limit must be \
delivered first",
);
settle_ack(delivered).await;
if refused {
assert!(
timeout(transport.quiet(), stream.next()).await.is_err(),
"refuses_like_the_server {transport}: the payload over the limit was refused, \
and delivered anyway",
);
}
refused
}
Refusal::Publish { name } => make_publisher(connected)
.publish(
OutgoingMessage::new(name.as_str(), b"refused".as_slice()),
None,
)
.await
.is_err(),
Refusal::Subscription { source } => source.clone().subscribe(connected).await.is_err(),
Refusal::Conflicting { open, refused } => {
let first = open.clone().subscribe(connected).await.unwrap_or_else(|err| {
panic!(
"refuses_like_the_server {transport}: the subscription {:?} opened first was \
refused: {err}",
open.name(),
)
});
let refused = refused.clone().subscribe(connected).await.is_err();
drop(first);
refused
}
}
}
#[cfg(all(test, feature = "memory"))]
mod tests;