#![cfg(all(feature = "memory", feature = "macros", feature = "json"))]
mod common;
use std::collections::{HashMap, HashSet};
use std::convert::Infallible;
use std::future::{Future, ready};
use std::pin::pin;
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::{Arc, Condvar, Mutex, PoisonError, mpsc};
use std::thread;
use std::time::{Duration, Instant};
use common::Order;
use futures::{Stream, StreamExt};
use ruststream::memory::prelude::*;
use ruststream::memory::{MemoryMessage, MemoryPublisher};
use ruststream::{
AckError, AddressedCopies, HeaderMap, IncomingMessage, Outgoing, RedeliveryAddress,
RedeliveryAddressed, Subscribe, Subscriber, SubscriptionSource,
};
use serde::{Deserialize, Serialize};
use tokio::runtime::{Builder, Runtime};
use tokio::sync::Notify;
use tokio::time::timeout;
const APP_THREADS: &str = "the-app-runtime";
const DEADLINE: Duration = Duration::from_secs(10);
fn app_runtime() -> Runtime {
Builder::new_multi_thread()
.worker_threads(2)
.thread_name(APP_THREADS)
.enable_all()
.build()
.expect("the app runtime")
}
fn here() -> String {
thread::current().name().unwrap_or_default().to_owned()
}
#[derive(Debug, Clone, Outgoing, Serialize, Deserialize, schemars::JsonSchema)]
struct Trace {
id: u32,
start: String,
after_timer: String,
spawned: String,
on_main: String,
spawned_on_main: String,
spawned_on_main_handle: String,
}
#[subscriber("jobs", threads(2), reply("traces"))]
async fn traced(job: &Order, Ctx(main): Ctx<MainRuntime>) -> Trace {
let start = here();
tokio::time::sleep(Duration::from_millis(1)).await;
let after_timer = here();
let spawned = tokio::spawn(async { here() }).await.unwrap_or_default();
let on_main = main.run(async { here() }).await.unwrap_or_default();
let spawned_on_main = main.spawn(async { here() }).await.unwrap_or_default();
let spawned_on_main_handle = main
.as_handle()
.spawn(async { here() })
.await
.unwrap_or_default();
Trace {
id: job.id,
start,
after_timer,
spawned,
on_main,
spawned_on_main,
spawned_on_main_handle,
}
}
async fn read<Model, Messages>(stream: &mut Messages, count: usize) -> Vec<Model>
where
Model: for<'de> Deserialize<'de>,
Messages: Stream<Item = Result<MemoryMessage, Infallible>> + Unpin,
{
let mut models = Vec::with_capacity(count);
for _ in 0..count {
let msg = timeout(DEADLINE, stream.next())
.await
.expect("a reply within the deadline")
.expect("the stream stays open")
.expect("a memory subscriber never errors");
models.push(serde_json::from_slice(msg.payload()).expect("a JSON reply"));
msg.ack().await.expect("ack");
}
models
}
#[test]
fn a_delivery_is_handled_on_a_dedicated_thread_alone() {
const JOBS: u32 = 6;
let broker = MemoryBroker::new();
let traces = app_runtime().block_on(async {
let mut subscriber = broker.subscribe("traces");
let mut replies = pin!(subscriber.stream());
let app =
RustStream::new(AppInfo::new("threads", "0.1.0")).with_broker(broker.clone(), |b| {
b.include(traced).out_reply(Publish);
});
let running = app.start().await.expect("startup");
let publisher = broker.publisher();
for id in 0..JOBS {
publisher
.message(&Order { id })
.to("jobs")
.publish()
.await
.expect("publish");
}
let traces: Vec<Trace> = read(&mut replies, JOBS as usize).await;
running.shutdown().await.expect("shutdown");
traces
});
let mut threads = HashSet::new();
for trace in &traces {
assert!(
!trace.start.starts_with(APP_THREADS),
"the handler ran on the app's runtime: {trace:?}"
);
assert_eq!(
trace.after_timer, trace.start,
"moved across a timer: {trace:?}"
);
assert_eq!(
trace.spawned, trace.start,
"a plain spawn left the thread: {trace:?}"
);
assert!(
trace.on_main.starts_with(APP_THREADS),
"the work run on the main runtime ran elsewhere: {trace:?}"
);
assert!(
trace.spawned_on_main.starts_with(APP_THREADS),
"the task spawned on the main runtime ran elsewhere: {trace:?}"
);
assert!(
trace.spawned_on_main_handle.starts_with(APP_THREADS),
"the task spawned through the main runtime's handle ran elsewhere: {trace:?}"
);
threads.insert(trace.start.clone());
}
assert!(
threads.len() <= 2,
"more threads than declared: {threads:?}"
);
}
#[derive(Debug, Clone, Outgoing, Serialize, Deserialize, schemars::JsonSchema)]
struct Placed {
id: u32,
thread: String,
}
#[subscriber("keyed", threads(3, by_key), reply("placed"))]
async fn placed(job: &Order) -> Placed {
tokio::task::yield_now().await;
Placed {
id: job.id,
thread: here(),
}
}
#[test]
fn a_key_stays_on_one_thread_and_in_order() {
const PER_KEY: u32 = 20;
let keys = ["alpha", "beta", "gamma", "delta"];
let broker = MemoryBroker::new();
let seen: Vec<Placed> = app_runtime().block_on(async {
let mut subscriber = broker.subscribe("placed");
let mut replies = pin!(subscriber.stream());
let app =
RustStream::new(AppInfo::new("threads", "0.1.0")).with_broker(broker.clone(), |b| {
b.include(placed).out_reply(Publish);
});
let running = app.start().await.expect("startup");
let publisher = broker.publisher();
for sequence in 0..PER_KEY {
for (band, key) in (0u32..).zip(keys) {
let mut headers = HeaderMap::new();
headers.insert("partition-key", key);
publisher
.message(&Order {
id: band * 1000 + sequence,
})
.with_headers(headers)
.to("keyed")
.publish()
.await
.expect("publish");
}
}
let seen = read(&mut replies, (PER_KEY as usize) * keys.len()).await;
running.shutdown().await.expect("shutdown");
seen
});
let mut bands: HashMap<u32, Vec<&Placed>> = HashMap::new();
for reply in &seen {
assert!(
!reply.thread.starts_with(APP_THREADS),
"a keyed delivery ran on the app's runtime: {reply:?}"
);
bands.entry(reply.id / 1000).or_default().push(reply);
}
for (band, replies) in bands {
let threads: HashSet<&str> = replies.iter().map(|reply| reply.thread.as_str()).collect();
assert_eq!(
threads.len(),
1,
"key {band} moved between threads: {threads:?}"
);
let ids: Vec<u32> = replies.iter().map(|reply| reply.id).collect();
assert!(
ids.windows(2).all(|pair| pair[0] < pair[1]),
"key {band} out of order: {ids:?}"
);
}
}
#[subscriber("unkeyed", threads(2, by_key), reply("unkeyed.done"))]
async fn unkeyed(job: &Order) -> Placed {
Placed {
id: job.id,
thread: here(),
}
}
#[test]
fn keyless_deliveries_of_a_keyed_subscription_take_the_threads_in_turn() {
const JOBS: u32 = 8;
let broker = MemoryBroker::new();
let seen: Vec<Placed> = app_runtime().block_on(async {
let mut subscriber = broker.subscribe("unkeyed.done");
let mut replies = pin!(subscriber.stream());
let app =
RustStream::new(AppInfo::new("threads", "0.1.0")).with_broker(broker.clone(), |b| {
b.include(unkeyed).out_reply(Publish);
});
let running = app.start().await.expect("startup");
let publisher = broker.publisher();
for id in 0..JOBS {
publisher
.message(&Order { id })
.to("unkeyed")
.publish()
.await
.expect("publish");
}
let seen = read(&mut replies, JOBS as usize).await;
running.shutdown().await.expect("shutdown");
seen
});
let mut by_id: Vec<&Placed> = seen.iter().collect();
by_id.sort_by_key(|reply| reply.id);
let threads: HashSet<&str> = by_id.iter().map(|reply| reply.thread.as_str()).collect();
assert_eq!(
threads.len(),
2,
"keyless deliveries piled on one lane: {seen:?}"
);
assert!(
by_id
.windows(2)
.all(|pair| pair[0].thread != pair[1].thread),
"keyless deliveries did not take the threads in turn: {by_id:?}"
);
}
const PAST_THE_HELD_RING: u32 = 20;
#[derive(Default)]
struct Held {
open: Mutex<bool>,
opened: Condvar,
others: AtomicU32,
holding: Notify,
}
#[derive(Debug, Clone, Outgoing, Serialize, Deserialize, schemars::JsonSchema)]
struct Released {
id: u32,
released: bool,
}
fn wait_at_gate(shared: &Held) -> bool {
let guard = shared.open.lock().unwrap_or_else(PoisonError::into_inner);
shared.holding.notify_one();
let (open, _) = shared
.opened
.wait_timeout_while(guard, DEADLINE, |open| !*open)
.unwrap_or_else(PoisonError::into_inner);
*open
}
#[subscriber("held", threads(2), reply("held.done"))]
async fn held(job: &Order, ctx: &mut Context<'_, (), Arc<Held>>) -> Released {
let shared = ctx.state();
if job.id == 0 {
return Released {
id: 0,
released: wait_at_gate(shared),
};
}
if shared.others.fetch_add(1, Ordering::SeqCst) + 1 == PAST_THE_HELD_RING {
*shared.open.lock().unwrap_or_else(PoisonError::into_inner) = true;
shared.opened.notify_all();
}
Released {
id: job.id,
released: true,
}
}
#[test]
fn a_held_thread_holds_up_none_of_the_others() {
const OTHERS: u32 = 40;
let broker = MemoryBroker::new();
let shared = Arc::new(Held::default());
let seen: Vec<Released> = app_runtime().block_on(async {
let mut subscriber = broker.subscribe("held.done");
let mut replies = pin!(subscriber.stream());
let state = Arc::clone(&shared);
let app = RustStream::new(AppInfo::new("threads", "0.1.0"))
.on_startup(async move |()| Ok::<_, Infallible>(state))
.with_broker(broker.clone(), |b| {
b.include(held).out_reply(Publish);
});
let running = app.start().await.expect("startup");
let publisher = broker.publisher();
publisher
.message(&Order { id: 0 })
.to("held")
.publish()
.await
.expect("publish");
timeout(DEADLINE, shared.holding.notified())
.await
.expect("order 0 reaches its gate");
for id in 1..=OTHERS {
publisher
.message(&Order { id })
.to("held")
.publish()
.await
.expect("publish");
}
let seen = read(&mut replies, OTHERS as usize + 1).await;
running.shutdown().await.expect("shutdown");
seen
});
let first = seen
.iter()
.find(|reply| reply.id == 0)
.expect("order 0 answers");
assert!(
first.released,
"the other thread did not get past the held thread's ring"
);
}
struct Report {
publisher: MemoryPublisher,
handled: Notify,
}
impl Report {
fn new(broker: &MemoryBroker) -> Arc<Self> {
Arc::new(Self {
publisher: broker.publisher(),
handled: Notify::new(),
})
}
}
const LATER: Duration = Duration::from_millis(200);
#[subscriber("continued", threads(1))]
async fn continued(job: &Order, ctx: &mut Context<'_, (), Arc<Report>>) -> HandlerOutcome {
let id = job.id;
let publisher = ctx.state().publisher.clone();
ctx.after(HandlerOutcome::ack()).then(async move {
tokio::time::sleep(LATER).await;
let reply = Placed { id, thread: here() };
let _ = publisher
.message(&reply)
.to("continued.done")
.publish()
.await;
});
let publisher = ctx.state().publisher.clone();
ctx.state().handled.notify_one();
HandlerOutcome::ack().and_after(async move {
tokio::time::sleep(LATER).await;
let reply = Placed {
id: id + 1000,
thread: here(),
};
let _ = publisher
.message(&reply)
.to("continued.done")
.publish()
.await;
})
}
fn continuations(early: bool) -> Vec<Placed> {
let broker = MemoryBroker::new();
app_runtime().block_on(async {
let mut subscriber = broker.subscribe("continued.done");
let mut replies = pin!(subscriber.stream());
let report = Report::new(&broker);
let state = Arc::clone(&report);
let app = RustStream::new(AppInfo::new("threads", "0.1.0"))
.on_startup(async move |()| Ok::<_, Infallible>(state))
.with_broker(broker.clone(), |b| {
b.include(continued);
});
let running = app.start().await.expect("startup");
broker
.publisher()
.message(&Order { id: 1 })
.to("continued")
.publish()
.await
.expect("publish");
if early {
timeout(DEADLINE, report.handled.notified())
.await
.expect("the handler returns");
running.shutdown().await.expect("shutdown");
read(&mut replies, 2).await
} else {
let seen = read(&mut replies, 2).await;
running.shutdown().await.expect("shutdown");
seen
}
})
}
#[test]
fn continuations_run_on_the_delivery_thread() {
for reply in continuations(false) {
assert!(
reply.thread.starts_with("continued/"),
"a continuation left the delivery's thread: {reply:?}"
);
}
}
#[test]
fn a_continuation_pending_at_shutdown_completes() {
let seen = continuations(true);
assert_eq!(seen.len(), 2, "both continuations report: {seen:?}");
}
struct Cancelled(mpsc::Sender<()>);
impl Drop for Cancelled {
fn drop(&mut self) {
let _ = self.0.send(());
}
}
struct Stuck {
started: Notify,
cancelled: Mutex<mpsc::Sender<()>>,
}
#[subscriber("stuck", threads(1))]
async fn stuck(_job: &Order, ctx: &mut Context<'_, (), Arc<Stuck>>) -> HandlerOutcome {
let sender = ctx
.state()
.cancelled
.lock()
.unwrap_or_else(PoisonError::into_inner)
.clone();
let _guard = Cancelled(sender);
ctx.state().started.notify_one();
std::future::pending::<()>().await;
HandlerOutcome::ack()
}
#[test]
fn the_shutdown_timeout_cancels_a_delivery_on_its_thread() {
let (cancelled, told) = mpsc::channel();
let broker = MemoryBroker::new();
app_runtime().block_on(async {
let shared = Arc::new(Stuck {
started: Notify::new(),
cancelled: Mutex::new(cancelled),
});
let state = Arc::clone(&shared);
let app = RustStream::new(AppInfo::new("threads", "0.1.0"))
.shutdown_timeout(Duration::from_millis(100))
.on_startup(async move |()| Ok::<_, Infallible>(state))
.with_broker(broker.clone(), |b| {
b.include(stuck);
});
let running = app.start().await.expect("startup");
broker
.publisher()
.message(&Order { id: 1 })
.to("stuck")
.publish()
.await
.expect("publish");
timeout(DEADLINE, shared.started.notified())
.await
.expect("the delivery starts");
timeout(DEADLINE, running.shutdown())
.await
.expect("the shutdown timeout bounds the wait")
.expect("shutdown");
});
assert!(
told.recv_timeout(DEADLINE).is_ok(),
"the delivery kept running on its thread after the shutdown timeout"
);
}
#[derive(Debug, Clone)]
struct CopiedSubscription;
impl CopiedSubscription {
const fn new() -> Self {
Self
}
}
impl<C: Subscribe> SubscriptionSource<C> for CopiedSubscription {
type Subscriber = UnsettledSubscriber<C::Subscriber>;
type Copies = AddressedCopies;
#[allow(clippy::unnecessary_literal_bound)]
fn name(&self) -> &str {
"retried"
}
async fn subscribe(self, connected: &C) -> Result<Self::Subscriber, C::Error> {
Ok(UnsettledSubscriber(connected.subscribe("retried").await?))
}
}
impl<C: Subscribe> RedeliveryAddressed<C> for CopiedSubscription {
fn redelivery_address(
&self,
_connected: &C,
) -> impl Future<Output = Result<RedeliveryAddress, C::Error>> + Send {
ready(Ok(RedeliveryAddress::new("copies")))
}
}
struct UnsettledSubscriber<S>(S);
impl<S: Subscriber> Subscriber for UnsettledSubscriber<S> {
type Message = UnsettledMessage<S::Message>;
type Error = S::Error;
fn stream(&mut self) -> impl Stream<Item = Result<Self::Message, Self::Error>> + Send + '_ {
self.0.stream().map(|item| item.map(UnsettledMessage))
}
}
struct UnsettledMessage<M>(M);
impl<M: IncomingMessage> IncomingMessage for UnsettledMessage<M> {
fn payload(&self) -> &[u8] {
self.0.payload()
}
fn headers(&self) -> &HeaderMap {
self.0.headers()
}
async fn ack(self) -> Result<(), AckError> {
self.0.ack().await
}
async fn nack(self, requeue: bool) -> Result<(), AckError> {
self.0.nack(requeue).await
}
}
const COMPUTATION: Duration = Duration::from_secs(1);
#[derive(Default)]
struct Deferred {
handled: Notify,
}
#[subscriber(CopiedSubscription::new(), threads(1))]
async fn retried(order: &Order, ctx: &mut Context<'_, (), Arc<Deferred>>) -> HandlerOutcome {
if order.id == 1 {
ctx.state().handled.notify_one();
return HandlerOutcome::retry_after(Duration::from_millis(50));
}
thread::sleep(COMPUTATION);
HandlerOutcome::ack()
}
fn copy_arrival(orders: &[u32], shut: bool) -> Option<Duration> {
let broker = MemoryBroker::new();
app_runtime().block_on(async {
let mut copy_subscriber = broker.subscribe("copies");
let mut arrivals = pin!(copy_subscriber.stream());
let deferred = Arc::new(Deferred::default());
let state = Arc::clone(&deferred);
let app = RustStream::new(AppInfo::new("threads", "0.1.0"))
.on_startup(async move |()| Ok::<_, Infallible>(state))
.with_broker(broker.clone(), |b| {
b.include(retried).out_retry(Publish);
});
let running = app.start().await.expect("startup");
let publisher = broker.publisher();
for &id in orders {
publisher
.message(&Order { id })
.to("retried")
.publish()
.await
.expect("publish");
}
timeout(DEADLINE, deferred.handled.notified())
.await
.expect("the handler defers order 1");
let since = Instant::now();
let running = if shut {
running.shutdown().await.expect("shutdown");
None
} else {
Some(running)
};
let arrived = timeout(DEADLINE, arrivals.next())
.await
.ok()
.map(|_| since.elapsed());
if let Some(running) = running {
running.shutdown().await.expect("shutdown");
}
arrived
})
}
#[test]
fn a_delayed_retry_copy_waits_for_the_thread() {
let arrived = copy_arrival(&[1, 2], false).expect("the copy arrives");
assert!(
arrived >= COMPUTATION / 2,
"the copy left before the thread's computation ended: {arrived:?}"
);
}
#[test]
fn a_retry_copy_pending_at_shutdown_is_published() {
assert!(
copy_arrival(&[1], true).is_some(),
"the copy pending at shutdown was lost"
);
}
#[subscriber("batched", threads(2))]
async fn batched(orders: &[Order], ctx: &mut Context<'_, (), Arc<Report>>) -> HandlerOutcome {
let reply = Placed {
id: orders.first().map_or(0, |order| order.id),
thread: here(),
};
let _ = ctx
.state()
.publisher
.message(&reply)
.to("batched.done")
.publish()
.await;
HandlerOutcome::ack()
}
#[test]
fn a_batch_is_handled_on_a_dedicated_thread() {
let broker = MemoryBroker::new();
let seen: Vec<Placed> = app_runtime().block_on(async {
let mut subscriber = broker.subscribe("batched.done");
let mut replies = pin!(subscriber.stream());
let report = Report::new(&broker);
let app = RustStream::new(AppInfo::new("threads", "0.1.0"))
.on_startup(async move |()| Ok::<_, Infallible>(report))
.with_broker(broker.clone(), |b| {
b.include(batched.batch(nonzero!(4)));
});
let running = app.start().await.expect("startup");
broker
.publisher()
.message(&Order { id: 1 })
.to("batched")
.publish()
.await
.expect("publish");
let seen = read(&mut replies, 1).await;
running.shutdown().await.expect("shutdown");
seen
});
for reply in &seen {
assert!(
reply.thread.starts_with("batched/"),
"the batch ran off the subscription's threads: {reply:?}"
);
}
}
#[cfg(feature = "poll-diagnostics")]
#[subscriber("sampled", threads(2), reply("sampled.done"))]
async fn sampled(job: &Order) -> Placed {
Placed {
id: job.id,
thread: here(),
}
}
#[cfg(feature = "poll-diagnostics")]
#[test]
fn every_thread_samples_into_the_subscription_report() {
use ruststream::runtime::PollDiagnostics;
const JOBS: u32 = 8;
let broker = MemoryBroker::new();
let diagnostics = PollDiagnostics::new().sample_every(ruststream::nonzero!(1u32));
app_runtime().block_on(async {
let mut subscriber = broker.subscribe("sampled.done");
let mut replies = pin!(subscriber.stream());
let app = RustStream::new(AppInfo::new("threads", "0.1.0"))
.poll_diagnostics(diagnostics.clone())
.with_broker(broker.clone(), |b| {
b.include(sampled).out_reply(Publish);
});
let running = app.start().await.expect("startup");
let publisher = broker.publisher();
for id in 0..JOBS {
publisher
.message(&Order { id })
.to("sampled")
.publish()
.await
.expect("publish");
}
read::<Placed, _>(&mut replies, JOBS as usize).await;
running.shutdown().await.expect("shutdown");
});
let report = diagnostics
.report("sampled")
.expect("the subscription is sampled");
assert_eq!(report.samples(), u64::from(JOBS));
}