#![cfg(all(
feature = "memory",
feature = "macros",
feature = "json",
feature = "cbor",
feature = "testing",
feature = "logging"
))]
mod common;
use std::convert::Infallible;
use std::future::{pending, ready};
use std::io;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, OnceLock};
use std::time::Duration;
use ruststream::codec::CborCodec;
use ruststream::memory::prelude::*;
use ruststream::memory::{ConnectedMemoryBroker, MemorySubscriber};
use ruststream::runtime::{
ContextKind, HealthState, IntoSource, Outgoing, PublishError, PublishTransform, Reads,
RustStreamError, subscriber as subscriber_def,
};
use ruststream::testing::TestApp;
use ruststream::{
AddressedCopies, ConnectedBroker, DescribeServer, RedeliveryAddress, RedeliveryAddressed,
ServerSpec, SubscriptionSource,
};
use tokio::sync::Notify;
use tokio::time::timeout;
use tracing_subscriber::EnvFilter;
use tracing_subscriber::fmt::MakeWriter;
use common::{Order, Receipt, Wire};
#[derive(Deserialized)]
struct Frame<'a>(&'a [u8]);
#[derive(Outgoing, Serialized)]
struct Export(Vec<u8>);
#[derive(Clone)]
struct LogBuffer(Arc<Mutex<Vec<u8>>>);
impl LogBuffer {
fn text(&self) -> String {
let bytes = self.0.lock().expect("log buffer").clone();
String::from_utf8(bytes).expect("utf-8 logs")
}
}
struct EventWriter {
shared: Arc<Mutex<Vec<u8>>>,
event: Vec<u8>,
}
impl io::Write for EventWriter {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
self.event.extend_from_slice(buf);
Ok(buf.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
impl Drop for EventWriter {
fn drop(&mut self) {
if let Ok(mut shared) = self.shared.lock() {
shared.extend_from_slice(&self.event);
}
}
}
impl<'a> MakeWriter<'a> for LogBuffer {
type Writer = EventWriter;
fn make_writer(&'a self) -> Self::Writer {
EventWriter {
shared: Arc::clone(&self.0),
event: Vec::new(),
}
}
}
static LOGS: OnceLock<LogBuffer> = OnceLock::new();
fn logs() -> &'static LogBuffer {
LOGS.get_or_init(|| {
let buffer = LogBuffer(Arc::new(Mutex::new(Vec::new())));
let subscriber = tracing_subscriber::fmt()
.with_env_filter(EnvFilter::new("ruststream=debug"))
.with_ansi(false)
.with_writer(buffer.clone())
.finish();
tracing::subscriber::set_global_default(subscriber)
.expect("this binary installs no other subscriber");
buffer
})
}
fn line_with<'a>(text: &'a str, markers: &[&str]) -> &'a str {
text.lines()
.find(|line| markers.iter().all(|marker| line.contains(marker)))
.unwrap_or_else(|| panic!("no logged line with {markers:?} in:\n{text}"))
}
#[subscriber("cov.logged")]
async fn logged(_order: &Order) -> HandlerOutcome {
HandlerOutcome::ack()
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn the_lifecycle_logs_name_the_service_brokers_and_subscribers() {
let logs = logs();
let app = RustStream::new(AppInfo::new("cov-logged-service", "0.3.0"))
.register_broker(MemoryBroker::new())
.with_broker_labeled("cov-logged-ingress", MemoryBroker::new(), |b| {
b.include(logged);
});
let running = app.start().await.expect("startup failed");
running.shutdown().await.expect("graceful shutdown failed");
let text = logs.text();
let starting = line_with(&text, &["starting service", "cov-logged-service"]);
assert!(starting.contains("version=0.3.0"), "{starting}");
assert!(starting.contains("brokers=2"), "{starting}");
assert!(starting.contains("subscribers=1"), "{starting}");
line_with(
&text,
&["broker connected", r#"broker="cov-logged-ingress""#],
);
line_with(&text, &["broker connected", "MemoryBroker"]);
line_with(&text, &["subscriber started", "subscriber=cov.logged"]);
line_with(
&text,
&["broker shut down", r#"broker="cov-logged-ingress""#],
);
}
static FAIL_FAST_READY: Notify = Notify::const_new();
#[subscriber("cov.failfast")]
async fn explodes(order: &Order) -> HandlerOutcome {
assert_eq!(order.id, u32::MAX, "handler exploded");
HandlerOutcome::ack()
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn run_until_returns_when_the_service_tears_itself_down() {
let broker = MemoryBroker::new();
let publisher = broker.publisher();
let app = RustStream::new(AppInfo::new("cov-failfast", "0.1.0"))
.shutdown_timeout(Duration::from_secs(5))
.after_startup(async move |_state| {
FAIL_FAST_READY.notify_one();
Ok::<(), io::Error>(())
})
.with_broker(broker, |b| {
b.include(explodes);
});
let run = tokio::spawn(app.run_until(pending()));
FAIL_FAST_READY.notified().await;
publisher
.message(&Order { id: 1 })
.to("cov.failfast")
.publish()
.await
.expect("publish failed");
let outcome = timeout(Duration::from_secs(10), run)
.await
.expect("run_until never returned on the fail-fast teardown")
.expect("join failed");
let err = outcome.expect_err("the fail-fast reason must surface");
assert!(matches!(err, RustStreamError::Dispatch(_)), "got: {err:?}");
}
static SIGNAL_READY: Notify = Notify::const_new();
#[subscriber("cov.signalled")]
async fn signalled(_order: &Order) -> HandlerOutcome {
HandlerOutcome::ack()
}
#[cfg(unix)]
fn raise_sigterm() {
let status = std::process::Command::new("sh")
.arg("-c")
.arg(format!("kill -TERM {}", std::process::id()))
.status()
.expect("sending a signal to this process");
assert!(status.success(), "kill -TERM failed: {status:?}");
}
#[cfg(unix)]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn run_shuts_down_gracefully_on_a_termination_signal() {
let _guard = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
.expect("SIGTERM registration");
let app = RustStream::new(AppInfo::new("cov-signal", "0.1.0"))
.shutdown_timeout(Duration::from_secs(5))
.after_startup(async move |_state| {
SIGNAL_READY.notify_one();
Ok::<(), io::Error>(())
})
.with_broker(MemoryBroker::new(), |b| {
b.include(signalled);
});
let mut run = tokio::spawn(app.run());
SIGNAL_READY.notified().await;
let outcome = timeout(Duration::from_secs(30), async {
loop {
raise_sigterm();
if let Ok(joined) = timeout(Duration::from_millis(50), &mut run).await {
break joined;
}
}
})
.await
.expect("run ignored the termination signal")
.expect("join failed");
outcome.expect("the signalled shutdown must be graceful");
}
#[derive(Clone)]
struct RefusedSubscription;
impl SubscriptionSource<ConnectedMemoryBroker> for RefusedSubscription {
type Subscriber = MemorySubscriber;
type Copies = AddressedCopies;
#[allow(clippy::unnecessary_literal_bound)]
fn name(&self) -> &str {
"cov.refused"
}
fn subscribe(
self,
_connected: &ConnectedMemoryBroker,
) -> impl Future<Output = Result<Self::Subscriber, MemoryError>> {
ready(Err(MemoryError::ShutDown))
}
}
impl RedeliveryAddressed<ConnectedMemoryBroker> for RefusedSubscription {
fn redelivery_address(
&self,
_connected: &ConnectedMemoryBroker,
) -> impl Future<Output = Result<RedeliveryAddress, MemoryError>> + Send {
ready(Ok(RedeliveryAddress::new("cov.refused")))
}
}
impl IntoSource for RefusedSubscription {
type Source = Self;
fn into_source(self) -> Self {
self
}
}
struct NeverDelivered;
impl<'p> Handle<Frame<'p>> for NeverDelivered {
fn handle(
&self,
_frame: &Frame<'p>,
_outs: &(),
_ctx: &mut Context<'_>,
) -> impl Future<Output = Result<(), HandlerOutcome>> {
ready(Ok(()))
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_refused_subscription_aborts_startup_and_unwinds_the_broker() {
let logs = logs();
let broker = MemoryBroker::new();
let publisher = broker.publisher();
let app = RustStream::new(AppInfo::new("cov-refused", "0.1.0")).with_broker_labeled(
"cov-refused-ingress",
broker,
|b| {
b.include(subscriber_def(RefusedSubscription, NeverDelivered).build());
},
);
let err = app
.start()
.await
.expect_err("the refused subscription must abort startup");
assert!(matches!(err, RustStreamError::Subscribe(_)), "got: {err:?}");
let refused = publisher
.message(&Wire::of(b"x"))
.to("cov.refused")
.publish()
.await
.expect_err("the connected broker must not survive the failed startup");
assert!(
matches!(refused, PublishError::Publish(MemoryError::ShutDown)),
"got: {refused:?}",
);
line_with(
&logs.text(),
&[
"broker shut down after a startup failure",
r#"broker="cov-refused-ingress""#,
],
);
}
struct StubbornBroker;
struct ConnectedStubborn;
impl Broker for StubbornBroker {
type Error = io::Error;
type Connected = ConnectedStubborn;
fn connect(self) -> impl Future<Output = Result<Self::Connected, Self::Error>> {
ready(Ok(ConnectedStubborn))
}
}
impl ConnectedBroker for ConnectedStubborn {
type Error = io::Error;
type Closed = ();
fn shutdown(self) -> impl Future<Output = Result<(), Self::Error>> {
ready(Err(io::Error::other("teardown refused")))
}
}
impl DescribeServer for StubbornBroker {
fn describe_server(&self) -> ServerSpec {
ServerSpec::new("stubborn:0", "memory")
}
}
struct UnreachableBroker;
enum NeverConnected {}
impl Broker for UnreachableBroker {
type Error = io::Error;
type Connected = NeverConnected;
fn connect(self) -> impl Future<Output = Result<Self::Connected, Self::Error>> {
ready(Err(io::Error::other("dial refused")))
}
}
impl ConnectedBroker for NeverConnected {
type Error = io::Error;
type Closed = ();
#[allow(clippy::unused_async_trait_impl)]
async fn shutdown(self) -> Result<(), Self::Error> {
match self {}
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_broker_that_refuses_to_shut_down_fails_the_teardown_and_the_probe() {
let app =
RustStream::new(AppInfo::new("cov-stubborn", "0.1.0")).register_broker(StubbornBroker);
let running = app.start().await.expect("startup failed");
let health = running.health();
assert!(health.is_running());
let err = running
.shutdown()
.await
.expect_err("a refused broker shutdown must surface");
assert!(matches!(err, RustStreamError::Shutdown(_)), "got: {err:?}");
let state = health.state();
assert!(
matches!(&state, HealthState::Failed { reason } if reason.contains("teardown refused")),
"got: {state:?}",
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_refused_broker_shutdown_still_closes_the_others_and_runs_after_shutdown() {
let ingress = MemoryBroker::new();
let publisher = ingress.publisher();
let after_shutdown_ran = Arc::new(AtomicBool::new(false));
let ran = Arc::clone(&after_shutdown_ran);
let app = RustStream::new(AppInfo::new("cov-stubborn-last", "0.1.0"))
.after_shutdown(async move |_: Arc<()>| {
ran.store(true, Ordering::SeqCst);
Ok::<_, io::Error>(())
})
.register_broker(ingress)
.register_broker(StubbornBroker);
let err = app
.start()
.await
.expect("startup failed")
.shutdown()
.await
.expect_err("a refused broker shutdown must surface");
assert!(matches!(err, RustStreamError::Shutdown(_)), "got: {err:?}");
let refused = publisher
.message(&Wire::of(b"x"))
.to("cov.stubborn")
.publish()
.await
.expect_err("the memory broker must be shut down too");
assert!(
matches!(refused, PublishError::Publish(MemoryError::ShutDown)),
"got: {refused:?}",
);
assert!(after_shutdown_ran.load(Ordering::SeqCst));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_refused_shutdown_during_the_startup_unwind_is_logged_not_returned() {
let logs = logs();
let app = RustStream::new(AppInfo::new("cov-unwind", "0.1.0"))
.with_broker_labeled("cov-unwind-egress", StubbornBroker, |_b| {})
.register_broker(UnreachableBroker);
let err = app
.start()
.await
.expect_err("the second broker cannot connect");
assert!(matches!(err, RustStreamError::Connect(_)), "got: {err:?}");
line_with(
&logs.text(),
&[
"broker shutdown failed during the startup unwind",
r#"broker="cov-unwind-egress""#,
"teardown refused",
],
);
}
static HUNG_HANDLER: Notify = Notify::const_new();
#[subscriber("cov.hung")]
async fn hung(_order: &Order) -> HandlerOutcome {
HUNG_HANDLER.notify_one();
pending::<()>().await;
HandlerOutcome::ack()
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn the_shutdown_timeout_aborts_a_handler_that_never_returns() {
let broker = MemoryBroker::new();
let publisher = broker.publisher();
let app = RustStream::new(AppInfo::new("cov-hung", "0.1.0"))
.shutdown_timeout(Duration::from_millis(50))
.with_broker(broker, |b| {
b.include(hung);
});
let running = app.start().await.expect("startup failed");
publisher
.message(&Order { id: 1 })
.to("cov.hung")
.publish()
.await
.expect("publish failed");
HUNG_HANDLER.notified().await;
timeout(Duration::from_secs(10), running.shutdown())
.await
.expect("the drain ignored its timeout and hung on the handler")
.expect("an aborted handler is not a shutdown failure");
}
static HUNG_CONTINUATION: Notify = Notify::const_new();
#[subscriber("cov.continuation")]
async fn with_continuation(_order: &Order) -> HandlerOutcome {
HandlerOutcome::ack().and_after(async {
HUNG_CONTINUATION.notify_one();
pending::<()>().await;
})
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn the_shutdown_timeout_abandons_a_continuation_that_never_returns() {
let broker = MemoryBroker::new();
let publisher = broker.publisher();
let app = RustStream::new(AppInfo::new("cov-continuation", "0.1.0"))
.shutdown_timeout(Duration::from_millis(50))
.with_broker(broker, |b| {
b.include(with_continuation);
});
let running = app.start().await.expect("startup failed");
publisher
.message(&Order { id: 1 })
.to("cov.continuation")
.publish()
.await
.expect("publish failed");
HUNG_CONTINUATION.notified().await;
timeout(Duration::from_secs(10), running.shutdown())
.await
.expect("the drain ignored its timeout and hung on the continuation")
.expect("an abandoned continuation is not a shutdown failure");
}
#[derive(Default)]
struct Drain {
started: Notify,
finished: AtomicBool,
}
const BRIEF_WORK: Duration = Duration::from_secs(1);
#[subscriber("cov.continuation.brief")]
async fn with_brief_continuation(
_order: &Order,
ctx: &mut Context<'_, (), Arc<Drain>>,
) -> HandlerOutcome {
let drain = Arc::clone(ctx.state());
HandlerOutcome::ack().and_after(async move {
drain.started.notify_one();
tokio::time::sleep(BRIEF_WORK).await;
drain.finished.store(true, Ordering::SeqCst);
})
}
#[tokio::test(start_paused = true)]
async fn the_shutdown_timeout_waits_for_a_continuation_that_finishes_in_time() {
let broker = MemoryBroker::new();
let publisher = broker.publisher();
let drain = Arc::new(Drain::default());
let state = Arc::clone(&drain);
let app = RustStream::new(AppInfo::new("cov-continuation-brief", "0.1.0"))
.shutdown_timeout(Duration::from_secs(5))
.on_startup(async move |()| Ok::<_, Infallible>(state))
.with_broker(broker, |b| {
b.include(with_brief_continuation);
});
let running = app.start().await.expect("startup failed");
publisher
.message(&Order { id: 1 })
.to("cov.continuation.brief")
.publish()
.await
.expect("publish failed");
drain.started.notified().await;
running.shutdown().await.expect("shutdown failed");
assert!(
drain.finished.load(Ordering::SeqCst),
"the shutdown returned before the continuation finished",
);
}
#[subscriber("cov.labeled")]
async fn labeled(order: &Order) -> HandlerOutcome {
let _ = order.id;
HandlerOutcome::ack()
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_labeled_scope_records_its_server_and_decodes_with_its_own_codec() {
let app = RustStream::new(AppInfo::new("cov-labeled", "0.1.0")).with_broker_labeled_codec(
"cov-egress",
MemoryBroker::new(),
CborCodec,
|b| {
b.include(labeled);
},
);
assert!(
app.servers().contains_key("cov-egress"),
"the label registers the broker's server: {:?}",
app.servers().keys().collect::<Vec<_>>(),
);
let rendered = format!("{app:?}");
assert!(rendered.contains("brokers: 1"), "{rendered}");
assert!(rendered.contains("handlers: 1"), "{rendered}");
let tb = TestApp::start(app).await.expect("harness start");
tb.broker::<MemoryBroker>()
.message(&Order { id: 11 })
.with_codec(CborCodec)
.to("cov.labeled")
.publish()
.await
.expect("publish");
tb.broker::<MemoryBroker>()
.subscriber("cov.labeled")
.assert_called_once()
.with_codec(&CborCodec, &Order { id: 11 })
.settled(HandlerOutcome::ack());
}
struct Envelope;
impl<K: ContextKind, Options> PublishTransform<K, Options> for Envelope {
type Destination = Reads;
fn apply(&self, out: &mut Outgoing<'_>, _options: &mut Option<Options>, _cx: &K::View<'_>) {
out.headers_mut().insert("x-envelope", b"1".to_vec());
}
}
#[subscriber("cov.audit.in", reply("cov.audit.out"))]
async fn audited_relay(frame: &Frame<'_>, Out(audit): Out<impl Publisher>) -> Export {
audit
.message(&Wire::of(frame.0))
.to("cov.audit.copy")
.publish()
.await
.expect("the slot publisher is live");
Export(frame.0.to_vec())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_raw_reply_handler_with_a_slot_defaults_its_reply_publisher() {
let app =
RustStream::new(AppInfo::new("cov-audit", "0.1.0")).with_broker(MemoryBroker::new(), |b| {
b.include(audited_relay).out(DefaultSlot, Publish).build();
});
let tb = TestApp::start(app).await.expect("harness start");
tb.broker::<MemoryBroker>()
.message(&Wire::of(b"frame"))
.to("cov.audit.in")
.publish()
.await
.expect("publish");
tb.out::<DefaultSlot>()
.assert_called_once()
.with_raw(b"frame");
let replies = tb
.broker::<MemoryBroker>()
.published::<Export>("cov.audit.out");
replies.assert_called_once().with_raw(b"frame");
}
#[subscriber("cov.gate.in", reply("cov.gate.out"))]
async fn gate(order: &Order, Out(audit): Out<impl Publisher>) -> Receipt {
audit
.message(order)
.to("cov.gate.copy")
.publish()
.await
.expect("the slot publisher is live");
Receipt { id: order.id }
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_publishing_handler_with_a_slot_takes_an_explicit_reply_publisher() {
let app =
RustStream::new(AppInfo::new("cov-gate", "0.1.0")).with_broker(MemoryBroker::new(), |b| {
b.include(gate)
.out(Reply, Publish)
.transform(Envelope)
.out(DefaultSlot, Publish)
.build();
});
let tb = TestApp::start(app).await.expect("harness start");
tb.message(&Order { id: 3 })
.to("cov.gate.in")
.publish()
.await
.expect("publish");
tb.out::<DefaultSlot>().assert_called_once();
let replies = tb
.broker::<MemoryBroker>()
.published::<Receipt>("cov.gate.out");
let replies = replies.assert_called_once();
assert_eq!(
replies.messages()[0].headers().get("x-envelope"),
Some(b"1".as_slice()),
"the reply must leave through the publisher named in the chain",
);
}
#[subscriber("cov.debug.slot")]
async fn debug_slot(_order: &Order, Out(out): Out<impl Publisher>) -> HandlerOutcome {
let _ = out;
HandlerOutcome::ack()
}
#[test]
#[should_panic(expected = "included but never mounted")]
fn a_slot_chain_dropped_before_build_refuses_to_vanish() {
let _app = RustStream::new(AppInfo::new("cov-unbuilt", "0.1.0")).with_broker(
MemoryBroker::new(),
|b| {
let _ = b.include(debug_slot).out(DefaultSlot, Publish);
},
);
}