use std::{
sync::Arc,
time::{Duration, Instant},
};
use rumqttc::{
AsyncClient, Event, EventLoop, Incoming, NetworkOptions, Outgoing, QoS, SubAck,
SubscribeFilter, SubscribeReasonCode,
};
use tracing::{debug, info, warn};
use crate::{
core::{CoreError, Service, ServiceFuture, ShutdownToken},
mqtt::{
MqttConfig, MqttConnectionState, MqttContext, MqttError, MqttHandler, MqttMessage,
MqttPublisher, MqttReadiness, MqttResult,
},
};
pub struct MqttService<H> {
name: String,
config: MqttConfig,
handler: Arc<H>,
readiness: MqttReadiness,
connack_timeout: Duration,
subscribe_timeout: Duration,
}
const DEFAULT_CONNACK_TIMEOUT: Duration = Duration::from_secs(10);
const DEFAULT_SUBSCRIBE_TIMEOUT: Duration = Duration::from_secs(30);
impl<H> MqttService<H>
where
H: MqttHandler,
{
pub fn new(config: MqttConfig, handler: H) -> Self {
Self {
name: "mqtt".to_string(),
config,
handler: Arc::new(handler),
readiness: MqttReadiness::new(),
connack_timeout: DEFAULT_CONNACK_TIMEOUT,
subscribe_timeout: DEFAULT_SUBSCRIBE_TIMEOUT,
}
}
pub fn named(name: impl Into<String>, config: MqttConfig, handler: H) -> Self {
Self {
name: name.into(),
config,
handler: Arc::new(handler),
readiness: MqttReadiness::new(),
connack_timeout: DEFAULT_CONNACK_TIMEOUT,
subscribe_timeout: DEFAULT_SUBSCRIBE_TIMEOUT,
}
}
pub fn with_readiness(mut self, readiness: MqttReadiness) -> Self {
self.readiness = readiness;
self
}
pub fn readiness(&self) -> MqttReadiness {
self.readiness.clone()
}
pub fn with_setup_timeouts(
mut self,
connack_timeout: Duration,
subscribe_timeout: Duration,
) -> Self {
self.connack_timeout = connack_timeout;
self.subscribe_timeout = subscribe_timeout;
self
}
}
impl<H> Service for MqttService<H>
where
H: MqttHandler,
{
fn name(&self) -> &str {
&self.name
}
fn start(&self, shutdown: ShutdownToken) -> ServiceFuture<'_> {
Box::pin(async move {
if shutdown.is_cancelled() {
self.readiness.set_state(MqttConnectionState::Disconnected);
return Ok(());
}
self.config
.validate()
.map_err(|error| CoreError::Service(error.to_string()))?;
run_with_reconnect(
&self.name,
&self.config,
self.handler.clone(),
self.readiness.clone(),
shutdown,
self.connack_timeout,
self.subscribe_timeout,
)
.await
.map_err(|error| CoreError::Service(error.to_string()))
})
}
}
async fn run_with_reconnect<H>(
service_name: &str,
config: &MqttConfig,
handler: Arc<H>,
readiness: MqttReadiness,
shutdown: ShutdownToken,
connack_timeout: Duration,
subscribe_timeout: Duration,
) -> MqttResult<()>
where
H: MqttHandler,
{
let mut attempt = 0;
loop {
let generation = readiness.begin_generation();
match run_once(
service_name,
config,
handler.clone(),
readiness.clone(),
shutdown.clone(),
generation,
connack_timeout,
subscribe_timeout,
)
.await
{
Ok(()) => return Ok(()),
Err(_failure) if shutdown.is_cancelled() => {
readiness.set_state_for_generation(generation, MqttConnectionState::Disconnected);
return Ok(());
}
Err(failure) => {
if failure.reached_ready {
attempt = 0;
}
readiness.set_state_for_generation(generation, MqttConnectionState::Disconnected);
readiness.set_state_for_generation(generation, MqttConnectionState::Reconnecting);
let base_delay = config.reconnect.delay_for_attempt(attempt);
let delay = jittered_delay(base_delay, generation, attempt);
record_event(config, service_name, "reconnect", "scheduled");
warn!(
service = service_name,
generation,
attempt,
?delay,
error = %failure.error,
"mqtt reconnect scheduled"
);
attempt = attempt.saturating_add(1);
tokio::select! {
_ = shutdown.cancelled() => {
readiness.set_state_for_generation(
generation,
MqttConnectionState::Disconnected,
);
return Ok(());
}
_ = tokio::time::sleep(delay) => {}
}
}
}
}
}
async fn run_once<H>(
service_name: &str,
config: &MqttConfig,
handler: Arc<H>,
readiness: MqttReadiness,
shutdown: ShutdownToken,
generation: u64,
connack_timeout: Duration,
subscribe_timeout: Duration,
) -> Result<(), GenerationFailure>
where
H: MqttHandler,
{
let options = config.to_mqtt_options().map_err(GenerationFailure::setup)?;
let (client, mut eventloop) = AsyncClient::new(options, config.inflight);
let mut network_options = NetworkOptions::new();
network_options.set_connection_timeout(duration_seconds_ceil(connack_timeout));
eventloop.set_network_options(network_options);
let context = MqttContext::new(MqttPublisher::new(client.clone()));
match wait_for_connack(&mut eventloop, &shutdown, connack_timeout).await {
Ok(true) => {
readiness.set_state_for_generation(generation, MqttConnectionState::Connected);
record_event(config, service_name, "connect", "success");
info!(
service = service_name,
generation, "mqtt broker connection accepted"
);
}
Ok(false) => {
readiness.set_state_for_generation(generation, MqttConnectionState::Disconnected);
record_event(config, service_name, "connect", "shutdown");
return Ok(());
}
Err(error) => {
readiness.set_state_for_generation(generation, MqttConnectionState::Disconnected);
record_event(config, service_name, "connect", "error");
warn!(service = service_name, generation, phase = "connack", error = %error, "mqtt setup failed");
return Err(GenerationFailure::setup(error));
}
}
if config.subscriptions.is_empty() {
readiness.set_state_for_generation(generation, MqttConnectionState::Subscribed);
} else {
match wait_for_suback(
&client,
&mut eventloop,
config,
&shutdown,
subscribe_timeout,
)
.await
{
Ok(true) => {
readiness.set_state_for_generation(generation, MqttConnectionState::Subscribed);
record_event(config, service_name, "subscribe", "success");
info!(
service = service_name,
generation,
subscription_count = config.subscriptions.len(),
"mqtt subscriptions acknowledged"
);
}
Ok(false) => {
readiness.set_state_for_generation(generation, MqttConnectionState::Disconnected);
record_event(config, service_name, "subscribe", "shutdown");
return Ok(());
}
Err(error) => {
readiness.set_state_for_generation(generation, MqttConnectionState::Disconnected);
record_event(config, service_name, "subscribe", "error");
warn!(service = service_name, generation, phase = "suback", error = %error, "mqtt setup failed");
return Err(GenerationFailure::setup(error));
}
}
}
poll_loop(
service_name,
config,
handler,
context,
readiness,
&mut eventloop,
shutdown,
generation,
)
.await
.map_err(GenerationFailure::ready)
}
fn duration_seconds_ceil(duration: Duration) -> u64 {
let seconds = duration.as_secs();
if duration.subsec_nanos() == 0 {
seconds.max(1)
} else {
seconds.saturating_add(1).max(1)
}
}
struct GenerationFailure {
error: MqttError,
reached_ready: bool,
}
impl GenerationFailure {
fn setup(error: MqttError) -> Self {
Self {
error,
reached_ready: false,
}
}
fn ready(error: MqttError) -> Self {
Self {
error,
reached_ready: true,
}
}
}
#[doc(hidden)]
pub async fn handle_message_for_test<H>(
service_name: &str,
config: &MqttConfig,
handler: &H,
message: MqttMessage,
) -> MqttResult<()>
where
H: MqttHandler,
{
handle_message(service_name, config, handler, message).await
}
#[doc(hidden)]
pub async fn handle_message_with_context_for_test<H>(
service_name: &str,
config: &MqttConfig,
handler: &H,
context: MqttContext,
message: MqttMessage,
) -> MqttResult<()>
where
H: MqttHandler,
{
handle_message_with_context(service_name, config, handler, context, message).await
}
async fn wait_for_connack(
eventloop: &mut EventLoop,
shutdown: &ShutdownToken,
timeout: Duration,
) -> MqttResult<bool> {
let deadline = tokio::time::sleep(timeout);
tokio::pin!(deadline);
tokio::select! {
biased;
_ = shutdown.cancelled() => Ok(false),
_ = &mut deadline => Err(MqttError::Connection(format!(
"CONNACK deadline expired after {timeout:?}"
))),
event = eventloop.poll() => match event {
Ok(Event::Incoming(Incoming::ConnAck(_))) => Ok(true),
Ok(other) => Err(MqttError::Connection(format!(
"expected accepted CONNACK, received {other:?}"
))),
Err(error) => Err(MqttError::Connection(error.to_string())),
},
}
}
async fn wait_for_suback(
client: &AsyncClient,
eventloop: &mut EventLoop,
config: &MqttConfig,
shutdown: &ShutdownToken,
timeout: Duration,
) -> MqttResult<bool> {
let filters = config
.subscriptions
.iter()
.map(|subscription| {
SubscribeFilter::new(subscription.topic.clone(), subscription.qos.into())
})
.collect::<Vec<_>>();
tokio::select! {
biased;
_ = shutdown.cancelled() => return Ok(false),
result = client.subscribe_many(filters) => {
result.map_err(|error| MqttError::Subscribe(error.to_string()))?;
}
}
let expected_qos = config
.subscriptions
.iter()
.map(|subscription| subscription.qos.into())
.collect::<Vec<QoS>>();
let deadline = tokio::time::sleep(timeout);
tokio::pin!(deadline);
let mut packet_id = None;
loop {
tokio::select! {
biased;
_ = shutdown.cancelled() => return Ok(false),
_ = &mut deadline => {
return Err(MqttError::Subscribe(format!(
"SUBACK deadline expired after {timeout:?}"
)));
}
event = eventloop.poll() => {
match event.map_err(|error| MqttError::Subscribe(error.to_string()))? {
Event::Outgoing(Outgoing::Subscribe(observed)) => {
if packet_id.replace(observed).is_some() {
return Err(MqttError::Subscribe(
"multiple outgoing SUBSCRIBE packets observed".to_string(),
));
}
}
Event::Incoming(Incoming::SubAck(ack)) => {
let expected = packet_id.ok_or_else(|| MqttError::Subscribe(
"SUBACK arrived before the outgoing SUBSCRIBE packet identifier"
.to_string(),
))?;
validate_suback(expected, &expected_qos, &ack)?;
return Ok(true);
}
other => debug!(event = ?other, phase = "subscribe", "mqtt setup event observed"),
}
}
}
}
}
fn validate_suback(expected_pkid: u16, expected_qos: &[QoS], ack: &SubAck) -> MqttResult<()> {
if ack.pkid != expected_pkid {
return Err(MqttError::Subscribe(format!(
"SUBACK packet identifier mismatch: expected {expected_pkid}, received {}",
ack.pkid
)));
}
if ack.return_codes.len() != expected_qos.len() {
return Err(MqttError::Subscribe(format!(
"SUBACK result count mismatch: expected {}, received {}",
expected_qos.len(),
ack.return_codes.len()
)));
}
for (index, (result, expected)) in ack.return_codes.iter().zip(expected_qos).enumerate() {
match result {
SubscribeReasonCode::Success(actual) if actual == expected => {}
SubscribeReasonCode::Success(actual) => {
return Err(MqttError::Subscribe(format!(
"SUBACK QoS mismatch at index {index}: expected {expected:?}, received {actual:?}"
)));
}
SubscribeReasonCode::Failure => {
return Err(MqttError::Subscribe(format!(
"SUBACK rejected subscription at index {index}"
)));
}
}
}
Ok(())
}
async fn poll_loop<H>(
service_name: &str,
config: &MqttConfig,
handler: Arc<H>,
context: MqttContext,
readiness: MqttReadiness,
eventloop: &mut EventLoop,
shutdown: ShutdownToken,
generation: u64,
) -> MqttResult<()>
where
H: MqttHandler,
{
loop {
tokio::select! {
_ = shutdown.cancelled() => {
readiness.set_state_for_generation(generation, MqttConnectionState::Disconnected);
record_event(config, service_name, "connection", "shutdown");
info!("mqtt service shutdown requested");
return Ok(());
}
event = eventloop.poll() => {
let event = match event {
Ok(event) => event,
Err(error) => {
readiness.set_state_for_generation(generation, MqttConnectionState::Disconnected);
record_event(config, service_name, "connection", "error");
return Err(MqttError::EventLoop(error.to_string()));
}
};
match event {
Event::Incoming(Incoming::Publish(publish)) => {
let result = handle_message_with_context(
service_name,
config,
handler.as_ref(),
context.clone(),
MqttMessage::from(publish),
)
.await;
if let Err(error) = result {
readiness.set_state_for_generation(
generation,
MqttConnectionState::Disconnected,
);
if shutdown.is_cancelled() {
record_event(config, service_name, "connection", "shutdown");
return Ok(());
}
record_event(config, service_name, "connection", "error");
return Err(error);
}
}
other => {
debug!(event = ?other, "mqtt event observed");
}
}
}
}
}
}
fn jittered_delay(base: Duration, generation: u64, attempt: u32) -> Duration {
if base.is_zero() {
return base;
}
let mut value = generation
.wrapping_mul(0x9e37_79b9_7f4a_7c15)
.wrapping_add(u64::from(attempt) + 1);
value ^= value >> 30;
value = value.wrapping_mul(0xbf58_476d_1ce4_e5b9);
value ^= value >> 27;
value = value.wrapping_mul(0x94d0_49bb_1331_11eb);
value ^= value >> 31;
let per_mille = 500u128 + u128::from(value % 501);
let nanos = base.as_nanos().saturating_mul(per_mille) / 1000;
Duration::from_nanos(nanos.min(u64::MAX as u128) as u64)
}
#[cfg(test)]
mod tests {
use std::time::Duration;
#[test]
fn connection_timeout_rounds_up_to_whole_seconds() {
assert_eq!(super::duration_seconds_ceil(Duration::ZERO), 1);
assert_eq!(super::duration_seconds_ceil(Duration::from_millis(500)), 1);
assert_eq!(super::duration_seconds_ceil(Duration::from_secs(10)), 10);
assert_eq!(
super::duration_seconds_ceil(Duration::from_millis(10_001)),
11
);
}
#[test]
fn reconnect_jitter_stays_between_half_and_full_backoff() {
let base = Duration::from_secs(5);
let delays = (1..=32)
.map(|generation| super::jittered_delay(base, generation, 3))
.collect::<Vec<_>>();
assert!(delays.iter().all(|delay| *delay >= base / 2));
assert!(delays.iter().all(|delay| *delay <= base));
assert!(delays.windows(2).any(|pair| pair[0] != pair[1]));
}
}
async fn handle_message<H>(
service_name: &str,
config: &MqttConfig,
handler: &H,
message: MqttMessage,
) -> MqttResult<()>
where
H: MqttHandler,
{
let mut options = rumqttc::MqttOptions::new("rs-zero-test-publisher", "127.0.0.1", 1883);
options.set_keep_alive(std::time::Duration::from_secs(5));
let (client, _eventloop) = AsyncClient::new(options, 1);
let context = MqttContext::new(MqttPublisher::new(client));
handle_message_with_context(service_name, config, handler, context, message).await
}
async fn handle_message_with_context<H>(
service_name: &str,
config: &MqttConfig,
handler: &H,
context: MqttContext,
message: MqttMessage,
) -> MqttResult<()>
where
H: MqttHandler,
{
let topic = message.topic().to_string();
let topic_label = config.topic_label(&topic);
let started = Instant::now();
let result = if let Some(timeout) = config.handler_timeout {
tokio::time::timeout(timeout, handler.handle_with_context(context, message))
.await
.map_err(|_| MqttError::HandlerTimeout)?
} else {
handler.handle_with_context(context, message).await
};
if let Err(error) = &result {
warn!(topic, error = %error, "mqtt handler failed");
}
record_message(
config,
service_name,
&topic_label,
if result.is_ok() { "success" } else { "error" },
started.elapsed(),
);
result
}
#[cfg(feature = "observability")]
fn record_event(config: &MqttConfig, service: &str, event: &str, outcome: &str) {
if let Some(metrics) = &config.metrics_registry {
metrics.record_mqtt_event(service, event, outcome);
}
}
#[cfg(not(feature = "observability"))]
fn record_event(_config: &MqttConfig, _service: &str, _event: &str, _outcome: &str) {}
#[cfg(feature = "observability")]
fn record_message(
config: &MqttConfig,
service: &str,
topic: &str,
outcome: &str,
duration: std::time::Duration,
) {
if let Some(metrics) = &config.metrics_registry {
metrics.record_mqtt_message(service, topic, outcome, duration);
}
}
#[cfg(not(feature = "observability"))]
fn record_message(
_config: &MqttConfig,
_service: &str,
_topic: &str,
_outcome: &str,
_duration: std::time::Duration,
) {
}