use std::collections::VecDeque;
use std::pin::pin;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use rumqttc::Outgoing;
use rumqttc::v5::mqttbytes::v5::{
ConnAck, ConnectReturnCode, Filter, Packet, RetainForwardRule, SubscribeProperties,
SubscribeReasonCode,
};
use rumqttc::v5::mqttbytes::{QoS, matches};
use rumqttc::v5::{AsyncClient, ConnectionError, Event, EventLoop, Request, StateError};
use tokio::sync::{Notify, oneshot};
use crate::broker::Link;
use crate::error::MqttError;
use crate::message::MqttMessage;
use crate::registry::{DeliverySender, Registry, SubscribeRequest};
enum Awaiting {
Open {
member: u64,
done: oneshot::Sender<Result<(), MqttError>>,
},
Identify,
Resubscribe,
}
struct PendingSub {
filter: String,
pkid: Option<u16>,
awaiting: Awaiting,
}
const HELD_DELIVERIES: usize = 1024;
const REQUEST_QUEUE_RETRY: Duration = Duration::from_millis(10);
pub(crate) struct Shared {
registry: Mutex<Registry>,
pending: Mutex<VecDeque<PendingSub>>,
held: Mutex<VecDeque<(MqttMessage, Vec<usize>)>>,
last_error: Mutex<Option<String>>,
pub(crate) closed: AtomicBool,
exited: AtomicBool,
stopped: Notify,
identifiers: AtomicBool,
}
impl Shared {
pub(crate) fn new() -> Self {
Self {
registry: Mutex::new(Registry::default()),
pending: Mutex::new(VecDeque::new()),
held: Mutex::new(VecDeque::new()),
last_error: Mutex::new(None),
closed: AtomicBool::new(false),
exited: AtomicBool::new(false),
stopped: Notify::new(),
identifiers: AtomicBool::new(true),
}
}
fn hold(&self, message: MqttMessage, identifiers: &[usize]) {
#[cfg(feature = "testing")]
let message = message.uncounted();
let mut held = self.held.lock().expect("mqtt held mutex poisoned");
if held.len() >= HELD_DELIVERIES
&& let Some((dropped, _)) = held.pop_front()
{
tracing::warn!(
topic = %dropped.topic(),
capacity = HELD_DELIVERIES,
"mqtt delivery dropped: no subscription matches its topic and the hold buffer is full"
);
}
held.push_back((message, identifiers.to_vec()));
}
fn record_error(&self, err: &ConnectionError) {
*self.last_error.lock().expect("mqtt error mutex poisoned") = Some(err.to_string());
}
pub(crate) fn last_error(&self) -> Option<String> {
self.last_error
.lock()
.expect("mqtt error mutex poisoned")
.clone()
}
pub(crate) fn held(&self) -> usize {
self.held.lock().expect("mqtt held mutex poisoned").len()
}
pub(crate) fn ensure_open(&self) -> Result<(), MqttError> {
if self.closed.load(Ordering::Acquire) {
return Err(MqttError::NotConnected);
}
Ok(())
}
pub(crate) async fn stopped(&self) {
let notified = self.stopped.notified();
let mut notified = pin!(notified);
notified.as_mut().enable();
if self.exited.load(Ordering::Acquire) {
return;
}
notified.await;
}
fn gone(&self) -> bool {
self.closed.load(Ordering::Acquire) || self.exited.load(Ordering::Acquire)
}
#[allow(clippy::significant_drop_tightening)]
#[cfg_attr(
not(feature = "testing"),
allow(clippy::infallible_destructuring_match)
)]
pub(crate) async fn open(
&self,
link: &Link,
wire_filter: &str,
match_filter: &str,
qos: QoS,
tx: DeliverySender,
) -> Result<u64, MqttError> {
self.ensure_open()?;
let joined = {
let mut registry = self.registry.lock().expect("mqtt registry mutex poisoned");
let joined = registry.join(
wire_filter,
match_filter,
qos,
self.identifiers.load(Ordering::Relaxed),
tx.clone(),
)?;
let mut held = self.held.lock().expect("mqtt held mutex poisoned");
let mut unclaimed = VecDeque::with_capacity(held.len());
while let Some((message, identifiers)) = held.pop_front() {
if matches(message.topic(), match_filter)
&& registry.claims(wire_filter, &identifiers)
{
#[cfg(feature = "testing")]
let message = message.recounted();
let _ = tx.send(Ok(message));
} else {
unclaimed.push_back((message, identifiers));
}
}
*held = unclaimed;
joined
};
let client = match link {
Link::Wire(client) => client,
#[cfg(feature = "testing")]
Link::InProcess(bus) => {
for request in &joined.requests {
if request.refresh {
if !self
.registry
.lock()
.expect("mqtt registry mutex poisoned")
.holds(&request.filter)
{
continue;
}
bus.subscribe(self, request);
self.registry
.lock()
.expect("mqtt registry mutex poisoned")
.identified(&request.filter, true);
} else {
bus.subscribe(self, request);
}
}
return Ok(joined.member);
}
};
let (done, wait) = oneshot::channel();
let mut done = Some(done);
for request in &joined.requests {
let awaiting = if request.refresh {
Awaiting::Identify
} else {
Awaiting::Open {
member: joined.member,
done: done.take().expect("one request opens the member"),
}
};
let mut pending = PendingSub {
filter: request.filter.clone(),
pkid: None,
awaiting,
};
loop {
let sent = {
let registry = self.registry.lock().expect("mqtt registry mutex poisoned");
if request.refresh && !registry.holds(&request.filter) {
break;
}
self.send_subscribe(client, request, pending)
};
match sent {
Ok(()) => break,
Err(_) if self.gone() => {
self.release(joined.member, link);
return Err(MqttError::Subscribe {
filter: match_filter.to_owned(),
reason: "the mqtt connection task has shut down".to_owned(),
});
}
Err(back) => {
pending = back;
tokio::time::sleep(REQUEST_QUEUE_RETRY).await;
}
}
}
}
wait.await.map_err(|_| MqttError::Subscribe {
filter: match_filter.to_owned(),
reason: "the mqtt connection task has shut down".to_owned(),
})??;
Ok(joined.member)
}
#[allow(clippy::significant_drop_tightening)]
fn send_subscribe(
&self,
client: &AsyncClient,
request: &SubscribeRequest,
record: PendingSub,
) -> Result<(), PendingSub> {
let mut filter = Filter::new(request.filter.clone(), request.qos);
if request.refresh {
filter.retain_forward_rule = RetainForwardRule::OnNewSubscribe;
}
let properties = request.identifier.map(|id| SubscribeProperties {
id: Some(id),
user_properties: Vec::new(),
});
let mut pending = self.pending.lock().expect("mqtt pending mutex poisoned");
let sent = match properties {
Some(properties) => client.try_subscribe_many_with_properties([filter], properties),
None => client.try_subscribe_many([filter]),
};
match sent {
Ok(()) => {
pending.push_back(record);
Ok(())
}
Err(_) => Err(record),
}
}
pub(crate) fn release(&self, member: u64, link: &Link) {
match link {
Link::Wire(client) => self.release_with(member, |filter| unsubscribe(client, &filter)),
#[cfg(feature = "testing")]
Link::InProcess(bus) => bus.release(self, member),
}
}
pub(crate) fn leave(&self, member: u64) {
self.release_with(member, |_| {});
}
pub(crate) fn release_with(&self, member: u64, take_off: impl FnOnce(String)) {
let mut registry = self.registry.lock().expect("mqtt registry mutex poisoned");
if let Some(filter) = registry.leave(member) {
take_off(filter);
}
}
#[cfg(feature = "testing")]
pub(crate) fn next_takers(&self, match_filter: &str) -> Vec<usize> {
let registry = self.registry.lock().expect("mqtt registry mutex poisoned");
let wires = || {
registry
.wires
.iter()
.filter(|wire| wire.match_filter == match_filter)
};
let mut opened: Vec<u64> = wires()
.flat_map(|wire| wire.members.iter().map(|member| member.id))
.collect();
opened.sort_unstable();
wires()
.filter_map(|wire| opened.binary_search(&wire.upcoming()).ok())
.collect()
}
fn forget(&self, member: u64) {
self.registry
.lock()
.expect("mqtt registry mutex poisoned")
.leave(member);
}
fn connected(&self, client: &AsyncClient, connack: &ConnAck) {
let identifiers = connack
.properties
.as_ref()
.and_then(|properties| properties.subscription_identifiers_available)
.is_none_or(|available| available != 0);
self.identifiers.store(identifiers, Ordering::Relaxed);
if connack.session_present {
return;
}
let mut registry = self.registry.lock().expect("mqtt registry mutex poisoned");
registry.resubscribing(identifiers);
for wire in ®istry.wires {
if !identifiers && wire.identifier.is_some() {
tracing::warn!(
filter = %wire.filter,
"mqtt server offers no subscription identifiers after a reconnect: a publish \
this filter shares with another one on the connection may be delivered to \
each of them more than once"
);
}
let request = SubscribeRequest {
filter: wire.filter.clone(),
qos: wire.qos,
identifier: wire.identifier.filter(|_| identifiers),
refresh: false,
};
let record = PendingSub {
filter: wire.filter.clone(),
pkid: None,
awaiting: Awaiting::Resubscribe,
};
if self.send_subscribe(client, &request, record).is_err() {
tracing::warn!(filter = %wire.filter, "mqtt resubscribe failed: the request queue is full");
}
}
}
fn broadcast_error(&self, reason: &str) {
{
let registry = self.registry.lock().expect("mqtt registry mutex poisoned");
for member in registry.wires.iter().flat_map(|wire| &wire.members) {
let _ = member.tx.send(Err(MqttError::Receive(reason.to_owned())));
}
}
let mut pending = self.pending.lock().expect("mqtt pending mutex poisoned");
for sub in pending.drain(..) {
if let Awaiting::Open { done, .. } = sub.awaiting {
let _ = done.send(Err(MqttError::Subscribe {
filter: sub.filter,
reason: reason.to_owned(),
}));
}
}
}
}
fn fatal_reason(err: &ConnectionError) -> Option<String> {
match err {
ConnectionError::ConnectionRefused(code) => match code {
ConnectReturnCode::ServerUnavailable
| ConnectReturnCode::ServerBusy
| ConnectReturnCode::ConnectionRateExceeded
| ConnectReturnCode::QuotaExceeded => None,
other => Some(format!("broker refused the connection: {other:?}")),
},
ConnectionError::MqttState(StateError::ServerDisconnect {
reason_code,
reason_string,
}) => Some(format!(
"broker disconnected the session: {reason_code:?} {reason_string:?}"
)),
ConnectionError::NotConnAck(_) => Some("the peer is not an MQTT broker".to_owned()),
ConnectionError::Tls(err) => Some(format!("tls handshake failed: {err}")),
_ => None,
}
}
pub(crate) struct Conn {
pub(crate) client: AsyncClient,
pub(crate) eventloop: EventLoop,
pub(crate) shared: Arc<Shared>,
pub(crate) first_connack: Option<oneshot::Sender<Result<(), MqttError>>>,
pub(crate) parked: VecDeque<Request>,
}
impl Conn {
fn park(&mut self) {
self.parked.append(&mut self.eventloop.pending);
}
fn unpark(&mut self, session_present: bool) {
if session_present {
self.eventloop.pending.append(&mut self.parked);
return;
}
let publishes = self.parked.drain(..).filter_map(|request| match request {
Request::Publish(mut publish) => {
publish.pkid = 0;
publish.dup = false;
Some(Request::Publish(publish))
}
_ => None,
});
self.eventloop.pending.extend(publishes);
}
}
pub(crate) async fn run(mut conn: Conn) {
let mut backoff = Duration::from_millis(100);
loop {
match conn.eventloop.poll().await {
Ok(Event::Incoming(packet)) => {
backoff = Duration::from_millis(100);
handle_incoming(&mut conn, packet);
}
Ok(Event::Outgoing(Outgoing::Subscribe(pkid))) => {
let mut pending = conn
.shared
.pending
.lock()
.expect("mqtt pending mutex poisoned");
if let Some(sub) = pending.iter_mut().find(|sub| sub.pkid.is_none()) {
sub.pkid = Some(pkid);
}
}
Ok(Event::Outgoing(Outgoing::Disconnect)) => break,
Ok(Event::Outgoing(_)) => {}
Err(err) => {
if conn.shared.closed.load(Ordering::Acquire) {
break;
}
if let Some(reason) = fatal_reason(&err) {
if let Some(done) = conn.first_connack.take() {
let _ = done.send(Err(MqttError::Connect(Box::from(reason.clone()))));
}
conn.shared.broadcast_error(&reason);
break;
}
tracing::debug!(error = %err, "mqtt connection error; backing off");
conn.park();
conn.shared.record_error(&err);
tokio::time::sleep(backoff).await;
backoff = (backoff * 2).min(Duration::from_secs(5));
}
}
}
conn.shared.exited.store(true, Ordering::Release);
conn.shared.stopped.notify_waiters();
}
fn handle_incoming(conn: &mut Conn, packet: Packet) {
match packet {
Packet::ConnAck(connack) => {
conn.unpark(connack.session_present);
conn.shared.connected(&conn.client, &connack);
if let Some(done) = conn.first_connack.take() {
let _ = done.send(Ok(()));
}
}
Packet::SubAck(suback) => {
let pending_sub = {
let mut pending = conn
.shared
.pending
.lock()
.expect("mqtt pending mutex poisoned");
pending
.iter()
.position(|sub| sub.pkid == Some(suback.pkid))
.and_then(|index| pending.remove(index))
};
let Some(sub) = pending_sub else { return };
let refused = match suback.return_codes.first() {
Some(SubscribeReasonCode::Success(_)) => None,
other => Some(format!("broker rejected the subscription: {other:?}")),
};
match sub.awaiting {
Awaiting::Open { member, done } => {
let outcome = match refused {
None => Ok(()),
Some(reason) => {
conn.shared.forget(member);
Err(MqttError::Subscribe {
filter: sub.filter,
reason,
})
}
};
let _ = done.send(outcome);
}
Awaiting::Identify => {
conn.shared
.registry
.lock()
.expect("mqtt registry mutex poisoned")
.identified(&sub.filter, refused.is_none());
if let Some(reason) = refused {
tracing::warn!(
filter = %sub.filter,
reason = %reason,
"mqtt server refused a subscription identifier: a publish this filter \
shares with another one on the connection may reach both of them twice"
);
}
}
Awaiting::Resubscribe => {
if let Some(reason) = refused {
tracing::warn!(filter = %sub.filter, reason = %reason, "mqtt resubscribe refused");
}
}
}
}
Packet::Publish(publish) => {
let Ok(topic) = std::str::from_utf8(&publish.topic) else {
tracing::warn!("mqtt publish with non-utf8 topic dropped");
return;
};
let identifiers = publish
.properties
.as_ref()
.map_or(&[][..], |properties| &properties.subscription_identifiers);
let client = &conn.client;
let dead = demultiplex(&conn.shared, topic, identifiers, |acknowledges| {
MqttMessage::new(
topic.to_owned(),
&publish,
acknowledges.then(|| Link::Wire(client.clone())),
)
});
for member in dead {
conn.shared
.release_with(member, |filter| unsubscribe(client, &filter));
}
}
_ => {}
}
}
pub(crate) fn demultiplex(
shared: &Shared,
topic: &str,
identifiers: &[usize],
message: impl FnMut(bool) -> MqttMessage,
) -> Vec<u64> {
let mut dead = Vec::new();
let mut registry = shared
.registry
.lock()
.expect("mqtt registry mutex poisoned");
if let Some(message) = registry.route(topic, identifiers, message, &mut dead) {
shared.hold(message, identifiers);
}
drop(registry);
dead
}
fn unsubscribe(client: &AsyncClient, filter: &str) {
if let Err(err) = client.try_unsubscribe(filter) {
tracing::warn!(filter = %filter, error = %err, "mqtt unsubscribe failed");
}
}
#[cfg(test)]
mod tests {
use rumqttc::v5::MqttOptions;
use rumqttc::v5::mqttbytes::v5::{PubRel, Publish, Subscribe};
use super::*;
fn after_a_lost_connection(left: impl IntoIterator<Item = Request>) -> Conn {
let (client, eventloop) = AsyncClient::new(MqttOptions::new("parked", "127.0.0.1", 1), 8);
let mut conn = Conn {
client,
eventloop,
shared: Arc::new(Shared::new()),
first_connack: None,
parked: VecDeque::new(),
};
conn.eventloop.pending.extend(left);
conn.park();
conn
}
fn unconfirmed(pkid: u16) -> Request {
let mut publish = Publish::new("readings", QoS::AtLeastOnce, "reading", None);
publish.pkid = pkid;
publish.dup = true;
Request::Publish(publish)
}
fn left_behind() -> Vec<Request> {
vec![
unconfirmed(7),
Request::PubRel(PubRel::new(3, None)),
Request::Subscribe(Subscribe::new(
Filter::new("readings", QoS::AtLeastOnce),
None,
)),
unconfirmed(0),
]
}
#[test]
fn a_new_session_gets_the_publishes_again_as_fresh_ones() {
let mut conn = after_a_lost_connection(left_behind());
assert!(
conn.eventloop.pending.is_empty(),
"nothing is left for the client to clear"
);
conn.unpark(false);
let resent: Vec<_> = conn.eventloop.pending.drain(..).collect();
let mut fresh = Publish::new("readings", QoS::AtLeastOnce, "reading", None);
fresh.pkid = 0;
fresh.dup = false;
assert_eq!(
resent,
vec![Request::Publish(fresh.clone()), Request::Publish(fresh)]
);
assert!(conn.parked.is_empty());
}
#[test]
fn a_resumed_session_gets_everything_back_as_it_was() {
let mut conn = after_a_lost_connection(left_behind());
conn.unpark(true);
assert_eq!(
conn.eventloop.pending.drain(..).collect::<Vec<_>>(),
left_behind()
);
assert!(conn.parked.is_empty());
}
}