use std::sync::Arc;
use std::io::Result;
use std::time::{Duration, Instant};
use std::collections::{HashSet, BTreeMap,
btree_map::Entry};
use futures::future::LocalBoxFuture;
use pi_async_rt::{rt::{AsyncRuntime, AsyncRuntimeBuilder,
worker_thread::WorkerRuntime},
lock::spin_lock::SpinLock};
use parking_lot::RwLock;
use dashmap::DashMap;
use futures::AsyncWriteExt;
use mqtt311::{TopicPath, Publish};
use tcp::Socket;
use log::info;
use crate::{server::MqttBrokerProtocol,
session::{MqttConnect, QosZeroSession},
utils::{PathTree, BrokerSession}};
lazy_static! {
pub static ref MQTT_RESPONSE_SYS_TOPIC: String = "$r".to_string();
}
lazy_static! {
static ref PUBLIC_BROKER_RUNTIME: WorkerRuntime<()> = {
let rt = AsyncRuntimeBuilder::default_worker_thread(Some("MQTT-BROKER-RT"), None, None, Some(Some(1)));
#[cfg(all(feature = "runtime_metrics", not(target_arch = "wasm32")))]
{
let metric_name = format!("pi_mqtt-worker-{}", rt.get_id());
if let Err(error) = pi_async_rt::runtime_metrics::register_runtime_metrics(&rt, &metric_name) {
log::error!("MQTT 运行时指标注册失败, 名称: {}, 原因: {}", metric_name, error);
}
}
rt
};
}
pub trait MqttBrokerListener<S: Socket>: Send + Sync + 'static {
fn connected(&self,
protocol: MqttBrokerProtocol,
connect: Arc<dyn MqttConnect<S>>) -> LocalBoxFuture<'static, Result<()>>;
fn closed(&self,
protocol: MqttBrokerProtocol,
connect: Arc<dyn MqttConnect<S>>,
context: BrokerSession,
reason: Result<()>) -> LocalBoxFuture<'static, ()>;
}
pub trait MqttBrokerService<S: Socket>: Send + Sync + 'static {
fn subscribe(&self,
protocol: MqttBrokerProtocol,
connect: Arc<dyn MqttConnect<S>>,
topics: Vec<(String, u8)>) -> LocalBoxFuture<'static, Result<()>>;
fn unsubscribe(&self,
protocol: MqttBrokerProtocol,
connect: Arc<dyn MqttConnect<S>>,
topics: Vec<String>) -> LocalBoxFuture<'static, Result<()>>;
fn publish(&self,
protocol: MqttBrokerProtocol,
connect: Arc<dyn MqttConnect<S>>,
topic: String,
payload: Arc<Vec<u8>>) -> LocalBoxFuture<'static, Result<()>>;
}
pub enum Retain {
Single(Publish), Mutil(Vec<Publish>), }
struct SubCache<S: Socket> {
retain: Option<Publish>, first: Option<Arc<QosZeroSession<S>>>, sessions: Vec<Arc<QosZeroSession<S>>>, }
unsafe impl<S: Socket> Send for SubCache<S> {}
impl<S: Socket> SubCache<S> {
pub fn with_session(first: Option<Arc<QosZeroSession<S>>>,
retain: Option<Publish>) -> Self {
SubCache {
retain,
first,
sessions: Vec::new(),
}
}
}
pub struct MqttListener<S: Socket>(pub Arc<dyn MqttBrokerListener<S>>);
unsafe impl<S: Socket> Send for MqttListener<S> {}
unsafe impl<S: Socket> Sync for MqttListener<S> {}
pub struct MqttService<S: Socket>(pub Arc<dyn MqttBrokerService<S>>);
unsafe impl<S: Socket> Send for MqttService<S> {}
unsafe impl<S: Socket> Sync for MqttService<S> {}
pub struct MqttBroker<S: Socket> {
listener: Arc<RwLock<Option<Arc<dyn MqttBrokerListener<S>>>>>, service: Arc<RwLock<Option<Arc<dyn MqttBrokerService<S>>>>>, services: Arc<DashMap<String, Arc<dyn MqttBrokerService<S>>>>, sessions: Arc<DashMap<String, Arc<QosZeroSession<S>>>>, sub_tab: Arc<DashMap<String, Arc<RwLock<SubCache<S>>>>>, patterns: Arc<RwLock<PathTree<Arc<QosZeroSession<S>>>>>, publics: Arc<RwLock<BTreeMap<String, u8>>>, topics: Arc<DashMap<Arc<QosZeroSession<S>>, Vec<String>>>, clock: Instant, topic_expiry: Arc<SpinLock<BTreeMap<Duration, HashSet<String>>>>, }
unsafe impl<S: Socket> Send for MqttBroker<S> {}
unsafe impl<S: Socket> Sync for MqttBroker<S> {}
impl<S: Socket> Clone for MqttBroker<S> {
fn clone(&self) -> Self {
MqttBroker {
listener: self.listener.clone(),
service: self.service.clone(),
services: self.services.clone(),
sessions: self.sessions.clone(),
sub_tab: self.sub_tab.clone(),
patterns: self.patterns.clone(),
publics: self.publics.clone(),
topics: self.topics.clone(),
clock: self.clock.clone(),
topic_expiry: self.topic_expiry.clone(),
}
}
}
impl<S: Socket> MqttBroker<S> {
pub fn new() -> Self {
MqttBroker {
listener: Arc::new(RwLock::new(None)),
service: Arc::new(RwLock::new(None)),
services: Arc::new(DashMap::default()),
sessions: Arc::new(DashMap::default()),
sub_tab: Arc::new(DashMap::default()),
patterns: Arc::new(RwLock::new(PathTree::empty())),
publics: Arc::new(RwLock::new(BTreeMap::new())),
topics: Arc::new(DashMap::default()),
clock: Instant::now(),
topic_expiry: Arc::new(SpinLock::new(BTreeMap::new())),
}
}
pub fn get_listener(&self) -> Option<MqttListener<S>> {
if let Some(listener) = self.listener.read().as_ref() {
return Some(MqttListener(listener.clone()));
}
None
}
pub fn register_listener(&self, listener: Arc<dyn MqttBrokerListener<S>>) {
*self.listener.write() = Some(listener);
}
pub fn get_service(&self) -> Option<MqttService<S>> {
if let Some(service) = self.service.read().as_ref() {
return Some(MqttService(service.clone()));
}
None
}
pub fn register_service(&self, service: Arc<dyn MqttBrokerService<S>>) {
*self.service.write() = Some(service);
}
pub fn get_topic_service(&self, topic: &String) -> Option<MqttService<S>> {
if let Some(item) = self.services.get(topic) {
return Some(MqttService(item.value().clone()));
}
None
}
pub fn register_topic_service(&self, topic: String,
service: Arc<dyn MqttBrokerService<S>>) {
self.services.insert(topic, service);
}
pub fn unregister_service(&self, topic: &String) {
let _ = self.services.remove(topic);
}
pub fn session_size(&self) -> usize {
self.sessions.len()
}
pub fn sub_size(&self) -> usize {
self.sub_tab.len()
}
pub fn pattern_size(&self) -> usize {
self.patterns.read().len()
}
pub fn is_exist_session(&self, client_id: &String) -> bool {
self.sessions.contains_key(client_id)
}
pub fn get_session(&self, client_id: &String) -> Option<Arc<QosZeroSession<S>>> {
if let Some(item) = self.sessions.get(client_id) {
return Some(item.value().clone());
}
None
}
pub fn insert_session(&self, client_id: String,
session: QosZeroSession<S>) -> Arc<QosZeroSession<S>> {
let connect = Arc::new(session);
self.sessions.insert(client_id, connect.clone());
connect
}
pub fn remove_session(&self, client_id: &String) -> Option<Arc<QosZeroSession<S>>> {
if let Some((_key, value)) = self.sessions.remove(client_id) {
Some(value)
} else {
None
}
}
pub(crate) fn insert_unsubscribed_topic(&self, topic: String, timeout: Duration) {
let broker = self.clone();
let key = self.clock.elapsed() + timeout;
let _ = PUBLIC_BROKER_RUNTIME.spawn(async move {
broker
.topic_expiry
.lock()
.entry(key)
.or_insert(HashSet::from([topic.clone()]))
.insert(topic);
});
}
pub fn startup_expire_unsubscribed_topic_loop(&self, interval: Duration) {
let broker = self.clone();
let _ = PUBLIC_BROKER_RUNTIME.spawn(async move {
loop {
PUBLIC_BROKER_RUNTIME.timeout(interval.as_millis() as usize).await;
let time = Instant::now();
let mut count = 0;
let mut removed_count = 0;
let mut timestamps = Vec::new();
let mut locked = broker.topic_expiry.lock();
count = locked.len();
let now = broker.clock.elapsed();
for (key, _value) in locked.range(Duration::default()..=now) {
timestamps.push(key.clone());
}
for timestamp in ×tamps {
if let Some(topics) = locked.remove(×tamp) {
for topic in topics.iter() {
let b = if let Some(cache) = broker.sub_tab.get(topic) {
let locked_ = cache.value().read();
locked_.first.is_none() && locked_.sessions.is_empty()
} else {
true
};
if b {
let _ = broker.sub_tab.remove(topic);
removed_count += 1;
}
}
}
}
info!("Expire unsubscribed topic successful, count: {:?}, removed: {:?}, time: {:?}",
count,
removed_count,
time.elapsed());
}
});
}
pub fn subscribed(&self, is_public: bool,
topic: &String,
qos: u8,
retain: Option<Publish>) -> Option<Vec<Arc<QosZeroSession<S>>>> {
let mut is_new_public = false; if is_public {
if let Entry::Vacant(v) = self.publics.write().entry(topic.clone()) {
v.insert(qos);
is_new_public = true;
}
}
let cache = if let Some(item) = self
.sub_tab
.get(topic) {
Some(item.value().clone())
} else {
None
};
if let Some(cache) = cache {
if let Some(publish) = &retain {
cache.write().retain = Some(publish.clone());
}
if is_new_public {
if let Some(mut sessions) = self
.patterns
.read()
.lookup(TopicPath::from(topic.as_str())) {
let first = cache.read().first.clone();
match first {
None => {
cache.write().sessions.extend_from_slice(&sessions[..]);
sessions = cache.read().sessions.clone();
},
Some(session) => {
sessions.push(session);
if sessions.len() > 0 {
cache.write().first = None;
cache.write().sessions = sessions.clone();
}
},
}
return Some(sessions);
}
}
if let Some(session) = &cache.read().first {
return Some(vec![session.clone()]);
}
let sessions = cache.read().sessions.clone();
return Some(sessions);
} else {
if let Some(sessions) = self
.patterns
.read()
.lookup(TopicPath::from(topic.as_str())) {
let len = sessions.len();
match len {
0 => {
let retain_copy = retain.clone();
self.sub_tab.entry(topic.clone()).or_insert_with(move || {
Arc::new(RwLock::new(SubCache::with_session(None,
retain_copy)))
});
self.insert_unsubscribed_topic(topic.clone(), Duration::from_secs(3600));
return Some(sessions);
},
1 => {
let retain_copy = retain.clone();
let session = sessions[0].clone();
self.sub_tab.entry(topic.clone()).or_insert_with(move || {
Arc::new(RwLock::new(SubCache::with_session(Some(session),
retain_copy)))
});
return Some(sessions);
},
_ => {
let retain_copy = retain.clone();
let sessions_copy = sessions.clone();
self.sub_tab.entry(topic.clone()).or_insert_with(move || {
let mut cache = SubCache::with_session(None,
retain_copy);
for session in &sessions_copy {
cache.sessions.push(session.clone());
}
Arc::new(RwLock::new(cache))
});
return Some(sessions);
},
}
self.subscribed(is_public, topic, qos, retain)
} else {
None
}
}
}
pub fn subscribe(&self, session: Arc<QosZeroSession<S>>, topic: String) -> Option<Retain> {
let path = TopicPath::from(topic.clone());
if path.wildcards {
let mut keys = Vec::new();
for (key, _) in self.publics.read().iter() {
if is_match(&path, &TopicPath::from(key.as_str())) {
keys.push(key.clone()); save_topic(self, &session, key); }
}
save_topic(self, &session, &topic);
let mut vec = Vec::with_capacity(keys.len());
for key in keys {
if let Some(Retain::Single(publish)) = self.subscribe(session.clone(),
key) {
vec.push(publish);
}
}
self.patterns.write().insert(path, session);
if vec.len() == 0 {
None
} else {
Some(Retain::Mutil(vec))
}
} else {
if !self.sub_tab.contains_key(&topic) {
let session_copy = session.clone();
self.sub_tab.entry(topic.clone()).or_insert_with(move || {
Arc::new(RwLock::new(SubCache::with_session(Some(session_copy),
None)))
});
save_topic(self, &session, &topic);
return None;
}
let cache = if let Some(item) = self
.sub_tab
.get(&topic) {
item.value().clone()
} else {
return None;
};
let mut w = cache.write();
if let Some(session) = w.first.take() {
w.sessions.push(session);
}
if let Err(index) = w.sessions.binary_search_by(|s| {
s.cmp(&session)
}) {
save_topic(self, &session, &topic); w.sessions.insert(index, session); }
if let Some(retain) = w.retain.as_ref() {
return Some(Retain::Single(retain.clone()));
}
None
}
}
pub fn unsubscribe(&self, session: &Arc<QosZeroSession<S>>, topic: String) {
let path = TopicPath::from(topic.clone());
if path.wildcards {
let mut vec = Vec::with_capacity(self.sub_tab.len());
for item in self.sub_tab.iter() {
vec.push((item.key().clone(), item.value().clone()));
}
let mut keys = Vec::new();
for (key, cache) in vec {
if is_match(&path, &TopicPath::from(key.as_str())) {
let mut w = cache.write();
if w.first.is_some() {
if Arc::ptr_eq(session, &w.first.as_ref().unwrap()) {
w.first = None;
keys.push(key.clone());
}
} else {
if let Ok(index) = w.sessions.binary_search_by(|s| {
s.cmp(&session)
}) {
w.sessions.remove(index);
}
if w.sessions.len() == 0 {
keys.push(key.clone());
}
}
}
}
for key in keys {
let _ = self.sub_tab.remove(&key);
}
self.patterns.write().remove(path, session.clone());
} else {
if self.sub_tab.contains_key(&topic) {
let mut is_remove = false;
let cache = {
let cache = self
.sub_tab
.get(&topic);
if cache.is_none() {
return;
}
cache
.unwrap()
.value()
.clone()
};
{
let mut w = cache.write();
if w.first.is_some() {
if Arc::ptr_eq(session, &w.first.as_ref().unwrap()) {
w.first = None;
is_remove = true;
}
} else {
if let Ok(index) = w.sessions.binary_search_by(|s| {
s.cmp(&session)
}) {
w.sessions.remove(index);
}
if w.sessions.len() == 0 {
is_remove = true;
}
}
}
if is_remove {
let _ = self.sub_tab.remove(&topic);
}
}
}
}
pub fn unsubscribe_all(&self, session: &Arc<QosZeroSession<S>>) {
let opt = self.topics.remove(session);
match opt {
None => {
()
},
Some((_key, topics)) => {
for topic in topics {
self.unsubscribe(session, topic);
}
},
}
}
}
fn is_match(pattern: &TopicPath, path: &TopicPath) -> bool {
let path_len = path.len();
let pattern_len = pattern.len();
if path_len < pattern_len {
let level_diff = pattern_len - path_len; if level_diff > 1 {
return false;
} else if level_diff == 1 {
if pattern.is_multi(pattern_len - 1) {
let index = pattern_len - 2;
return pattern.get(index).unwrap().fit(path.get(index).unwrap());
} else {
return false;
}
} else {
return false;
}
} else {
for index in 0..path_len {
if pattern.is_final(index) {
if pattern.is_multi(index) {
return true;
}
}
if let Some(p) = pattern.get(index) {
if p.fit(path.get(index).unwrap()) {
continue;
}
}
return false;
}
}
true
}
fn save_topic<S: Socket>(broker: &MqttBroker<S>,
session: &Arc<QosZeroSession<S>>,
topic: &String) {
if broker.topics.contains_key(session) {
if let Some(mut item) = broker.topics.get_mut(session) {
item.value_mut().push(topic.clone());
}
} else {
broker.topics.insert(session.clone(), vec![topic.clone()]);
}
}
#[cfg(all(test, not(target_arch = "wasm32")))]
pub(crate) mod runtime_metrics_tests {
use super::*;
use opentelemetry_sdk::metrics::{ManualReader, SdkMeterProvider};
use pi_async_rt::runtime_metrics::{collect_and_report_runtime_metrics,
init_runtime_metrics_reporting, RuntimeMetricCollectionOptions,
RuntimeMetricInstanceStatus, RuntimeMetricKind};
use std::sync::OnceLock;
pub(crate) fn initialize_reporting() {
static PROVIDER: OnceLock<SdkMeterProvider> = OnceLock::new();
PROVIDER.get_or_init(|| {
let provider = SdkMeterProvider::builder()
.with_reader(ManualReader::default()).build();
opentelemetry::global::set_meter_provider(provider.clone());
init_runtime_metrics_reporting("pi_mqtt-单元宿主").expect("测试记录器初始化失败");
provider
});
}
#[test]
#[ignore = "需测试宿主显式开启pi-async-rt/trace,串行执行此专项"]
fn test_broker_runtime_metrics_name_and_reuse() {
initialize_reporting();
let start = std::sync::Barrier::new(2);
let ids = std::thread::scope(|scope| {
let first = scope.spawn(|| { start.wait(); PUBLIC_BROKER_RUNTIME.get_id() });
let second = scope.spawn(|| { start.wait(); PUBLIC_BROKER_RUNTIME.get_id() });
(first.join().unwrap(), second.join().unwrap())
});
assert_eq!(ids.0, ids.1, "并发首次使用只能构建同一个原Worker");
let rt = &*PUBLIC_BROKER_RUNTIME;
assert_eq!(rt.get_id(), ids.0);
let name = format!("pi_mqtt-worker-{}", rt.get_id());
let first = collect_and_report_runtime_metrics(RuntimeMetricCollectionOptions::default())
.expect("采集测试Worker失败");
let matches: Vec<_> = first.instances().iter()
.filter(|entry| entry.names().any(|value| value == name)).collect();
if cfg!(feature = "runtime_metrics") {
assert_eq!(matches.len(), 1, "MQTT原初始化块必须自动登记一次");
let entry = matches[0];
assert_eq!(entry.name(), name);
assert_eq!(entry.names().collect::<Vec<_>>(), vec![name.as_str()]);
assert_eq!(entry.kind(), RuntimeMetricKind::Worker);
assert_eq!(entry.status(), RuntimeMetricInstanceStatus::RecordInvoked);
assert!(entry.id().as_u64() > 0);
assert_eq!(entry.counts(), Some((rt.len(), Some(rt.wait_len()))));
} else {
assert!(matches.is_empty(), "关闭feature不得自动登记MQTT单例");
}
let cloned = rt.clone();
assert_eq!(cloned.get_id(), rt.get_id());
assert_eq!(PUBLIC_BROKER_RUNTIME.get_id(), rt.get_id());
let second = collect_and_report_runtime_metrics(RuntimeMetricCollectionOptions::default())
.expect("重复采集失败");
let again: Vec<_> = second.instances().iter()
.filter(|entry| entry.names().any(|value| value == name)).collect();
assert_eq!(again.len(), matches.len());
if let Some(entry) = matches.first() {
assert_eq!(again[0].id(), entry.id());
assert_eq!(again[0].names().collect::<Vec<_>>(), vec![name.as_str()]);
}
}
}