use std::collections::{HashMap, VecDeque};
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use bytes::Bytes;
use dashmap::{DashMap, DashSet};
use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
pub(crate) struct Link {
pub dialer: String,
pub session: u64,
pub cancel: CancellationToken,
pub activity: Arc<AtomicU64>,
}
const LINK_SILENT_MS: u64 = super::HEARTBEAT_INTERVAL_SECS * 2 * 1000;
pub(crate) type Links = DashMap<String, Link>;
#[derive(Debug, PartialEq, Eq)]
pub(crate) enum LinkRefused {
SelfLink,
Duplicate,
}
pub(crate) fn claim_link(
links: &Links,
local: &str,
peer: &str,
link: Link,
) -> Result<(), LinkRefused> {
use dashmap::mapref::entry::Entry;
if peer == local {
return Err(LinkRefused::SelfLink);
}
match links.entry(peer.to_string()) {
Entry::Vacant(e) => {
e.insert(link);
Ok(())
}
Entry::Occupied(mut e) => {
let existing = e.get();
let replace = if existing.dialer == link.dialer {
super::epoch_ms().saturating_sub(existing.activity.load(Ordering::Relaxed))
> LINK_SILENT_MS
} else {
link.dialer < existing.dialer
};
if !replace {
return Err(LinkRefused::Duplicate);
}
e.get().cancel.cancel();
e.insert(link);
Ok(())
}
}
}
#[derive(Default)]
pub(crate) struct PeerInterest {
exact: ahash::AHashSet<String>,
patterns: Vec<String>,
}
impl PeerInterest {
fn is_pattern(topic: &str) -> bool {
topic.contains('*') || topic.contains('?')
}
pub fn len(&self) -> usize {
self.exact.len() + self.patterns.len()
}
pub fn insert(&mut self, topic: String, cap: usize) -> bool {
if Self::is_pattern(&topic) {
if self.patterns.contains(&topic) {
return true;
}
if self.len() >= cap {
return false;
}
self.patterns.push(topic);
} else {
if self.exact.contains(&topic) {
return true;
}
if self.len() >= cap {
return false;
}
self.exact.insert(topic);
}
true
}
pub fn remove(&mut self, topic: &str) {
if Self::is_pattern(topic) {
self.patterns.retain(|p| p != topic);
} else {
self.exact.remove(topic);
}
}
pub fn wants(&self, topic: &str) -> bool {
self.exact.contains(topic)
|| self
.patterns
.iter()
.any(|p| crate::server::glob_match(p, topic))
}
}
pub(crate) fn release_link(links: &Links, peer: &str, session: u64) {
links.remove_if(peer, |_, link| link.session == session);
}
#[derive(Clone)]
pub(crate) struct ClusterContext {
pub interest_tx: mpsc::Sender<InterestUpdate>,
pub fanout: crate::server::Fanout,
pub metrics: Arc<ClusterMetrics>,
pub new_peer_tx: mpsc::UnboundedSender<String>,
pub cmd_tx: mpsc::UnboundedSender<ClusterCommand>,
pub presence_tx: mpsc::Sender<PresencePeerFrame>,
pub links: Arc<Links>,
pub known_peers: Arc<DashSet<String>>,
pub local_topic_refcount: Arc<std::sync::Mutex<HashMap<String, usize>>>,
pub cluster_addr: Option<String>,
pub max_outbound: usize,
pub slow_drops: Arc<crate::server::Tally>,
pub recovery: Option<Arc<crate::recovery::RecoveryManager>>,
pub topic_message_counts: Arc<DashMap<String, AtomicU64>>,
pub queue_groups: Arc<DashMap<String, DashMap<String, crate::server::QueueGroup>>>,
pub producers: Arc<crate::server::Producers>,
pub producer_links: Arc<super::ProducerLinks>,
pub heard: Arc<DashMap<String, Arc<Heard>>>,
pub incarnations: Arc<DashMap<String, String>>,
pub leaving: CancellationToken,
pub reached_at: String,
}
#[derive(Default)]
pub(crate) struct Heard {
through: AtomicU64,
shed: std::sync::atomic::AtomicBool,
shut_down: std::sync::atomic::AtomicBool,
links: std::sync::atomic::AtomicUsize,
unclean: std::sync::atomic::AtomicBool,
}
impl Heard {
pub(crate) fn live(&self, seq: u64) {
if !self.shed.load(Ordering::Relaxed) {
self.through.fetch_max(seq, Ordering::Relaxed);
}
}
pub(crate) fn shed(&self) {
self.shed.store(true, Ordering::Relaxed);
}
pub(crate) fn caught_up(&self, through: u64) {
self.unclean.store(false, Ordering::Relaxed);
if !self.shed.load(Ordering::Relaxed) {
self.through.fetch_max(through, Ordering::Relaxed);
}
}
pub(crate) fn shut_down(&self) {
self.shut_down.store(true, Ordering::Relaxed);
}
pub(crate) fn linked(&self) {
self.links.fetch_add(1, Ordering::Relaxed);
}
pub(crate) fn ended(&self) {
let _ = self
.links
.try_update(Ordering::Relaxed, Ordering::Relaxed, |links| {
Some(links.saturating_sub(1))
});
let clean = self.shut_down.swap(false, Ordering::Relaxed);
self.unclean.store(!clean, Ordering::Relaxed);
}
pub(crate) fn unclean(&self) -> bool {
self.unclean.load(Ordering::Relaxed)
}
pub(crate) fn gone_unclean(&self) -> bool {
self.unclean()
|| (self.links.load(Ordering::Relaxed) > 0 && !self.shut_down.load(Ordering::Relaxed))
}
pub(crate) fn asked(&self) -> u64 {
self.shed.store(false, Ordering::Relaxed);
self.through.load(Ordering::Relaxed)
}
}
pub(crate) enum ClusterCommand {
Publish {
topic: String,
payload: String,
position: Option<(u32, u64)>,
seq: u64,
},
Sub {
topic: String,
},
Unsub {
topic: String,
},
Shutdown,
Drain,
GossipPeerAnnounce {
addr: String,
exclude_peer: String,
},
PresenceUpdate {
topic: String,
user_id: String,
action: u8, data: String,
updated_at: u64,
},
}
pub(crate) enum PeerRegistration {
Register {
peer_addr: String,
write_tx: mpsc::Sender<Bytes>,
negotiated_caps: u32,
taken: Option<tokio::sync::oneshot::Sender<()>>,
link: tokio_util::sync::CancellationToken,
},
Deregister {
peer_addr: String,
},
}
pub(crate) enum PresencePeerFrame {
Update {
origin: String,
topic: String,
user_id: String,
action: u8,
data: String,
updated_at: u64,
generation: u64,
},
Full {
origin: String,
entries: String,
round: Option<RoundPart>,
generation: u64,
},
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct RoundPart {
pub first: bool,
pub last: bool,
}
pub(crate) enum InterestUpdate {
Sub {
peer_addr: String,
topic: String,
},
Unsub {
peer_addr: String,
topic: String,
},
Resync {
peer_addr: String,
topics: Vec<String>,
is_continuation: bool,
more: bool,
generation: u64,
},
PeerDisconnected {
peer_addr: String,
generation: u64,
},
Drain {
peer_addr: String,
},
}
pub(crate) struct ClusterMetrics {
pub messages_sent: AtomicU64,
pub messages_delivered: AtomicU64,
pub messages_dropped: AtomicU64,
pub bytes_sent: AtomicU64,
pub bytes_received: AtomicU64,
pub reconnect_count: AtomicU64,
pub connected_peers: AtomicU64,
pub unknown_message_types: AtomicU64,
pub peer_info: DashMap<String, PeerInfo>,
}
#[derive(Clone)]
pub(crate) struct PeerInfo {
pub address: String,
pub instance_id: String,
pub capabilities: u32,
pub connected_at: u64,
}
impl ClusterMetrics {
pub fn new() -> Self {
Self {
messages_sent: AtomicU64::new(0),
messages_delivered: AtomicU64::new(0),
messages_dropped: AtomicU64::new(0),
bytes_sent: AtomicU64::new(0),
bytes_received: AtomicU64::new(0),
reconnect_count: AtomicU64::new(0),
connected_peers: AtomicU64::new(0),
unknown_message_types: AtomicU64::new(0),
peer_info: DashMap::new(),
}
}
}
impl ClusterMetrics {
pub(crate) fn add_delivered(&self, count: u64) {
self.messages_delivered.fetch_add(count, Ordering::Relaxed);
}
pub(crate) fn add_dropped(&self, count: u64) {
self.messages_dropped.fetch_add(count, Ordering::Relaxed);
}
}
pub(crate) struct ClusterDlqEntry {
pub topic: String,
pub payload: String,
pub peer_addr: String,
pub error: String,
}
pub(crate) struct ClusterDlq {
entries: VecDeque<ClusterDlqEntry>,
max_entries: usize,
}
impl ClusterDlq {
pub fn new(max_entries: usize) -> Self {
Self {
entries: VecDeque::new(),
max_entries,
}
}
pub fn push(&mut self, entry: ClusterDlqEntry) {
if self.entries.len() >= self.max_entries {
self.entries.pop_front();
}
self.entries.push_back(entry);
}
pub fn len(&self) -> usize {
self.entries.len()
}
pub fn drain_all(&mut self) -> Vec<ClusterDlqEntry> {
self.entries.drain(..).collect()
}
}