wse 3.2.0

WebSocket engine for reactive user interfaces: topics with message recovery, presence, clustering without a broker, JWT, end-to-end encryption and signed messages.
Documentation
//! The cluster mesh: server instances forward topic messages, interest and
//! presence to each other over a framed TCP (optionally mTLS) protocol.
//!
//! # Module map
//!
//! - `mod.rs`: protocol constants and limits.
//! - `protocol`: the wire format: message types, frame encoding and decoding,
//!   the length-prefix framing, version / capability negotiation.
//! - `tls`: the cluster mTLS configuration.
//! - `types`: what the cluster tasks share and exchange: `ClusterContext`,
//!   commands, registrations, interest updates, metrics, the dead-letter queue.
//! - `peer`: one peer link: writer, reader, dispatch, heartbeat, the outbound
//!   session and its reconnect loop.
//! - `manager`: the cluster manager that owns every peer link.
//! - `inbound`: inbound peer links, on the cluster port or the main port.
//!
//! The whole module is engine code: it uses no PyO3. The crate-visible items
//! are re-exported below, so `cluster::X` paths stay as they were (`MsgType` is
//! used inside `protocol` only and is not re-exported).

mod inbound;
mod manager;
mod peer;
mod producers;
mod protocol;
mod tls;
mod types;

pub(crate) use inbound::InboundLinks;
pub(crate) use manager::cluster_manager;
pub(crate) use producers::ProducerLinks;
#[cfg(test)]
pub(crate) use protocol::encode_presence_full;
pub(crate) use protocol::{
    ClusterFrame, PRESENCE_ROUND, PRESENCE_ROUND_FIRST, PRESENCE_ROUND_LAST, ResumePoint,
    ResumeTopic, decode_frame, encode_drain, encode_hello, encode_msg_full, encode_peer_announce,
    encode_peer_list, encode_ping, encode_pong, encode_presence_round, encode_presence_update,
    encode_replayed, encode_resume, encode_resync, encode_shutdown, encode_sub, encode_unsub,
    negotiate_capabilities, negotiate_version,
};
#[cfg(test)]
pub(crate) use protocol::{encode_msg, encode_msg_compressed, encode_msg_with_recovery};
pub(crate) use tls::{ClusterTlsConfig, build_cluster_tls, build_cluster_tls_from_secret};
#[cfg(test)]
use types::Links;
pub(crate) use types::{
    ClusterCommand, ClusterContext, ClusterDlq, ClusterDlqEntry, ClusterMetrics, Heard,
    InterestUpdate, Link, LinkRefused, PeerInfo, PeerInterest, PeerRegistration, PresencePeerFrame,
    RoundPart, claim_link, release_link,
};

use std::sync::atomic::AtomicU64;
use std::time::{Duration, SystemTime, UNIX_EPOCH};

use bytes::Bytes;

/// Wall-clock time in milliseconds since UNIX epoch (for lock-free heartbeat tracking).
fn epoch_ms() -> u64 {
    SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .unwrap_or_default()
        .as_millis() as u64
}

// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------

pub(crate) const MAGIC: &[u8; 4] = b"WSE\x00";
pub(crate) const PROTOCOL_VERSION: u16 = 1;
pub(crate) const MAX_FRAME_SIZE: usize = 1_048_576; // 1MB
pub(crate) const HEARTBEAT_INTERVAL_SECS: u64 = 5;
pub(crate) const HEARTBEAT_TIMEOUT_SECS: u64 = 15;
/// How often the manager re-sends this node's full interest (RESYNC) to every
/// peer, and a presence sync round to the peers that missed a presence
/// frame. SUB/UNSUB and presence deltas are best-effort (try_send drops under
/// back-pressure), so a dropped one would otherwise leave the peer's view of
/// this node wrong for good; the periodic full set heals it.
pub(crate) const CLUSTER_RESYNC_INTERVAL_SECS: u64 = 60;
/// Interest and presence updates the peer links hand to the manager, queued
/// at most: a peer sending them faster than the manager keeps up is slowed
/// down at its socket instead of growing the queue.
pub(crate) const MANAGER_QUEUE_CAP: usize = 65_536;
pub(crate) const MAX_BATCH_BYTES: usize = 65_536;

/// Monotonic generation counter for peer sessions. Prevents stale RESYNC
/// continuation frames from a prior connection corrupting the new session.
static PEER_SESSION_GENERATION: AtomicU64 = AtomicU64::new(0);
pub(crate) const MAX_CLUSTER_PEERS: usize = 256;
/// Peer addresses a node remembers (several addresses may name one node):
/// what gossip adds past it is ignored, so a peer cannot grow the list, or
/// the PEER_LIST every new link gets, without end.
pub(crate) const MAX_KNOWN_PEERS: usize = MAX_CLUSTER_PEERS * 4;
// Cluster peer channels are bounded (10K messages) to prevent OOM from slow peers.
// This applies to BOTH directions: the per-peer writer channel (peer_write_tx) AND
// the inbound MSG dispatch channel (dispatch_tx). When a peer publishes faster than
// local fan-out can drain, the bounded dispatch queue fills and excess messages are
// shed (try_send) with metrics.messages_dropped incremented, rather than growing an
// unbounded backlog. Shedding keeps the single reader loop responsive to control and
// heartbeat frames. A shed message is not stored for recovery here: a recovery
// across it reports recovered: false.
pub(crate) const PEER_DISPATCH_QUEUE_CAP: usize = 10_000;
/// Max distinct topics tracked in one peer's remote-interest set. Bounds memory
/// so a buggy or compromised (but authenticated) peer streaming distinct SUB
/// frames cannot grow the set without limit. Generous -- a peer aggregates the
/// interest of all its local connections.
pub(crate) const MAX_REMOTE_INTEREST_PER_PEER: usize = 1_000_000;

pub(crate) const PROTOCOL_VERSION_MIN: u16 = 1;

// Capability flags (negotiated via bitwise AND of both peers' values)
pub(crate) const CAP_INTEREST_ROUTING: u32 = 1 << 0; // SUB/UNSUB interest routing
pub(crate) const CAP_COMPRESSION: u32 = 1 << 1; // zstd frame compression
pub(crate) const CAP_PRESENCE: u32 = 1 << 2; // Presence sync
pub(crate) const CAP_RECOVERY: u32 = 1 << 3; // Cross-node recovery
/// Catch-up on a new link: each node says where it stands in the streams the
/// other publishes (RESUME), and gets what it missed before anything live.
/// Offered by a node that keeps recovery buffers.
pub(crate) const CAP_RESUME: u32 = 1 << 4;
/// A node asks another what it took of a client's producer, when the client
/// comes to it from that node (PRODUCER_QUERY / PRODUCER_STATE).
pub(crate) const CAP_PRODUCERS: u32 = 1 << 5;
/// The catch-up covers the streams a node does not hold too: its RESUME
/// request names the topics it has, each with how long ago its interest
/// began, and the other sends what it published to them since (a stream
/// begun while the link was down). Offered with CAP_RESUME.
pub(crate) const CAP_RESUME_TOPICS: u32 = 1 << 6;
/// The catch-up starts where the node left off: each message it relays
/// carries its number in the arrival order of the node that published it
/// (MSG flag `0x08`); a node names the last one it heard of the other
/// (RESUME), and gets what came after it, what of that is no longer held
/// (LOST), and up to where the catch-up covers (CAUGHT_UP). Offered with
/// CAP_RESUME_TOPICS.
pub(crate) const CAP_ARRIVAL: u32 = 1 << 7;

/// Our capabilities bitmask
pub(crate) const LOCAL_CAPABILITIES: u32 =
    CAP_INTEREST_ROUTING | CAP_COMPRESSION | CAP_PRESENCE | CAP_RECOVERY | CAP_PRODUCERS;

/// What this node offers in its HELLO: catch-up too when it keeps recovery
/// buffers (both nodes need them, one to say where it stands and the other
/// to send what came after).
pub(crate) fn local_capabilities(ctx: &ClusterContext) -> u32 {
    if ctx.recovery.is_some() {
        LOCAL_CAPABILITIES | CAP_RESUME | CAP_RESUME_TOPICS | CAP_ARRIVAL
    } else {
        LOCAL_CAPABILITIES
    }
}

// Per-frame flags of MSG frames (the flags byte of the frame header)
// bit 0: compressed (zstd), bit 1: has recovery trailer, bit 2: sent again
// for a peer that missed it (catch-up), bit 3: has the origin's arrival
// number (8 bytes after the recovery trailer), bits 4-7: reserved.
// PRESENCE_FULL frames carry their sync round bits there (protocol.rs).
const FLAG_COMPRESSED: u8 = 0x01;
const FLAG_HAS_RECOVERY: u8 = 0x02;
const FLAG_REPLAYED: u8 = 0x04;
const FLAG_ARRIVAL: u8 = 0x08;
/// How long a new link waits for the peer's RESUME request.
const RESUME_TIMEOUT: Duration = Duration::from_secs(5);
/// How long a node that leaves the cluster waits for its links to say
/// SHUTDOWN and close, before its run ends.
pub(crate) const LEAVE_WITHIN: Duration = Duration::from_secs(2);
/// Positions a RESUME request may name: more than any node's recovery
/// buffers hold within their memory budget.
const MAX_RESUME_POINTS: usize = 1 << 20;

/// What a peer's reader hands its dispatch task, in the order it read it.
enum DispatchItem {
    /// A message of a topic, relayed or replayed: with its origin's
    /// recovery position when it has one, and a relayed one with its number
    /// in the origin's arrival order (CAP_ARRIVAL).
    Message(String, Bytes, Option<(u32, u64)>, Option<u64>),
    /// Messages of a topic (or of each topic a pattern matches) were
    /// published on the peer that no node holds: after what was replayed
    /// before it.
    Lost(String),
    /// The peer's catch-up covers what it published up to this arrival:
    /// once what came before it is delivered.
    CaughtUp(u64),
    /// The peer said SHUTDOWN: it published nothing more, and its end is
    /// clean once what came before it is delivered.
    Shutdown,
}

// Minimum payload size for compression to be worthwhile (bytes).
// Below this threshold, zstd overhead exceeds savings.
const COMPRESSION_THRESHOLD: usize = 256;

// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------

#[cfg(test)]
mod tests;