wse 3.0.1

WebSocket engine for reactive user interfaces: topics with message recovery, presence, clustering without a broker, JWT, end-to-end encryption and signed messages.
Documentation
//! The cluster mesh: server instances forward topic messages, interest and
//! presence to each other over a framed TCP (optionally mTLS) protocol.
//!
//! # Module map
//!
//! - `mod.rs`: protocol constants and limits.
//! - `protocol`: the wire format: message types, frame encoding and decoding,
//!   the length-prefix framing, version / capability negotiation.
//! - `tls`: the cluster mTLS configuration.
//! - `types`: what the cluster tasks share and exchange: `ClusterContext`,
//!   commands, registrations, interest updates, metrics, the dead-letter queue.
//! - `peer`: one peer link: writer, reader, dispatch, heartbeat, the outbound
//!   session and its reconnect loop.
//! - `manager`: the cluster manager that owns every peer link.
//! - `inbound`: inbound peer links, on the cluster port or the main port.
//!
//! The whole module is engine code: it uses no PyO3. The crate-visible items
//! are re-exported below, so `cluster::X` paths stay as they were (`MsgType` is
//! used inside `protocol` only and is not re-exported).

mod inbound;
mod manager;
mod peer;
mod protocol;
mod tls;
mod types;

pub(crate) use inbound::InboundLinks;
pub(crate) use manager::cluster_manager;
#[cfg(test)]
pub(crate) use protocol::encode_presence_full;
pub(crate) use protocol::{
    ClusterFrame, PRESENCE_ROUND, PRESENCE_ROUND_FIRST, PRESENCE_ROUND_LAST, ResumePoint,
    decode_frame, encode_drain, encode_hello, encode_msg, encode_msg_compressed,
    encode_msg_with_recovery, encode_peer_announce, encode_peer_list, encode_ping, encode_pong,
    encode_presence_round, encode_presence_update, encode_replayed, encode_resume, encode_resync,
    encode_shutdown, encode_sub, encode_unsub, negotiate_capabilities, negotiate_version,
};
pub(crate) use tls::{ClusterTlsConfig, build_cluster_tls, build_cluster_tls_from_secret};
#[cfg(test)]
use types::Links;
pub(crate) use types::{
    ClusterCommand, ClusterContext, ClusterDlq, ClusterDlqEntry, ClusterMetrics, InterestUpdate,
    Link, LinkRefused, PeerInfo, PeerInterest, PeerRegistration, PresencePeerFrame, RoundPart,
    claim_link, release_link,
};

use std::sync::atomic::AtomicU64;
use std::time::{Duration, SystemTime, UNIX_EPOCH};

use bytes::Bytes;

/// Wall-clock time in milliseconds since UNIX epoch (for lock-free heartbeat tracking).
fn epoch_ms() -> u64 {
    SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .unwrap_or_default()
        .as_millis() as u64
}

// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------

pub(crate) const MAGIC: &[u8; 4] = b"WSE\x00";
pub(crate) const PROTOCOL_VERSION: u16 = 1;
pub(crate) const MAX_FRAME_SIZE: usize = 1_048_576; // 1MB
pub(crate) const HEARTBEAT_INTERVAL_SECS: u64 = 5;
pub(crate) const HEARTBEAT_TIMEOUT_SECS: u64 = 15;
/// How often the manager re-sends this node's full interest (RESYNC) to every
/// peer, and a presence sync round to the peers that missed a presence
/// frame. SUB/UNSUB and presence deltas are best-effort (try_send drops under
/// back-pressure), so a dropped one would otherwise leave the peer's view of
/// this node wrong for good; the periodic full set heals it.
pub(crate) const CLUSTER_RESYNC_INTERVAL_SECS: u64 = 60;
/// Interest and presence updates the peer links hand to the manager, queued
/// at most: a peer sending them faster than the manager keeps up is slowed
/// down at its socket instead of growing the queue.
pub(crate) const MANAGER_QUEUE_CAP: usize = 65_536;
pub(crate) const MAX_BATCH_BYTES: usize = 65_536;

/// Monotonic generation counter for peer sessions. Prevents stale RESYNC
/// continuation frames from a prior connection corrupting the new session.
static PEER_SESSION_GENERATION: AtomicU64 = AtomicU64::new(0);
pub(crate) const MAX_CLUSTER_PEERS: usize = 256;
/// Peer addresses a node remembers (several addresses may name one node):
/// what gossip adds past it is ignored, so a peer cannot grow the list, or
/// the PEER_LIST every new link gets, without end.
pub(crate) const MAX_KNOWN_PEERS: usize = MAX_CLUSTER_PEERS * 4;
// Cluster peer channels are bounded (10K messages) to prevent OOM from slow peers.
// This applies to BOTH directions: the per-peer writer channel (peer_write_tx) AND
// the inbound MSG dispatch channel (dispatch_tx). When a peer publishes faster than
// local fan-out can drain, the bounded dispatch queue fills and excess messages are
// shed (try_send) with metrics.messages_dropped incremented, rather than growing an
// unbounded backlog. Shedding keeps the single reader loop responsive to control and
// heartbeat frames. A shed message is not stored for recovery here: a recovery
// across it reports recovered: false.
pub(crate) const PEER_DISPATCH_QUEUE_CAP: usize = 10_000;
/// Max distinct topics tracked in one peer's remote-interest set. Bounds memory
/// so a buggy or compromised (but authenticated) peer streaming distinct SUB
/// frames cannot grow the set without limit. Generous -- a peer aggregates the
/// interest of all its local connections.
pub(crate) const MAX_REMOTE_INTEREST_PER_PEER: usize = 1_000_000;

pub(crate) const PROTOCOL_VERSION_MIN: u16 = 1;

// Capability flags (negotiated via bitwise AND of both peers' values)
pub(crate) const CAP_INTEREST_ROUTING: u32 = 1 << 0; // SUB/UNSUB interest routing
pub(crate) const CAP_COMPRESSION: u32 = 1 << 1; // zstd frame compression
pub(crate) const CAP_PRESENCE: u32 = 1 << 2; // Presence sync
pub(crate) const CAP_RECOVERY: u32 = 1 << 3; // Cross-node recovery
/// Catch-up on a new link: each node says where it stands in the streams the
/// other publishes (RESUME), and gets what it missed before anything live.
/// Offered by a node that keeps recovery buffers.
pub(crate) const CAP_RESUME: u32 = 1 << 4;

/// Our capabilities bitmask
pub(crate) const LOCAL_CAPABILITIES: u32 =
    CAP_INTEREST_ROUTING | CAP_COMPRESSION | CAP_PRESENCE | CAP_RECOVERY;

/// What this node offers in its HELLO: catch-up too when it keeps recovery
/// buffers (both nodes need them, one to say where it stands and the other
/// to send what came after).
pub(crate) fn local_capabilities(ctx: &ClusterContext) -> u32 {
    if ctx.recovery.is_some() {
        LOCAL_CAPABILITIES | CAP_RESUME
    } else {
        LOCAL_CAPABILITIES
    }
}

// Per-frame flags of MSG frames (the flags byte of the frame header)
// bit 0: compressed (zstd), bit 1: has recovery trailer, bit 2: sent again
// for a peer that missed it (catch-up), bits 3-7: reserved.
// PRESENCE_FULL frames carry their sync round bits there (protocol.rs).
const FLAG_COMPRESSED: u8 = 0x01;
const FLAG_HAS_RECOVERY: u8 = 0x02;
const FLAG_REPLAYED: u8 = 0x04;
/// How long a new link waits for the peer's RESUME request.
const RESUME_TIMEOUT: Duration = Duration::from_secs(5);
/// Positions a RESUME request may name: more than any node's recovery
/// buffers hold within their memory budget.
const MAX_RESUME_POINTS: usize = 1 << 20;

/// Dispatch tuple: (topic, payload, optional origin recovery position).
type DispatchItem = (String, Bytes, Option<(u32, u64)>);

// Minimum payload size for compression to be worthwhile (bytes).
// Below this threshold, zstd overhead exceeds savings.
const COMPRESSION_THRESHOLD: usize = 256;

// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------

#[cfg(test)]
mod tests;