1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
//! The cluster mesh: server instances forward topic messages, interest and
//! presence to each other over a framed TCP (optionally mTLS) protocol.
//!
//! # Module map
//!
//! - `mod.rs`: protocol constants and limits.
//! - `protocol`: the wire format: message types, frame encoding and decoding,
//! the length-prefix framing, version / capability negotiation.
//! - `tls`: the cluster mTLS configuration.
//! - `types`: what the cluster tasks share and exchange: `ClusterContext`,
//! commands, registrations, interest updates, metrics, the dead-letter queue.
//! - `peer`: one peer link: writer, reader, dispatch, heartbeat, the outbound
//! session and its reconnect loop.
//! - `manager`: the cluster manager that owns every peer link.
//! - `inbound`: inbound peer links, on the cluster port or the main port.
//!
//! The whole module is engine code: it uses no PyO3. The crate-visible items
//! are re-exported below, so `cluster::X` paths stay as they were (`MsgType` is
//! used inside `protocol` only and is not re-exported).
pub use InboundLinks;
pub use cluster_manager;
pub use encode_presence_full;
pub use ;
pub use ;
use Links;
pub use ;
use AtomicU64;
use ;
use Bytes;
/// Wall-clock time in milliseconds since UNIX epoch (for lock-free heartbeat tracking).
// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------
pub const MAGIC: & = b"WSE\x00";
pub const PROTOCOL_VERSION: u16 = 1;
pub const MAX_FRAME_SIZE: usize = 1_048_576; // 1MB
pub const HEARTBEAT_INTERVAL_SECS: u64 = 5;
pub const HEARTBEAT_TIMEOUT_SECS: u64 = 15;
/// How often the manager re-sends this node's full interest (RESYNC) to every
/// peer, and a presence sync round to the peers that missed a presence
/// frame. SUB/UNSUB and presence deltas are best-effort (try_send drops under
/// back-pressure), so a dropped one would otherwise leave the peer's view of
/// this node wrong for good; the periodic full set heals it.
pub const CLUSTER_RESYNC_INTERVAL_SECS: u64 = 60;
/// Interest and presence updates the peer links hand to the manager, queued
/// at most: a peer sending them faster than the manager keeps up is slowed
/// down at its socket instead of growing the queue.
pub const MANAGER_QUEUE_CAP: usize = 65_536;
pub const MAX_BATCH_BYTES: usize = 65_536;
/// Monotonic generation counter for peer sessions. Prevents stale RESYNC
/// continuation frames from a prior connection corrupting the new session.
static PEER_SESSION_GENERATION: AtomicU64 = new;
pub const MAX_CLUSTER_PEERS: usize = 256;
/// Peer addresses a node remembers (several addresses may name one node):
/// what gossip adds past it is ignored, so a peer cannot grow the list, or
/// the PEER_LIST every new link gets, without end.
pub const MAX_KNOWN_PEERS: usize = MAX_CLUSTER_PEERS * 4;
// Cluster peer channels are bounded (10K messages) to prevent OOM from slow peers.
// This applies to BOTH directions: the per-peer writer channel (peer_write_tx) AND
// the inbound MSG dispatch channel (dispatch_tx). When a peer publishes faster than
// local fan-out can drain, the bounded dispatch queue fills and excess messages are
// shed (try_send) with metrics.messages_dropped incremented, rather than growing an
// unbounded backlog. Shedding keeps the single reader loop responsive to control and
// heartbeat frames. A shed message is not stored for recovery here: a recovery
// across it reports recovered: false.
pub const PEER_DISPATCH_QUEUE_CAP: usize = 10_000;
/// Max distinct topics tracked in one peer's remote-interest set. Bounds memory
/// so a buggy or compromised (but authenticated) peer streaming distinct SUB
/// frames cannot grow the set without limit. Generous -- a peer aggregates the
/// interest of all its local connections.
pub const MAX_REMOTE_INTEREST_PER_PEER: usize = 1_000_000;
pub const PROTOCOL_VERSION_MIN: u16 = 1;
// Capability flags (negotiated via bitwise AND of both peers' values)
pub const CAP_INTEREST_ROUTING: u32 = 1 << 0; // SUB/UNSUB interest routing
pub const CAP_COMPRESSION: u32 = 1 << 1; // zstd frame compression
pub const CAP_PRESENCE: u32 = 1 << 2; // Presence sync
pub const CAP_RECOVERY: u32 = 1 << 3; // Cross-node recovery
/// Catch-up on a new link: each node says where it stands in the streams the
/// other publishes (RESUME), and gets what it missed before anything live.
/// Offered by a node that keeps recovery buffers.
pub const CAP_RESUME: u32 = 1 << 4;
/// Our capabilities bitmask
pub const LOCAL_CAPABILITIES: u32 =
CAP_INTEREST_ROUTING | CAP_COMPRESSION | CAP_PRESENCE | CAP_RECOVERY;
/// What this node offers in its HELLO: catch-up too when it keeps recovery
/// buffers (both nodes need them, one to say where it stands and the other
/// to send what came after).
pub
// Per-frame flags of MSG frames (the flags byte of the frame header)
// bit 0: compressed (zstd), bit 1: has recovery trailer, bit 2: sent again
// for a peer that missed it (catch-up), bits 3-7: reserved.
// PRESENCE_FULL frames carry their sync round bits there (protocol.rs).
const FLAG_COMPRESSED: u8 = 0x01;
const FLAG_HAS_RECOVERY: u8 = 0x02;
const FLAG_REPLAYED: u8 = 0x04;
/// How long a new link waits for the peer's RESUME request.
const RESUME_TIMEOUT: Duration = from_secs;
/// Positions a RESUME request may name: more than any node's recovery
/// buffers hold within their memory budget.
const MAX_RESUME_POINTS: usize = 1 << 20;
/// Dispatch tuple: (topic, payload, optional origin recovery position).
type DispatchItem = ;
// Minimum payload size for compression to be worthwhile (bytes).
// Below this threshold, zstd overhead exceeds savings.
const COMPRESSION_THRESHOLD: usize = 256;
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------