1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
//! The cluster mesh: server instances forward topic messages, interest and
//! presence to each other over a framed TCP (optionally mTLS) protocol.
//!
//! # Module map
//!
//! - `mod.rs`: protocol constants and limits.
//! - `protocol`: the wire format: message types, frame encoding and decoding,
//! the length-prefix framing, version / capability negotiation.
//! - `tls`: the cluster mTLS configuration.
//! - `types`: what the cluster tasks share and exchange: `ClusterContext`,
//! commands, registrations, interest updates, metrics, the dead-letter queue.
//! - `peer`: one peer link: writer, reader, dispatch, heartbeat, the outbound
//! session and its reconnect loop.
//! - `manager`: the cluster manager that owns every peer link.
//! - `inbound`: inbound peer links, on the cluster port or the main port.
//!
//! The whole module is engine code: it uses no PyO3. The crate-visible items
//! are re-exported below, so `cluster::X` paths stay as they were (`MsgType` is
//! used inside `protocol` only and is not re-exported).
pub use InboundLinks;
pub use cluster_manager;
pub use ProducerLinks;
pub use encode_presence_full;
pub use ;
pub use ;
pub use ;
use Links;
pub use ;
use AtomicU64;
use ;
use Bytes;
/// Wall-clock time in milliseconds since UNIX epoch (for lock-free heartbeat tracking).
// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------
pub const MAGIC: & = b"WSE\x00";
pub const PROTOCOL_VERSION: u16 = 1;
pub const MAX_FRAME_SIZE: usize = 1_048_576; // 1MB
pub const HEARTBEAT_INTERVAL_SECS: u64 = 5;
pub const HEARTBEAT_TIMEOUT_SECS: u64 = 15;
/// How often the manager re-sends this node's full interest (RESYNC) to every
/// peer, and a presence sync round to the peers that missed a presence
/// frame. SUB/UNSUB and presence deltas are best-effort (try_send drops under
/// back-pressure), so a dropped one would otherwise leave the peer's view of
/// this node wrong for good; the periodic full set heals it.
pub const CLUSTER_RESYNC_INTERVAL_SECS: u64 = 60;
/// Interest and presence updates the peer links hand to the manager, queued
/// at most: a peer sending them faster than the manager keeps up is slowed
/// down at its socket instead of growing the queue.
pub const MANAGER_QUEUE_CAP: usize = 65_536;
pub const MAX_BATCH_BYTES: usize = 65_536;
/// Monotonic generation counter for peer sessions. Prevents stale RESYNC
/// continuation frames from a prior connection corrupting the new session.
static PEER_SESSION_GENERATION: AtomicU64 = new;
pub const MAX_CLUSTER_PEERS: usize = 256;
/// Peer addresses a node remembers (several addresses may name one node):
/// what gossip adds past it is ignored, so a peer cannot grow the list, or
/// the PEER_LIST every new link gets, without end.
pub const MAX_KNOWN_PEERS: usize = MAX_CLUSTER_PEERS * 4;
// Cluster peer channels are bounded (10K messages) to prevent OOM from slow peers.
// This applies to BOTH directions: the per-peer writer channel (peer_write_tx) AND
// the inbound MSG dispatch channel (dispatch_tx). When a peer publishes faster than
// local fan-out can drain, the bounded dispatch queue fills and excess messages are
// shed (try_send) with metrics.messages_dropped incremented, rather than growing an
// unbounded backlog. Shedding keeps the single reader loop responsive to control and
// heartbeat frames. A shed message is not stored for recovery here: a recovery
// across it reports recovered: false.
pub const PEER_DISPATCH_QUEUE_CAP: usize = 10_000;
/// Max distinct topics tracked in one peer's remote-interest set. Bounds memory
/// so a buggy or compromised (but authenticated) peer streaming distinct SUB
/// frames cannot grow the set without limit. Generous -- a peer aggregates the
/// interest of all its local connections.
pub const MAX_REMOTE_INTEREST_PER_PEER: usize = 1_000_000;
pub const PROTOCOL_VERSION_MIN: u16 = 1;
// Capability flags (negotiated via bitwise AND of both peers' values)
pub const CAP_INTEREST_ROUTING: u32 = 1 << 0; // SUB/UNSUB interest routing
pub const CAP_COMPRESSION: u32 = 1 << 1; // zstd frame compression
pub const CAP_PRESENCE: u32 = 1 << 2; // Presence sync
pub const CAP_RECOVERY: u32 = 1 << 3; // Cross-node recovery
/// Catch-up on a new link: each node says where it stands in the streams the
/// other publishes (RESUME), and gets what it missed before anything live.
/// Offered by a node that keeps recovery buffers.
pub const CAP_RESUME: u32 = 1 << 4;
/// A node asks another what it took of a client's producer, when the client
/// comes to it from that node (PRODUCER_QUERY / PRODUCER_STATE).
pub const CAP_PRODUCERS: u32 = 1 << 5;
/// The catch-up covers the streams a node does not hold too: its RESUME
/// request names the topics it has, each with how long ago its interest
/// began, and the other sends what it published to them since (a stream
/// begun while the link was down). Offered with CAP_RESUME.
pub const CAP_RESUME_TOPICS: u32 = 1 << 6;
/// The catch-up starts where the node left off: each message it relays
/// carries its number in the arrival order of the node that published it
/// (MSG flag `0x08`); a node names the last one it heard of the other
/// (RESUME), and gets what came after it, what of that is no longer held
/// (LOST), and up to where the catch-up covers (CAUGHT_UP). Offered with
/// CAP_RESUME_TOPICS.
pub const CAP_ARRIVAL: u32 = 1 << 7;
/// Our capabilities bitmask
pub const LOCAL_CAPABILITIES: u32 =
CAP_INTEREST_ROUTING | CAP_COMPRESSION | CAP_PRESENCE | CAP_RECOVERY | CAP_PRODUCERS;
/// What this node offers in its HELLO: catch-up too when it keeps recovery
/// buffers (both nodes need them, one to say where it stands and the other
/// to send what came after).
pub
// Per-frame flags of MSG frames (the flags byte of the frame header)
// bit 0: compressed (zstd), bit 1: has recovery trailer, bit 2: sent again
// for a peer that missed it (catch-up), bit 3: has the origin's arrival
// number (8 bytes after the recovery trailer), bits 4-7: reserved.
// PRESENCE_FULL frames carry their sync round bits there (protocol.rs).
const FLAG_COMPRESSED: u8 = 0x01;
const FLAG_HAS_RECOVERY: u8 = 0x02;
const FLAG_REPLAYED: u8 = 0x04;
const FLAG_ARRIVAL: u8 = 0x08;
/// How long a new link waits for the peer's RESUME request.
const RESUME_TIMEOUT: Duration = from_secs;
/// How long a node that leaves the cluster waits for its links to say
/// SHUTDOWN and close, before its run ends.
pub const LEAVE_WITHIN: Duration = from_secs;
/// Positions a RESUME request may name: more than any node's recovery
/// buffers hold within their memory budget.
const MAX_RESUME_POINTS: usize = 1 << 20;
/// What a peer's reader hands its dispatch task, in the order it read it.
// Minimum payload size for compression to be worthwhile (bytes).
// Below this threshold, zstd overhead exceeds savings.
const COMPRESSION_THRESHOLD: usize = 256;
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------