1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
//! A running server: its delivery lanes, its command task, its periodic
//! tasks and the accept loop.
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::time::Duration;
use ahash::AHashSet;
use tokio::net::TcpListener;
use tokio::sync::mpsc;
use tokio_tungstenite::tungstenite::protocol::Message;
use tokio_util::sync::CancellationToken;
use tokio_util::task::TaskTracker;
use super::commands::process_commands;
use super::connection::handle_connection;
use super::handler::{Handler, guarded};
use super::handshake::{HANDSHAKE_TIMEOUT, HandshakeGuard, MAX_PENDING_HANDSHAKES};
use super::{
CLOSE_IDLE, CLOSE_TRY_AGAIN_LATER, Commands, ConnectionHandle, Fanout, ServerCommand,
SharedState,
};
/// TCP keepalive of accepted connections: the first probe after a minute of
/// silence, then every 10 seconds.
fn tcp_keepalive() -> socket2::TcpKeepalive {
let keepalive = socket2::TcpKeepalive::new().with_time(Duration::from_secs(60));
#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
let keepalive = keepalive.with_interval(Duration::from_secs(10));
keepalive
}
/// What a server's start makes before it serves: the command queue, and a
/// delivery task for each lane, running before the first command.
pub(crate) struct Started {
commands_rx: mpsc::UnboundedReceiver<ServerCommand>,
fanout: Fanout,
}
/// Start `shared`'s server on the current runtime: its lanes run from here
/// on, its commands queue until `serve` takes them.
pub(crate) fn start<H: Handler>(shared: &Arc<SharedState>, handler: &Arc<H>) -> Started {
// A drain of the last run is over.
shared.draining.store(false, Ordering::SeqCst);
*shared.started_at.lock() = Some(std::time::Instant::now());
let fanout = Fanout::start(
shared.topics.clone(),
shared.max_outbound_queue_bytes,
Arc::clone(&shared.slow_consumer_drops),
);
*shared.fanout.write().unwrap() = Some(fanout.clone());
let (tx, commands_rx) = mpsc::unbounded_channel::<ServerCommand>();
shared.commands_queued.store(0, Ordering::Relaxed);
let commands = Commands::new(tx, Arc::clone(&shared.commands_queued));
// The handler hears of presence changes.
let presence_handler = Arc::clone(handler);
shared
.presence_events
.store(Some(Arc::new(Box::new(move |event| {
if guarded(|| presence_handler.presence(event)).is_err() {
tracing::error!("[WSE] Handler::presence panicked");
}
}))));
// For presence broadcasts from the disconnect cleanup paths.
shared.commands.store(Some(Arc::new(commands)));
Started {
commands_rx,
fanout,
}
}
/// Serve on `listener` until `running` is false: the command task, the
/// periodic tasks (recovery cleanup, topic counter sweep, signed-message
/// window, presence TTL sweep, ping / idle / slow-consumer check) and the
/// accept loop. Returns once the command task has ended (`Shutdown`) and
/// every task of this run with it, the connections' cleanup and their
/// disconnect events included; what the run set up is gone then.
pub(crate) async fn serve<H: Handler>(
shared: Arc<SharedState>,
started: Started,
listener: TcpListener,
handler: Arc<H>,
running: Arc<AtomicBool>,
) {
let Started {
commands_rx: cmd_rx,
fanout,
..
} = started;
let cmd_handle = tokio::spawn(process_commands(cmd_rx, shared.clone(), fanout));
// This run's tasks: ended by `stop`, and waited for before the run ends.
let stop = CancellationToken::new();
let tasks = TaskTracker::new();
// Spawn periodic recovery cleanup if recovery is enabled
if let Some(ref recovery) = shared.recovery {
let rm = recovery.clone();
let cleanup_state = shared.clone();
let stop = stop.clone();
tasks.spawn(async move {
let mut interval = tokio::time::interval(crate::recovery::CLEANUP_INTERVAL);
while tick(&mut interval, &stop).await {
rm.cleanup();
cleanup_state.sweep_interest_linger();
}
});
}
// Per-topic delivery counters of topics nobody subscribes
// to any more: dropped once there are many (user- or
// UUID-scoped topics), whether or not metrics are read.
{
let counts_state = shared.clone();
let stop = stop.clone();
tasks.spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(60));
while tick(&mut interval, &stop).await {
let counts = &counts_state.topic_message_counts;
if counts.len() > 10_000 {
counts.retain(|topic, _| counts_state.topics.subscribers.contains_key(topic));
}
}
});
}
// Signed messages: a (kid, n) is forgotten once its time
// is out of the window, as it could not be taken again.
if shared.signer.is_some() {
let seen_state = shared.clone();
let stop = stop.clone();
tasks.spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(5));
while tick(&mut interval, &stop).await {
let now = crate::server::signing::now_ms();
seen_state.signed_seen.retain(|_, until| *until >= now);
}
});
}
// Presence TTL sweep (every 30 seconds)
if let Some(ref pm) = shared.presence {
let sweep_pm = pm.clone();
let sweep_state = shared.clone();
let stop = stop.clone();
tasks.spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(30));
while tick(&mut interval, &stop).await {
let removed = sweep_pm.sweep_dead_connections(|conn_id| {
if conn_id.starts_with("__remote__") {
return true;
}
sweep_state.conn_last_activity.contains_key(conn_id)
});
if removed.is_empty() {
continue;
}
for (topic, user_id, data) in removed {
sweep_state.announce_presence_leave(topic, user_id, data, true);
}
}
});
}
// Server-initiated ping + zombie detection + slow consumer detection
{
let ping_state = shared.clone();
let stop = stop.clone();
tasks.spawn(async move {
let mut interval =
tokio::time::interval(Duration::from_secs(ping_state.ping_interval_secs));
let idle_timeout = Duration::from_secs(ping_state.idle_timeout_secs);
let max_pending = ping_state.max_outbound_queue_bytes;
// Two-phase slow consumer detection: connections suspected
// of being slow (queue full on first check). If still full
// on the next tick, they get disconnected.
let mut suspected_slow: AHashSet<String> = AHashSet::new();
while tick(&mut interval, &stop).await {
let now = std::time::Instant::now();
let snapshot: Vec<(String, ConnectionHandle)> = {
let conns = ping_state.connections.read();
conns
.iter()
.map(|(cid, h)| (cid.clone(), h.clone()))
.collect()
};
// Forget suspects that disconnected meanwhile.
if !suspected_slow.is_empty() {
let live: AHashSet<&str> =
snapshot.iter().map(|(cid, _)| cid.as_str()).collect();
suspected_slow.retain(|id| live.contains(id.as_str()));
}
for (cid, handle) in &snapshot {
// A slow consumer: its queue is still full on a
// second tick. It gets a close frame; if it does
// not read that either, the close grace drops it.
if max_pending > 0 && handle.out.pending.load(Ordering::Relaxed) >= max_pending
{
if suspected_slow.remove(cid) {
handle.close(CLOSE_TRY_AGAIN_LATER, "too slow to read");
} else {
suspected_slow.insert(cid.clone());
}
continue;
}
suspected_slow.remove(cid);
// Copied out: holding the map entry while the
// connection's cleanup removes it would deadlock
// on its shard lock.
let last_seen = ping_state
.conn_last_activity
.get(cid)
.map(|entry| entry.load(Ordering::Relaxed));
let Some(last) = last_seen else {
continue; // gone since the snapshot
};
let idle_ms = ping_state.activity_ms(now).saturating_sub(last);
if u128::from(idle_ms) > idle_timeout.as_millis() {
handle.close(CLOSE_IDLE, "idle timeout");
continue;
}
// Not admitted yet (an end-to-end session or its proof
// under way, or the application's accept): nothing goes
// to it but its handshake.
if handle.closer.is_closing() || !handle.out.admitted.load(Ordering::Acquire) {
continue;
}
let ts =
chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true);
let ping_msg = format!(
"{{\"c\":\"WSE\",\"t\":\"ping\",\"p\":{{\"server_time\":\"{ts}\"}}}}"
);
// Sealed in an E2E session.
handle.send_control(Message::Text(ping_msg.into()));
}
}
});
}
// Bounds the number of connections in the pre-handshake phase
// (slowloris): max_connections only counts registered ones.
let handshakes_in_flight = Arc::new(AtomicUsize::new(0));
let mut refused_handshakes: u64 = 0;
let mut last_refused_log = tokio::time::Instant::now() - Duration::from_secs(1);
let mut accept_pause = Duration::ZERO;
loop {
tokio::select! {
res = listener.accept() => {
match res {
Ok((stream, addr)) => {
accept_pause = Duration::ZERO;
if handshakes_in_flight.load(Ordering::Relaxed)
>= MAX_PENDING_HANDSHAKES
{
// Logged once a second at most: a flood
// of sockets is not a flood of lines.
refused_handshakes += 1;
if last_refused_log.elapsed() >= Duration::from_secs(1) {
tracing::warn!(
"[WSE] Pre-handshake cap reached ({MAX_PENDING_HANDSHAKES}), dropped {refused_handshakes} socket(s), the last from {addr}"
);
refused_handshakes = 0;
last_refused_log = tokio::time::Instant::now();
}
continue; // drop the stream
}
let hs_guard =
HandshakeGuard::new(handshakes_in_flight.clone());
let _ = stream.set_nodelay(true);
// Larger send buffer for broadcast throughput (default 16KB -> 256KB)
let sock = socket2::SockRef::from(&stream);
let _ = sock.set_send_buffer_size(262_144);
// A peer that vanished without a FIN is found
// by the kernel too, not only by idle_timeout.
let _ = sock.set_tcp_keepalive(&tcp_keepalive());
let shared2 = shared.clone();
let handler = Arc::clone(&handler);
// Cluster links come on this port only in
// single-port mode: the peek is for that.
let single_port =
shared.cluster_main_port.read().unwrap().is_some();
let stop = stop.clone();
tasks.spawn(async move {
// Protocol detection: peek first 4 bytes to distinguish
// cluster binary frames (4-byte big-endian length prefix:
// 0x00 0x00 0x00 XX for frames < 256 bytes) from HTTP/WS
// (first byte is ASCII letter like 'G' for GET) or TLS
// (first byte 0x16 for ClientHello).
let mut peek = [0u8; 4];
let peeked = if single_port {
tokio::select! {
peeked = tokio::time::timeout(
HANDSHAKE_TIMEOUT,
stream.peek(&mut peek),
) => Some(peeked),
// The server stops: a socket not yet
// read just goes.
() = stop.cancelled() => return,
}
} else {
None
};
// A cluster link starts with a TLS ClientHello (a
// handshake record, 0x16 0x03): the WebSocket port
// speaks plain HTTP itself, which starts with a
// letter (GET).
let is_cluster = match peeked {
Some(Ok(Ok(n))) if n >= 2 => peek[0] == 0x16 && peek[1] == 0x03,
_ => false,
};
if is_cluster {
// Only in single-port mode, and then the
// cluster manager takes the link over.
drop(hs_guard);
let main_port = shared2
.cluster_main_port
.read()
.unwrap()
.clone();
match main_port {
Some(manager) => {
let _ = manager.send((stream, addr));
}
None => tracing::debug!(
"[WSE] Refused a cluster link on the WebSocket port from {addr}"
),
}
} else {
handle_connection(stream, addr, shared2, hs_guard, handler, stop)
.await;
}
});
}
Err(e) => {
if !running.load(Ordering::SeqCst) { break; }
// Out of file descriptors, say: retrying at
// once would spin.
accept_pause = (accept_pause * 2)
.clamp(Duration::from_millis(10), Duration::from_secs(1));
tracing::error!("[WSE] Accept error: {e} (retrying in {accept_pause:?})");
tokio::time::sleep(accept_pause).await;
continue;
}
}
}
_ = tokio::time::sleep(std::time::Duration::from_millis(100)) => {
if !running.load(Ordering::SeqCst) { break; }
}
}
}
let _ = cmd_handle.await;
// The command task closed every connection: the periodic tasks end, and
// the connections finish their cleanup.
stop.cancel();
tasks.close();
tasks.wait().await;
running.store(false, Ordering::SeqCst);
// Lanes, queues and the handler go with the run, however it was stopped
// (`stop`, or the server dropped).
stopped(&shared);
tracing::info!("[WSE] Shut down");
}
/// The next tick of a periodic task; false once its run is stopping.
async fn tick(interval: &mut tokio::time::Interval, stop: &CancellationToken) -> bool {
tokio::select! {
() = stop.cancelled() => false,
_ = interval.tick() => true,
}
}
/// What a stopped server leaves: no lanes, no command queue, no cluster.
pub(crate) fn stopped(shared: &SharedState) {
*shared.started_at.lock() = None;
shared.commands.store(None);
shared.presence_events.store(None);
*shared.fanout.write().unwrap() = None;
shared.cluster_cmd_tx.store(None);
*shared.cluster_main_port.write().unwrap() = None;
*shared.cluster_instance_id.lock().unwrap() = None;
}