offline_protocol/protocol/mesh_relay.rs
1//! Admission control and pacing for mesh forwarding.
2//!
3//! Forwarding is what turns a set of nearby devices into a network: a frame
4//! addressed to someone out of radio range is carried by the devices in
5//! between. The hazard is that the same property which gives coverage —
6//! every node repeating what it hears — also multiplies traffic, and in a
7//! dense room the multiplication can outrun the radio. This module is the
8//! governor on that: it decides whether a frame is forwarded at all, when,
9//! and how fast frames may leave.
10//!
11//! Four mechanisms, each covering something the others cannot:
12//!
13//! 1. **Suppression** ([`RelaySeenCache`]) — an id is forwarded once. Without
14//! it, copies circulate until their hop limit runs out, and every node
15//! repeats every copy.
16//! 2. **Jitter with cancellation** — a forward waits a short randomized delay
17//! before going out, and is dropped if the same id arrives again while it
18//! waits. In a dense cluster the first neighbor to fire covers everyone who
19//! can hear it, and the rest stand down. This is what makes cost scale with
20//! *coverage* rather than with the number of links, and it adapts on its own
21//! as the room fills up — no threshold to tune.
22//! 3. **Hop limits** — an arriving frame's remaining hop budget is clamped
23//! before use, so a frame that claims an implausible budget (nothing
24//! authenticates it at this layer) cannot circulate longer than one our own
25//! policy would have issued.
26//! 4. **Budgets** — a hard ceiling on frames per second, in total and per
27//! neighbor. The first three assume frames are what they appear to be;
28//! budgets hold even when they are not. Whatever arrives, and however it is
29//! shaped, a node cannot be made to transmit faster than its budget: that
30//! invariant is what bounds the worst case, so the failure mode under attack
31//! or overload is added delay rather than a radio that never goes quiet.
32//!
33//! Ordering matters. Suppression runs before anything expensive, admission
34//! before queueing (so a flood cannot grow the queue), and pacing at the moment
35//! of transmission (so a forward cancelled while waiting costs no budget).
36
37use offline_protocol_core::{Message, MessagePriority};
38use offline_protocol_reliability::{RelaySeenCache, RelaySeenConfig};
39use std::collections::hash_map::DefaultHasher;
40use std::collections::HashMap;
41use std::hash::{Hash, Hasher};
42use std::time::{Duration, Instant};
43
44/// Hop budget a forwarded frame is clamped to under normal density.
45pub const DEFAULT_RELAY_MAX_TTL: u8 = 8;
46
47/// Hop budget used once the local neighborhood is dense.
48///
49/// With many neighbors, coverage is reached in fewer hops — the same message
50/// travels further per hop — so a lower ceiling reaches the same devices for
51/// less traffic.
52pub const DEFAULT_RELAY_DENSE_MAX_TTL: u8 = 5;
53
54/// Neighbor count at or above which the dense hop budget applies.
55pub const DEFAULT_RELAY_DENSE_DEGREE: usize = 6;
56
57/// Absolute ceiling on how many times a frame may be forwarded, regardless of
58/// the hop budget it carries. A backstop against a frame whose hop fields have
59/// been shaped to evade the clamp.
60pub const RELAY_MAX_HOP_COUNT: u8 = 16;
61
62/// Neighbors a single frame is forwarded to.
63pub const DEFAULT_RELAY_FANOUT: usize = 3;
64
65/// Shortest delay before a queued forward is transmitted.
66pub const DEFAULT_RELAY_JITTER_MIN_MS: u64 = 20;
67
68/// Longest delay before a queued forward is transmitted, at low density.
69pub const DEFAULT_RELAY_JITTER_MAX_MS: u64 = 200;
70
71/// Sustained ceiling on forwarded frames leaving this node, per second.
72pub const DEFAULT_RELAY_RATE_PER_SEC: f32 = 10.0;
73
74/// Burst allowance on top of the sustained forwarding rate.
75pub const DEFAULT_RELAY_BURST: f32 = 30.0;
76
77/// Transmissions held back from carrying other people's traffic so this device
78/// can always get its own frames out.
79///
80/// Own traffic and forwarded traffic share one budget, because they share one
81/// radio. Without a reserve they also share one queue discipline, and a device
82/// forwarding at its ceiling would starve its *own* sends behind strangers'
83/// — including the delivery acknowledgement for a message it just received,
84/// whose absence makes a delivered message look lost. Carving the reserve out
85/// of the burst rather than adding a second bucket keeps the ceiling exactly
86/// where it was: this changes who may spend the last few tokens, not how many
87/// there are.
88///
89/// It costs forwarding no throughput at saturation. Tokens refill continuously,
90/// so a forward simply waits for the level to climb back above the reserve —
91/// which at the sustained rate is a fraction of a second.
92pub const RELAY_OWN_TRAFFIC_RESERVE: f32 = 5.0;
93
94/// Sustained ceiling on frames accepted for forwarding from any one neighbor.
95///
96/// Keeps a single noisy or hostile link from consuming the whole node's
97/// forwarding budget: its frames are refused at the door, leaving the budget
98/// for everyone else's traffic.
99pub const DEFAULT_RELAY_PEER_RATE_PER_SEC: f32 = 5.0;
100
101/// Burst allowance for a single neighbor.
102pub const DEFAULT_RELAY_PEER_BURST: f32 = 15.0;
103
104/// Maximum forwards waiting for their delay to elapse.
105pub const DEFAULT_RELAY_QUEUE_CAPACITY: usize = 256;
106
107/// Maximum neighbors tracked for per-neighbor rate limiting.
108pub const MAX_RELAY_RATE_PEERS: usize = 256;
109
110/// How long a queued forward may sit past its due time before being abandoned.
111///
112/// If a node is so far over budget that a frame waits this long, the frame's
113/// usefulness has passed — other paths have carried it or the sender has
114/// retransmitted — and holding it only displaces newer traffic.
115const RELAY_QUEUE_MAX_OVERDUE: Duration = Duration::from_secs(5);
116
117/// Tunables for mesh forwarding.
118#[derive(Debug, Clone)]
119pub struct MeshRelayConfig {
120 /// Hop budget a forwarded frame is clamped to.
121 pub max_ttl: u8,
122 /// Hop budget applied once the neighborhood is dense.
123 pub dense_max_ttl: u8,
124 /// Neighbor count at which the dense budget applies.
125 pub dense_degree: usize,
126 /// Neighbors a frame is forwarded to.
127 pub fanout: usize,
128 /// Shortest pre-transmit delay.
129 pub jitter_min: Duration,
130 /// Longest pre-transmit delay at low density.
131 pub jitter_max: Duration,
132 /// Sustained forwarding rate, frames per second.
133 pub rate_per_sec: f32,
134 /// Burst allowance above the sustained rate.
135 pub burst: f32,
136 /// Sustained per-neighbor acceptance rate.
137 pub peer_rate_per_sec: f32,
138 /// Per-neighbor burst allowance.
139 pub peer_burst: f32,
140 /// Maximum forwards awaiting transmission.
141 pub queue_capacity: usize,
142 /// Suppression cache sizing.
143 pub seen: RelaySeenConfig,
144}
145
146impl Default for MeshRelayConfig {
147 fn default() -> Self {
148 Self {
149 max_ttl: DEFAULT_RELAY_MAX_TTL,
150 dense_max_ttl: DEFAULT_RELAY_DENSE_MAX_TTL,
151 dense_degree: DEFAULT_RELAY_DENSE_DEGREE,
152 fanout: DEFAULT_RELAY_FANOUT,
153 jitter_min: Duration::from_millis(DEFAULT_RELAY_JITTER_MIN_MS),
154 jitter_max: Duration::from_millis(DEFAULT_RELAY_JITTER_MAX_MS),
155 rate_per_sec: DEFAULT_RELAY_RATE_PER_SEC,
156 burst: DEFAULT_RELAY_BURST,
157 peer_rate_per_sec: DEFAULT_RELAY_PEER_RATE_PER_SEC,
158 peer_burst: DEFAULT_RELAY_PEER_BURST,
159 queue_capacity: DEFAULT_RELAY_QUEUE_CAPACITY,
160 seen: RelaySeenConfig::default(),
161 }
162 }
163}
164
165/// Why a frame was not queued for forwarding.
166#[derive(Debug, Clone, Copy, PartialEq, Eq)]
167pub enum RelayRejection {
168 /// Already handled; this is another copy of a frame we have dealt with.
169 AlreadySeen,
170 /// Another copy arrived while this one was waiting, so the pending forward
171 /// was cancelled — a neighbor has covered it.
172 SupersededByDuplicate,
173 /// The frame has no hop budget left.
174 HopLimitReached,
175 /// The neighbor that sent it is over its acceptance rate.
176 PeerRateLimited,
177 /// No room in the pending queue.
178 QueueFull,
179}
180
181/// The outcome of offering an inbound frame to the governor.
182#[derive(Debug, Clone, Copy, PartialEq, Eq)]
183pub enum RelayAdmission {
184 /// Queued; it will be transmitted once its delay elapses.
185 Queued,
186 /// Not queued.
187 Rejected(RelayRejection),
188}
189
190/// A forward waiting for its delay to elapse.
191#[derive(Debug, Clone)]
192pub struct PendingRelay {
193 /// The frame to forward, with hop fields already adjusted.
194 pub message: Message,
195 /// The neighbor it arrived from, which must not receive it back.
196 pub arrival_peer: Option<String>,
197 /// When it becomes eligible to transmit.
198 pub due_at: Instant,
199}
200
201/// Running totals, for telemetry and for tests that assert a flood stayed
202/// bounded.
203#[derive(Debug, Clone, Default, PartialEq, Eq)]
204pub struct MeshRelayCounters {
205 /// Frames queued for forwarding.
206 pub queued: u64,
207 /// Times a frame was put on a link, counting each link separately.
208 ///
209 /// This is the airtime meter — what the send budget actually bounds — so it
210 /// counts this device handing over its *own* messages too. For "how much
211 /// have I carried for other people", see [`Self::forwarded`].
212 pub transmissions: u64,
213 /// Third-party frames that crossed at least one link on someone's behalf.
214 ///
215 /// Counted once per frame carried, not once per link, and never for this
216 /// device's own traffic.
217 pub forwarded: u64,
218 /// Frames put back on the queue because they reached no neighbor at all —
219 /// the budget ran out, the links chosen for them went away, or there was
220 /// no usable link to choose.
221 pub requeued: u64,
222 /// Queued forwards displaced to make room for a higher-priority frame.
223 pub queue_evicted: u64,
224 /// Arrivals suppressed as already handled.
225 pub duplicates_suppressed: u64,
226 /// Pending forwards cancelled because a neighbor covered them.
227 pub cancelled_by_duplicate: u64,
228 /// Frames refused because their sender was over its rate.
229 pub peer_rate_limited: u64,
230 /// Frames refused because the pending queue was full.
231 pub queue_full: u64,
232 /// Frames dropped for having no hop budget left.
233 pub hop_limit_reached: u64,
234 /// Transmissions deferred because the node was at its forwarding rate.
235 pub rate_deferred: u64,
236 /// Queued forwards abandoned after waiting too long past their due time.
237 pub abandoned_overdue: u64,
238 /// Hop budgets reduced because the frame claimed more than policy allows.
239 pub ttl_clamped: u64,
240}
241
242/// What this device has carried for other people.
243///
244/// A snapshot, safe to poll: every field counts since start-up.
245#[derive(Debug, Clone, Default, PartialEq, Eq)]
246pub struct MeshRelayStats {
247 /// Frames carried onward on someone else's behalf, counted once each.
248 pub forwarded: u64,
249 /// Times this device put a frame on a link, counting each link separately
250 /// and including hand-offs of its own messages.
251 ///
252 /// This is what the per-second forwarding budget bounds, so it is the
253 /// number to compare against that ceiling — [`Self::forwarded`] is the
254 /// contribution figure to show a user.
255 pub transmissions: u64,
256 /// Frames accepted for forwarding.
257 pub queued: u64,
258 /// Frames waiting to go out right now.
259 pub awaiting_transmission: usize,
260 /// Arrivals ignored because the frame had already been handled.
261 pub duplicates_suppressed: u64,
262 /// Forwards dropped because a neighbor transmitted the same frame first —
263 /// the saving that keeps a crowded room from repeating itself.
264 pub covered_by_a_neighbor: u64,
265 /// Frames refused because the neighbor sending them was over its share.
266 pub peer_rate_limited: u64,
267 /// Transmissions held back because this device was at its forwarding rate.
268 ///
269 /// A frame counted here is delayed, not dropped: it goes back on the queue
270 /// and is tried again on the next tick, until it is either sent or has
271 /// waited so long past its turn that carrying it no longer helps.
272 pub rate_deferred: u64,
273 /// Frames that had travelled as far as they were allowed to.
274 pub hop_limit_reached: u64,
275 /// Frames whose claimed reach was cut down to local policy.
276 pub reach_clamped: u64,
277 /// Ids forgotten for lack of room rather than age.
278 ///
279 /// Expected to stay at zero. A non-zero value means this device is seeing
280 /// more traffic than it can remember having handled, and a frame it forgets
281 /// can be forwarded a second time.
282 pub dropped_for_capacity: u64,
283}
284
285/// A refilling allowance, in frames.
286#[derive(Debug, Clone)]
287struct TokenBucket {
288 tokens: f32,
289 capacity: f32,
290 refill_per_sec: f32,
291 last_refill: Instant,
292}
293
294impl TokenBucket {
295 fn new(capacity: f32, refill_per_sec: f32, now: Instant) -> Self {
296 Self {
297 tokens: capacity.max(0.0),
298 capacity: capacity.max(0.0),
299 refill_per_sec: refill_per_sec.max(0.0),
300 last_refill: now,
301 }
302 }
303
304 fn refill(&mut self, now: Instant) {
305 let elapsed = now
306 .saturating_duration_since(self.last_refill)
307 .as_secs_f32();
308 if elapsed > 0.0 {
309 self.tokens = (self.tokens + elapsed * self.refill_per_sec).min(self.capacity);
310 self.last_refill = now;
311 }
312 }
313
314 /// Spends a token, but only while more than `floor` would remain
315 /// untouched. A `floor` of zero spends down to empty.
316 fn try_take_above(&mut self, floor: f32, now: Instant) -> bool {
317 self.refill(now);
318 if self.tokens >= 1.0 + floor {
319 self.tokens -= 1.0;
320 true
321 } else {
322 false
323 }
324 }
325}
326
327/// Decides what gets forwarded, when, and how fast.
328///
329/// Holds no transports and sends nothing itself: it answers questions and hands
330/// back due work, so its behavior is testable without a radio and its policy
331/// stays in one place rather than spread through the receive path.
332#[derive(Debug)]
333pub struct MeshRelayGovernor {
334 config: MeshRelayConfig,
335 seen: RelaySeenCache,
336 /// Forwards awaiting their delay, in the order they were queued.
337 pending: Vec<PendingRelay>,
338 send_budget: TokenBucket,
339 /// Tokens of [`Self::send_budget`] that only this device's own frames may
340 /// spend. Clamped to half the burst so a small configured burst cannot
341 /// reserve the whole bucket and stop forwarding outright.
342 own_reserve: f32,
343 peer_budgets: HashMap<String, TokenBucket>,
344 counters: MeshRelayCounters,
345 /// Seeds the per-frame delay so two nodes holding the same frame pick
346 /// different delays.
347 local_id: String,
348}
349
350impl MeshRelayGovernor {
351 /// Creates a governor with explicit tunables.
352 pub fn with_config(local_id: impl Into<String>, config: MeshRelayConfig) -> Self {
353 let now = Instant::now();
354 let seen = RelaySeenCache::with_config(config.seen.clone());
355 Self {
356 send_budget: TokenBucket::new(config.burst, config.rate_per_sec, now),
357 own_reserve: RELAY_OWN_TRAFFIC_RESERVE.min(config.burst.max(0.0) / 2.0),
358 seen,
359 pending: Vec::new(),
360 peer_budgets: HashMap::new(),
361 counters: MeshRelayCounters::default(),
362 local_id: local_id.into(),
363 config,
364 }
365 }
366
367 /// Re-points the governor at this device's real identity once it is known.
368 ///
369 /// The governor is built before `initialize_mls` derives the address, so it
370 /// starts out seeded with the profile. Both uses are hashes — fan-out
371 /// spread and send jitter — so the change is only a reseed: it shifts which
372 /// frames this node prefers, never whether a frame is relayed.
373 pub fn set_local_id(&mut self, local_id: impl Into<String>) {
374 self.local_id = local_id.into();
375 }
376
377 /// The tunables in force. Apps read these from
378 /// `ProtocolConfig::mesh_relay`; this is for the tests that assert the
379 /// defaults hold together.
380 #[cfg(test)]
381 pub fn config(&self) -> &MeshRelayConfig {
382 &self.config
383 }
384
385 /// Running totals.
386 pub fn counters(&self) -> &MeshRelayCounters {
387 &self.counters
388 }
389
390 /// Forwards awaiting transmission.
391 pub fn pending_len(&self) -> usize {
392 self.pending.len()
393 }
394
395 /// Ids dropped from the suppression cache for capacity rather than age.
396 /// Expected to stay zero; see [`RelaySeenCache::capacity_evictions`].
397 pub fn seen_capacity_evictions(&self) -> u64 {
398 self.seen.capacity_evictions()
399 }
400
401 /// Offers an inbound third-party frame for forwarding.
402 ///
403 /// Runs the full admission sequence and, when the frame is accepted,
404 /// queues an adjusted copy for later transmission. `degree` is the current
405 /// neighbor count, which sets both the hop budget and the delay spread.
406 /// `is_last_hop` says whether the frame's recipient is one of our own
407 /// neighbors, which changes what a duplicate means — see below.
408 pub fn admit(
409 &mut self,
410 message: &Message,
411 arrival_peer: Option<&str>,
412 degree: usize,
413 is_last_hop: bool,
414 ) -> RelayAdmission {
415 let now = Instant::now();
416 let message_id = message.id.as_str();
417
418 // Has this id been dealt with? Checked without recording, because a
419 // frame that is about to be *refused* must not be remembered as
420 // handled: another copy of it arriving over a healthy link a moment
421 // later is the alternate path that should still carry it. Recording on
422 // rejection would leave the id suppressed for the whole retention
423 // window, so a single link running hot — or one frame with its hop
424 // budget rewritten to nothing, which nothing here can authenticate —
425 // would blank every path through this device.
426 if self.seen.contains(&message_id) {
427 self.counters.duplicates_suppressed =
428 self.counters.duplicates_suppressed.saturating_add(1);
429
430 // Our own copy is still waiting, so a neighbor has transmitted
431 // this frame already.
432 if let Some(index) = self
433 .pending
434 .iter()
435 .position(|p| p.message.id.as_str() == message_id)
436 {
437 // Standing down is right in the middle of a network, where a
438 // neighbor's transmission reaches the same region ours would.
439 // It is wrong at the last hop: our pending copy is addressed
440 // to the recipient themselves, and no other device can be
441 // assumed to hold that link. Dropping it there loses the
442 // delivery outright — and leaves the id suppressed, so the
443 // sender's retries cannot rescue it either.
444 if is_last_hop {
445 return RelayAdmission::Rejected(RelayRejection::AlreadySeen);
446 }
447
448 self.pending.remove(index);
449 self.counters.cancelled_by_duplicate =
450 self.counters.cancelled_by_duplicate.saturating_add(1);
451 return RelayAdmission::Rejected(RelayRejection::SupersededByDuplicate);
452 }
453
454 return RelayAdmission::Rejected(RelayRejection::AlreadySeen);
455 }
456
457 // Per-neighbor admission, before the frame can occupy queue space.
458 //
459 // A frame whose arrival link the carrier did not identify skips this
460 // step: there is no link to charge it to, and inventing a shared bucket
461 // for "unknown" would let one such carrier throttle every other. Those
462 // frames are still bounded by the queue capacity and the per-second
463 // send budget below, which is what makes skipping it acceptable — the
464 // per-neighbor limit shapes *fairness between links*, and a frame with
465 // no known link has no share to exceed.
466 if let Some(peer) = arrival_peer {
467 if !self.take_peer_token(peer, now) {
468 self.counters.peer_rate_limited = self.counters.peer_rate_limited.saturating_add(1);
469 return RelayAdmission::Rejected(RelayRejection::PeerRateLimited);
470 }
471 }
472
473 // Hop accounting. The arriving budget is clamped to what our own policy
474 // would have issued before it is spent, so an inflated claim buys at
475 // most this one hop.
476 let Some(forwarded) = self.prepare_hop(message, degree) else {
477 self.counters.hop_limit_reached = self.counters.hop_limit_reached.saturating_add(1);
478 return RelayAdmission::Rejected(RelayRejection::HopLimitReached);
479 };
480
481 if self.pending.len() >= self.config.queue_capacity && !self.evict_for(&forwarded, now) {
482 self.counters.queue_full = self.counters.queue_full.saturating_add(1);
483 return RelayAdmission::Rejected(RelayRejection::QueueFull);
484 }
485
486 // Accepted: now it is genuinely handled, and further copies can be
487 // suppressed. Recording here rather than at the top also means the
488 // cache fills at the rate frames are *taken on*, which the per-neighbor
489 // and per-device limits bound — not at the rate they arrive, which
490 // nothing bounds.
491 self.seen.observe(&message_id);
492
493 let due_at = now + self.jitter_for(&message_id, degree);
494 self.pending.push(PendingRelay {
495 message: forwarded,
496 arrival_peer: arrival_peer.map(str::to_string),
497 due_at,
498 });
499 self.counters.queued = self.counters.queued.saturating_add(1);
500
501 RelayAdmission::Queued
502 }
503
504 /// Returns the forwards whose delay has elapsed, removing them from the
505 /// queue.
506 ///
507 /// Frames whose delay has not elapsed stay queued, as do frames the device
508 /// has no budget left to transmit. A frame that has waited far past its
509 /// turn is abandoned instead: by then other paths have carried it or the
510 /// sender has retransmitted, and holding it only displaces newer traffic.
511 ///
512 /// Budget is *checked* here and *spent* in [`Self::take_send_token`], once
513 /// per link a frame actually crosses. Spending it here instead would
514 /// undercount by the fan-out — one release from this queue can put the
515 /// frame on several links — and the whole point of the ceiling is that it
516 /// bounds what reaches the radio.
517 ///
518 /// The check is therefore not a reservation, and cannot be: a released
519 /// frame may still find the budget gone by the time it reaches the radio,
520 /// because the frames released alongside it spend from the same bucket. A
521 /// caller that gets nothing onto a link for that reason must hand the frame
522 /// back with [`Self::requeue`] rather than drop it — the id is already
523 /// recorded as handled here, so dropping it would lose this copy *and*
524 /// refuse the copies and retransmissions that follow.
525 pub fn take_due(&mut self, now: Instant) -> Vec<PendingRelay> {
526 self.seen.expire(now);
527
528 let mut due = Vec::new();
529 let mut keep: Vec<PendingRelay> = Vec::with_capacity(self.pending.len());
530
531 for relay in std::mem::take(&mut self.pending) {
532 if relay.due_at > now {
533 keep.push(relay);
534 continue;
535 }
536
537 if now.saturating_duration_since(relay.due_at) > RELAY_QUEUE_MAX_OVERDUE {
538 // Never made it onto a link, so its id must not stay
539 // suppressed: the sender's retransmissions carry the same id,
540 // and refusing them would close this route for the whole
541 // retention window over a few seconds of congestion.
542 self.seen.forget(&relay.message.id.as_str());
543 self.counters.abandoned_overdue = self.counters.abandoned_overdue.saturating_add(1);
544 continue;
545 }
546
547 if self.has_send_budget(now) {
548 due.push(relay);
549 } else {
550 self.counters.rate_deferred = self.counters.rate_deferred.saturating_add(1);
551 keep.push(relay);
552 }
553 }
554
555 self.pending = keep;
556 due
557 }
558
559 /// Whether any budget remains to forward right now.
560 ///
561 /// Measured against the same reserve [`Self::take_send_token`] respects, so
562 /// `take_due` does not release a batch the flush is about to refuse.
563 fn has_send_budget(&mut self, now: Instant) -> bool {
564 self.send_budget.refill(now);
565 self.send_budget.tokens >= 1.0 + self.own_reserve
566 }
567
568 /// Claims budget for putting one **forwarded** frame on one link.
569 ///
570 /// Stops short of the own-traffic reserve: at the ceiling, other people's
571 /// traffic waits so this device can still send its own. See
572 /// [`RELAY_OWN_TRAFFIC_RESERVE`] and [`Self::take_own_send_token`].
573 pub fn take_send_token(&mut self) -> bool {
574 self.claim_transmission(self.own_reserve)
575 }
576
577 /// Claims budget for putting one of **this device's own** frames on one
578 /// link.
579 ///
580 /// Metered against the same ceiling — it is the same radio — but allowed
581 /// into the reserve, so a device forwarding at its limit can still get its
582 /// own messages and acknowledgements out.
583 pub fn take_own_send_token(&mut self) -> bool {
584 self.claim_transmission(0.0)
585 }
586
587 fn claim_transmission(&mut self, floor: f32) -> bool {
588 if self.send_budget.try_take_above(floor, Instant::now()) {
589 self.counters.transmissions = self.counters.transmissions.saturating_add(1);
590 true
591 } else {
592 self.counters.rate_deferred = self.counters.rate_deferred.saturating_add(1);
593 false
594 }
595 }
596
597 /// Records that a third-party frame was carried onward.
598 ///
599 /// Separate from [`Self::take_send_token`], which meters airtime for every
600 /// frame this device transmits including its own. Counted once per frame
601 /// carried rather than once per link, so it reads as "messages I moved for
602 /// other people".
603 pub fn record_forwarded(&mut self) {
604 self.counters.forwarded = self.counters.forwarded.saturating_add(1);
605 }
606
607 /// Puts a released forward back on the queue, keeping its original due
608 /// time.
609 ///
610 /// For the frame that reached no neighbor at all between [`Self::take_due`]
611 /// releasing it and the radio being asked — because the budget ran out,
612 /// because every link picked for it had gone away, or because there was no
613 /// usable link to pick. All three end the same way: the frame has travelled
614 /// nowhere, its id is already recorded as handled, and dropping it would
615 /// refuse the copies and retransmissions behind it for the whole retention
616 /// window.
617 ///
618 /// It keeps its due time deliberately: the overdue cut-off in `take_due` is
619 /// measured from that, so a frame that stays stuck is eventually abandoned
620 /// instead of being retried forever.
621 ///
622 /// Refused only if the queue has filled meanwhile, which is the same
623 /// bound every other queued frame is subject to.
624 pub fn requeue(&mut self, relay: PendingRelay) {
625 if self.pending.len() >= self.config.queue_capacity {
626 // Refused room on the way back, so this frame is being dropped
627 // having reached nobody. Release its id for the same reason the
628 // overdue cut-off does — a copy behind it, or the sender's own
629 // retransmission, is now the only way it travels.
630 self.seen.forget(&relay.message.id.as_str());
631 self.counters.queue_full = self.counters.queue_full.saturating_add(1);
632 return;
633 }
634 self.counters.requeued = self.counters.requeued.saturating_add(1);
635 self.pending.push(relay);
636 }
637
638 /// Records an id as handled without queueing a forward for it.
639 ///
640 /// Used for frames this device **originates** and hands to its neighbors:
641 /// a copy circling back to us through the mesh is then recognized rather
642 /// than carried onward as though it were someone else's.
643 ///
644 /// Deliberately not called for frames addressed to us. A later copy of one
645 /// of those is still addressed to us, so it never reaches the forwarding
646 /// path at all — recording it would only fill the cache at delivery rate,
647 /// which the capacity is not sized for and which would make
648 /// [`Self::seen_capacity_evictions`] climb for reasons unrelated to relay
649 /// load.
650 pub fn mark_handled(&mut self, message_id: &str) {
651 self.seen.observe(message_id);
652 }
653
654 /// Absorbs an arrival that is a copy of a frame already dealt with and
655 /// needs no further decision, reporting whether it did.
656 ///
657 /// This is [`Self::admit`]'s suppression check, split out so a caller can
658 /// reach it without first assembling the arguments `admit` needs. In a
659 /// dense neighborhood most third-party arrivals are copies, and the caller's
660 /// `degree` and `is_last_hop` cost a status snapshot across every transport
661 /// and an enumeration of every link — work the suppression check would only
662 /// throw away.
663 ///
664 /// Deliberately answers `false` while we still hold a pending copy of the
665 /// id, even though that is also a duplicate: standing down for a neighbor
666 /// is the one duplicate outcome that depends on the neighbor set, so it has
667 /// to go through the full path. Those are rare — the pending window is one
668 /// jitter delay wide.
669 pub fn absorb_settled_duplicate(&mut self, message_id: &str) -> bool {
670 if !self.seen.contains(message_id) {
671 return false;
672 }
673 if self
674 .pending
675 .iter()
676 .any(|p| p.message.id.as_str() == message_id)
677 {
678 return false;
679 }
680 self.counters.duplicates_suppressed = self.counters.duplicates_suppressed.saturating_add(1);
681 true
682 }
683
684 /// Neighbors a frame should be forwarded to, given the current neighbor
685 /// set and the peers it must not go back to.
686 ///
687 /// Excludes the neighbor it arrived from and the peer that wrote it —
688 /// both already have it, and returning it is how a frame ends up bouncing
689 /// between two nodes.
690 pub fn select_targets<'a>(
691 &self,
692 neighbors: impl IntoIterator<Item = (&'a str, u8)>,
693 exclude: &[&str],
694 message_id: &str,
695 ) -> Vec<String> {
696 let mut candidates: Vec<(&str, u8)> = neighbors
697 .into_iter()
698 .filter(|(peer, _)| !exclude.contains(peer))
699 .collect();
700
701 if candidates.len() <= self.config.fanout {
702 return candidates
703 .into_iter()
704 .map(|(peer, _)| peer.to_string())
705 .collect();
706 }
707
708 // Strongest links first: a fan-out slot spent on a barely-connected
709 // neighbor is one the frame probably does not survive, and there are
710 // only a few slots.
711 //
712 // Ties break on a hash of (id, peer, self), which does two things.
713 // Within a group of equally good links it spreads different messages
714 // across different neighbors rather than pinning every frame to the
715 // same one. And it is stable per frame: a second copy of the same
716 // message picks the same neighbors instead of a fresh set, so a
717 // retransmission does not widen its own footprint.
718 candidates.sort_by(|(a_peer, a_quality), (b_peer, b_quality)| {
719 b_quality.cmp(a_quality).then_with(|| {
720 Self::hash_of(&[message_id, a_peer, &self.local_id]).cmp(&Self::hash_of(&[
721 message_id,
722 b_peer,
723 &self.local_id,
724 ]))
725 })
726 });
727 candidates
728 .into_iter()
729 .take(self.config.fanout)
730 .map(|(peer, _)| peer.to_string())
731 .collect()
732 }
733
734 /// Drops per-neighbor rate state for a peer that has gone away.
735 pub fn forget_peer(&mut self, peer_id: &str) {
736 self.peer_budgets.remove(peer_id);
737 }
738
739 /// Releases expired suppression entries. Safe to call from a periodic tick.
740 pub fn maintain(&mut self, now: Instant) {
741 self.seen.expire(now);
742 }
743
744 /// Builds the copy to forward, or `None` when the frame has no hop budget
745 /// left.
746 fn prepare_hop(&mut self, message: &Message, degree: usize) -> Option<Message> {
747 if message.hop_count.value() >= RELAY_MAX_HOP_COUNT {
748 return None;
749 }
750
751 let ceiling = if degree >= self.config.dense_degree {
752 self.config.dense_max_ttl
753 } else {
754 self.config.max_ttl
755 };
756
757 let claimed = message.ttl.value();
758 let effective = claimed.min(ceiling);
759 if effective < claimed {
760 self.counters.ttl_clamped = self.counters.ttl_clamped.saturating_add(1);
761 }
762
763 // A budget of 1 or less is spent: this node is the last that may hold
764 // the frame, matching `TTL::is_exhausted`.
765 let remaining = offline_protocol_core::TTL::from_value(effective).decrement()?;
766
767 let mut forwarded = message.clone();
768 // Rewrite rather than decrement in place: the frame leaves with a
769 // budget this node vouches for, so an inflated claim cannot survive
770 // past this hop.
771 forwarded.ttl = remaining;
772 let _ = forwarded.increment_hop();
773 Some(forwarded)
774 }
775
776 /// Makes room for a higher-priority forward by displacing the
777 /// lowest-priority one that is not yet due.
778 ///
779 /// Frames already past their due time are left alone: they are on their way
780 /// to the radio this tick, so taking one costs a transmission that was
781 /// about to happen while the frame replacing it still has to wait out its
782 /// hold. Only when every lower-priority frame is already due does it
783 /// displace one of those instead of refusing outright.
784 fn evict_for(&mut self, candidate: &Message, now: Instant) -> bool {
785 let candidate_rank = priority_rank(candidate.priority);
786 let lower_priority = |p: &PendingRelay| priority_rank(p.message.priority) < candidate_rank;
787
788 let victim = self
789 .pending
790 .iter()
791 .enumerate()
792 .filter(|(_, p)| lower_priority(p) && p.due_at > now)
793 .min_by_key(|(_, p)| priority_rank(p.message.priority))
794 .map(|(index, _)| index)
795 .or_else(|| {
796 self.pending
797 .iter()
798 .enumerate()
799 .filter(|(_, p)| lower_priority(p))
800 .min_by_key(|(_, p)| priority_rank(p.message.priority))
801 .map(|(index, _)| index)
802 });
803
804 match victim {
805 Some(index) => {
806 let displaced = self.pending.remove(index);
807 // Displaced before it ever reached a link. Its id goes back to
808 // being unknown, so the copy behind it — or the sender's
809 // retransmission — can still be carried by this device once
810 // there is room again.
811 self.seen.forget(&displaced.message.id.as_str());
812 self.counters.queue_evicted = self.counters.queue_evicted.saturating_add(1);
813 true
814 }
815 None => false,
816 }
817 }
818
819 fn take_peer_token(&mut self, peer_id: &str, now: Instant) -> bool {
820 if !self.peer_budgets.contains_key(peer_id) {
821 // Bound the map: an attacker cannot grow it with invented peer ids
822 // because entries are only created for peers we hold a link to,
823 // but churn over a long session can still accumulate.
824 if self.peer_budgets.len() >= MAX_RELAY_RATE_PEERS {
825 // Make room by forgetting the link that is spending least,
826 // rather than by clearing the table. A wholesale reset hands
827 // every neighbor a fresh burst — including the noisy one the
828 // per-link limit exists to hold back, which is the entry it can
829 // least afford to lose. The bucket that has recovered the most
830 // tokens is the one whose absence changes the least: at full
831 // capacity it is indistinguishable from a link never tracked.
832 //
833 // Costs a scan of the table, but only once the ceiling is
834 // reached and only when a genuinely new link appears.
835 for bucket in self.peer_budgets.values_mut() {
836 bucket.refill(now);
837 }
838 let most_recovered = self
839 .peer_budgets
840 .iter()
841 .max_by(|(_, a), (_, b)| a.tokens.total_cmp(&b.tokens))
842 .map(|(peer, _)| peer.clone());
843 if let Some(peer) = most_recovered {
844 self.peer_budgets.remove(&peer);
845 }
846 }
847 self.peer_budgets.insert(
848 peer_id.to_string(),
849 TokenBucket::new(self.config.peer_burst, self.config.peer_rate_per_sec, now),
850 );
851 }
852
853 self.peer_budgets
854 .get_mut(peer_id)
855 .map(|bucket| bucket.try_take_above(0.0, now))
856 .unwrap_or(true)
857 }
858
859 /// The delay before a queued forward is transmitted.
860 ///
861 /// Derived from the frame and this node's identity, so two nodes holding
862 /// the same frame wait different amounts and one of them gets to cancel.
863 /// The spread widens with density, because that is where the contention is.
864 fn jitter_for(&self, message_id: &str, degree: usize) -> Duration {
865 let min_ms = self.config.jitter_min.as_millis() as u64;
866 let max_ms = (self.config.jitter_max.as_millis() as u64).max(min_ms);
867
868 // More neighbors means more nodes racing to forward the same frame, so
869 // give them more room to separate.
870 let density_scale = 1 + (degree / self.config.dense_degree.max(1)) as u64;
871 let span = (max_ms - min_ms).saturating_mul(density_scale).max(1);
872
873 let hash = Self::hash_of(&[message_id, &self.local_id]);
874 Duration::from_millis(min_ms + (hash % span))
875 }
876
877 /// Stable hash used to spread delays and fan-out choices.
878 ///
879 /// `DefaultHasher` is unseeded, so this is predictable to anyone who knows
880 /// the inputs — both of which (a message id and a user id) are visible on
881 /// the wire. That is accepted: knowing it buys only the ability to guess
882 /// which neighbor fires first or which three links a frame takes, and every
883 /// path it could steer a frame down is already bounded by the fan-out cap
884 /// and the send budget. It must stay *stable*, which a randomly-seeded
885 /// hasher would not be — a second copy of a frame has to pick the same
886 /// neighbors, or a retransmission widens its own footprint.
887 fn hash_of(parts: &[&str]) -> u64 {
888 let mut hasher = DefaultHasher::new();
889 for part in parts {
890 part.hash(&mut hasher);
891 }
892 hasher.finish()
893 }
894}
895
896fn priority_rank(priority: MessagePriority) -> u8 {
897 match priority {
898 MessagePriority::Low => 0,
899 MessagePriority::Medium => 1,
900 MessagePriority::High => 2,
901 MessagePriority::Critical => 3,
902 }
903}
904
905#[cfg(test)]
906mod tests {
907 use super::*;
908 use offline_protocol_core::{AppId, UserId, TTL};
909
910 /// Zero jitter so `admit` queues a forward that is immediately due, which
911 /// keeps the tests deterministic without a clock abstraction.
912 fn immediate_config() -> MeshRelayConfig {
913 MeshRelayConfig {
914 jitter_min: Duration::from_millis(0),
915 jitter_max: Duration::from_millis(0),
916 ..Default::default()
917 }
918 }
919
920 fn governor() -> MeshRelayGovernor {
921 MeshRelayGovernor::with_config("relay-node", immediate_config())
922 }
923
924 fn frame_with(ttl: u8, priority: MessagePriority) -> Message {
925 Message::builder(
926 UserId::new("alice").unwrap(),
927 UserId::new("bob").unwrap(),
928 AppId::new("app").unwrap(),
929 )
930 .content("hello")
931 .priority(priority)
932 .ttl(TTL::new(ttl).unwrap())
933 .build()
934 }
935
936 fn frame() -> Message {
937 frame_with(8, MessagePriority::Medium)
938 }
939
940 #[test]
941 fn a_frame_is_forwarded_once_however_many_copies_arrive() {
942 let mut gov = governor();
943 let msg = frame();
944
945 assert_eq!(gov.admit(&msg, Some("b"), 3, false), RelayAdmission::Queued);
946
947 // Copies arriving after ours has gone out are suppressed, not forwarded.
948 let due = gov.take_due(Instant::now());
949 assert_eq!(due.len(), 1);
950
951 for _ in 0..5 {
952 assert_eq!(
953 gov.admit(&msg, Some("c"), 3, false),
954 RelayAdmission::Rejected(RelayRejection::AlreadySeen)
955 );
956 }
957 assert_eq!(gov.counters().queued, 1);
958 assert_eq!(gov.counters().duplicates_suppressed, 5);
959 }
960
961 #[test]
962 fn a_neighbor_beating_us_to_it_cancels_our_pending_forward() {
963 // The density control: in a cluster where everyone hears everyone, the
964 // first node to transmit covers the rest, and they stand down instead
965 // of each sending their own copy.
966 let mut gov = MeshRelayGovernor::with_config(
967 "relay-node",
968 MeshRelayConfig {
969 jitter_min: Duration::from_millis(50),
970 jitter_max: Duration::from_millis(200),
971 ..Default::default()
972 },
973 );
974 let msg = frame();
975
976 assert_eq!(gov.admit(&msg, Some("b"), 6, false), RelayAdmission::Queued);
977 assert_eq!(gov.pending_len(), 1);
978
979 // The same frame arrives again while ours is still waiting.
980 assert_eq!(
981 gov.admit(&msg, Some("c"), 6, false),
982 RelayAdmission::Rejected(RelayRejection::SupersededByDuplicate)
983 );
984
985 assert_eq!(gov.pending_len(), 0);
986 assert_eq!(gov.counters().cancelled_by_duplicate, 1);
987 assert!(gov
988 .take_due(Instant::now() + Duration::from_secs(1))
989 .is_empty());
990 assert_eq!(gov.counters().transmissions, 0);
991 }
992
993 #[test]
994 fn an_inflated_hop_budget_is_cut_to_local_policy() {
995 // Nothing authenticates the hop budget at this layer, so a frame can
996 // claim any value. It must not buy more than one hop past our own
997 // ceiling.
998 let mut gov = governor();
999 let msg = frame_with(255, MessagePriority::Medium);
1000
1001 assert_eq!(gov.admit(&msg, Some("b"), 2, false), RelayAdmission::Queued);
1002 let due = gov.take_due(Instant::now());
1003
1004 assert_eq!(due.len(), 1);
1005 assert_eq!(due[0].message.ttl.value(), DEFAULT_RELAY_MAX_TTL - 1);
1006 assert_eq!(gov.counters().ttl_clamped, 1);
1007 }
1008
1009 #[test]
1010 fn a_dense_neighborhood_uses_the_lower_hop_ceiling() {
1011 let mut gov = governor();
1012 let msg = frame_with(255, MessagePriority::Medium);
1013
1014 assert_eq!(
1015 gov.admit(&msg, Some("b"), DEFAULT_RELAY_DENSE_DEGREE, false),
1016 RelayAdmission::Queued
1017 );
1018 let due = gov.take_due(Instant::now());
1019 assert_eq!(due[0].message.ttl.value(), DEFAULT_RELAY_DENSE_MAX_TTL - 1);
1020 }
1021
1022 #[test]
1023 fn a_frame_out_of_hops_is_not_forwarded() {
1024 let mut gov = governor();
1025 // A budget of 1 is spent: this node is the last that may hold it.
1026 let msg = frame_with(1, MessagePriority::Medium);
1027
1028 assert_eq!(
1029 gov.admit(&msg, Some("b"), 3, false),
1030 RelayAdmission::Rejected(RelayRejection::HopLimitReached)
1031 );
1032 assert_eq!(gov.counters().hop_limit_reached, 1);
1033 }
1034
1035 #[test]
1036 fn the_last_hop_is_forwarded_and_the_one_after_it_is_not() {
1037 let mut gov = governor();
1038
1039 // Two hops left: goes out with one, which the next node will refuse to
1040 // forward. Verifies the budget lands exactly on `TTL::is_exhausted`
1041 // rather than a hop early or late.
1042 assert_eq!(
1043 gov.admit(&frame_with(2, MessagePriority::Medium), Some("b"), 3, false),
1044 RelayAdmission::Queued
1045 );
1046 let due = gov.take_due(Instant::now());
1047 assert_eq!(due[0].message.ttl.value(), 1);
1048 assert!(due[0].message.is_ttl_exhausted());
1049 }
1050
1051 #[test]
1052 fn the_hop_ceiling_stops_a_frame_that_evaded_the_budget() {
1053 let mut gov = governor();
1054 let mut msg = frame_with(255, MessagePriority::Medium);
1055 for _ in 0..RELAY_MAX_HOP_COUNT {
1056 let _ = msg.increment_hop();
1057 }
1058
1059 assert_eq!(
1060 gov.admit(&msg, Some("b"), 3, false),
1061 RelayAdmission::Rejected(RelayRejection::HopLimitReached)
1062 );
1063 }
1064
1065 #[test]
1066 fn one_noisy_neighbor_cannot_consume_the_whole_node() {
1067 let mut gov = governor();
1068
1069 // Spend the noisy neighbor's allowance on unique frames — the shape
1070 // suppression cannot help with, since every id is new.
1071 let mut accepted_from_noisy = 0;
1072 for _ in 0..200 {
1073 if gov.admit(&frame(), Some("noisy"), 3, false) == RelayAdmission::Queued {
1074 accepted_from_noisy += 1;
1075 }
1076 }
1077
1078 assert!(
1079 accepted_from_noisy <= DEFAULT_RELAY_PEER_BURST as usize + 2,
1080 "accepted {accepted_from_noisy} frames from one neighbor"
1081 );
1082 assert!(gov.counters().peer_rate_limited > 0);
1083
1084 // A different neighbor still gets through: the limit is per link, so
1085 // one bad actor degrades only itself.
1086 assert_eq!(
1087 gov.admit(&frame(), Some("quiet"), 3, false),
1088 RelayAdmission::Queued
1089 );
1090 }
1091
1092 #[test]
1093 fn transmission_never_exceeds_the_node_budget() {
1094 // The invariant the whole design rests on: whatever arrives, what the
1095 // radio is asked to send is capped. Asserted on transmissions rather
1096 // than on queue releases, because one release can put a frame on
1097 // several links and it is the links that cost airtime.
1098 let mut gov = MeshRelayGovernor::with_config(
1099 "relay-node",
1100 MeshRelayConfig {
1101 queue_capacity: 512,
1102 // Admission is not what is under test here.
1103 peer_rate_per_sec: 10_000.0,
1104 peer_burst: 10_000.0,
1105 ..immediate_config()
1106 },
1107 );
1108
1109 for _ in 0..200 {
1110 gov.admit(&frame(), Some("b"), 3, false);
1111 }
1112 let due = gov.take_due(Instant::now());
1113 assert!(!due.is_empty());
1114
1115 // Ask for far more transmissions than the budget allows.
1116 let granted = (0..1000).filter(|_| gov.take_send_token()).count();
1117
1118 assert!(
1119 granted <= DEFAULT_RELAY_BURST as usize + 1,
1120 "granted {granted} transmissions against a burst of {DEFAULT_RELAY_BURST}"
1121 );
1122 assert_eq!(gov.counters().transmissions as usize, granted);
1123 assert!(gov.counters().rate_deferred > 0);
1124 }
1125
1126 #[test]
1127 fn a_frame_that_reached_no_neighbor_for_budget_goes_back_on_the_queue() {
1128 // `take_due` checks the budget without reserving it, so a batch of due
1129 // frames can be released against a single token: the first spends it
1130 // and the rest reach the radio with nothing left. Those frames have
1131 // been forwarded nowhere, and their ids are already recorded as
1132 // handled — dropping one would lose this copy and refuse both the
1133 // copies behind it and the sender's retransmissions for the whole
1134 // retention window. It has to go back on the queue.
1135 let mut gov = MeshRelayGovernor::with_config(
1136 "relay-node",
1137 MeshRelayConfig {
1138 peer_rate_per_sec: 10_000.0,
1139 peer_burst: 10_000.0,
1140 ..immediate_config()
1141 },
1142 );
1143
1144 for _ in 0..40 {
1145 gov.admit(&frame(), Some("b"), 3, false);
1146 }
1147 let due = gov.take_due(Instant::now());
1148 assert!(
1149 due.len() > DEFAULT_RELAY_BURST as usize,
1150 "the batch must outrun the budget for this to be the case under test"
1151 );
1152
1153 // Drain the budget the way a fan-out does, then hand back everything
1154 // that got nowhere.
1155 let mut requeued = 0;
1156 for relay in due {
1157 if gov.take_send_token() {
1158 continue;
1159 }
1160 gov.requeue(relay);
1161 requeued += 1;
1162 }
1163
1164 assert!(requeued > 0, "the budget must have run out mid-batch");
1165 assert_eq!(gov.pending_len(), requeued, "every starved frame is kept");
1166 assert_eq!(gov.counters().requeued as usize, requeued);
1167 }
1168
1169 #[test]
1170 fn a_requeued_frame_is_still_abandoned_once_it_is_far_past_its_turn() {
1171 // Re-queueing must not become an unbounded retry: a frame kept for
1172 // budget keeps its original due time, so the overdue cut-off still
1173 // reaches it.
1174 let mut gov = governor();
1175 gov.admit(&frame(), Some("b"), 3, false);
1176 let mut due = gov.take_due(Instant::now());
1177 assert_eq!(due.len(), 1);
1178
1179 gov.requeue(due.remove(0));
1180 assert_eq!(gov.pending_len(), 1);
1181
1182 let later = gov.take_due(Instant::now() + RELAY_QUEUE_MAX_OVERDUE + Duration::from_secs(1));
1183 assert!(later.is_empty());
1184 assert_eq!(gov.counters().abandoned_overdue, 1);
1185 assert_eq!(gov.pending_len(), 0);
1186 }
1187
1188 #[test]
1189 fn a_frame_abandoned_for_waiting_too_long_releases_its_id() {
1190 // A frame dropped for waiting past its turn never reached a link, so
1191 // nothing is circulating that a second forward could duplicate. Its id
1192 // must go back to being unknown, or the sender's retransmissions —
1193 // which carry the same id — are refused for the whole retention window
1194 // because of a few seconds of congestion.
1195 let mut gov = governor();
1196 let msg = frame();
1197 gov.admit(&msg, Some("b"), 3, false);
1198
1199 let due = gov.take_due(Instant::now() + RELAY_QUEUE_MAX_OVERDUE + Duration::from_secs(1));
1200 assert!(due.is_empty());
1201 assert_eq!(gov.counters().abandoned_overdue, 1);
1202
1203 assert_eq!(
1204 gov.admit(&msg, Some("c"), 3, false),
1205 RelayAdmission::Queued,
1206 "the sender's retransmission must still be carried"
1207 );
1208 }
1209
1210 #[test]
1211 fn a_frame_displaced_from_the_queue_releases_its_id() {
1212 // Same rule for a frame evicted to make room for an urgent one: it was
1213 // never transmitted, so the route through this device must stay open.
1214 let mut gov = MeshRelayGovernor::with_config(
1215 "relay-node",
1216 MeshRelayConfig {
1217 queue_capacity: 1,
1218 peer_rate_per_sec: 10_000.0,
1219 peer_burst: 10_000.0,
1220 ..immediate_config()
1221 },
1222 );
1223
1224 let routine = frame_with(8, MessagePriority::Low);
1225 gov.admit(&routine, Some("b"), 3, false);
1226 gov.admit(
1227 &frame_with(8, MessagePriority::Critical),
1228 Some("b"),
1229 3,
1230 false,
1231 );
1232 assert_eq!(gov.counters().queue_evicted, 1);
1233
1234 // The urgent frame goes out, leaving room again. A later copy of the
1235 // displaced one — or the sender's retransmission — must be carryable.
1236 gov.take_due(Instant::now());
1237 assert_eq!(
1238 gov.admit(&routine, Some("c"), 3, false),
1239 RelayAdmission::Queued,
1240 "the displaced frame's id must not be blackholed"
1241 );
1242 }
1243
1244 #[test]
1245 fn a_frame_refused_room_on_the_way_back_releases_its_id() {
1246 // The third drop that happens after the id was recorded: a frame that
1247 // reached no neighbor and finds the queue full when handed back.
1248 let mut gov = MeshRelayGovernor::with_config(
1249 "relay-node",
1250 MeshRelayConfig {
1251 queue_capacity: 1,
1252 peer_rate_per_sec: 10_000.0,
1253 peer_burst: 10_000.0,
1254 ..immediate_config()
1255 },
1256 );
1257
1258 let starved = frame();
1259 gov.admit(&starved, Some("b"), 3, false);
1260 let mut due = gov.take_due(Instant::now());
1261 assert_eq!(due.len(), 1);
1262
1263 // The queue fills while the frame is out being transmitted, so there is
1264 // no room for it on the way back.
1265 gov.admit(&frame(), Some("b"), 3, false);
1266 gov.requeue(due.remove(0));
1267 assert_eq!(gov.counters().queue_full, 1);
1268 assert_eq!(gov.pending_len(), 1, "the dropped frame is not held");
1269
1270 // Once the queue drains, the copy behind it must still be carryable.
1271 gov.take_due(Instant::now());
1272 assert_eq!(
1273 gov.admit(&starved, Some("c"), 3, false),
1274 RelayAdmission::Queued,
1275 "a frame that got nowhere and could not be kept must stay carryable"
1276 );
1277 }
1278
1279 #[test]
1280 fn forwarding_leaves_room_for_this_devices_own_traffic() {
1281 // One radio, one budget — but a device saturated with other people's
1282 // traffic must still be able to send its own, above all the delivery
1283 // acknowledgement whose absence makes a delivered message look lost.
1284 let mut gov = MeshRelayGovernor::with_config(
1285 "relay-node",
1286 MeshRelayConfig {
1287 queue_capacity: 512,
1288 peer_rate_per_sec: 10_000.0,
1289 peer_burst: 10_000.0,
1290 ..immediate_config()
1291 },
1292 );
1293
1294 for _ in 0..200 {
1295 gov.admit(&frame(), Some("b"), 3, false);
1296 }
1297 gov.take_due(Instant::now());
1298
1299 // Drain the budget the way a saturated forwarder does.
1300 let forwarded = (0..1000).filter(|_| gov.take_send_token()).count();
1301 assert!(forwarded > 0, "forwarding must get its share first");
1302 assert!(
1303 !gov.take_send_token(),
1304 "forwarding must stop at the reserve, not at empty"
1305 );
1306
1307 // The reserve is still there for us.
1308 let own = (0..1000).filter(|_| gov.take_own_send_token()).count();
1309 assert!(
1310 own >= RELAY_OWN_TRAFFIC_RESERVE as usize,
1311 "own traffic got {own} transmissions against a reserve of {RELAY_OWN_TRAFFIC_RESERVE}"
1312 );
1313
1314 // And the ceiling still holds across both: the reserve is carved out of
1315 // the burst, not added to it.
1316 assert!(
1317 (forwarded + own) <= DEFAULT_RELAY_BURST as usize + 1,
1318 "{forwarded} forwarded + {own} own exceeds the burst of {DEFAULT_RELAY_BURST}"
1319 );
1320 }
1321
1322 #[test]
1323 fn a_small_configured_burst_still_forwards() {
1324 // The reserve is clamped to half the burst, so a device configured with
1325 // a burst smaller than the reserve does not silently stop forwarding
1326 // altogether.
1327 let mut gov = MeshRelayGovernor::with_config(
1328 "relay-node",
1329 MeshRelayConfig {
1330 burst: 2.0,
1331 rate_per_sec: 1.0,
1332 peer_rate_per_sec: 10_000.0,
1333 peer_burst: 10_000.0,
1334 ..immediate_config()
1335 },
1336 );
1337
1338 gov.admit(&frame(), Some("b"), 3, false);
1339 assert!(
1340 !gov.take_due(Instant::now()).is_empty(),
1341 "a small burst must still release forwards"
1342 );
1343 assert!(gov.take_send_token(), "and still allow transmitting them");
1344 }
1345
1346 #[test]
1347 fn a_settled_duplicate_is_absorbed_without_the_full_admission_path() {
1348 let mut gov = governor();
1349 let msg = frame();
1350
1351 assert!(
1352 !gov.absorb_settled_duplicate(&msg.id.as_str()),
1353 "an id never seen is not a duplicate"
1354 );
1355
1356 gov.admit(&msg, Some("b"), 3, false);
1357 // While our copy is still pending, standing down is a decision that
1358 // needs the neighbor set, so the cheap path must decline to make it.
1359 assert!(!gov.absorb_settled_duplicate(&msg.id.as_str()));
1360
1361 // Once it has gone out, further copies are settled duplicates.
1362 gov.take_due(Instant::now());
1363 assert!(gov.absorb_settled_duplicate(&msg.id.as_str()));
1364 assert_eq!(gov.counters().duplicates_suppressed, 1);
1365 }
1366
1367 #[test]
1368 fn eviction_prefers_a_frame_that_has_not_yet_gone_out() {
1369 // A due frame is on its way to the radio this tick; taking it costs a
1370 // transmission that was about to happen, while the frame replacing it
1371 // still has to wait out its hold.
1372 let mut gov = MeshRelayGovernor::with_config(
1373 "relay-node",
1374 MeshRelayConfig {
1375 queue_capacity: 2,
1376 jitter_min: Duration::from_millis(0),
1377 jitter_max: Duration::from_millis(0),
1378 peer_rate_per_sec: 10_000.0,
1379 peer_burst: 10_000.0,
1380 ..Default::default()
1381 },
1382 );
1383
1384 // One low-priority frame due immediately.
1385 let due_now = frame_with(8, MessagePriority::Low);
1386 gov.admit(&due_now, Some("b"), 3, false);
1387
1388 // A second low-priority frame that is not due for a while.
1389 gov.config.jitter_min = Duration::from_millis(500);
1390 gov.config.jitter_max = Duration::from_millis(500);
1391 let waiting = frame_with(8, MessagePriority::Low);
1392 gov.admit(&waiting, Some("b"), 3, false);
1393
1394 // An urgent frame needs the room.
1395 gov.admit(
1396 &frame_with(8, MessagePriority::Critical),
1397 Some("b"),
1398 3,
1399 false,
1400 );
1401
1402 assert_eq!(gov.counters().queue_evicted, 1);
1403 assert_eq!(
1404 gov.counters().queue_full,
1405 0,
1406 "making room is not the same as refusing"
1407 );
1408 assert!(
1409 gov.pending
1410 .iter()
1411 .any(|p| p.message.id.as_str() == due_now.id.as_str()),
1412 "the frame already due must survive"
1413 );
1414 assert!(
1415 !gov.pending
1416 .iter()
1417 .any(|p| p.message.id.as_str() == waiting.id.as_str()),
1418 "the frame still waiting is the one displaced"
1419 );
1420 }
1421
1422 #[test]
1423 fn a_full_peer_table_keeps_the_links_that_are_spending() {
1424 // Clearing the table wholesale hands every neighbor a fresh burst,
1425 // including the noisy one the per-link limit exists to hold back.
1426 let mut gov = MeshRelayGovernor::with_config("relay-node", immediate_config());
1427
1428 // One neighbor spends most of its allowance.
1429 for _ in 0..(DEFAULT_RELAY_PEER_BURST as usize - 1) {
1430 gov.admit(&frame(), Some("noisy"), 3, false);
1431 }
1432
1433 // Fill the table with links that arrive once and go quiet.
1434 for i in 0..MAX_RELAY_RATE_PEERS {
1435 gov.admit(&frame(), Some(&format!("idle-{i}")), 3, false);
1436 }
1437
1438 assert!(gov.peer_budgets.len() <= MAX_RELAY_RATE_PEERS);
1439 // Still tracked at all: the entry the limit exists to hold back is the
1440 // one it can least afford to forget, and it never sends again here, so
1441 // an eviction would leave it absent rather than rebuilt.
1442 let noisy = gov
1443 .peer_budgets
1444 .get("noisy")
1445 .expect("a link mid-spend must not be forgotten to make room for idle ones");
1446 assert!(
1447 noisy.tokens < noisy.capacity,
1448 "the noisy link kept its spent allowance rather than being handed a fresh burst"
1449 );
1450 }
1451
1452 #[test]
1453 fn a_refused_frame_leaves_the_way_open_for_another_copy() {
1454 // A frame turned away must not be remembered as handled: the copy
1455 // arriving over a healthy link a moment later is the alternate path,
1456 // and suppressing it would blank every route through this device for
1457 // the whole retention window.
1458 let mut gov = governor();
1459
1460 // Refused for having no hops left — the cheapest thing an attacker can
1461 // forge, since nothing authenticates the claim.
1462 let msg = frame_with(1, MessagePriority::Medium);
1463 assert_eq!(
1464 gov.admit(&msg, Some("hostile"), 3, false),
1465 RelayAdmission::Rejected(RelayRejection::HopLimitReached)
1466 );
1467
1468 // The same message, arriving intact by another route.
1469 let mut healthy = msg.clone();
1470 healthy.ttl = TTL::new(8).unwrap();
1471 assert_eq!(
1472 gov.admit(&healthy, Some("good"), 3, false),
1473 RelayAdmission::Queued,
1474 "a refused frame must not blackhole its id"
1475 );
1476 }
1477
1478 #[test]
1479 fn a_rate_limited_neighbor_does_not_blackhole_the_ids_it_sent() {
1480 let mut gov = governor();
1481
1482 // Exhaust one neighbor's allowance with distinct frames.
1483 let mut refused = Vec::new();
1484 for _ in 0..200 {
1485 let msg = frame();
1486 if gov.admit(&msg, Some("noisy"), 3, false)
1487 == RelayAdmission::Rejected(RelayRejection::PeerRateLimited)
1488 {
1489 refused.push(msg);
1490 }
1491 }
1492 assert!(!refused.is_empty());
1493
1494 // Those same messages, reaching us through someone else, must still be
1495 // carried.
1496 let victim = &refused[0];
1497 assert_eq!(
1498 gov.admit(victim, Some("quiet"), 3, false),
1499 RelayAdmission::Queued
1500 );
1501 }
1502
1503 #[test]
1504 fn the_last_hop_does_not_stand_down_for_a_neighbor() {
1505 // Standing down assumes a neighbor's transmission covers the same
1506 // ground ours would. At the last hop it does not: our copy is going to
1507 // the recipient over a link only we are known to hold, so dropping it
1508 // loses the delivery.
1509 let mut gov = MeshRelayGovernor::with_config(
1510 "relay-node",
1511 MeshRelayConfig {
1512 jitter_min: Duration::from_millis(50),
1513 jitter_max: Duration::from_millis(200),
1514 ..Default::default()
1515 },
1516 );
1517 let msg = frame();
1518
1519 assert_eq!(gov.admit(&msg, Some("b"), 3, true), RelayAdmission::Queued);
1520 // Another forwarder hands us the same frame.
1521 assert_eq!(
1522 gov.admit(&msg, Some("c"), 3, true),
1523 RelayAdmission::Rejected(RelayRejection::AlreadySeen)
1524 );
1525
1526 assert_eq!(gov.pending_len(), 1, "the delivering copy must survive");
1527 assert_eq!(gov.counters().cancelled_by_duplicate, 0);
1528 }
1529
1530 #[test]
1531 fn a_forward_waiting_far_past_its_turn_is_abandoned() {
1532 let mut gov = governor();
1533 gov.admit(&frame(), Some("b"), 3, false);
1534
1535 // Long enough that other paths have carried it or the sender has
1536 // retransmitted; holding it would only displace newer traffic.
1537 let due = gov.take_due(Instant::now() + RELAY_QUEUE_MAX_OVERDUE + Duration::from_secs(1));
1538 assert!(due.is_empty());
1539 assert_eq!(gov.counters().abandoned_overdue, 1);
1540 assert_eq!(gov.pending_len(), 0);
1541 }
1542
1543 #[test]
1544 fn a_full_queue_refuses_rather_than_growing() {
1545 let mut gov = MeshRelayGovernor::with_config(
1546 "relay-node",
1547 MeshRelayConfig {
1548 queue_capacity: 4,
1549 jitter_min: Duration::from_millis(500),
1550 jitter_max: Duration::from_millis(500),
1551 peer_rate_per_sec: 10_000.0,
1552 peer_burst: 10_000.0,
1553 ..Default::default()
1554 },
1555 );
1556
1557 for _ in 0..4 {
1558 assert_eq!(
1559 gov.admit(&frame(), Some("b"), 3, false),
1560 RelayAdmission::Queued
1561 );
1562 }
1563 assert_eq!(
1564 gov.admit(&frame(), Some("b"), 3, false),
1565 RelayAdmission::Rejected(RelayRejection::QueueFull)
1566 );
1567 assert_eq!(gov.pending_len(), 4);
1568 }
1569
1570 #[test]
1571 fn a_full_queue_still_takes_an_urgent_frame_over_a_routine_one() {
1572 let mut gov = MeshRelayGovernor::with_config(
1573 "relay-node",
1574 MeshRelayConfig {
1575 queue_capacity: 2,
1576 jitter_min: Duration::from_millis(500),
1577 jitter_max: Duration::from_millis(500),
1578 peer_rate_per_sec: 10_000.0,
1579 peer_burst: 10_000.0,
1580 ..Default::default()
1581 },
1582 );
1583
1584 gov.admit(&frame_with(8, MessagePriority::Low), Some("b"), 3, false);
1585 gov.admit(&frame_with(8, MessagePriority::Low), Some("b"), 3, false);
1586
1587 assert_eq!(
1588 gov.admit(
1589 &frame_with(8, MessagePriority::Critical),
1590 Some("b"),
1591 3,
1592 false
1593 ),
1594 RelayAdmission::Queued
1595 );
1596 assert_eq!(gov.pending_len(), 2);
1597 assert!(gov
1598 .take_due(Instant::now() + Duration::from_secs(1))
1599 .iter()
1600 .any(|r| r.message.priority == MessagePriority::Critical));
1601 }
1602
1603 #[test]
1604 fn a_frame_is_never_sent_back_where_it_came_from() {
1605 let gov = governor();
1606 let targets = gov.select_targets(
1607 [("alice", 80), ("b", 80), ("c", 80), ("d", 80)],
1608 &["b", "alice"], // arrival neighbor and the peer that wrote it
1609 "msg-1",
1610 );
1611
1612 assert!(!targets.contains(&"b".to_string()));
1613 assert!(!targets.contains(&"alice".to_string()));
1614 assert!(targets.contains(&"c".to_string()));
1615 assert!(targets.contains(&"d".to_string()));
1616 }
1617
1618 #[test]
1619 fn fanout_is_capped_and_stable_for_a_given_frame() {
1620 let gov = governor();
1621 // All equally good, so the tie-break is what decides — which is the
1622 // property under test.
1623 let neighbors = [
1624 ("n0", 70),
1625 ("n1", 70),
1626 ("n2", 70),
1627 ("n3", 70),
1628 ("n4", 70),
1629 ("n5", 70),
1630 ("n6", 70),
1631 ("n7", 70),
1632 ];
1633
1634 let first = gov.select_targets(neighbors, &[], "msg-1");
1635 let again = gov.select_targets(neighbors, &[], "msg-1");
1636
1637 assert_eq!(first.len(), DEFAULT_RELAY_FANOUT);
1638 assert_eq!(first, again, "a second copy must pick the same neighbors");
1639
1640 // Different frames spread across different neighbors rather than
1641 // hammering one set.
1642 let other = gov.select_targets(neighbors, &[], "msg-2");
1643 assert_eq!(other.len(), DEFAULT_RELAY_FANOUT);
1644 }
1645
1646 #[test]
1647 fn a_frame_we_originated_is_not_carried_back_for_someone_else() {
1648 // Our own message, handed to neighbors. When a neighbor's copy reaches
1649 // us again we must recognize it, not take it on as third-party traffic.
1650 let mut gov = governor();
1651 let msg = frame();
1652
1653 gov.mark_handled(&msg.id.as_str());
1654
1655 assert_eq!(
1656 gov.admit(&msg, Some("b"), 3, false),
1657 RelayAdmission::Rejected(RelayRejection::AlreadySeen)
1658 );
1659 }
1660
1661 #[test]
1662 fn the_suppression_cache_holds_a_saturated_node_for_its_full_window() {
1663 // Capacity has to outlast in-flight copies at the maximum rate the node
1664 // will accept traffic. If it did not, an id could be forgotten while
1665 // copies were still circulating and be forwarded a second time — by
1666 // every node, which is how a flood becomes a storm.
1667 let gov = governor();
1668 let max_admissions_per_sec = DEFAULT_RELAY_PEER_RATE_PER_SEC * MAX_RELAY_RATE_PEERS as f32;
1669 let sustained_ceiling = DEFAULT_RELAY_RATE_PER_SEC.min(max_admissions_per_sec);
1670 let ids_in_window = sustained_ceiling * gov.config().seen.retention.as_secs_f32();
1671
1672 assert!(
1673 (gov.config().seen.capacity as f32) >= ids_in_window,
1674 "cache holds {} ids but a saturated node sees {} within the retention window",
1675 gov.config().seen.capacity,
1676 ids_in_window
1677 );
1678 }
1679}