Skip to main content

offline_protocol/protocol/
mesh_relay.rs

1//! Admission control and pacing for mesh forwarding.
2//!
3//! Forwarding is what turns a set of nearby devices into a network: a frame
4//! addressed to someone out of radio range is carried by the devices in
5//! between. The hazard is that the same property which gives coverage —
6//! every node repeating what it hears — also multiplies traffic, and in a
7//! dense room the multiplication can outrun the radio. This module is the
8//! governor on that: it decides whether a frame is forwarded at all, when,
9//! and how fast frames may leave.
10//!
11//! Four mechanisms, each covering something the others cannot:
12//!
13//! 1. **Suppression** ([`RelaySeenCache`]) — an id is forwarded once. Without
14//!    it, copies circulate until their hop limit runs out, and every node
15//!    repeats every copy.
16//! 2. **Jitter with cancellation** — a forward waits a short randomized delay
17//!    before going out, and is dropped if the same id arrives again while it
18//!    waits. In a dense cluster the first neighbor to fire covers everyone who
19//!    can hear it, and the rest stand down. This is what makes cost scale with
20//!    *coverage* rather than with the number of links, and it adapts on its own
21//!    as the room fills up — no threshold to tune.
22//! 3. **Hop limits** — an arriving frame's remaining hop budget is clamped
23//!    before use, so a frame that claims an implausible budget (nothing
24//!    authenticates it at this layer) cannot circulate longer than one our own
25//!    policy would have issued.
26//! 4. **Budgets** — a hard ceiling on frames per second, in total and per
27//!    neighbor. The first three assume frames are what they appear to be;
28//!    budgets hold even when they are not. Whatever arrives, and however it is
29//!    shaped, a node cannot be made to transmit faster than its budget: that
30//!    invariant is what bounds the worst case, so the failure mode under attack
31//!    or overload is added delay rather than a radio that never goes quiet.
32//!
33//! Ordering matters. Suppression runs before anything expensive, admission
34//! before queueing (so a flood cannot grow the queue), and pacing at the moment
35//! of transmission (so a forward cancelled while waiting costs no budget).
36
37use offline_protocol_core::{Message, MessagePriority};
38use offline_protocol_reliability::{RelaySeenCache, RelaySeenConfig};
39use std::collections::hash_map::DefaultHasher;
40use std::collections::HashMap;
41use std::hash::{Hash, Hasher};
42use std::time::{Duration, Instant};
43
44/// Hop budget a forwarded frame is clamped to under normal density.
45pub const DEFAULT_RELAY_MAX_TTL: u8 = 8;
46
47/// Hop budget used once the local neighborhood is dense.
48///
49/// With many neighbors, coverage is reached in fewer hops — the same message
50/// travels further per hop — so a lower ceiling reaches the same devices for
51/// less traffic.
52pub const DEFAULT_RELAY_DENSE_MAX_TTL: u8 = 5;
53
54/// Neighbor count at or above which the dense hop budget applies.
55pub const DEFAULT_RELAY_DENSE_DEGREE: usize = 6;
56
57/// Absolute ceiling on how many times a frame may be forwarded, regardless of
58/// the hop budget it carries. A backstop against a frame whose hop fields have
59/// been shaped to evade the clamp.
60pub const RELAY_MAX_HOP_COUNT: u8 = 16;
61
62/// Neighbors a single frame is forwarded to.
63pub const DEFAULT_RELAY_FANOUT: usize = 3;
64
65/// Shortest delay before a queued forward is transmitted.
66pub const DEFAULT_RELAY_JITTER_MIN_MS: u64 = 20;
67
68/// Longest delay before a queued forward is transmitted, at low density.
69pub const DEFAULT_RELAY_JITTER_MAX_MS: u64 = 200;
70
71/// Sustained ceiling on forwarded frames leaving this node, per second.
72pub const DEFAULT_RELAY_RATE_PER_SEC: f32 = 10.0;
73
74/// Burst allowance on top of the sustained forwarding rate.
75pub const DEFAULT_RELAY_BURST: f32 = 30.0;
76
77/// Transmissions held back from carrying other people's traffic so this device
78/// can always get its own frames out.
79///
80/// Own traffic and forwarded traffic share one budget, because they share one
81/// radio. Without a reserve they also share one queue discipline, and a device
82/// forwarding at its ceiling would starve its *own* sends behind strangers'
83/// — including the delivery acknowledgement for a message it just received,
84/// whose absence makes a delivered message look lost. Carving the reserve out
85/// of the burst rather than adding a second bucket keeps the ceiling exactly
86/// where it was: this changes who may spend the last few tokens, not how many
87/// there are.
88///
89/// It costs forwarding no throughput at saturation. Tokens refill continuously,
90/// so a forward simply waits for the level to climb back above the reserve —
91/// which at the sustained rate is a fraction of a second.
92pub const RELAY_OWN_TRAFFIC_RESERVE: f32 = 5.0;
93
94/// Sustained ceiling on frames accepted for forwarding from any one neighbor.
95///
96/// Keeps a single noisy or hostile link from consuming the whole node's
97/// forwarding budget: its frames are refused at the door, leaving the budget
98/// for everyone else's traffic.
99pub const DEFAULT_RELAY_PEER_RATE_PER_SEC: f32 = 5.0;
100
101/// Burst allowance for a single neighbor.
102pub const DEFAULT_RELAY_PEER_BURST: f32 = 15.0;
103
104/// Maximum forwards waiting for their delay to elapse.
105pub const DEFAULT_RELAY_QUEUE_CAPACITY: usize = 256;
106
107/// Maximum neighbors tracked for per-neighbor rate limiting.
108pub const MAX_RELAY_RATE_PEERS: usize = 256;
109
110/// How long a queued forward may sit past its due time before being abandoned.
111///
112/// If a node is so far over budget that a frame waits this long, the frame's
113/// usefulness has passed — other paths have carried it or the sender has
114/// retransmitted — and holding it only displaces newer traffic.
115const RELAY_QUEUE_MAX_OVERDUE: Duration = Duration::from_secs(5);
116
117/// Tunables for mesh forwarding.
118#[derive(Debug, Clone)]
119pub struct MeshRelayConfig {
120    /// Hop budget a forwarded frame is clamped to.
121    pub max_ttl: u8,
122    /// Hop budget applied once the neighborhood is dense.
123    pub dense_max_ttl: u8,
124    /// Neighbor count at which the dense budget applies.
125    pub dense_degree: usize,
126    /// Neighbors a frame is forwarded to.
127    pub fanout: usize,
128    /// Shortest pre-transmit delay.
129    pub jitter_min: Duration,
130    /// Longest pre-transmit delay at low density.
131    pub jitter_max: Duration,
132    /// Sustained forwarding rate, frames per second.
133    pub rate_per_sec: f32,
134    /// Burst allowance above the sustained rate.
135    pub burst: f32,
136    /// Sustained per-neighbor acceptance rate.
137    pub peer_rate_per_sec: f32,
138    /// Per-neighbor burst allowance.
139    pub peer_burst: f32,
140    /// Maximum forwards awaiting transmission.
141    pub queue_capacity: usize,
142    /// Suppression cache sizing.
143    pub seen: RelaySeenConfig,
144}
145
146impl Default for MeshRelayConfig {
147    fn default() -> Self {
148        Self {
149            max_ttl: DEFAULT_RELAY_MAX_TTL,
150            dense_max_ttl: DEFAULT_RELAY_DENSE_MAX_TTL,
151            dense_degree: DEFAULT_RELAY_DENSE_DEGREE,
152            fanout: DEFAULT_RELAY_FANOUT,
153            jitter_min: Duration::from_millis(DEFAULT_RELAY_JITTER_MIN_MS),
154            jitter_max: Duration::from_millis(DEFAULT_RELAY_JITTER_MAX_MS),
155            rate_per_sec: DEFAULT_RELAY_RATE_PER_SEC,
156            burst: DEFAULT_RELAY_BURST,
157            peer_rate_per_sec: DEFAULT_RELAY_PEER_RATE_PER_SEC,
158            peer_burst: DEFAULT_RELAY_PEER_BURST,
159            queue_capacity: DEFAULT_RELAY_QUEUE_CAPACITY,
160            seen: RelaySeenConfig::default(),
161        }
162    }
163}
164
165/// Why a frame was not queued for forwarding.
166#[derive(Debug, Clone, Copy, PartialEq, Eq)]
167pub enum RelayRejection {
168    /// Already handled; this is another copy of a frame we have dealt with.
169    AlreadySeen,
170    /// Another copy arrived while this one was waiting, so the pending forward
171    /// was cancelled — a neighbor has covered it.
172    SupersededByDuplicate,
173    /// The frame has no hop budget left.
174    HopLimitReached,
175    /// The neighbor that sent it is over its acceptance rate.
176    PeerRateLimited,
177    /// No room in the pending queue.
178    QueueFull,
179}
180
181/// The outcome of offering an inbound frame to the governor.
182#[derive(Debug, Clone, Copy, PartialEq, Eq)]
183pub enum RelayAdmission {
184    /// Queued; it will be transmitted once its delay elapses.
185    Queued,
186    /// Not queued.
187    Rejected(RelayRejection),
188}
189
190/// A forward waiting for its delay to elapse.
191#[derive(Debug, Clone)]
192pub struct PendingRelay {
193    /// The frame to forward, with hop fields already adjusted.
194    pub message: Message,
195    /// The neighbor it arrived from, which must not receive it back.
196    pub arrival_peer: Option<String>,
197    /// When it becomes eligible to transmit.
198    pub due_at: Instant,
199}
200
201/// Running totals, for telemetry and for tests that assert a flood stayed
202/// bounded.
203#[derive(Debug, Clone, Default, PartialEq, Eq)]
204pub struct MeshRelayCounters {
205    /// Frames queued for forwarding.
206    pub queued: u64,
207    /// Times a frame was put on a link, counting each link separately.
208    ///
209    /// This is the airtime meter — what the send budget actually bounds — so it
210    /// counts this device handing over its *own* messages too. For "how much
211    /// have I carried for other people", see [`Self::forwarded`].
212    pub transmissions: u64,
213    /// Third-party frames that crossed at least one link on someone's behalf.
214    ///
215    /// Counted once per frame carried, not once per link, and never for this
216    /// device's own traffic.
217    pub forwarded: u64,
218    /// Frames put back on the queue because they reached no neighbor at all —
219    /// the budget ran out, the links chosen for them went away, or there was
220    /// no usable link to choose.
221    pub requeued: u64,
222    /// Queued forwards displaced to make room for a higher-priority frame.
223    pub queue_evicted: u64,
224    /// Arrivals suppressed as already handled.
225    pub duplicates_suppressed: u64,
226    /// Pending forwards cancelled because a neighbor covered them.
227    pub cancelled_by_duplicate: u64,
228    /// Frames refused because their sender was over its rate.
229    pub peer_rate_limited: u64,
230    /// Frames refused because the pending queue was full.
231    pub queue_full: u64,
232    /// Frames dropped for having no hop budget left.
233    pub hop_limit_reached: u64,
234    /// Transmissions deferred because the node was at its forwarding rate.
235    pub rate_deferred: u64,
236    /// Queued forwards abandoned after waiting too long past their due time.
237    pub abandoned_overdue: u64,
238    /// Hop budgets reduced because the frame claimed more than policy allows.
239    pub ttl_clamped: u64,
240}
241
242/// What this device has carried for other people.
243///
244/// A snapshot, safe to poll: every field counts since start-up.
245#[derive(Debug, Clone, Default, PartialEq, Eq)]
246pub struct MeshRelayStats {
247    /// Frames carried onward on someone else's behalf, counted once each.
248    pub forwarded: u64,
249    /// Times this device put a frame on a link, counting each link separately
250    /// and including hand-offs of its own messages.
251    ///
252    /// This is what the per-second forwarding budget bounds, so it is the
253    /// number to compare against that ceiling — [`Self::forwarded`] is the
254    /// contribution figure to show a user.
255    pub transmissions: u64,
256    /// Frames accepted for forwarding.
257    pub queued: u64,
258    /// Frames waiting to go out right now.
259    pub awaiting_transmission: usize,
260    /// Arrivals ignored because the frame had already been handled.
261    pub duplicates_suppressed: u64,
262    /// Forwards dropped because a neighbor transmitted the same frame first —
263    /// the saving that keeps a crowded room from repeating itself.
264    pub covered_by_a_neighbor: u64,
265    /// Frames refused because the neighbor sending them was over its share.
266    pub peer_rate_limited: u64,
267    /// Transmissions held back because this device was at its forwarding rate.
268    ///
269    /// A frame counted here is delayed, not dropped: it goes back on the queue
270    /// and is tried again on the next tick, until it is either sent or has
271    /// waited so long past its turn that carrying it no longer helps.
272    pub rate_deferred: u64,
273    /// Frames that had travelled as far as they were allowed to.
274    pub hop_limit_reached: u64,
275    /// Frames whose claimed reach was cut down to local policy.
276    pub reach_clamped: u64,
277    /// Ids forgotten for lack of room rather than age.
278    ///
279    /// Expected to stay at zero. A non-zero value means this device is seeing
280    /// more traffic than it can remember having handled, and a frame it forgets
281    /// can be forwarded a second time.
282    pub dropped_for_capacity: u64,
283}
284
285/// A refilling allowance, in frames.
286#[derive(Debug, Clone)]
287struct TokenBucket {
288    tokens: f32,
289    capacity: f32,
290    refill_per_sec: f32,
291    last_refill: Instant,
292}
293
294impl TokenBucket {
295    fn new(capacity: f32, refill_per_sec: f32, now: Instant) -> Self {
296        Self {
297            tokens: capacity.max(0.0),
298            capacity: capacity.max(0.0),
299            refill_per_sec: refill_per_sec.max(0.0),
300            last_refill: now,
301        }
302    }
303
304    fn refill(&mut self, now: Instant) {
305        let elapsed = now
306            .saturating_duration_since(self.last_refill)
307            .as_secs_f32();
308        if elapsed > 0.0 {
309            self.tokens = (self.tokens + elapsed * self.refill_per_sec).min(self.capacity);
310            self.last_refill = now;
311        }
312    }
313
314    /// Spends a token, but only while more than `floor` would remain
315    /// untouched. A `floor` of zero spends down to empty.
316    fn try_take_above(&mut self, floor: f32, now: Instant) -> bool {
317        self.refill(now);
318        if self.tokens >= 1.0 + floor {
319            self.tokens -= 1.0;
320            true
321        } else {
322            false
323        }
324    }
325}
326
327/// Decides what gets forwarded, when, and how fast.
328///
329/// Holds no transports and sends nothing itself: it answers questions and hands
330/// back due work, so its behavior is testable without a radio and its policy
331/// stays in one place rather than spread through the receive path.
332#[derive(Debug)]
333pub struct MeshRelayGovernor {
334    config: MeshRelayConfig,
335    seen: RelaySeenCache,
336    /// Forwards awaiting their delay, in the order they were queued.
337    pending: Vec<PendingRelay>,
338    send_budget: TokenBucket,
339    /// Tokens of [`Self::send_budget`] that only this device's own frames may
340    /// spend. Clamped to half the burst so a small configured burst cannot
341    /// reserve the whole bucket and stop forwarding outright.
342    own_reserve: f32,
343    peer_budgets: HashMap<String, TokenBucket>,
344    counters: MeshRelayCounters,
345    /// Seeds the per-frame delay so two nodes holding the same frame pick
346    /// different delays.
347    local_id: String,
348}
349
350impl MeshRelayGovernor {
351    /// Creates a governor with explicit tunables.
352    pub fn with_config(local_id: impl Into<String>, config: MeshRelayConfig) -> Self {
353        let now = Instant::now();
354        let seen = RelaySeenCache::with_config(config.seen.clone());
355        Self {
356            send_budget: TokenBucket::new(config.burst, config.rate_per_sec, now),
357            own_reserve: RELAY_OWN_TRAFFIC_RESERVE.min(config.burst.max(0.0) / 2.0),
358            seen,
359            pending: Vec::new(),
360            peer_budgets: HashMap::new(),
361            counters: MeshRelayCounters::default(),
362            local_id: local_id.into(),
363            config,
364        }
365    }
366
367    /// Re-points the governor at this device's real identity once it is known.
368    ///
369    /// The governor is built before `initialize_mls` derives the address, so it
370    /// starts out seeded with the profile. Both uses are hashes — fan-out
371    /// spread and send jitter — so the change is only a reseed: it shifts which
372    /// frames this node prefers, never whether a frame is relayed.
373    pub fn set_local_id(&mut self, local_id: impl Into<String>) {
374        self.local_id = local_id.into();
375    }
376
377    /// The tunables in force. Apps read these from
378    /// `ProtocolConfig::mesh_relay`; this is for the tests that assert the
379    /// defaults hold together.
380    #[cfg(test)]
381    pub fn config(&self) -> &MeshRelayConfig {
382        &self.config
383    }
384
385    /// Running totals.
386    pub fn counters(&self) -> &MeshRelayCounters {
387        &self.counters
388    }
389
390    /// Forwards awaiting transmission.
391    pub fn pending_len(&self) -> usize {
392        self.pending.len()
393    }
394
395    /// Ids dropped from the suppression cache for capacity rather than age.
396    /// Expected to stay zero; see [`RelaySeenCache::capacity_evictions`].
397    pub fn seen_capacity_evictions(&self) -> u64 {
398        self.seen.capacity_evictions()
399    }
400
401    /// Offers an inbound third-party frame for forwarding.
402    ///
403    /// Runs the full admission sequence and, when the frame is accepted,
404    /// queues an adjusted copy for later transmission. `degree` is the current
405    /// neighbor count, which sets both the hop budget and the delay spread.
406    /// `is_last_hop` says whether the frame's recipient is one of our own
407    /// neighbors, which changes what a duplicate means — see below.
408    pub fn admit(
409        &mut self,
410        message: &Message,
411        arrival_peer: Option<&str>,
412        degree: usize,
413        is_last_hop: bool,
414    ) -> RelayAdmission {
415        let now = Instant::now();
416        let message_id = message.id.as_str();
417
418        // Has this id been dealt with? Checked without recording, because a
419        // frame that is about to be *refused* must not be remembered as
420        // handled: another copy of it arriving over a healthy link a moment
421        // later is the alternate path that should still carry it. Recording on
422        // rejection would leave the id suppressed for the whole retention
423        // window, so a single link running hot — or one frame with its hop
424        // budget rewritten to nothing, which nothing here can authenticate —
425        // would blank every path through this device.
426        if self.seen.contains(&message_id) {
427            self.counters.duplicates_suppressed =
428                self.counters.duplicates_suppressed.saturating_add(1);
429
430            // Our own copy is still waiting, so a neighbor has transmitted
431            // this frame already.
432            if let Some(index) = self
433                .pending
434                .iter()
435                .position(|p| p.message.id.as_str() == message_id)
436            {
437                // Standing down is right in the middle of a network, where a
438                // neighbor's transmission reaches the same region ours would.
439                // It is wrong at the last hop: our pending copy is addressed
440                // to the recipient themselves, and no other device can be
441                // assumed to hold that link. Dropping it there loses the
442                // delivery outright — and leaves the id suppressed, so the
443                // sender's retries cannot rescue it either.
444                if is_last_hop {
445                    return RelayAdmission::Rejected(RelayRejection::AlreadySeen);
446                }
447
448                self.pending.remove(index);
449                self.counters.cancelled_by_duplicate =
450                    self.counters.cancelled_by_duplicate.saturating_add(1);
451                return RelayAdmission::Rejected(RelayRejection::SupersededByDuplicate);
452            }
453
454            return RelayAdmission::Rejected(RelayRejection::AlreadySeen);
455        }
456
457        // Per-neighbor admission, before the frame can occupy queue space.
458        //
459        // A frame whose arrival link the carrier did not identify skips this
460        // step: there is no link to charge it to, and inventing a shared bucket
461        // for "unknown" would let one such carrier throttle every other. Those
462        // frames are still bounded by the queue capacity and the per-second
463        // send budget below, which is what makes skipping it acceptable — the
464        // per-neighbor limit shapes *fairness between links*, and a frame with
465        // no known link has no share to exceed.
466        if let Some(peer) = arrival_peer {
467            if !self.take_peer_token(peer, now) {
468                self.counters.peer_rate_limited = self.counters.peer_rate_limited.saturating_add(1);
469                return RelayAdmission::Rejected(RelayRejection::PeerRateLimited);
470            }
471        }
472
473        // Hop accounting. The arriving budget is clamped to what our own policy
474        // would have issued before it is spent, so an inflated claim buys at
475        // most this one hop.
476        let Some(forwarded) = self.prepare_hop(message, degree) else {
477            self.counters.hop_limit_reached = self.counters.hop_limit_reached.saturating_add(1);
478            return RelayAdmission::Rejected(RelayRejection::HopLimitReached);
479        };
480
481        if self.pending.len() >= self.config.queue_capacity && !self.evict_for(&forwarded, now) {
482            self.counters.queue_full = self.counters.queue_full.saturating_add(1);
483            return RelayAdmission::Rejected(RelayRejection::QueueFull);
484        }
485
486        // Accepted: now it is genuinely handled, and further copies can be
487        // suppressed. Recording here rather than at the top also means the
488        // cache fills at the rate frames are *taken on*, which the per-neighbor
489        // and per-device limits bound — not at the rate they arrive, which
490        // nothing bounds.
491        self.seen.observe(&message_id);
492
493        let due_at = now + self.jitter_for(&message_id, degree);
494        self.pending.push(PendingRelay {
495            message: forwarded,
496            arrival_peer: arrival_peer.map(str::to_string),
497            due_at,
498        });
499        self.counters.queued = self.counters.queued.saturating_add(1);
500
501        RelayAdmission::Queued
502    }
503
504    /// Returns the forwards whose delay has elapsed, removing them from the
505    /// queue.
506    ///
507    /// Frames whose delay has not elapsed stay queued, as do frames the device
508    /// has no budget left to transmit. A frame that has waited far past its
509    /// turn is abandoned instead: by then other paths have carried it or the
510    /// sender has retransmitted, and holding it only displaces newer traffic.
511    ///
512    /// Budget is *checked* here and *spent* in [`Self::take_send_token`], once
513    /// per link a frame actually crosses. Spending it here instead would
514    /// undercount by the fan-out — one release from this queue can put the
515    /// frame on several links — and the whole point of the ceiling is that it
516    /// bounds what reaches the radio.
517    ///
518    /// The check is therefore not a reservation, and cannot be: a released
519    /// frame may still find the budget gone by the time it reaches the radio,
520    /// because the frames released alongside it spend from the same bucket. A
521    /// caller that gets nothing onto a link for that reason must hand the frame
522    /// back with [`Self::requeue`] rather than drop it — the id is already
523    /// recorded as handled here, so dropping it would lose this copy *and*
524    /// refuse the copies and retransmissions that follow.
525    pub fn take_due(&mut self, now: Instant) -> Vec<PendingRelay> {
526        self.seen.expire(now);
527
528        let mut due = Vec::new();
529        let mut keep: Vec<PendingRelay> = Vec::with_capacity(self.pending.len());
530
531        for relay in std::mem::take(&mut self.pending) {
532            if relay.due_at > now {
533                keep.push(relay);
534                continue;
535            }
536
537            if now.saturating_duration_since(relay.due_at) > RELAY_QUEUE_MAX_OVERDUE {
538                // Never made it onto a link, so its id must not stay
539                // suppressed: the sender's retransmissions carry the same id,
540                // and refusing them would close this route for the whole
541                // retention window over a few seconds of congestion.
542                self.seen.forget(&relay.message.id.as_str());
543                self.counters.abandoned_overdue = self.counters.abandoned_overdue.saturating_add(1);
544                continue;
545            }
546
547            if self.has_send_budget(now) {
548                due.push(relay);
549            } else {
550                self.counters.rate_deferred = self.counters.rate_deferred.saturating_add(1);
551                keep.push(relay);
552            }
553        }
554
555        self.pending = keep;
556        due
557    }
558
559    /// Whether any budget remains to forward right now.
560    ///
561    /// Measured against the same reserve [`Self::take_send_token`] respects, so
562    /// `take_due` does not release a batch the flush is about to refuse.
563    fn has_send_budget(&mut self, now: Instant) -> bool {
564        self.send_budget.refill(now);
565        self.send_budget.tokens >= 1.0 + self.own_reserve
566    }
567
568    /// Claims budget for putting one **forwarded** frame on one link.
569    ///
570    /// Stops short of the own-traffic reserve: at the ceiling, other people's
571    /// traffic waits so this device can still send its own. See
572    /// [`RELAY_OWN_TRAFFIC_RESERVE`] and [`Self::take_own_send_token`].
573    pub fn take_send_token(&mut self) -> bool {
574        self.claim_transmission(self.own_reserve)
575    }
576
577    /// Claims budget for putting one of **this device's own** frames on one
578    /// link.
579    ///
580    /// Metered against the same ceiling — it is the same radio — but allowed
581    /// into the reserve, so a device forwarding at its limit can still get its
582    /// own messages and acknowledgements out.
583    pub fn take_own_send_token(&mut self) -> bool {
584        self.claim_transmission(0.0)
585    }
586
587    fn claim_transmission(&mut self, floor: f32) -> bool {
588        if self.send_budget.try_take_above(floor, Instant::now()) {
589            self.counters.transmissions = self.counters.transmissions.saturating_add(1);
590            true
591        } else {
592            self.counters.rate_deferred = self.counters.rate_deferred.saturating_add(1);
593            false
594        }
595    }
596
597    /// Records that a third-party frame was carried onward.
598    ///
599    /// Separate from [`Self::take_send_token`], which meters airtime for every
600    /// frame this device transmits including its own. Counted once per frame
601    /// carried rather than once per link, so it reads as "messages I moved for
602    /// other people".
603    pub fn record_forwarded(&mut self) {
604        self.counters.forwarded = self.counters.forwarded.saturating_add(1);
605    }
606
607    /// Puts a released forward back on the queue, keeping its original due
608    /// time.
609    ///
610    /// For the frame that reached no neighbor at all between [`Self::take_due`]
611    /// releasing it and the radio being asked — because the budget ran out,
612    /// because every link picked for it had gone away, or because there was no
613    /// usable link to pick. All three end the same way: the frame has travelled
614    /// nowhere, its id is already recorded as handled, and dropping it would
615    /// refuse the copies and retransmissions behind it for the whole retention
616    /// window.
617    ///
618    /// It keeps its due time deliberately: the overdue cut-off in `take_due` is
619    /// measured from that, so a frame that stays stuck is eventually abandoned
620    /// instead of being retried forever.
621    ///
622    /// Refused only if the queue has filled meanwhile, which is the same
623    /// bound every other queued frame is subject to.
624    pub fn requeue(&mut self, relay: PendingRelay) {
625        if self.pending.len() >= self.config.queue_capacity {
626            // Refused room on the way back, so this frame is being dropped
627            // having reached nobody. Release its id for the same reason the
628            // overdue cut-off does — a copy behind it, or the sender's own
629            // retransmission, is now the only way it travels.
630            self.seen.forget(&relay.message.id.as_str());
631            self.counters.queue_full = self.counters.queue_full.saturating_add(1);
632            return;
633        }
634        self.counters.requeued = self.counters.requeued.saturating_add(1);
635        self.pending.push(relay);
636    }
637
638    /// Records an id as handled without queueing a forward for it.
639    ///
640    /// Used for frames this device **originates** and hands to its neighbors:
641    /// a copy circling back to us through the mesh is then recognized rather
642    /// than carried onward as though it were someone else's.
643    ///
644    /// Deliberately not called for frames addressed to us. A later copy of one
645    /// of those is still addressed to us, so it never reaches the forwarding
646    /// path at all — recording it would only fill the cache at delivery rate,
647    /// which the capacity is not sized for and which would make
648    /// [`Self::seen_capacity_evictions`] climb for reasons unrelated to relay
649    /// load.
650    pub fn mark_handled(&mut self, message_id: &str) {
651        self.seen.observe(message_id);
652    }
653
654    /// Absorbs an arrival that is a copy of a frame already dealt with and
655    /// needs no further decision, reporting whether it did.
656    ///
657    /// This is [`Self::admit`]'s suppression check, split out so a caller can
658    /// reach it without first assembling the arguments `admit` needs. In a
659    /// dense neighborhood most third-party arrivals are copies, and the caller's
660    /// `degree` and `is_last_hop` cost a status snapshot across every transport
661    /// and an enumeration of every link — work the suppression check would only
662    /// throw away.
663    ///
664    /// Deliberately answers `false` while we still hold a pending copy of the
665    /// id, even though that is also a duplicate: standing down for a neighbor
666    /// is the one duplicate outcome that depends on the neighbor set, so it has
667    /// to go through the full path. Those are rare — the pending window is one
668    /// jitter delay wide.
669    pub fn absorb_settled_duplicate(&mut self, message_id: &str) -> bool {
670        if !self.seen.contains(message_id) {
671            return false;
672        }
673        if self
674            .pending
675            .iter()
676            .any(|p| p.message.id.as_str() == message_id)
677        {
678            return false;
679        }
680        self.counters.duplicates_suppressed = self.counters.duplicates_suppressed.saturating_add(1);
681        true
682    }
683
684    /// Neighbors a frame should be forwarded to, given the current neighbor
685    /// set and the peers it must not go back to.
686    ///
687    /// Excludes the neighbor it arrived from and the peer that wrote it —
688    /// both already have it, and returning it is how a frame ends up bouncing
689    /// between two nodes.
690    pub fn select_targets<'a>(
691        &self,
692        neighbors: impl IntoIterator<Item = (&'a str, u8)>,
693        exclude: &[&str],
694        message_id: &str,
695    ) -> Vec<String> {
696        let mut candidates: Vec<(&str, u8)> = neighbors
697            .into_iter()
698            .filter(|(peer, _)| !exclude.contains(peer))
699            .collect();
700
701        if candidates.len() <= self.config.fanout {
702            return candidates
703                .into_iter()
704                .map(|(peer, _)| peer.to_string())
705                .collect();
706        }
707
708        // Strongest links first: a fan-out slot spent on a barely-connected
709        // neighbor is one the frame probably does not survive, and there are
710        // only a few slots.
711        //
712        // Ties break on a hash of (id, peer, self), which does two things.
713        // Within a group of equally good links it spreads different messages
714        // across different neighbors rather than pinning every frame to the
715        // same one. And it is stable per frame: a second copy of the same
716        // message picks the same neighbors instead of a fresh set, so a
717        // retransmission does not widen its own footprint.
718        candidates.sort_by(|(a_peer, a_quality), (b_peer, b_quality)| {
719            b_quality.cmp(a_quality).then_with(|| {
720                Self::hash_of(&[message_id, a_peer, &self.local_id]).cmp(&Self::hash_of(&[
721                    message_id,
722                    b_peer,
723                    &self.local_id,
724                ]))
725            })
726        });
727        candidates
728            .into_iter()
729            .take(self.config.fanout)
730            .map(|(peer, _)| peer.to_string())
731            .collect()
732    }
733
734    /// Drops per-neighbor rate state for a peer that has gone away.
735    pub fn forget_peer(&mut self, peer_id: &str) {
736        self.peer_budgets.remove(peer_id);
737    }
738
739    /// Releases expired suppression entries. Safe to call from a periodic tick.
740    pub fn maintain(&mut self, now: Instant) {
741        self.seen.expire(now);
742    }
743
744    /// Builds the copy to forward, or `None` when the frame has no hop budget
745    /// left.
746    fn prepare_hop(&mut self, message: &Message, degree: usize) -> Option<Message> {
747        if message.hop_count.value() >= RELAY_MAX_HOP_COUNT {
748            return None;
749        }
750
751        let ceiling = if degree >= self.config.dense_degree {
752            self.config.dense_max_ttl
753        } else {
754            self.config.max_ttl
755        };
756
757        let claimed = message.ttl.value();
758        let effective = claimed.min(ceiling);
759        if effective < claimed {
760            self.counters.ttl_clamped = self.counters.ttl_clamped.saturating_add(1);
761        }
762
763        // A budget of 1 or less is spent: this node is the last that may hold
764        // the frame, matching `TTL::is_exhausted`.
765        let remaining = offline_protocol_core::TTL::from_value(effective).decrement()?;
766
767        let mut forwarded = message.clone();
768        // Rewrite rather than decrement in place: the frame leaves with a
769        // budget this node vouches for, so an inflated claim cannot survive
770        // past this hop.
771        forwarded.ttl = remaining;
772        let _ = forwarded.increment_hop();
773        Some(forwarded)
774    }
775
776    /// Makes room for a higher-priority forward by displacing the
777    /// lowest-priority one that is not yet due.
778    ///
779    /// Frames already past their due time are left alone: they are on their way
780    /// to the radio this tick, so taking one costs a transmission that was
781    /// about to happen while the frame replacing it still has to wait out its
782    /// hold. Only when every lower-priority frame is already due does it
783    /// displace one of those instead of refusing outright.
784    fn evict_for(&mut self, candidate: &Message, now: Instant) -> bool {
785        let candidate_rank = priority_rank(candidate.priority);
786        let lower_priority = |p: &PendingRelay| priority_rank(p.message.priority) < candidate_rank;
787
788        let victim = self
789            .pending
790            .iter()
791            .enumerate()
792            .filter(|(_, p)| lower_priority(p) && p.due_at > now)
793            .min_by_key(|(_, p)| priority_rank(p.message.priority))
794            .map(|(index, _)| index)
795            .or_else(|| {
796                self.pending
797                    .iter()
798                    .enumerate()
799                    .filter(|(_, p)| lower_priority(p))
800                    .min_by_key(|(_, p)| priority_rank(p.message.priority))
801                    .map(|(index, _)| index)
802            });
803
804        match victim {
805            Some(index) => {
806                let displaced = self.pending.remove(index);
807                // Displaced before it ever reached a link. Its id goes back to
808                // being unknown, so the copy behind it — or the sender's
809                // retransmission — can still be carried by this device once
810                // there is room again.
811                self.seen.forget(&displaced.message.id.as_str());
812                self.counters.queue_evicted = self.counters.queue_evicted.saturating_add(1);
813                true
814            }
815            None => false,
816        }
817    }
818
819    fn take_peer_token(&mut self, peer_id: &str, now: Instant) -> bool {
820        if !self.peer_budgets.contains_key(peer_id) {
821            // Bound the map: an attacker cannot grow it with invented peer ids
822            // because entries are only created for peers we hold a link to,
823            // but churn over a long session can still accumulate.
824            if self.peer_budgets.len() >= MAX_RELAY_RATE_PEERS {
825                // Make room by forgetting the link that is spending least,
826                // rather than by clearing the table. A wholesale reset hands
827                // every neighbor a fresh burst — including the noisy one the
828                // per-link limit exists to hold back, which is the entry it can
829                // least afford to lose. The bucket that has recovered the most
830                // tokens is the one whose absence changes the least: at full
831                // capacity it is indistinguishable from a link never tracked.
832                //
833                // Costs a scan of the table, but only once the ceiling is
834                // reached and only when a genuinely new link appears.
835                for bucket in self.peer_budgets.values_mut() {
836                    bucket.refill(now);
837                }
838                let most_recovered = self
839                    .peer_budgets
840                    .iter()
841                    .max_by(|(_, a), (_, b)| a.tokens.total_cmp(&b.tokens))
842                    .map(|(peer, _)| peer.clone());
843                if let Some(peer) = most_recovered {
844                    self.peer_budgets.remove(&peer);
845                }
846            }
847            self.peer_budgets.insert(
848                peer_id.to_string(),
849                TokenBucket::new(self.config.peer_burst, self.config.peer_rate_per_sec, now),
850            );
851        }
852
853        self.peer_budgets
854            .get_mut(peer_id)
855            .map(|bucket| bucket.try_take_above(0.0, now))
856            .unwrap_or(true)
857    }
858
859    /// The delay before a queued forward is transmitted.
860    ///
861    /// Derived from the frame and this node's identity, so two nodes holding
862    /// the same frame wait different amounts and one of them gets to cancel.
863    /// The spread widens with density, because that is where the contention is.
864    fn jitter_for(&self, message_id: &str, degree: usize) -> Duration {
865        let min_ms = self.config.jitter_min.as_millis() as u64;
866        let max_ms = (self.config.jitter_max.as_millis() as u64).max(min_ms);
867
868        // More neighbors means more nodes racing to forward the same frame, so
869        // give them more room to separate.
870        let density_scale = 1 + (degree / self.config.dense_degree.max(1)) as u64;
871        let span = (max_ms - min_ms).saturating_mul(density_scale).max(1);
872
873        let hash = Self::hash_of(&[message_id, &self.local_id]);
874        Duration::from_millis(min_ms + (hash % span))
875    }
876
877    /// Stable hash used to spread delays and fan-out choices.
878    ///
879    /// `DefaultHasher` is unseeded, so this is predictable to anyone who knows
880    /// the inputs — both of which (a message id and a user id) are visible on
881    /// the wire. That is accepted: knowing it buys only the ability to guess
882    /// which neighbor fires first or which three links a frame takes, and every
883    /// path it could steer a frame down is already bounded by the fan-out cap
884    /// and the send budget. It must stay *stable*, which a randomly-seeded
885    /// hasher would not be — a second copy of a frame has to pick the same
886    /// neighbors, or a retransmission widens its own footprint.
887    fn hash_of(parts: &[&str]) -> u64 {
888        let mut hasher = DefaultHasher::new();
889        for part in parts {
890            part.hash(&mut hasher);
891        }
892        hasher.finish()
893    }
894}
895
896fn priority_rank(priority: MessagePriority) -> u8 {
897    match priority {
898        MessagePriority::Low => 0,
899        MessagePriority::Medium => 1,
900        MessagePriority::High => 2,
901        MessagePriority::Critical => 3,
902    }
903}
904
905#[cfg(test)]
906mod tests {
907    use super::*;
908    use offline_protocol_core::{AppId, UserId, TTL};
909
910    /// Zero jitter so `admit` queues a forward that is immediately due, which
911    /// keeps the tests deterministic without a clock abstraction.
912    fn immediate_config() -> MeshRelayConfig {
913        MeshRelayConfig {
914            jitter_min: Duration::from_millis(0),
915            jitter_max: Duration::from_millis(0),
916            ..Default::default()
917        }
918    }
919
920    fn governor() -> MeshRelayGovernor {
921        MeshRelayGovernor::with_config("relay-node", immediate_config())
922    }
923
924    fn frame_with(ttl: u8, priority: MessagePriority) -> Message {
925        Message::builder(
926            UserId::new("alice").unwrap(),
927            UserId::new("bob").unwrap(),
928            AppId::new("app").unwrap(),
929        )
930        .content("hello")
931        .priority(priority)
932        .ttl(TTL::new(ttl).unwrap())
933        .build()
934    }
935
936    fn frame() -> Message {
937        frame_with(8, MessagePriority::Medium)
938    }
939
940    #[test]
941    fn a_frame_is_forwarded_once_however_many_copies_arrive() {
942        let mut gov = governor();
943        let msg = frame();
944
945        assert_eq!(gov.admit(&msg, Some("b"), 3, false), RelayAdmission::Queued);
946
947        // Copies arriving after ours has gone out are suppressed, not forwarded.
948        let due = gov.take_due(Instant::now());
949        assert_eq!(due.len(), 1);
950
951        for _ in 0..5 {
952            assert_eq!(
953                gov.admit(&msg, Some("c"), 3, false),
954                RelayAdmission::Rejected(RelayRejection::AlreadySeen)
955            );
956        }
957        assert_eq!(gov.counters().queued, 1);
958        assert_eq!(gov.counters().duplicates_suppressed, 5);
959    }
960
961    #[test]
962    fn a_neighbor_beating_us_to_it_cancels_our_pending_forward() {
963        // The density control: in a cluster where everyone hears everyone, the
964        // first node to transmit covers the rest, and they stand down instead
965        // of each sending their own copy.
966        let mut gov = MeshRelayGovernor::with_config(
967            "relay-node",
968            MeshRelayConfig {
969                jitter_min: Duration::from_millis(50),
970                jitter_max: Duration::from_millis(200),
971                ..Default::default()
972            },
973        );
974        let msg = frame();
975
976        assert_eq!(gov.admit(&msg, Some("b"), 6, false), RelayAdmission::Queued);
977        assert_eq!(gov.pending_len(), 1);
978
979        // The same frame arrives again while ours is still waiting.
980        assert_eq!(
981            gov.admit(&msg, Some("c"), 6, false),
982            RelayAdmission::Rejected(RelayRejection::SupersededByDuplicate)
983        );
984
985        assert_eq!(gov.pending_len(), 0);
986        assert_eq!(gov.counters().cancelled_by_duplicate, 1);
987        assert!(gov
988            .take_due(Instant::now() + Duration::from_secs(1))
989            .is_empty());
990        assert_eq!(gov.counters().transmissions, 0);
991    }
992
993    #[test]
994    fn an_inflated_hop_budget_is_cut_to_local_policy() {
995        // Nothing authenticates the hop budget at this layer, so a frame can
996        // claim any value. It must not buy more than one hop past our own
997        // ceiling.
998        let mut gov = governor();
999        let msg = frame_with(255, MessagePriority::Medium);
1000
1001        assert_eq!(gov.admit(&msg, Some("b"), 2, false), RelayAdmission::Queued);
1002        let due = gov.take_due(Instant::now());
1003
1004        assert_eq!(due.len(), 1);
1005        assert_eq!(due[0].message.ttl.value(), DEFAULT_RELAY_MAX_TTL - 1);
1006        assert_eq!(gov.counters().ttl_clamped, 1);
1007    }
1008
1009    #[test]
1010    fn a_dense_neighborhood_uses_the_lower_hop_ceiling() {
1011        let mut gov = governor();
1012        let msg = frame_with(255, MessagePriority::Medium);
1013
1014        assert_eq!(
1015            gov.admit(&msg, Some("b"), DEFAULT_RELAY_DENSE_DEGREE, false),
1016            RelayAdmission::Queued
1017        );
1018        let due = gov.take_due(Instant::now());
1019        assert_eq!(due[0].message.ttl.value(), DEFAULT_RELAY_DENSE_MAX_TTL - 1);
1020    }
1021
1022    #[test]
1023    fn a_frame_out_of_hops_is_not_forwarded() {
1024        let mut gov = governor();
1025        // A budget of 1 is spent: this node is the last that may hold it.
1026        let msg = frame_with(1, MessagePriority::Medium);
1027
1028        assert_eq!(
1029            gov.admit(&msg, Some("b"), 3, false),
1030            RelayAdmission::Rejected(RelayRejection::HopLimitReached)
1031        );
1032        assert_eq!(gov.counters().hop_limit_reached, 1);
1033    }
1034
1035    #[test]
1036    fn the_last_hop_is_forwarded_and_the_one_after_it_is_not() {
1037        let mut gov = governor();
1038
1039        // Two hops left: goes out with one, which the next node will refuse to
1040        // forward. Verifies the budget lands exactly on `TTL::is_exhausted`
1041        // rather than a hop early or late.
1042        assert_eq!(
1043            gov.admit(&frame_with(2, MessagePriority::Medium), Some("b"), 3, false),
1044            RelayAdmission::Queued
1045        );
1046        let due = gov.take_due(Instant::now());
1047        assert_eq!(due[0].message.ttl.value(), 1);
1048        assert!(due[0].message.is_ttl_exhausted());
1049    }
1050
1051    #[test]
1052    fn the_hop_ceiling_stops_a_frame_that_evaded_the_budget() {
1053        let mut gov = governor();
1054        let mut msg = frame_with(255, MessagePriority::Medium);
1055        for _ in 0..RELAY_MAX_HOP_COUNT {
1056            let _ = msg.increment_hop();
1057        }
1058
1059        assert_eq!(
1060            gov.admit(&msg, Some("b"), 3, false),
1061            RelayAdmission::Rejected(RelayRejection::HopLimitReached)
1062        );
1063    }
1064
1065    #[test]
1066    fn one_noisy_neighbor_cannot_consume_the_whole_node() {
1067        let mut gov = governor();
1068
1069        // Spend the noisy neighbor's allowance on unique frames — the shape
1070        // suppression cannot help with, since every id is new.
1071        let mut accepted_from_noisy = 0;
1072        for _ in 0..200 {
1073            if gov.admit(&frame(), Some("noisy"), 3, false) == RelayAdmission::Queued {
1074                accepted_from_noisy += 1;
1075            }
1076        }
1077
1078        assert!(
1079            accepted_from_noisy <= DEFAULT_RELAY_PEER_BURST as usize + 2,
1080            "accepted {accepted_from_noisy} frames from one neighbor"
1081        );
1082        assert!(gov.counters().peer_rate_limited > 0);
1083
1084        // A different neighbor still gets through: the limit is per link, so
1085        // one bad actor degrades only itself.
1086        assert_eq!(
1087            gov.admit(&frame(), Some("quiet"), 3, false),
1088            RelayAdmission::Queued
1089        );
1090    }
1091
1092    #[test]
1093    fn transmission_never_exceeds_the_node_budget() {
1094        // The invariant the whole design rests on: whatever arrives, what the
1095        // radio is asked to send is capped. Asserted on transmissions rather
1096        // than on queue releases, because one release can put a frame on
1097        // several links and it is the links that cost airtime.
1098        let mut gov = MeshRelayGovernor::with_config(
1099            "relay-node",
1100            MeshRelayConfig {
1101                queue_capacity: 512,
1102                // Admission is not what is under test here.
1103                peer_rate_per_sec: 10_000.0,
1104                peer_burst: 10_000.0,
1105                ..immediate_config()
1106            },
1107        );
1108
1109        for _ in 0..200 {
1110            gov.admit(&frame(), Some("b"), 3, false);
1111        }
1112        let due = gov.take_due(Instant::now());
1113        assert!(!due.is_empty());
1114
1115        // Ask for far more transmissions than the budget allows.
1116        let granted = (0..1000).filter(|_| gov.take_send_token()).count();
1117
1118        assert!(
1119            granted <= DEFAULT_RELAY_BURST as usize + 1,
1120            "granted {granted} transmissions against a burst of {DEFAULT_RELAY_BURST}"
1121        );
1122        assert_eq!(gov.counters().transmissions as usize, granted);
1123        assert!(gov.counters().rate_deferred > 0);
1124    }
1125
1126    #[test]
1127    fn a_frame_that_reached_no_neighbor_for_budget_goes_back_on_the_queue() {
1128        // `take_due` checks the budget without reserving it, so a batch of due
1129        // frames can be released against a single token: the first spends it
1130        // and the rest reach the radio with nothing left. Those frames have
1131        // been forwarded nowhere, and their ids are already recorded as
1132        // handled — dropping one would lose this copy and refuse both the
1133        // copies behind it and the sender's retransmissions for the whole
1134        // retention window. It has to go back on the queue.
1135        let mut gov = MeshRelayGovernor::with_config(
1136            "relay-node",
1137            MeshRelayConfig {
1138                peer_rate_per_sec: 10_000.0,
1139                peer_burst: 10_000.0,
1140                ..immediate_config()
1141            },
1142        );
1143
1144        for _ in 0..40 {
1145            gov.admit(&frame(), Some("b"), 3, false);
1146        }
1147        let due = gov.take_due(Instant::now());
1148        assert!(
1149            due.len() > DEFAULT_RELAY_BURST as usize,
1150            "the batch must outrun the budget for this to be the case under test"
1151        );
1152
1153        // Drain the budget the way a fan-out does, then hand back everything
1154        // that got nowhere.
1155        let mut requeued = 0;
1156        for relay in due {
1157            if gov.take_send_token() {
1158                continue;
1159            }
1160            gov.requeue(relay);
1161            requeued += 1;
1162        }
1163
1164        assert!(requeued > 0, "the budget must have run out mid-batch");
1165        assert_eq!(gov.pending_len(), requeued, "every starved frame is kept");
1166        assert_eq!(gov.counters().requeued as usize, requeued);
1167    }
1168
1169    #[test]
1170    fn a_requeued_frame_is_still_abandoned_once_it_is_far_past_its_turn() {
1171        // Re-queueing must not become an unbounded retry: a frame kept for
1172        // budget keeps its original due time, so the overdue cut-off still
1173        // reaches it.
1174        let mut gov = governor();
1175        gov.admit(&frame(), Some("b"), 3, false);
1176        let mut due = gov.take_due(Instant::now());
1177        assert_eq!(due.len(), 1);
1178
1179        gov.requeue(due.remove(0));
1180        assert_eq!(gov.pending_len(), 1);
1181
1182        let later = gov.take_due(Instant::now() + RELAY_QUEUE_MAX_OVERDUE + Duration::from_secs(1));
1183        assert!(later.is_empty());
1184        assert_eq!(gov.counters().abandoned_overdue, 1);
1185        assert_eq!(gov.pending_len(), 0);
1186    }
1187
1188    #[test]
1189    fn a_frame_abandoned_for_waiting_too_long_releases_its_id() {
1190        // A frame dropped for waiting past its turn never reached a link, so
1191        // nothing is circulating that a second forward could duplicate. Its id
1192        // must go back to being unknown, or the sender's retransmissions —
1193        // which carry the same id — are refused for the whole retention window
1194        // because of a few seconds of congestion.
1195        let mut gov = governor();
1196        let msg = frame();
1197        gov.admit(&msg, Some("b"), 3, false);
1198
1199        let due = gov.take_due(Instant::now() + RELAY_QUEUE_MAX_OVERDUE + Duration::from_secs(1));
1200        assert!(due.is_empty());
1201        assert_eq!(gov.counters().abandoned_overdue, 1);
1202
1203        assert_eq!(
1204            gov.admit(&msg, Some("c"), 3, false),
1205            RelayAdmission::Queued,
1206            "the sender's retransmission must still be carried"
1207        );
1208    }
1209
1210    #[test]
1211    fn a_frame_displaced_from_the_queue_releases_its_id() {
1212        // Same rule for a frame evicted to make room for an urgent one: it was
1213        // never transmitted, so the route through this device must stay open.
1214        let mut gov = MeshRelayGovernor::with_config(
1215            "relay-node",
1216            MeshRelayConfig {
1217                queue_capacity: 1,
1218                peer_rate_per_sec: 10_000.0,
1219                peer_burst: 10_000.0,
1220                ..immediate_config()
1221            },
1222        );
1223
1224        let routine = frame_with(8, MessagePriority::Low);
1225        gov.admit(&routine, Some("b"), 3, false);
1226        gov.admit(
1227            &frame_with(8, MessagePriority::Critical),
1228            Some("b"),
1229            3,
1230            false,
1231        );
1232        assert_eq!(gov.counters().queue_evicted, 1);
1233
1234        // The urgent frame goes out, leaving room again. A later copy of the
1235        // displaced one — or the sender's retransmission — must be carryable.
1236        gov.take_due(Instant::now());
1237        assert_eq!(
1238            gov.admit(&routine, Some("c"), 3, false),
1239            RelayAdmission::Queued,
1240            "the displaced frame's id must not be blackholed"
1241        );
1242    }
1243
1244    #[test]
1245    fn a_frame_refused_room_on_the_way_back_releases_its_id() {
1246        // The third drop that happens after the id was recorded: a frame that
1247        // reached no neighbor and finds the queue full when handed back.
1248        let mut gov = MeshRelayGovernor::with_config(
1249            "relay-node",
1250            MeshRelayConfig {
1251                queue_capacity: 1,
1252                peer_rate_per_sec: 10_000.0,
1253                peer_burst: 10_000.0,
1254                ..immediate_config()
1255            },
1256        );
1257
1258        let starved = frame();
1259        gov.admit(&starved, Some("b"), 3, false);
1260        let mut due = gov.take_due(Instant::now());
1261        assert_eq!(due.len(), 1);
1262
1263        // The queue fills while the frame is out being transmitted, so there is
1264        // no room for it on the way back.
1265        gov.admit(&frame(), Some("b"), 3, false);
1266        gov.requeue(due.remove(0));
1267        assert_eq!(gov.counters().queue_full, 1);
1268        assert_eq!(gov.pending_len(), 1, "the dropped frame is not held");
1269
1270        // Once the queue drains, the copy behind it must still be carryable.
1271        gov.take_due(Instant::now());
1272        assert_eq!(
1273            gov.admit(&starved, Some("c"), 3, false),
1274            RelayAdmission::Queued,
1275            "a frame that got nowhere and could not be kept must stay carryable"
1276        );
1277    }
1278
1279    #[test]
1280    fn forwarding_leaves_room_for_this_devices_own_traffic() {
1281        // One radio, one budget — but a device saturated with other people's
1282        // traffic must still be able to send its own, above all the delivery
1283        // acknowledgement whose absence makes a delivered message look lost.
1284        let mut gov = MeshRelayGovernor::with_config(
1285            "relay-node",
1286            MeshRelayConfig {
1287                queue_capacity: 512,
1288                peer_rate_per_sec: 10_000.0,
1289                peer_burst: 10_000.0,
1290                ..immediate_config()
1291            },
1292        );
1293
1294        for _ in 0..200 {
1295            gov.admit(&frame(), Some("b"), 3, false);
1296        }
1297        gov.take_due(Instant::now());
1298
1299        // Drain the budget the way a saturated forwarder does.
1300        let forwarded = (0..1000).filter(|_| gov.take_send_token()).count();
1301        assert!(forwarded > 0, "forwarding must get its share first");
1302        assert!(
1303            !gov.take_send_token(),
1304            "forwarding must stop at the reserve, not at empty"
1305        );
1306
1307        // The reserve is still there for us.
1308        let own = (0..1000).filter(|_| gov.take_own_send_token()).count();
1309        assert!(
1310            own >= RELAY_OWN_TRAFFIC_RESERVE as usize,
1311            "own traffic got {own} transmissions against a reserve of {RELAY_OWN_TRAFFIC_RESERVE}"
1312        );
1313
1314        // And the ceiling still holds across both: the reserve is carved out of
1315        // the burst, not added to it.
1316        assert!(
1317            (forwarded + own) <= DEFAULT_RELAY_BURST as usize + 1,
1318            "{forwarded} forwarded + {own} own exceeds the burst of {DEFAULT_RELAY_BURST}"
1319        );
1320    }
1321
1322    #[test]
1323    fn a_small_configured_burst_still_forwards() {
1324        // The reserve is clamped to half the burst, so a device configured with
1325        // a burst smaller than the reserve does not silently stop forwarding
1326        // altogether.
1327        let mut gov = MeshRelayGovernor::with_config(
1328            "relay-node",
1329            MeshRelayConfig {
1330                burst: 2.0,
1331                rate_per_sec: 1.0,
1332                peer_rate_per_sec: 10_000.0,
1333                peer_burst: 10_000.0,
1334                ..immediate_config()
1335            },
1336        );
1337
1338        gov.admit(&frame(), Some("b"), 3, false);
1339        assert!(
1340            !gov.take_due(Instant::now()).is_empty(),
1341            "a small burst must still release forwards"
1342        );
1343        assert!(gov.take_send_token(), "and still allow transmitting them");
1344    }
1345
1346    #[test]
1347    fn a_settled_duplicate_is_absorbed_without_the_full_admission_path() {
1348        let mut gov = governor();
1349        let msg = frame();
1350
1351        assert!(
1352            !gov.absorb_settled_duplicate(&msg.id.as_str()),
1353            "an id never seen is not a duplicate"
1354        );
1355
1356        gov.admit(&msg, Some("b"), 3, false);
1357        // While our copy is still pending, standing down is a decision that
1358        // needs the neighbor set, so the cheap path must decline to make it.
1359        assert!(!gov.absorb_settled_duplicate(&msg.id.as_str()));
1360
1361        // Once it has gone out, further copies are settled duplicates.
1362        gov.take_due(Instant::now());
1363        assert!(gov.absorb_settled_duplicate(&msg.id.as_str()));
1364        assert_eq!(gov.counters().duplicates_suppressed, 1);
1365    }
1366
1367    #[test]
1368    fn eviction_prefers_a_frame_that_has_not_yet_gone_out() {
1369        // A due frame is on its way to the radio this tick; taking it costs a
1370        // transmission that was about to happen, while the frame replacing it
1371        // still has to wait out its hold.
1372        let mut gov = MeshRelayGovernor::with_config(
1373            "relay-node",
1374            MeshRelayConfig {
1375                queue_capacity: 2,
1376                jitter_min: Duration::from_millis(0),
1377                jitter_max: Duration::from_millis(0),
1378                peer_rate_per_sec: 10_000.0,
1379                peer_burst: 10_000.0,
1380                ..Default::default()
1381            },
1382        );
1383
1384        // One low-priority frame due immediately.
1385        let due_now = frame_with(8, MessagePriority::Low);
1386        gov.admit(&due_now, Some("b"), 3, false);
1387
1388        // A second low-priority frame that is not due for a while.
1389        gov.config.jitter_min = Duration::from_millis(500);
1390        gov.config.jitter_max = Duration::from_millis(500);
1391        let waiting = frame_with(8, MessagePriority::Low);
1392        gov.admit(&waiting, Some("b"), 3, false);
1393
1394        // An urgent frame needs the room.
1395        gov.admit(
1396            &frame_with(8, MessagePriority::Critical),
1397            Some("b"),
1398            3,
1399            false,
1400        );
1401
1402        assert_eq!(gov.counters().queue_evicted, 1);
1403        assert_eq!(
1404            gov.counters().queue_full,
1405            0,
1406            "making room is not the same as refusing"
1407        );
1408        assert!(
1409            gov.pending
1410                .iter()
1411                .any(|p| p.message.id.as_str() == due_now.id.as_str()),
1412            "the frame already due must survive"
1413        );
1414        assert!(
1415            !gov.pending
1416                .iter()
1417                .any(|p| p.message.id.as_str() == waiting.id.as_str()),
1418            "the frame still waiting is the one displaced"
1419        );
1420    }
1421
1422    #[test]
1423    fn a_full_peer_table_keeps_the_links_that_are_spending() {
1424        // Clearing the table wholesale hands every neighbor a fresh burst,
1425        // including the noisy one the per-link limit exists to hold back.
1426        let mut gov = MeshRelayGovernor::with_config("relay-node", immediate_config());
1427
1428        // One neighbor spends most of its allowance.
1429        for _ in 0..(DEFAULT_RELAY_PEER_BURST as usize - 1) {
1430            gov.admit(&frame(), Some("noisy"), 3, false);
1431        }
1432
1433        // Fill the table with links that arrive once and go quiet.
1434        for i in 0..MAX_RELAY_RATE_PEERS {
1435            gov.admit(&frame(), Some(&format!("idle-{i}")), 3, false);
1436        }
1437
1438        assert!(gov.peer_budgets.len() <= MAX_RELAY_RATE_PEERS);
1439        // Still tracked at all: the entry the limit exists to hold back is the
1440        // one it can least afford to forget, and it never sends again here, so
1441        // an eviction would leave it absent rather than rebuilt.
1442        let noisy = gov
1443            .peer_budgets
1444            .get("noisy")
1445            .expect("a link mid-spend must not be forgotten to make room for idle ones");
1446        assert!(
1447            noisy.tokens < noisy.capacity,
1448            "the noisy link kept its spent allowance rather than being handed a fresh burst"
1449        );
1450    }
1451
1452    #[test]
1453    fn a_refused_frame_leaves_the_way_open_for_another_copy() {
1454        // A frame turned away must not be remembered as handled: the copy
1455        // arriving over a healthy link a moment later is the alternate path,
1456        // and suppressing it would blank every route through this device for
1457        // the whole retention window.
1458        let mut gov = governor();
1459
1460        // Refused for having no hops left — the cheapest thing an attacker can
1461        // forge, since nothing authenticates the claim.
1462        let msg = frame_with(1, MessagePriority::Medium);
1463        assert_eq!(
1464            gov.admit(&msg, Some("hostile"), 3, false),
1465            RelayAdmission::Rejected(RelayRejection::HopLimitReached)
1466        );
1467
1468        // The same message, arriving intact by another route.
1469        let mut healthy = msg.clone();
1470        healthy.ttl = TTL::new(8).unwrap();
1471        assert_eq!(
1472            gov.admit(&healthy, Some("good"), 3, false),
1473            RelayAdmission::Queued,
1474            "a refused frame must not blackhole its id"
1475        );
1476    }
1477
1478    #[test]
1479    fn a_rate_limited_neighbor_does_not_blackhole_the_ids_it_sent() {
1480        let mut gov = governor();
1481
1482        // Exhaust one neighbor's allowance with distinct frames.
1483        let mut refused = Vec::new();
1484        for _ in 0..200 {
1485            let msg = frame();
1486            if gov.admit(&msg, Some("noisy"), 3, false)
1487                == RelayAdmission::Rejected(RelayRejection::PeerRateLimited)
1488            {
1489                refused.push(msg);
1490            }
1491        }
1492        assert!(!refused.is_empty());
1493
1494        // Those same messages, reaching us through someone else, must still be
1495        // carried.
1496        let victim = &refused[0];
1497        assert_eq!(
1498            gov.admit(victim, Some("quiet"), 3, false),
1499            RelayAdmission::Queued
1500        );
1501    }
1502
1503    #[test]
1504    fn the_last_hop_does_not_stand_down_for_a_neighbor() {
1505        // Standing down assumes a neighbor's transmission covers the same
1506        // ground ours would. At the last hop it does not: our copy is going to
1507        // the recipient over a link only we are known to hold, so dropping it
1508        // loses the delivery.
1509        let mut gov = MeshRelayGovernor::with_config(
1510            "relay-node",
1511            MeshRelayConfig {
1512                jitter_min: Duration::from_millis(50),
1513                jitter_max: Duration::from_millis(200),
1514                ..Default::default()
1515            },
1516        );
1517        let msg = frame();
1518
1519        assert_eq!(gov.admit(&msg, Some("b"), 3, true), RelayAdmission::Queued);
1520        // Another forwarder hands us the same frame.
1521        assert_eq!(
1522            gov.admit(&msg, Some("c"), 3, true),
1523            RelayAdmission::Rejected(RelayRejection::AlreadySeen)
1524        );
1525
1526        assert_eq!(gov.pending_len(), 1, "the delivering copy must survive");
1527        assert_eq!(gov.counters().cancelled_by_duplicate, 0);
1528    }
1529
1530    #[test]
1531    fn a_forward_waiting_far_past_its_turn_is_abandoned() {
1532        let mut gov = governor();
1533        gov.admit(&frame(), Some("b"), 3, false);
1534
1535        // Long enough that other paths have carried it or the sender has
1536        // retransmitted; holding it would only displace newer traffic.
1537        let due = gov.take_due(Instant::now() + RELAY_QUEUE_MAX_OVERDUE + Duration::from_secs(1));
1538        assert!(due.is_empty());
1539        assert_eq!(gov.counters().abandoned_overdue, 1);
1540        assert_eq!(gov.pending_len(), 0);
1541    }
1542
1543    #[test]
1544    fn a_full_queue_refuses_rather_than_growing() {
1545        let mut gov = MeshRelayGovernor::with_config(
1546            "relay-node",
1547            MeshRelayConfig {
1548                queue_capacity: 4,
1549                jitter_min: Duration::from_millis(500),
1550                jitter_max: Duration::from_millis(500),
1551                peer_rate_per_sec: 10_000.0,
1552                peer_burst: 10_000.0,
1553                ..Default::default()
1554            },
1555        );
1556
1557        for _ in 0..4 {
1558            assert_eq!(
1559                gov.admit(&frame(), Some("b"), 3, false),
1560                RelayAdmission::Queued
1561            );
1562        }
1563        assert_eq!(
1564            gov.admit(&frame(), Some("b"), 3, false),
1565            RelayAdmission::Rejected(RelayRejection::QueueFull)
1566        );
1567        assert_eq!(gov.pending_len(), 4);
1568    }
1569
1570    #[test]
1571    fn a_full_queue_still_takes_an_urgent_frame_over_a_routine_one() {
1572        let mut gov = MeshRelayGovernor::with_config(
1573            "relay-node",
1574            MeshRelayConfig {
1575                queue_capacity: 2,
1576                jitter_min: Duration::from_millis(500),
1577                jitter_max: Duration::from_millis(500),
1578                peer_rate_per_sec: 10_000.0,
1579                peer_burst: 10_000.0,
1580                ..Default::default()
1581            },
1582        );
1583
1584        gov.admit(&frame_with(8, MessagePriority::Low), Some("b"), 3, false);
1585        gov.admit(&frame_with(8, MessagePriority::Low), Some("b"), 3, false);
1586
1587        assert_eq!(
1588            gov.admit(
1589                &frame_with(8, MessagePriority::Critical),
1590                Some("b"),
1591                3,
1592                false
1593            ),
1594            RelayAdmission::Queued
1595        );
1596        assert_eq!(gov.pending_len(), 2);
1597        assert!(gov
1598            .take_due(Instant::now() + Duration::from_secs(1))
1599            .iter()
1600            .any(|r| r.message.priority == MessagePriority::Critical));
1601    }
1602
1603    #[test]
1604    fn a_frame_is_never_sent_back_where_it_came_from() {
1605        let gov = governor();
1606        let targets = gov.select_targets(
1607            [("alice", 80), ("b", 80), ("c", 80), ("d", 80)],
1608            &["b", "alice"], // arrival neighbor and the peer that wrote it
1609            "msg-1",
1610        );
1611
1612        assert!(!targets.contains(&"b".to_string()));
1613        assert!(!targets.contains(&"alice".to_string()));
1614        assert!(targets.contains(&"c".to_string()));
1615        assert!(targets.contains(&"d".to_string()));
1616    }
1617
1618    #[test]
1619    fn fanout_is_capped_and_stable_for_a_given_frame() {
1620        let gov = governor();
1621        // All equally good, so the tie-break is what decides — which is the
1622        // property under test.
1623        let neighbors = [
1624            ("n0", 70),
1625            ("n1", 70),
1626            ("n2", 70),
1627            ("n3", 70),
1628            ("n4", 70),
1629            ("n5", 70),
1630            ("n6", 70),
1631            ("n7", 70),
1632        ];
1633
1634        let first = gov.select_targets(neighbors, &[], "msg-1");
1635        let again = gov.select_targets(neighbors, &[], "msg-1");
1636
1637        assert_eq!(first.len(), DEFAULT_RELAY_FANOUT);
1638        assert_eq!(first, again, "a second copy must pick the same neighbors");
1639
1640        // Different frames spread across different neighbors rather than
1641        // hammering one set.
1642        let other = gov.select_targets(neighbors, &[], "msg-2");
1643        assert_eq!(other.len(), DEFAULT_RELAY_FANOUT);
1644    }
1645
1646    #[test]
1647    fn a_frame_we_originated_is_not_carried_back_for_someone_else() {
1648        // Our own message, handed to neighbors. When a neighbor's copy reaches
1649        // us again we must recognize it, not take it on as third-party traffic.
1650        let mut gov = governor();
1651        let msg = frame();
1652
1653        gov.mark_handled(&msg.id.as_str());
1654
1655        assert_eq!(
1656            gov.admit(&msg, Some("b"), 3, false),
1657            RelayAdmission::Rejected(RelayRejection::AlreadySeen)
1658        );
1659    }
1660
1661    #[test]
1662    fn the_suppression_cache_holds_a_saturated_node_for_its_full_window() {
1663        // Capacity has to outlast in-flight copies at the maximum rate the node
1664        // will accept traffic. If it did not, an id could be forgotten while
1665        // copies were still circulating and be forwarded a second time — by
1666        // every node, which is how a flood becomes a storm.
1667        let gov = governor();
1668        let max_admissions_per_sec = DEFAULT_RELAY_PEER_RATE_PER_SEC * MAX_RELAY_RATE_PEERS as f32;
1669        let sustained_ceiling = DEFAULT_RELAY_RATE_PER_SEC.min(max_admissions_per_sec);
1670        let ids_in_window = sustained_ceiling * gov.config().seen.retention.as_secs_f32();
1671
1672        assert!(
1673            (gov.config().seen.capacity as f32) >= ids_in_window,
1674            "cache holds {} ids but a saturated node sees {} within the retention window",
1675            gov.config().seen.capacity,
1676            ids_in_window
1677        );
1678    }
1679}