Skip to main content

pb_mapper_server/
routing.rs

1//! The routing model: registered control connections by service, pending
2//! streams, namespace quotas and rate limits, and registration leases.
3
4use std::num::NonZeroUsize;
5use std::sync::Arc;
6use std::time::{Duration, Instant};
7
8use pb_mapper_auth::AuthContext;
9use pb_mapper_core::config::server_lease_timeout;
10use pb_mapper_core::conn_id::RemoteConnId;
11use pb_mapper_core::time::saturating_millis;
12use pb_mapper_protocol::command::{CONTROL_PROTOCOL_V2, PbServiceConnStatus};
13use tokio_util::sync::CancellationToken;
14
15use crate::server::LEGACY_SERVER_IDLE_TIMEOUT;
16use crate::service_key::split_scoped_service_key;
17use crate::task::{ConnTask, ConnTaskSender};
18
19pub type ImutableKey = Arc<str>;
20
21/// The retryable refusal code a subscribe gets when no registration of its
22/// service can serve it: none is registered, none answered in time, or none
23/// did before its setup bound.
24pub(crate) const SERVICE_NOT_AVAILABLE: &str = "service_not_available";
25
26#[derive(Debug, Clone, Copy, PartialEq, Eq)]
27pub enum ServerConnHealth {
28    Healthy,
29    Suspect,
30}
31
32#[derive(Debug, Clone)]
33pub struct ServerConnInfo {
34    pub conn_id: RemoteConnId,
35    pub generation: u64,
36    pub health: ServerConnHealth,
37    pub need_codec: bool,
38    pub is_datagram: bool,
39    pub protocol_version: u16,
40    pub last_rx_at: Instant,
41    /// Stream acknowledgement budget learned from completed requests and timeouts.
42    pub stream_ack_budget: Duration,
43    /// The suspicion the one confirmation scheduled for this connection will
44    /// settle, when one is: the latest, if activity answered an earlier one.
45    pub suspect_check: Option<Instant>,
46    /// Cancelled to tear this connection's socket down; see
47    /// [`ManagerTask::Register::retire_token`](crate::ManagerTask::Register::retire_token).
48    ///
49    /// Held here rather than in the task manager's sender map so that retiring a
50    /// connection reaches the socket even when its work queue is wedged. This is
51    /// also why the struct is `Clone` and no longer `Copy`.
52    pub retire_token: CancellationToken,
53    pub direct: Option<AuthContext>,
54}
55
56/// A confirmation to schedule: once `due`, look for authenticated control
57/// activity since the suspicion at `observed_at`.
58#[derive(Debug, Clone, Copy, PartialEq, Eq)]
59pub(crate) struct SuspectCheck {
60    pub(crate) observed_at: Instant,
61    pub(crate) due: Instant,
62}
63
64/// What a confirmation found when it came due.
65#[derive(Debug, Clone, Copy, PartialEq, Eq)]
66pub(crate) enum Confirmation {
67    /// Not the confirmation this registration waits on.
68    Stale,
69    /// Authenticated control activity answered the suspicion.
70    Cleared,
71    /// Silent since the suspicion: retire the registration.
72    Retire,
73    /// Activity answered the suspicion this confirmation was for, and a later
74    /// one renewed it while it waited: schedule this in its place.
75    Rearm(SuspectCheck),
76}
77
78impl ServerConnInfo {
79    /// Count authenticated control activity at `now`, which answers any
80    /// suspicion; whether that put this registration back in rotation.
81    pub(crate) fn record_activity(&mut self, now: Instant) -> bool {
82        self.last_rx_at = now;
83        self.restore()
84    }
85
86    /// Put this registration back in rotation; whether it was suspected.
87    pub(crate) fn restore(&mut self) -> bool {
88        std::mem::replace(&mut self.health, ServerConnHealth::Healthy) == ServerConnHealth::Suspect
89    }
90
91    /// Suspect this registration at `now`, after a stream it was asked for
92    /// timed out, doubling its acknowledgement budget up to `max_ack_budget`.
93    ///
94    /// The confirmation to schedule, or `None` when the one already scheduled
95    /// will settle this suspicion.
96    pub(crate) fn suspect(
97        &mut self,
98        now: Instant,
99        max_ack_budget: Duration,
100    ) -> Option<SuspectCheck> {
101        self.stream_ack_budget = self.stream_ack_budget.saturating_mul(2).min(max_ack_budget);
102        // A timeout affects candidate selection immediately, but a late
103        // authenticated ACK/heartbeat can restore it without a reconnect.
104        self.health = ServerConnHealth::Suspect;
105        match self.suspect_check {
106            // Nothing has answered the pending suspicion, so whatever answers
107            // this one answers that one too, and its confirmation settles both.
108            Some(pending) if self.last_rx_at <= pending => None,
109            // Activity answered the pending suspicion, and this one renews it.
110            // Its confirmation, already scheduled and too soon for this one,
111            // goes on to this one when it comes due (see `confirm`).
112            Some(_) => {
113                self.suspect_check = Some(now);
114                None
115            }
116            None => {
117                self.suspect_check = Some(now);
118                Some(self.check(now))
119            }
120        }
121    }
122
123    /// Settle the confirmation scheduled for the suspicion at `observed_at`.
124    pub(crate) fn confirm(&mut self, observed_at: Instant) -> Confirmation {
125        match self.suspect_check {
126            Some(pending) if pending == observed_at => {}
127            Some(pending) if pending > observed_at => {
128                return Confirmation::Rearm(self.check(pending));
129            }
130            _ => return Confirmation::Stale,
131        }
132        self.suspect_check = None;
133        if self.last_rx_at > observed_at {
134            Confirmation::Cleared
135        } else {
136            Confirmation::Retire
137        }
138    }
139
140    /// How long a suspicion waits for authenticated control activity.
141    pub(crate) fn suspect_grace(&self) -> Duration {
142        pb_mapper_core::config::control_suspect_grace().max(self.stream_ack_budget)
143    }
144
145    fn check(&self, observed_at: Instant) -> SuspectCheck {
146        SuspectCheck {
147            observed_at,
148            due: observed_at + self.suspect_grace(),
149        }
150    }
151}
152
153#[cfg(test)]
154impl ServerConnInfo {
155    /// A healthy registration last heard from now, on the stream budget it
156    /// starts with.
157    pub(crate) fn for_test(
158        conn_id: u32,
159        generation: u64,
160        protocol_version: u16,
161        direct: Option<AuthContext>,
162    ) -> Self {
163        Self {
164            conn_id: RemoteConnId::from(conn_id),
165            generation,
166            health: ServerConnHealth::Healthy,
167            need_codec: false,
168            is_datagram: false,
169            protocol_version,
170            last_rx_at: Instant::now(),
171            stream_ack_budget: pb_mapper_core::config::stream_ack_timeout(),
172            suspect_check: None,
173            retire_token: CancellationToken::new(),
174            direct,
175        }
176    }
177}
178
179pub type ServerConnMap = hashbrown::HashMap<ImutableKey, Vec<ServerConnInfo>>;
180
181/// A subscriber's stream request: the server connection it was routed to, that
182/// connection's generation, the service key it asked for, and when.
183#[derive(Clone)]
184pub(crate) struct PendingStream {
185    pub(crate) server_conn_id: RemoteConnId,
186    pub(crate) server_generation: u64,
187    pub(crate) key: ImutableKey,
188    pub(crate) requested_at: Instant,
189}
190
191/// Subscribers' stream requests by their own connection ID, counted per
192/// namespace as they are added and removed.
193#[derive(Default)]
194pub(crate) struct PendingStreams {
195    by_client: hashbrown::HashMap<RemoteConnId, PendingStream>,
196    per_namespace: hashbrown::HashMap<u64, usize>,
197}
198
199impl PendingStreams {
200    pub(crate) fn get(&self, client_id: &RemoteConnId) -> Option<&PendingStream> {
201        self.by_client.get(client_id)
202    }
203
204    pub(crate) fn contains(&self, client_id: &RemoteConnId) -> bool {
205        self.by_client.contains_key(client_id)
206    }
207
208    pub(crate) fn iter(&self) -> impl Iterator<Item = (&RemoteConnId, &PendingStream)> {
209        self.by_client.iter()
210    }
211
212    /// Record a client's stream request, replacing the one it made before.
213    pub(crate) fn insert(&mut self, client_id: RemoteConnId, stream: PendingStream) {
214        let namespace = split_scoped_service_key(&stream.key).0;
215        match self.by_client.insert(client_id, stream) {
216            Some(replaced) if split_scoped_service_key(&replaced.key).0 == namespace => return,
217            Some(replaced) => self.uncount(split_scoped_service_key(&replaced.key).0),
218            None => {}
219        }
220        *self.per_namespace.entry(namespace).or_default() += 1;
221    }
222
223    pub(crate) fn remove(&mut self, client_id: &RemoteConnId) -> Option<PendingStream> {
224        let removed = self.by_client.remove(client_id)?;
225        self.uncount(split_scoped_service_key(&removed.key).0);
226        Some(removed)
227    }
228
229    /// How many requests in `namespace` are pending.
230    pub(crate) fn count_in(&self, namespace: u64) -> usize {
231        self.per_namespace
232            .get(&namespace)
233            .copied()
234            .unwrap_or_default()
235    }
236
237    /// Whether `namespace` can take `extra` more streams than it has pending.
238    pub(crate) fn has_room(&self, namespace: u64, extra: usize, limit: NonZeroUsize) -> bool {
239        extra + self.count_in(namespace) < limit.get()
240    }
241
242    fn uncount(&mut self, namespace: u64) {
243        let Some(count) = self.per_namespace.get_mut(&namespace) else {
244            return;
245        };
246        *count = count.saturating_sub(1);
247        if *count == 0 {
248            self.per_namespace.remove(&namespace);
249        }
250    }
251}
252
253/// Per-namespace new-stream rate limiters, keyed by the namespace's key ID.
254pub(crate) struct NamespaceRateLimits {
255    buckets: hashbrown::HashMap<u64, NamespaceRateLimit>,
256    rate_per_second: NonZeroUsize,
257    burst: NonZeroUsize,
258}
259
260impl NamespaceRateLimits {
261    pub(crate) fn new(rate_per_second: NonZeroUsize, burst: NonZeroUsize) -> Self {
262        Self {
263            buckets: hashbrown::HashMap::new(),
264            rate_per_second,
265            burst,
266        }
267    }
268
269    /// Take a token for one new stream in `namespace`.
270    pub(crate) fn try_take(&mut self, namespace: u64) -> bool {
271        self.buckets
272            .entry(namespace)
273            .or_insert_with(|| NamespaceRateLimit::new(self.rate_per_second, self.burst))
274            .allow()
275    }
276
277    /// Forget `namespace`'s bucket, as nothing in it is registered or pending.
278    pub(crate) fn release(&mut self, namespace: u64) {
279        self.buckets.remove(&namespace);
280    }
281}
282
283struct NamespaceRateLimit {
284    tokens: f64,
285    last_refill: Instant,
286    rate_per_second: f64,
287    burst: f64,
288}
289
290impl NamespaceRateLimit {
291    fn new(rate_per_second: NonZeroUsize, burst: NonZeroUsize) -> Self {
292        Self {
293            tokens: burst.get() as f64,
294            last_refill: Instant::now(),
295            rate_per_second: rate_per_second.get() as f64,
296            burst: burst.get() as f64,
297        }
298    }
299
300    fn allow(&mut self) -> bool {
301        let now = Instant::now();
302        self.tokens = (self.tokens
303            + now.duration_since(self.last_refill).as_secs_f64() * self.rate_per_second)
304            .min(self.burst);
305        self.last_refill = now;
306        if self.tokens < 1.0 {
307            false
308        } else {
309            self.tokens -= 1.0;
310            true
311        }
312    }
313}
314
315/// Routing limits and timings, read from the environment once as the relay
316/// starts.
317pub(crate) struct RoutingConfig {
318    pub(crate) max_services_per_namespace: NonZeroUsize,
319    pub(crate) max_register_connections_per_service: NonZeroUsize,
320    pub(crate) max_streams_per_namespace: NonZeroUsize,
321    pub(crate) new_streams_per_second: NonZeroUsize,
322    pub(crate) new_streams_burst: NonZeroUsize,
323    /// The stream acknowledgement budget a registration starts with, and the
324    /// floor its learned budget never drops below.
325    pub(crate) stream_ack_timeout: Duration,
326    /// The lease a protocol-v2 registration is granted.
327    pub(crate) lease_timeout: Duration,
328}
329
330impl RoutingConfig {
331    pub(crate) fn from_env() -> Self {
332        Self {
333            max_services_per_namespace: env_limit(
334                "PB_MAPPER_MAX_SERVICES_PER_NAMESPACE",
335                const { limit(256) },
336            ),
337            max_register_connections_per_service: env_limit(
338                "PB_MAPPER_MAX_REGISTER_CONNECTIONS_PER_SERVICE",
339                const { limit(16) },
340            ),
341            max_streams_per_namespace: env_limit(
342                "PB_MAPPER_MAX_STREAMS_PER_NAMESPACE",
343                const { limit(1024) },
344            ),
345            new_streams_per_second: pb_mapper_core::config::new_streams_per_second(),
346            new_streams_burst: env_limit("PB_MAPPER_NEW_STREAMS_BURST", const { limit(200) }),
347            stream_ack_timeout: pb_mapper_core::config::stream_ack_timeout(),
348            lease_timeout: server_lease_timeout(),
349        }
350    }
351
352    /// The ceiling a learned stream acknowledgement budget never exceeds.
353    pub(crate) fn max_stream_ack_budget(&self) -> Duration {
354        Duration::from_secs(5).max(self.stream_ack_timeout)
355    }
356}
357
358/// `name`'s value when it is a positive integer, otherwise `default`.
359fn env_limit(name: &str, default: NonZeroUsize) -> NonZeroUsize {
360    std::env::var(name)
361        .ok()
362        .and_then(|value| value.parse().ok())
363        .unwrap_or(default)
364}
365
366/// A default limit; evaluated in a `const` block, so zero fails the build.
367const fn limit(value: usize) -> NonZeroUsize {
368    match NonZeroUsize::new(value) {
369        Some(limit) => limit,
370        None => panic!("a routing limit is never zero"),
371    }
372}
373
374/// How long a registered control connection may go quiet before the relay treats
375/// it as gone.
376///
377/// Two very different numbers, because the two protocol versions keep their
378/// registrations alive in very different ways. A v2 client renews a lease it was
379/// told the length of, every [`control_heartbeat_interval`] — 2s against a 15s
380/// lease. A v1 client has no lease and pings on a schedule of its own, minutes
381/// apart, so its threshold has to be generous enough that jitter never costs it a
382/// healthy registration.
383///
384/// Both the per-connection reader timeout and the periodic sweep read the
385/// threshold from here: they are two views of one rule, and a sweep stricter than
386/// the reader would retire connections that are behaving exactly as designed.
387///
388/// [`control_heartbeat_interval`]: pb_mapper_core::config::control_heartbeat_interval
389pub(crate) fn server_conn_idle_timeout(protocol_version: u16) -> Duration {
390    idle_timeout(protocol_version, server_lease_timeout)
391}
392
393/// [`server_conn_idle_timeout`], taking the lease from `lease_timeout` and only
394/// when the protocol has one.
395fn idle_timeout(protocol_version: u16, lease_timeout: impl FnOnce() -> Duration) -> Duration {
396    if protocol_version >= CONTROL_PROTOCOL_V2 {
397        lease_timeout()
398    } else {
399        LEGACY_SERVER_IDLE_TIMEOUT
400    }
401}
402
403/// Drop one connection from the service map, returning what was registered.
404///
405/// The removed [`ServerConnInfo`] is returned rather than a bare `bool` because it
406/// carries the connection's [`ServerConnInfo::retire_token`], and the map is the
407/// only place that holds it — a caller tearing the connection down needs it after
408/// the entry is gone.
409pub(crate) fn remove_server_conn(
410    server_conn_map: &mut ServerConnMap,
411    key: &ImutableKey,
412    conn_id: RemoteConnId,
413) -> Option<ServerConnInfo> {
414    let ids = server_conn_map.get_mut(key)?;
415    let idx = ids.iter().position(|info| info.conn_id == conn_id)?;
416    let removed = ids.remove(idx);
417    if ids.is_empty() {
418        server_conn_map.remove(key);
419    }
420    Some(removed)
421}
422
423/// Drop a connection from the service map and tear its socket down.
424///
425/// For the callers that found a registration unusable and have nothing further to
426/// say to it — a subscribe whose sender was gone, or whose stream request would not
427/// send. Removing the entry alone would leave the socket task running until its own
428/// reader timed out, holding its connection ID and its slot in the service's quota.
429pub(crate) fn cancel_and_remove_server_conn(
430    server_conn_map: &mut ServerConnMap,
431    key: &ImutableKey,
432    conn_id: RemoteConnId,
433) -> bool {
434    match remove_server_conn(server_conn_map, key, conn_id) {
435        Some(info) => {
436            info.retire_token.cancel();
437            true
438        }
439        None => false,
440    }
441}
442
443pub(crate) fn registered_server_conn_count(server_conn_map: &ServerConnMap) -> usize {
444    server_conn_map.values().map(Vec::len).sum()
445}
446
447pub(crate) fn service_conn_count(server_conn_map: &ServerConnMap, key: &ImutableKey) -> usize {
448    server_conn_map.get(key).map(Vec::len).unwrap_or_default()
449}
450
451pub(crate) fn service_status_connections(
452    server_conn_map: &ServerConnMap,
453    key: &ImutableKey,
454) -> Vec<PbServiceConnStatus> {
455    let now = Instant::now();
456    server_conn_map
457        .get(key)
458        .map(|infos| {
459            infos
460                .iter()
461                .map(|info| PbServiceConnStatus {
462                    conn_id: info.conn_id.into(),
463                    generation: info.generation,
464                    protocol_version: info.protocol_version,
465                    healthy: info.health == ServerConnHealth::Healthy,
466                    last_rx_age_ms: saturating_millis(now.duration_since(info.last_rx_at)),
467                })
468                .collect()
469        })
470        .unwrap_or_default()
471}
472
473/// One registration the sweep decided to retire.
474pub(crate) struct StaleServerConn {
475    pub(crate) key: ImutableKey,
476    pub(crate) conn_id: RemoteConnId,
477    pub(crate) protocol_version: u16,
478    pub(crate) idle_for: Duration,
479}
480
481/// Advance every registration's health by one sweep, and report the ones that ran
482/// out of grace.
483///
484/// Two stages, because retiring on the first sweep would be both premature and
485/// less useful:
486///
487/// 1. A [`ServerConnHealth::Healthy`] connection past its idle threshold becomes
488///    [`ServerConnHealth::Suspect`]. It stays registered but drops out of the
489///    subscribe candidate filter, so a subscriber stops being handed a
490///    registration that has gone quiet — the fix that matters most, and the one
491///    that costs nothing if the connection turns out to be alive.
492/// 2. A connection still `Suspect` and still past its threshold on a later sweep
493///    is returned for retirement.
494///
495/// Any activity resets health to `Healthy` (see [`record_server_conn_activity`]),
496/// so a connection only reaches stage 2 by being quiet across the whole span. That
497/// span also gives the connection's own reader timeout a chance to fire first,
498/// which is the tidier teardown: the socket task unwinds and deregisters itself.
499/// The sweep exists for the case where it does not — a task wedged on a socket
500/// that will never return, which is how three services sat at a full 16/16
501/// connection quota for seventeen hours.
502pub(crate) fn sweep_server_conn_leases(
503    server_conn_map: &mut ServerConnMap,
504    lease_timeout: Duration,
505) -> Vec<StaleServerConn> {
506    let now = Instant::now();
507    let mut stale = Vec::new();
508    for (key, infos) in server_conn_map.iter_mut() {
509        for info in infos.iter_mut() {
510            let idle_for = now.duration_since(info.last_rx_at);
511            if idle_for <= idle_timeout(info.protocol_version, || lease_timeout) {
512                continue;
513            }
514            match info.health {
515                ServerConnHealth::Healthy => info.health = ServerConnHealth::Suspect,
516                ServerConnHealth::Suspect => stale.push(StaleServerConn {
517                    key: key.clone(),
518                    conn_id: info.conn_id,
519                    protocol_version: info.protocol_version,
520                    idle_for,
521                }),
522            }
523        }
524    }
525    stale
526}
527
528/// Count authenticated control activity of `conn_id`'s registration of
529/// `key`: `None` when there is none, otherwise whether that put it back in
530/// rotation.
531pub(crate) fn record_server_conn_activity(
532    server_conn_map: &mut ServerConnMap,
533    key: &ImutableKey,
534    conn_id: RemoteConnId,
535) -> Option<bool> {
536    let info = server_conn_map
537        .get_mut(key)?
538        .iter_mut()
539        .find(|info| info.conn_id == conn_id)?;
540    Some(info.record_activity(Instant::now()))
541}
542
543/// [`record_server_conn_activity`] for `conn_id`'s registration of whichever
544/// service: its key, and whether it is back in rotation.
545pub(crate) fn record_server_conn_activity_by_conn_id(
546    server_conn_map: &mut ServerConnMap,
547    conn_id: RemoteConnId,
548) -> Option<(ImutableKey, bool)> {
549    server_conn_map.iter_mut().find_map(|(key, infos)| {
550        let info = infos.iter_mut().find(|info| info.conn_id == conn_id)?;
551        Some((key.clone(), info.record_activity(Instant::now())))
552    })
553}
554
555/// Subscribes waiting for a registration of their service that they have not
556/// passed over, by service. Each is told to ask again, with a `SubcribeRetry`
557/// while it has no request outstanding, when what it would find changes: a
558/// registration comes, one is back in rotation, or the service has none left.
559/// None asks while nothing changes, and one that still finds none waits anew.
560#[derive(Default)]
561pub(crate) struct ReplacementWaiters {
562    by_key: hashbrown::HashMap<ImutableKey, Vec<(RemoteConnId, ConnTaskSender)>>,
563}
564
565impl ReplacementWaiters {
566    /// Note that the subscribe `conn_id` waits for a registration of `key`.
567    pub(crate) fn wait(
568        &mut self,
569        key: &ImutableKey,
570        conn_id: RemoteConnId,
571        sender: &ConnTaskSender,
572    ) {
573        let waiting = self.by_key.entry(key.clone()).or_default();
574        if waiting.iter().all(|(waiter, _)| *waiter != conn_id) {
575            waiting.push((conn_id, sender.clone()));
576        }
577    }
578
579    /// Tell every subscribe waiting for a registration of `key` to ask again.
580    pub(crate) fn wake(&mut self, key: &ImutableKey) {
581        for (_, sender) in self.by_key.remove(key).into_iter().flatten() {
582            // A waiting subscribe reads its mailbox as things come, so it is
583            // not full; one that is closed has gone.
584            let _ = sender.try_send(ConnTask::SubcribeRetry {
585                reason: format!("a registration of `{key}` may serve it"),
586            });
587        }
588    }
589
590    /// Forget the subscribe `conn_id`, which has ended, if it waits on `key`.
591    pub(crate) fn leave(&mut self, key: &ImutableKey, conn_id: RemoteConnId) {
592        if let Some(waiting) = self.by_key.get_mut(key) {
593            waiting.retain(|(waiter, _)| *waiter != conn_id);
594            if waiting.is_empty() {
595                self.by_key.remove(key);
596            }
597        }
598    }
599}
600
601#[cfg(test)]
602mod lease_tests {
603    use super::*;
604
605    const LEASE: Duration = Duration::from_secs(15);
606
607    fn conn(conn_id: u32, protocol_version: u16, idle_for: Duration) -> ServerConnInfo {
608        ServerConnInfo {
609            last_rx_at: Instant::now() - idle_for,
610            ..ServerConnInfo::for_test(conn_id, 1, protocol_version, None)
611        }
612    }
613
614    fn map(infos: Vec<ServerConnInfo>) -> ServerConnMap {
615        let mut map = ServerConnMap::new();
616        map.insert(ImutableKey::from("sf-backend"), infos);
617        map
618    }
619
620    fn health(map: &ServerConnMap, conn_id: u32) -> ServerConnHealth {
621        map.values()
622            .flatten()
623            .find(|info| info.conn_id == RemoteConnId::from(conn_id))
624            .expect("connection is still registered")
625            .health
626    }
627
628    #[test]
629    fn sweep_marks_before_it_retires() {
630        let mut map = map(vec![conn(1, CONTROL_PROTOCOL_V2, Duration::from_secs(60))]);
631
632        // First sweep only withdraws it from the subscribe candidates.
633        assert!(sweep_server_conn_leases(&mut map, LEASE).is_empty());
634        assert_eq!(health(&map, 1), ServerConnHealth::Suspect);
635
636        let stale = sweep_server_conn_leases(&mut map, LEASE);
637        assert_eq!(stale.len(), 1);
638        assert_eq!(stale[0].conn_id, RemoteConnId::from(1));
639        assert_eq!(stale[0].protocol_version, CONTROL_PROTOCOL_V2);
640    }
641
642    #[test]
643    fn activity_between_sweeps_clears_suspicion() {
644        let key = ImutableKey::from("sf-backend");
645        let mut map = map(vec![conn(1, CONTROL_PROTOCOL_V2, Duration::from_secs(60))]);
646
647        assert!(sweep_server_conn_leases(&mut map, LEASE).is_empty());
648        // Back in rotation, which wakes the subscribes waiting on its service.
649        assert_eq!(
650            record_server_conn_activity(&mut map, &key, RemoteConnId::from(1)),
651            Some(true)
652        );
653
654        assert!(sweep_server_conn_leases(&mut map, LEASE).is_empty());
655        assert_eq!(health(&map, 1), ServerConnHealth::Healthy);
656    }
657
658    #[test]
659    fn a_legacy_registration_keeps_its_far_longer_grace() {
660        // Well past a v2 lease, nowhere near the v1 ping interval.
661        let idle_for = Duration::from_secs(60);
662        let mut map = map(vec![
663            conn(1, 1, idle_for),
664            conn(2, CONTROL_PROTOCOL_V2, idle_for),
665        ]);
666
667        assert!(sweep_server_conn_leases(&mut map, LEASE).is_empty());
668        assert_eq!(health(&map, 1), ServerConnHealth::Healthy);
669        assert_eq!(health(&map, 2), ServerConnHealth::Suspect);
670    }
671
672    /// A provider suspected, answered by a heartbeat and suspected again, a
673    /// few times within one grace, then silent for good, is retired one grace
674    /// after its last suspicion, with one confirmation scheduled at a time.
675    ///
676    /// The clock is the instants passed in, and the router's timers are played
677    /// by delivering each scheduled confirmation once it is due.
678    #[test]
679    fn a_renewed_suspicion_is_confirmed_a_grace_after_it() {
680        let max_ack_budget = Duration::from_secs(5);
681        let mut info = ServerConnInfo::for_test(1, 1, CONTROL_PROTOCOL_V2, None);
682        let start = info.last_rx_at;
683        let at = |ms| start + Duration::from_millis(ms);
684        let mut scheduled = Vec::new();
685
686        scheduled.extend(info.suspect(at(1_000), max_ack_budget));
687        let mut last_suspicion = at(1_000);
688        for flap in 1..=3 {
689            info.record_activity(at(1_000 + flap * 100));
690            last_suspicion = at(1_050 + flap * 100);
691            scheduled.extend(info.suspect(last_suspicion, max_ack_budget));
692            assert!(
693                scheduled.len() <= 1,
694                "a flapping provider is confirmed once at a time"
695            );
696        }
697        let grace = info.suspect_grace();
698
699        let mut retired_at = None;
700        while let Some(next) = (0..scheduled.len()).min_by_key(|&i| scheduled[i].due) {
701            let SuspectCheck { observed_at, due } = scheduled.swap_remove(next);
702            match info.confirm(observed_at) {
703                Confirmation::Retire => {
704                    retired_at = Some(due);
705                    break;
706                }
707                Confirmation::Rearm(check) => scheduled.push(check),
708                Confirmation::Cleared | Confirmation::Stale => {}
709            }
710            assert!(
711                scheduled.len() <= 1,
712                "a flapping provider is confirmed once at a time"
713            );
714        }
715        assert_eq!(
716            retired_at,
717            Some(last_suspicion + grace),
718            "the last suspicion was never confirmed; the provider stays suspect until its lease runs out"
719        );
720    }
721
722    #[test]
723    fn sweep_reports_every_stale_connection_of_a_service() {
724        let idle_for = Duration::from_secs(60);
725        // The third is renewing its lease.
726        let mut map = map(vec![
727            conn(1, CONTROL_PROTOCOL_V2, idle_for),
728            conn(2, CONTROL_PROTOCOL_V2, idle_for),
729            conn(3, CONTROL_PROTOCOL_V2, Duration::from_secs(1)),
730        ]);
731
732        assert!(sweep_server_conn_leases(&mut map, LEASE).is_empty());
733        let mut stale: Vec<u32> = sweep_server_conn_leases(&mut map, LEASE)
734            .into_iter()
735            .map(|stale| stale.conn_id.into())
736            .collect();
737        stale.sort_unstable();
738        assert_eq!(stale, vec![1, 2]);
739        assert_eq!(health(&map, 3), ServerConnHealth::Healthy);
740    }
741}
742
743#[cfg(test)]
744mod pending_stream_tests {
745    use super::*;
746    use crate::service_key::compose_service_key;
747
748    /// The per-namespace count is what admission and rate-limit release read in
749    /// place of scanning every pending stream, so it must always equal the scan.
750    #[test]
751    fn namespace_counts_match_a_scan_of_the_pending_streams() {
752        let keys = [
753            ImutableKey::from("plain"),
754            compose_service_key(1, "echo"),
755            compose_service_key(1, "other"),
756            compose_service_key(2, "echo"),
757        ];
758        let mut pending = PendingStreams::default();
759        // A fixed pseudo-random walk over inserts, replacements and removals.
760        let mut state = 0x2545_f491_u32;
761        for _ in 0..2000 {
762            state ^= state << 13;
763            state ^= state >> 17;
764            state ^= state << 5;
765            let client_id = RemoteConnId::from(state % 16);
766            if state & 0x100 == 0 {
767                pending.remove(&client_id);
768            } else {
769                let key = &keys[(state >> 9) as usize % keys.len()];
770                pending.insert(
771                    client_id,
772                    PendingStream {
773                        server_conn_id: RemoteConnId::from(0),
774                        server_generation: 1,
775                        key: key.clone(),
776                        requested_at: Instant::now(),
777                    },
778                );
779            }
780            for namespace in [0, 1, 2, 3] {
781                let scanned = pending
782                    .iter()
783                    .filter(|(_, stream)| split_scoped_service_key(&stream.key).0 == namespace)
784                    .count();
785                assert_eq!(pending.count_in(namespace), scanned);
786                assert_eq!(pending.per_namespace.contains_key(&namespace), scanned > 0);
787            }
788        }
789    }
790}