1use std::num::NonZeroUsize;
5use std::sync::Arc;
6use std::time::{Duration, Instant};
7
8use pb_mapper_auth::AuthContext;
9use pb_mapper_core::config::server_lease_timeout;
10use pb_mapper_core::conn_id::RemoteConnId;
11use pb_mapper_core::time::saturating_millis;
12use pb_mapper_protocol::command::{CONTROL_PROTOCOL_V2, PbServiceConnStatus};
13use tokio_util::sync::CancellationToken;
14
15use crate::server::LEGACY_SERVER_IDLE_TIMEOUT;
16use crate::service_key::split_scoped_service_key;
17use crate::task::{ConnTask, ConnTaskSender};
18
19pub type ImutableKey = Arc<str>;
20
21pub(crate) const SERVICE_NOT_AVAILABLE: &str = "service_not_available";
25
26#[derive(Debug, Clone, Copy, PartialEq, Eq)]
27pub enum ServerConnHealth {
28 Healthy,
29 Suspect,
30}
31
32#[derive(Debug, Clone)]
33pub struct ServerConnInfo {
34 pub conn_id: RemoteConnId,
35 pub generation: u64,
36 pub health: ServerConnHealth,
37 pub need_codec: bool,
38 pub is_datagram: bool,
39 pub protocol_version: u16,
40 pub last_rx_at: Instant,
41 pub stream_ack_budget: Duration,
43 pub suspect_check: Option<Instant>,
46 pub retire_token: CancellationToken,
53 pub direct: Option<AuthContext>,
54}
55
56#[derive(Debug, Clone, Copy, PartialEq, Eq)]
59pub(crate) struct SuspectCheck {
60 pub(crate) observed_at: Instant,
61 pub(crate) due: Instant,
62}
63
64#[derive(Debug, Clone, Copy, PartialEq, Eq)]
66pub(crate) enum Confirmation {
67 Stale,
69 Cleared,
71 Retire,
73 Rearm(SuspectCheck),
76}
77
78impl ServerConnInfo {
79 pub(crate) fn record_activity(&mut self, now: Instant) -> bool {
82 self.last_rx_at = now;
83 self.restore()
84 }
85
86 pub(crate) fn restore(&mut self) -> bool {
88 std::mem::replace(&mut self.health, ServerConnHealth::Healthy) == ServerConnHealth::Suspect
89 }
90
91 pub(crate) fn suspect(
97 &mut self,
98 now: Instant,
99 max_ack_budget: Duration,
100 ) -> Option<SuspectCheck> {
101 self.stream_ack_budget = self.stream_ack_budget.saturating_mul(2).min(max_ack_budget);
102 self.health = ServerConnHealth::Suspect;
105 match self.suspect_check {
106 Some(pending) if self.last_rx_at <= pending => None,
109 Some(_) => {
113 self.suspect_check = Some(now);
114 None
115 }
116 None => {
117 self.suspect_check = Some(now);
118 Some(self.check(now))
119 }
120 }
121 }
122
123 pub(crate) fn confirm(&mut self, observed_at: Instant) -> Confirmation {
125 match self.suspect_check {
126 Some(pending) if pending == observed_at => {}
127 Some(pending) if pending > observed_at => {
128 return Confirmation::Rearm(self.check(pending));
129 }
130 _ => return Confirmation::Stale,
131 }
132 self.suspect_check = None;
133 if self.last_rx_at > observed_at {
134 Confirmation::Cleared
135 } else {
136 Confirmation::Retire
137 }
138 }
139
140 pub(crate) fn suspect_grace(&self) -> Duration {
142 pb_mapper_core::config::control_suspect_grace().max(self.stream_ack_budget)
143 }
144
145 fn check(&self, observed_at: Instant) -> SuspectCheck {
146 SuspectCheck {
147 observed_at,
148 due: observed_at + self.suspect_grace(),
149 }
150 }
151}
152
153#[cfg(test)]
154impl ServerConnInfo {
155 pub(crate) fn for_test(
158 conn_id: u32,
159 generation: u64,
160 protocol_version: u16,
161 direct: Option<AuthContext>,
162 ) -> Self {
163 Self {
164 conn_id: RemoteConnId::from(conn_id),
165 generation,
166 health: ServerConnHealth::Healthy,
167 need_codec: false,
168 is_datagram: false,
169 protocol_version,
170 last_rx_at: Instant::now(),
171 stream_ack_budget: pb_mapper_core::config::stream_ack_timeout(),
172 suspect_check: None,
173 retire_token: CancellationToken::new(),
174 direct,
175 }
176 }
177}
178
179pub type ServerConnMap = hashbrown::HashMap<ImutableKey, Vec<ServerConnInfo>>;
180
181#[derive(Clone)]
184pub(crate) struct PendingStream {
185 pub(crate) server_conn_id: RemoteConnId,
186 pub(crate) server_generation: u64,
187 pub(crate) key: ImutableKey,
188 pub(crate) requested_at: Instant,
189}
190
191#[derive(Default)]
194pub(crate) struct PendingStreams {
195 by_client: hashbrown::HashMap<RemoteConnId, PendingStream>,
196 per_namespace: hashbrown::HashMap<u64, usize>,
197}
198
199impl PendingStreams {
200 pub(crate) fn get(&self, client_id: &RemoteConnId) -> Option<&PendingStream> {
201 self.by_client.get(client_id)
202 }
203
204 pub(crate) fn contains(&self, client_id: &RemoteConnId) -> bool {
205 self.by_client.contains_key(client_id)
206 }
207
208 pub(crate) fn iter(&self) -> impl Iterator<Item = (&RemoteConnId, &PendingStream)> {
209 self.by_client.iter()
210 }
211
212 pub(crate) fn insert(&mut self, client_id: RemoteConnId, stream: PendingStream) {
214 let namespace = split_scoped_service_key(&stream.key).0;
215 match self.by_client.insert(client_id, stream) {
216 Some(replaced) if split_scoped_service_key(&replaced.key).0 == namespace => return,
217 Some(replaced) => self.uncount(split_scoped_service_key(&replaced.key).0),
218 None => {}
219 }
220 *self.per_namespace.entry(namespace).or_default() += 1;
221 }
222
223 pub(crate) fn remove(&mut self, client_id: &RemoteConnId) -> Option<PendingStream> {
224 let removed = self.by_client.remove(client_id)?;
225 self.uncount(split_scoped_service_key(&removed.key).0);
226 Some(removed)
227 }
228
229 pub(crate) fn count_in(&self, namespace: u64) -> usize {
231 self.per_namespace
232 .get(&namespace)
233 .copied()
234 .unwrap_or_default()
235 }
236
237 pub(crate) fn has_room(&self, namespace: u64, extra: usize, limit: NonZeroUsize) -> bool {
239 extra + self.count_in(namespace) < limit.get()
240 }
241
242 fn uncount(&mut self, namespace: u64) {
243 let Some(count) = self.per_namespace.get_mut(&namespace) else {
244 return;
245 };
246 *count = count.saturating_sub(1);
247 if *count == 0 {
248 self.per_namespace.remove(&namespace);
249 }
250 }
251}
252
253pub(crate) struct NamespaceRateLimits {
255 buckets: hashbrown::HashMap<u64, NamespaceRateLimit>,
256 rate_per_second: NonZeroUsize,
257 burst: NonZeroUsize,
258}
259
260impl NamespaceRateLimits {
261 pub(crate) fn new(rate_per_second: NonZeroUsize, burst: NonZeroUsize) -> Self {
262 Self {
263 buckets: hashbrown::HashMap::new(),
264 rate_per_second,
265 burst,
266 }
267 }
268
269 pub(crate) fn try_take(&mut self, namespace: u64) -> bool {
271 self.buckets
272 .entry(namespace)
273 .or_insert_with(|| NamespaceRateLimit::new(self.rate_per_second, self.burst))
274 .allow()
275 }
276
277 pub(crate) fn release(&mut self, namespace: u64) {
279 self.buckets.remove(&namespace);
280 }
281}
282
283struct NamespaceRateLimit {
284 tokens: f64,
285 last_refill: Instant,
286 rate_per_second: f64,
287 burst: f64,
288}
289
290impl NamespaceRateLimit {
291 fn new(rate_per_second: NonZeroUsize, burst: NonZeroUsize) -> Self {
292 Self {
293 tokens: burst.get() as f64,
294 last_refill: Instant::now(),
295 rate_per_second: rate_per_second.get() as f64,
296 burst: burst.get() as f64,
297 }
298 }
299
300 fn allow(&mut self) -> bool {
301 let now = Instant::now();
302 self.tokens = (self.tokens
303 + now.duration_since(self.last_refill).as_secs_f64() * self.rate_per_second)
304 .min(self.burst);
305 self.last_refill = now;
306 if self.tokens < 1.0 {
307 false
308 } else {
309 self.tokens -= 1.0;
310 true
311 }
312 }
313}
314
315pub(crate) struct RoutingConfig {
318 pub(crate) max_services_per_namespace: NonZeroUsize,
319 pub(crate) max_register_connections_per_service: NonZeroUsize,
320 pub(crate) max_streams_per_namespace: NonZeroUsize,
321 pub(crate) new_streams_per_second: NonZeroUsize,
322 pub(crate) new_streams_burst: NonZeroUsize,
323 pub(crate) stream_ack_timeout: Duration,
326 pub(crate) lease_timeout: Duration,
328}
329
330impl RoutingConfig {
331 pub(crate) fn from_env() -> Self {
332 Self {
333 max_services_per_namespace: env_limit(
334 "PB_MAPPER_MAX_SERVICES_PER_NAMESPACE",
335 const { limit(256) },
336 ),
337 max_register_connections_per_service: env_limit(
338 "PB_MAPPER_MAX_REGISTER_CONNECTIONS_PER_SERVICE",
339 const { limit(16) },
340 ),
341 max_streams_per_namespace: env_limit(
342 "PB_MAPPER_MAX_STREAMS_PER_NAMESPACE",
343 const { limit(1024) },
344 ),
345 new_streams_per_second: pb_mapper_core::config::new_streams_per_second(),
346 new_streams_burst: env_limit("PB_MAPPER_NEW_STREAMS_BURST", const { limit(200) }),
347 stream_ack_timeout: pb_mapper_core::config::stream_ack_timeout(),
348 lease_timeout: server_lease_timeout(),
349 }
350 }
351
352 pub(crate) fn max_stream_ack_budget(&self) -> Duration {
354 Duration::from_secs(5).max(self.stream_ack_timeout)
355 }
356}
357
358fn env_limit(name: &str, default: NonZeroUsize) -> NonZeroUsize {
360 std::env::var(name)
361 .ok()
362 .and_then(|value| value.parse().ok())
363 .unwrap_or(default)
364}
365
366const fn limit(value: usize) -> NonZeroUsize {
368 match NonZeroUsize::new(value) {
369 Some(limit) => limit,
370 None => panic!("a routing limit is never zero"),
371 }
372}
373
374pub(crate) fn server_conn_idle_timeout(protocol_version: u16) -> Duration {
390 idle_timeout(protocol_version, server_lease_timeout)
391}
392
393fn idle_timeout(protocol_version: u16, lease_timeout: impl FnOnce() -> Duration) -> Duration {
396 if protocol_version >= CONTROL_PROTOCOL_V2 {
397 lease_timeout()
398 } else {
399 LEGACY_SERVER_IDLE_TIMEOUT
400 }
401}
402
403pub(crate) fn remove_server_conn(
410 server_conn_map: &mut ServerConnMap,
411 key: &ImutableKey,
412 conn_id: RemoteConnId,
413) -> Option<ServerConnInfo> {
414 let ids = server_conn_map.get_mut(key)?;
415 let idx = ids.iter().position(|info| info.conn_id == conn_id)?;
416 let removed = ids.remove(idx);
417 if ids.is_empty() {
418 server_conn_map.remove(key);
419 }
420 Some(removed)
421}
422
423pub(crate) fn cancel_and_remove_server_conn(
430 server_conn_map: &mut ServerConnMap,
431 key: &ImutableKey,
432 conn_id: RemoteConnId,
433) -> bool {
434 match remove_server_conn(server_conn_map, key, conn_id) {
435 Some(info) => {
436 info.retire_token.cancel();
437 true
438 }
439 None => false,
440 }
441}
442
443pub(crate) fn registered_server_conn_count(server_conn_map: &ServerConnMap) -> usize {
444 server_conn_map.values().map(Vec::len).sum()
445}
446
447pub(crate) fn service_conn_count(server_conn_map: &ServerConnMap, key: &ImutableKey) -> usize {
448 server_conn_map.get(key).map(Vec::len).unwrap_or_default()
449}
450
451pub(crate) fn service_status_connections(
452 server_conn_map: &ServerConnMap,
453 key: &ImutableKey,
454) -> Vec<PbServiceConnStatus> {
455 let now = Instant::now();
456 server_conn_map
457 .get(key)
458 .map(|infos| {
459 infos
460 .iter()
461 .map(|info| PbServiceConnStatus {
462 conn_id: info.conn_id.into(),
463 generation: info.generation,
464 protocol_version: info.protocol_version,
465 healthy: info.health == ServerConnHealth::Healthy,
466 last_rx_age_ms: saturating_millis(now.duration_since(info.last_rx_at)),
467 })
468 .collect()
469 })
470 .unwrap_or_default()
471}
472
473pub(crate) struct StaleServerConn {
475 pub(crate) key: ImutableKey,
476 pub(crate) conn_id: RemoteConnId,
477 pub(crate) protocol_version: u16,
478 pub(crate) idle_for: Duration,
479}
480
481pub(crate) fn sweep_server_conn_leases(
503 server_conn_map: &mut ServerConnMap,
504 lease_timeout: Duration,
505) -> Vec<StaleServerConn> {
506 let now = Instant::now();
507 let mut stale = Vec::new();
508 for (key, infos) in server_conn_map.iter_mut() {
509 for info in infos.iter_mut() {
510 let idle_for = now.duration_since(info.last_rx_at);
511 if idle_for <= idle_timeout(info.protocol_version, || lease_timeout) {
512 continue;
513 }
514 match info.health {
515 ServerConnHealth::Healthy => info.health = ServerConnHealth::Suspect,
516 ServerConnHealth::Suspect => stale.push(StaleServerConn {
517 key: key.clone(),
518 conn_id: info.conn_id,
519 protocol_version: info.protocol_version,
520 idle_for,
521 }),
522 }
523 }
524 }
525 stale
526}
527
528pub(crate) fn record_server_conn_activity(
532 server_conn_map: &mut ServerConnMap,
533 key: &ImutableKey,
534 conn_id: RemoteConnId,
535) -> Option<bool> {
536 let info = server_conn_map
537 .get_mut(key)?
538 .iter_mut()
539 .find(|info| info.conn_id == conn_id)?;
540 Some(info.record_activity(Instant::now()))
541}
542
543pub(crate) fn record_server_conn_activity_by_conn_id(
546 server_conn_map: &mut ServerConnMap,
547 conn_id: RemoteConnId,
548) -> Option<(ImutableKey, bool)> {
549 server_conn_map.iter_mut().find_map(|(key, infos)| {
550 let info = infos.iter_mut().find(|info| info.conn_id == conn_id)?;
551 Some((key.clone(), info.record_activity(Instant::now())))
552 })
553}
554
555#[derive(Default)]
561pub(crate) struct ReplacementWaiters {
562 by_key: hashbrown::HashMap<ImutableKey, Vec<(RemoteConnId, ConnTaskSender)>>,
563}
564
565impl ReplacementWaiters {
566 pub(crate) fn wait(
568 &mut self,
569 key: &ImutableKey,
570 conn_id: RemoteConnId,
571 sender: &ConnTaskSender,
572 ) {
573 let waiting = self.by_key.entry(key.clone()).or_default();
574 if waiting.iter().all(|(waiter, _)| *waiter != conn_id) {
575 waiting.push((conn_id, sender.clone()));
576 }
577 }
578
579 pub(crate) fn wake(&mut self, key: &ImutableKey) {
581 for (_, sender) in self.by_key.remove(key).into_iter().flatten() {
582 let _ = sender.try_send(ConnTask::SubcribeRetry {
585 reason: format!("a registration of `{key}` may serve it"),
586 });
587 }
588 }
589
590 pub(crate) fn leave(&mut self, key: &ImutableKey, conn_id: RemoteConnId) {
592 if let Some(waiting) = self.by_key.get_mut(key) {
593 waiting.retain(|(waiter, _)| *waiter != conn_id);
594 if waiting.is_empty() {
595 self.by_key.remove(key);
596 }
597 }
598 }
599}
600
601#[cfg(test)]
602mod lease_tests {
603 use super::*;
604
605 const LEASE: Duration = Duration::from_secs(15);
606
607 fn conn(conn_id: u32, protocol_version: u16, idle_for: Duration) -> ServerConnInfo {
608 ServerConnInfo {
609 last_rx_at: Instant::now() - idle_for,
610 ..ServerConnInfo::for_test(conn_id, 1, protocol_version, None)
611 }
612 }
613
614 fn map(infos: Vec<ServerConnInfo>) -> ServerConnMap {
615 let mut map = ServerConnMap::new();
616 map.insert(ImutableKey::from("sf-backend"), infos);
617 map
618 }
619
620 fn health(map: &ServerConnMap, conn_id: u32) -> ServerConnHealth {
621 map.values()
622 .flatten()
623 .find(|info| info.conn_id == RemoteConnId::from(conn_id))
624 .expect("connection is still registered")
625 .health
626 }
627
628 #[test]
629 fn sweep_marks_before_it_retires() {
630 let mut map = map(vec![conn(1, CONTROL_PROTOCOL_V2, Duration::from_secs(60))]);
631
632 assert!(sweep_server_conn_leases(&mut map, LEASE).is_empty());
634 assert_eq!(health(&map, 1), ServerConnHealth::Suspect);
635
636 let stale = sweep_server_conn_leases(&mut map, LEASE);
637 assert_eq!(stale.len(), 1);
638 assert_eq!(stale[0].conn_id, RemoteConnId::from(1));
639 assert_eq!(stale[0].protocol_version, CONTROL_PROTOCOL_V2);
640 }
641
642 #[test]
643 fn activity_between_sweeps_clears_suspicion() {
644 let key = ImutableKey::from("sf-backend");
645 let mut map = map(vec![conn(1, CONTROL_PROTOCOL_V2, Duration::from_secs(60))]);
646
647 assert!(sweep_server_conn_leases(&mut map, LEASE).is_empty());
648 assert_eq!(
650 record_server_conn_activity(&mut map, &key, RemoteConnId::from(1)),
651 Some(true)
652 );
653
654 assert!(sweep_server_conn_leases(&mut map, LEASE).is_empty());
655 assert_eq!(health(&map, 1), ServerConnHealth::Healthy);
656 }
657
658 #[test]
659 fn a_legacy_registration_keeps_its_far_longer_grace() {
660 let idle_for = Duration::from_secs(60);
662 let mut map = map(vec![
663 conn(1, 1, idle_for),
664 conn(2, CONTROL_PROTOCOL_V2, idle_for),
665 ]);
666
667 assert!(sweep_server_conn_leases(&mut map, LEASE).is_empty());
668 assert_eq!(health(&map, 1), ServerConnHealth::Healthy);
669 assert_eq!(health(&map, 2), ServerConnHealth::Suspect);
670 }
671
672 #[test]
679 fn a_renewed_suspicion_is_confirmed_a_grace_after_it() {
680 let max_ack_budget = Duration::from_secs(5);
681 let mut info = ServerConnInfo::for_test(1, 1, CONTROL_PROTOCOL_V2, None);
682 let start = info.last_rx_at;
683 let at = |ms| start + Duration::from_millis(ms);
684 let mut scheduled = Vec::new();
685
686 scheduled.extend(info.suspect(at(1_000), max_ack_budget));
687 let mut last_suspicion = at(1_000);
688 for flap in 1..=3 {
689 info.record_activity(at(1_000 + flap * 100));
690 last_suspicion = at(1_050 + flap * 100);
691 scheduled.extend(info.suspect(last_suspicion, max_ack_budget));
692 assert!(
693 scheduled.len() <= 1,
694 "a flapping provider is confirmed once at a time"
695 );
696 }
697 let grace = info.suspect_grace();
698
699 let mut retired_at = None;
700 while let Some(next) = (0..scheduled.len()).min_by_key(|&i| scheduled[i].due) {
701 let SuspectCheck { observed_at, due } = scheduled.swap_remove(next);
702 match info.confirm(observed_at) {
703 Confirmation::Retire => {
704 retired_at = Some(due);
705 break;
706 }
707 Confirmation::Rearm(check) => scheduled.push(check),
708 Confirmation::Cleared | Confirmation::Stale => {}
709 }
710 assert!(
711 scheduled.len() <= 1,
712 "a flapping provider is confirmed once at a time"
713 );
714 }
715 assert_eq!(
716 retired_at,
717 Some(last_suspicion + grace),
718 "the last suspicion was never confirmed; the provider stays suspect until its lease runs out"
719 );
720 }
721
722 #[test]
723 fn sweep_reports_every_stale_connection_of_a_service() {
724 let idle_for = Duration::from_secs(60);
725 let mut map = map(vec![
727 conn(1, CONTROL_PROTOCOL_V2, idle_for),
728 conn(2, CONTROL_PROTOCOL_V2, idle_for),
729 conn(3, CONTROL_PROTOCOL_V2, Duration::from_secs(1)),
730 ]);
731
732 assert!(sweep_server_conn_leases(&mut map, LEASE).is_empty());
733 let mut stale: Vec<u32> = sweep_server_conn_leases(&mut map, LEASE)
734 .into_iter()
735 .map(|stale| stale.conn_id.into())
736 .collect();
737 stale.sort_unstable();
738 assert_eq!(stale, vec![1, 2]);
739 assert_eq!(health(&map, 3), ServerConnHealth::Healthy);
740 }
741}
742
743#[cfg(test)]
744mod pending_stream_tests {
745 use super::*;
746 use crate::service_key::compose_service_key;
747
748 #[test]
751 fn namespace_counts_match_a_scan_of_the_pending_streams() {
752 let keys = [
753 ImutableKey::from("plain"),
754 compose_service_key(1, "echo"),
755 compose_service_key(1, "other"),
756 compose_service_key(2, "echo"),
757 ];
758 let mut pending = PendingStreams::default();
759 let mut state = 0x2545_f491_u32;
761 for _ in 0..2000 {
762 state ^= state << 13;
763 state ^= state >> 17;
764 state ^= state << 5;
765 let client_id = RemoteConnId::from(state % 16);
766 if state & 0x100 == 0 {
767 pending.remove(&client_id);
768 } else {
769 let key = &keys[(state >> 9) as usize % keys.len()];
770 pending.insert(
771 client_id,
772 PendingStream {
773 server_conn_id: RemoteConnId::from(0),
774 server_generation: 1,
775 key: key.clone(),
776 requested_at: Instant::now(),
777 },
778 );
779 }
780 for namespace in [0, 1, 2, 3] {
781 let scanned = pending
782 .iter()
783 .filter(|(_, stream)| split_scoped_service_key(&stream.key).0 == namespace)
784 .count();
785 assert_eq!(pending.count_in(namespace), scanned);
786 assert_eq!(pending.per_namespace.contains_key(&namespace), scanned > 0);
787 }
788 }
789 }
790}