1use std::{
2 collections::{BTreeMap, HashMap, HashSet},
3 error::Error,
4 fmt,
5 sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
6 time::Duration,
7};
8
9use subc_control::{ClientControlResponse, RouteCloseReason};
10use subc_protocol::{
11 manifest::Concurrency,
12 session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
13 ErrorBody, Flags, FrameType, Principal, Priority,
14};
15use tokio::sync::{oneshot, Semaphore};
16use tokio::time::Instant;
17use tracing::{debug, info, warn};
18
19use crate::{
20 control::{RouteBindBreakers, RouteBindConcurrency},
21 observability::DaemonCounters,
22 registry::ConnectionId,
23 router::FrameSink,
24 scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
25 Frame, ProjectRootId,
26};
27
28const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
30
31const STATELESS_PARALLEL_WINDOW: usize = 1024;
33
34const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
38
39#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
44pub struct ModuleEndpointId {
45 pub connection_id: ConnectionId,
46 pub generation: u64,
47}
48
49#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
51pub(crate) struct ClientRouteKey {
52 pub connection_id: ConnectionId,
53 pub channel: u16,
54}
55
56#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
58pub(crate) struct ModuleRouteKey {
59 pub endpoint: ModuleEndpointId,
60 pub channel: u16,
61}
62
63#[derive(Debug)]
64pub(crate) struct RouteBinding {
65 pub client_connection_id: ConnectionId,
66 pub client_sink: FrameSink,
67 pub client_negotiated_ver: u8,
68 pub client_channel: u16,
69 pub client_epoch: u32,
70 pub module_id: String,
71 pub module_endpoint: ModuleEndpointId,
72 pub module_sink: FrameSink,
73 pub module_negotiated_ver: u8,
74 pub module_channel: u16,
75 pub module_epoch: u32,
76 pub principal: Principal,
77 pub project_root: Option<ProjectRootId>,
78 pub bound_at: Instant,
79 pub flow: Arc<ChannelFlow>,
80 pub scope: Option<BoundScope>,
83}
84
85#[derive(Debug, Clone)]
86pub(crate) enum DataRoute {
87 Client(DataRouteState),
88 Module(DataRouteState),
89}
90
91#[derive(Debug, Clone)]
92pub(crate) enum DataRouteState {
93 Bound(Arc<RouteBinding>),
94 Reserved,
95 EpochMismatch,
96 Absent,
97}
98
99#[derive(Debug, Clone, Copy, PartialEq, Eq)]
128pub(crate) enum GoodbyeTargetKind {
129 Client,
130 Module,
131}
132
133#[derive(Debug, Clone)]
134pub(crate) struct GoodbyeTarget {
135 pub connection_id: ConnectionId,
136 pub sink: FrameSink,
137 pub negotiated_ver: u8,
138 pub channel: u16,
139 pub epoch: u32,
140 pub kind: GoodbyeTargetKind,
141 pub module_id: Option<String>,
146}
147
148#[derive(Debug, Clone, Copy)]
151pub(crate) struct UndeliveredFrame<'a> {
152 pub module_id: Option<&'a str>,
154 pub sink: &'a FrameSink,
156}
157
158pub(crate) fn principal_label(principal: &Principal) -> String {
160 match principal {
161 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
162 Principal::Direct => "direct".to_string(),
163 other => format!("{other:?}"),
164 }
165}
166
167fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
170 let labels = inner
171 .client_to_module
172 .iter()
173 .filter(|(key, _)| key.connection_id == connection_id)
174 .map(|(_, route)| principal_label(&route.principal))
175 .collect::<std::collections::BTreeSet<_>>();
176 if labels.is_empty() {
177 "none".to_string()
178 } else {
179 labels.into_iter().collect::<Vec<_>>().join(",")
180 }
181}
182
183impl GoodbyeTarget {
184 pub(crate) fn close_on_delivery_failure(&self) -> bool {
187 matches!(self.kind, GoodbyeTargetKind::Client)
188 }
189}
190
191pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
204
205pub(crate) fn send_module_route_goodbye(
217 counters: &DaemonCounters,
218 sink: &FrameSink,
219 frame: Frame,
220 module_id: Option<&str>,
221 context: &'static str,
222) {
223 let channel = frame.header.channel;
224 let epoch = frame.header.epoch;
225 let Err(err) = sink.try_send(frame.clone()) else {
226 return;
227 };
228 let runtime = match tokio::runtime::Handle::try_current() {
231 Ok(runtime) if !sink.is_closed() => runtime,
232 _ => {
233 counters.increment_goodbye_relay_module_dropped(module_id);
234 warn!(
235 module_id = module_id.unwrap_or("unknown"),
236 route_channel = channel,
237 route_epoch = epoch,
238 error = %err,
239 context,
240 "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
241 );
242 return;
243 }
244 };
245 debug!(
246 module_id = module_id.unwrap_or("unknown"),
247 route_channel = channel,
248 route_epoch = epoch,
249 error = %err,
250 context,
251 "module egress queue refused route GOODBYE; delivering it once the module frees room"
252 );
253 let counters = counters.clone();
254 let sink = sink.clone();
255 let module_id = module_id.map(str::to_string);
256 runtime.spawn(async move {
257 let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
258 let why = match outcome {
259 Ok(Ok(())) => {
260 debug!(
261 module_id = module_id.as_deref().unwrap_or("unknown"),
262 route_channel = channel,
263 route_epoch = epoch,
264 context,
265 "late route GOODBYE delivered to module"
266 );
267 return;
268 }
269 Ok(Err(err)) => err.to_string(),
270 Err(_) => format!(
271 "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
272 ),
273 };
274 counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
275 warn!(
276 module_id = module_id.as_deref().unwrap_or("unknown"),
277 route_channel = channel,
278 route_epoch = epoch,
279 error = %why,
280 context,
281 "route GOODBYE to module dropped under backpressure; not closing shared module connection"
282 );
283 });
284}
285
286#[derive(Debug, Clone)]
292pub(crate) struct EndpointRoute {
293 pub goodbye_target: GoodbyeTarget,
294 pub principal: Principal,
295 pub bound_at: Instant,
296 pub draining: bool,
297 pub drain_reason: Option<RouteCloseReason>,
301}
302
303#[derive(Debug, Clone)]
306pub(crate) struct ScopeDrainedRoute {
307 pub reason: RouteCloseReason,
308 pub scope: BoundScope,
309 pub module_id: String,
310 pub client: GoodbyeTarget,
311 pub module: GoodbyeTarget,
312}
313
314#[derive(Debug)]
315pub(crate) struct PendingRouteBindRelay {
316 pub endpoint: ModuleEndpointId,
317 pub module_sink: FrameSink,
318 pub negotiated_ver: u8,
319 pub client_channel: u16,
320 pub client_epoch: u32,
321 pub module_channel: u16,
322 pub module_epoch: u32,
323 pub corr: u64,
324 pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
325}
326
327#[derive(Debug, Clone)]
328pub(crate) struct ModuleDrainTarget {
329 pub endpoint: ModuleEndpointId,
330 pub sink: FrameSink,
331 pub negotiated_ver: u8,
332 pub abandoned_bindings: Vec<GoodbyeTarget>,
333 pub excluded_subscriptions: u32,
334}
335
336#[cfg(unix)]
339#[derive(Debug, Clone)]
340pub(crate) struct ModuleConnectionTarget {
341 pub module_id: String,
342 pub endpoint: ModuleEndpointId,
343 pub sink: FrameSink,
344 pub negotiated_ver: u8,
345}
346
347#[derive(Debug, Clone)]
348pub(crate) enum RouteBindRelayOutcome {
349 Accepted,
350 Rejected(ErrorBody),
351 ModuleGone(String),
352}
353
354#[derive(Debug, Clone, Copy, PartialEq, Eq)]
356pub(crate) struct ForwardingCutover {
357 pub promoted: ModuleEndpointId,
359 pub incumbent: Option<ModuleEndpointId>,
362}
363
364#[derive(Debug)]
366pub(crate) struct ConnectionCleanup {
367 pub released: Vec<GoodbyeTarget>,
369 pub abandoned_relays: u32,
372}
373
374#[derive(Debug, Clone)]
375pub(crate) struct PendingRelayCompletion {
376 pub settled: bool,
377 pub abandoned: Option<GoodbyeTarget>,
378}
379
380#[derive(Debug)]
381pub(crate) struct PendingModuleControlRpc {
382 pub endpoint: ModuleEndpointId,
383 pub module_sink: FrameSink,
384 pub negotiated_ver: u8,
385 pub corr: u64,
386 pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
387}
388
389#[derive(Debug, Clone)]
390pub(crate) enum ModuleControlRpcOutcome {
391 Response(ModuleControlResponse),
392 Rejected(ErrorBody),
393 ModuleGone(String),
394 MalformedResponse(String),
395 UnexpectedOp { expected: String, actual: String },
396 DeadlineElapsed,
397}
398
399#[derive(Debug, Clone, PartialEq, Eq)]
400pub(crate) enum ModuleControlRpcCompletion {
401 Unknown,
402 Settled,
403 LateHealthAnswer {
404 module_id: String,
405 latency: Duration,
406 },
407}
408
409#[derive(Debug)]
410struct PendingModuleControlRpcEntry {
411 expected_op: String,
412 deadline: Instant,
413 health_probe_started_at: Option<Instant>,
414 sender: oneshot::Sender<ModuleControlRpcOutcome>,
415}
416
417#[derive(Debug)]
418struct HealthProbeTombstone {
419 expected_op: String,
420 module_id: String,
421 probe_started_at: Instant,
422 expires_at: Instant,
423}
424
425#[derive(Debug, Clone)]
426struct RouteReservation {
427 client_key: ClientRouteKey,
428 module_key: ModuleRouteKey,
429 client_epoch: u32,
430 module_epoch: u32,
431 project_root: Option<ProjectRootId>,
432}
433
434#[derive(Debug)]
435struct PendingRouteBindRelayEntry {
436 reservation: RouteReservation,
437 client_sink: FrameSink,
438 client_negotiated_ver: u8,
439 client_permit: crate::router::EgressPermit,
440 route_open_frame: Frame,
441 principal: Principal,
442 scope: Option<BoundScope>,
445 deadline: Instant,
446 relay_enqueued: bool,
457 sender: oneshot::Sender<RouteBindRelayOutcome>,
458}
459
460#[derive(Debug, Clone, Copy, PartialEq, Eq)]
463pub(crate) enum RouteBindRelayMark {
464 Marked,
466 SettledByReply,
470 ReleasedWithoutReply,
474}
475
476#[derive(Debug, Clone)]
477pub(crate) enum RouteRelease {
478 Removed(GoodbyeTarget),
479 Stale,
480 Absent,
481}
482
483#[derive(Debug, Clone)]
484pub(crate) enum RoutePollSnapshot {
485 Bound {
486 module_id: String,
487 status: Option<String>,
488 },
489 Absent,
490}
491
492#[derive(Debug, Clone)]
493struct ModuleConnection {
494 endpoint: ModuleEndpointId,
495 sink: FrameSink,
496 negotiated_ver: u8,
497 concurrency: Concurrency,
498}
499
500#[derive(Debug, Default)]
501struct ForwardingInner {
502 operator_confirms: Arc<crate::operator_confirm::OperatorConfirms>,
503 daemon_draining: bool,
504 daemon_drain_results: HashMap<ModuleEndpointId, bool>,
509 modules_by_id: HashMap<String, ModuleConnection>,
513 candidates_by_id: HashMap<String, ModuleConnection>,
519 superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
525 endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
526 module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
527 draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
531 closing_connections: HashSet<ConnectionId>,
532 next_generation: u64,
533 reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
534 reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
535 next_client_channel: HashMap<ConnectionId, u16>,
536 next_module_channel: HashMap<ModuleEndpointId, u16>,
537 client_slot_epochs: HashMap<ClientRouteKey, u32>,
538 module_slot_epochs: HashMap<ModuleRouteKey, u32>,
539 last_published_epoch: HashMap<ClientRouteKey, u32>,
540 client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
541 module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
542 status: HashMap<(ClientRouteKey, u32), String>,
543 pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
544 relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
549 scope_tags: HashMap<(String, String), ScopeTag>,
554 next_control_corr: HashMap<ModuleEndpointId, u64>,
555 pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
556 health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
557}
558
559#[derive(Debug, Clone)]
560pub(crate) struct CloseReason {
561 code: &'static str,
562 message: String,
563}
564
565impl CloseReason {
566 pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
567 Self {
568 code,
569 message: message.into(),
570 }
571 }
572}
573
574impl fmt::Display for CloseReason {
575 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
576 write!(f, "{}: {}", self.code, self.message)
577 }
578}
579
580pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
581
582#[derive(Debug, Default)]
584pub struct ForwardingTable {
585 inner: Arc<RwLock<ForwardingInner>>,
586 #[cfg(feature = "test-support")]
587 route_closing_hold: Mutex<Option<oneshot::Receiver<()>>>,
588 close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
589 counters: DaemonCounters,
590 route_bind_breakers: RouteBindBreakers,
595 route_bind_concurrency: RouteBindConcurrency,
598 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
602}
603
604impl ForwardingTable {
605 #[cfg(feature = "test-support")]
647 pub fn configure_operator_script(
648 &self,
649 script: &std::path::Path,
650 events: Option<&std::path::Path>,
651 limits: Option<crate::operator_confirm::OperatorLimits>,
652 ) {
653 self.operator_confirms()
654 .configure_script(script, events, limits);
655 }
656
657 #[cfg(feature = "test-support")]
661 pub fn hold_route_closing_for_test(&self) -> oneshot::Sender<()> {
662 let (release, hold) = oneshot::channel();
663 *self.route_closing_hold.lock().unwrap() = Some(hold);
664 release
665 }
666
667 #[cfg(feature = "test-support")]
668 pub(crate) async fn wait_before_route_closing_for_test(&self) {
669 let hold = self.route_closing_hold.lock().unwrap().take();
670 if let Some(hold) = hold {
671 let _ = hold.await;
672 }
673 }
674
675 pub(crate) fn operator_confirms(&self) -> Arc<crate::operator_confirm::OperatorConfirms> {
676 Arc::clone(
677 &self
678 .inner
679 .read()
680 .unwrap_or_else(|p| p.into_inner())
681 .operator_confirms,
682 )
683 }
684
685 #[cfg(test)]
686 pub(crate) fn inject_operator_principal(&self, key: ModuleRouteKey, principal: Principal) {
687 let mut inner = self.write_inner().unwrap();
688 let old = inner.module_to_client.remove(&key).unwrap();
689 let client = ClientRouteKey {
690 connection_id: old.client_connection_id,
691 channel: old.client_channel,
692 };
693 inner.client_to_module.remove(&client);
694 let mut binding = Arc::try_unwrap(old).expect("test binding has no outstanding readers");
695 binding.principal = principal;
696 let binding = Arc::new(binding);
697 inner.client_to_module.insert(client, Arc::clone(&binding));
698 inner.module_to_client.insert(key, binding);
699 }
700
701 pub(crate) fn with_operator_route<T>(
705 &self,
706 connection_id: ConnectionId,
707 channel: u16,
708 epoch: u32,
709 admit: impl FnOnce(Option<&RouteBinding>) -> T,
710 ) -> Result<T, ForwardingError> {
711 let inner = self.read_inner()?;
712 let binding = inner
713 .endpoint_by_connection
714 .get(&connection_id)
715 .and_then(|endpoint| {
716 inner.module_to_client.get(&ModuleRouteKey {
717 endpoint: *endpoint,
718 channel,
719 })
720 })
721 .filter(|binding| binding.module_epoch == epoch);
722 Ok(admit(binding.map(Arc::as_ref)))
723 }
724
725 pub(crate) fn counters(&self) -> DaemonCounters {
726 self.counters.clone()
727 }
728
729 pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
730 self.route_bind_breakers.clone()
731 }
732
733 pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
734 self.route_bind_concurrency.clone()
735 }
736
737 pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
738 Arc::clone(&self.route_outages)
739 }
740
741 pub(crate) fn register_connection_close(
742 &self,
743 connection_id: ConnectionId,
744 ) -> ConnectionCloseReceiver {
745 let (sender, receiver) = oneshot::channel();
746 let replaced = self
747 .lock_close_registry()
748 .insert(connection_id, sender)
749 .is_some();
750 if replaced {
751 warn!(
752 connection_id = connection_id.get(),
753 "replaced existing connection close registration"
754 );
755 }
756 receiver
757 }
758
759 pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
760 self.lock_close_registry().remove(&connection_id);
761 }
762
763 #[cfg(unix)]
772 pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
773 let senders: Vec<_> = self.lock_close_registry().drain().collect();
774 let count = senders.len();
775 for (_, sender) in senders {
776 let _ = sender.send(reason.clone());
777 }
778 count
779 }
780
781 #[cfg(unix)]
785 pub(crate) fn module_connections(
786 &self,
787 ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
788 let inner = self.read_inner()?;
789 let mut seen = HashSet::new();
790 Ok(inner
791 .modules_by_id
792 .values()
793 .chain(inner.candidates_by_id.values())
794 .chain(inner.superseded_endpoints.values())
795 .filter(|module| seen.insert(module.endpoint))
796 .map(|module| ModuleConnectionTarget {
797 module_id: inner
798 .module_id_by_endpoint
799 .get(&module.endpoint)
800 .cloned()
801 .unwrap_or_default(),
802 endpoint: module.endpoint,
803 sink: module.sink.clone(),
804 negotiated_ver: module.negotiated_ver,
805 })
806 .collect())
807 }
808
809 pub(crate) fn request_connection_close(
813 &self,
814 connection_id: ConnectionId,
815 reason: CloseReason,
816 ) -> bool {
817 let sender = self.lock_close_registry().remove(&connection_id);
818 if let Some(sender) = sender {
819 debug!(
820 connection_id = connection_id.get(),
821 close_reason = %reason,
822 "requesting connection close"
823 );
824 let _ = sender.send(reason);
825 true
826 } else {
827 debug!(
828 connection_id = connection_id.get(),
829 close_reason = %reason,
830 "connection close request ignored for inactive connection"
831 );
832 false
833 }
834 }
835
836 pub fn register_module_connection(
837 &self,
838 connection_id: ConnectionId,
839 module_id: String,
840 negotiated_ver: u8,
841 concurrency: Concurrency,
842 sink: FrameSink,
843 ) -> Result<ModuleEndpointId, ForwardingError> {
844 self.register_module_connection_inner(
845 connection_id,
846 module_id,
847 negotiated_ver,
848 concurrency,
849 sink,
850 None,
851 )
852 }
853
854 pub(crate) fn register_module_connection_acked(
867 &self,
868 connection_id: ConnectionId,
869 module_id: String,
870 negotiated_ver: u8,
871 concurrency: Concurrency,
872 sink: FrameSink,
873 hello_ack: Frame,
874 ) -> Result<ModuleEndpointId, ForwardingError> {
875 self.register_module_connection_inner(
876 connection_id,
877 module_id,
878 negotiated_ver,
879 concurrency,
880 sink,
881 Some(hello_ack),
882 )
883 }
884
885 fn register_module_connection_inner(
886 &self,
887 connection_id: ConnectionId,
888 module_id: String,
889 negotiated_ver: u8,
890 concurrency: Concurrency,
891 sink: FrameSink,
892 hello_ack: Option<Frame>,
893 ) -> Result<ModuleEndpointId, ForwardingError> {
894 let mut inner = self.write_inner()?;
895 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
896 return Err(ForwardingError::ConnectionClosing { connection_id });
897 }
898 check_module_connection_role_locked(&inner, connection_id)?;
899 enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
902
903 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
904 let endpoint = ModuleEndpointId {
905 connection_id,
906 generation: inner.next_generation,
907 };
908 inner.endpoint_by_connection.insert(connection_id, endpoint);
909 inner
910 .module_id_by_endpoint
911 .insert(endpoint, module_id.clone());
912 inner.next_module_channel.insert(endpoint, 1);
913 inner.next_control_corr.insert(endpoint, 1);
914 inner.modules_by_id.insert(
915 module_id.clone(),
916 ModuleConnection {
917 endpoint,
918 sink,
919 negotiated_ver,
920 concurrency,
921 },
922 );
923 drop(inner);
924
925 if let Some(discarded) = self
938 .route_bind_breakers
939 .reset_for_new_module_connection(&module_id)
940 {
941 info!(
942 module_id = %module_id,
943 discarded_consecutive_timeouts = discarded,
944 "route.bind breaker state discarded: a new module connection replaced the process it described"
945 );
946 }
947 Ok(endpoint)
948 }
949
950 #[cfg(test)]
964 pub(crate) fn register_candidate_module_connection(
965 &self,
966 connection_id: ConnectionId,
967 module_id: String,
968 negotiated_ver: u8,
969 concurrency: Concurrency,
970 sink: FrameSink,
971 ) -> Result<ModuleEndpointId, ForwardingError> {
972 self.register_candidate_module_connection_inner(
973 connection_id,
974 module_id,
975 negotiated_ver,
976 concurrency,
977 sink,
978 None,
979 )
980 }
981
982 pub(crate) fn register_candidate_module_connection_acked(
989 &self,
990 connection_id: ConnectionId,
991 module_id: String,
992 negotiated_ver: u8,
993 concurrency: Concurrency,
994 sink: FrameSink,
995 hello_ack: Frame,
996 ) -> Result<ModuleEndpointId, ForwardingError> {
997 self.register_candidate_module_connection_inner(
998 connection_id,
999 module_id,
1000 negotiated_ver,
1001 concurrency,
1002 sink,
1003 Some(hello_ack),
1004 )
1005 }
1006
1007 fn register_candidate_module_connection_inner(
1008 &self,
1009 connection_id: ConnectionId,
1010 module_id: String,
1011 negotiated_ver: u8,
1012 concurrency: Concurrency,
1013 sink: FrameSink,
1014 hello_ack: Option<Frame>,
1015 ) -> Result<ModuleEndpointId, ForwardingError> {
1016 let mut inner = self.write_inner()?;
1017 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1018 return Err(ForwardingError::ConnectionClosing { connection_id });
1019 }
1020 check_module_connection_role_locked(&inner, connection_id)?;
1021 if inner.candidates_by_id.contains_key(&module_id) {
1022 return Err(ForwardingError::CandidateSlotOccupied { module_id });
1023 }
1024 enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1027
1028 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1029 let endpoint = ModuleEndpointId {
1030 connection_id,
1031 generation: inner.next_generation,
1032 };
1033 inner.endpoint_by_connection.insert(connection_id, endpoint);
1034 inner
1035 .module_id_by_endpoint
1036 .insert(endpoint, module_id.clone());
1037 inner.next_module_channel.insert(endpoint, 1);
1038 inner.next_control_corr.insert(endpoint, 1);
1039 inner.candidates_by_id.insert(
1040 module_id,
1041 ModuleConnection {
1042 endpoint,
1043 sink,
1044 negotiated_ver,
1045 concurrency,
1046 },
1047 );
1048 Ok(endpoint)
1049 }
1050
1051 pub(crate) fn cutover_candidate(
1069 &self,
1070 module_id: &str,
1071 ) -> Result<Option<ForwardingCutover>, ForwardingError> {
1072 let mut inner = self.write_inner()?;
1073 if inner.daemon_draining {
1074 return Err(ForwardingError::ModuleReloading {
1075 module_id: module_id.to_string(),
1076 });
1077 }
1078 let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
1079 return Ok(None);
1080 };
1081 let promoted = candidate.endpoint;
1082 let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
1083 let incumbent = incumbent.map(|incumbent| {
1084 let endpoint = incumbent.endpoint;
1085 inner.superseded_endpoints.insert(endpoint, incumbent);
1086 endpoint
1087 });
1088 drop(inner);
1089
1090 if let Some(discarded) = self
1094 .route_bind_breakers
1095 .reset_for_new_module_connection(module_id)
1096 {
1097 info!(
1098 module_id = %module_id,
1099 discarded_consecutive_timeouts = discarded,
1100 "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
1101 );
1102 }
1103 Ok(Some(ForwardingCutover {
1104 promoted,
1105 incumbent,
1106 }))
1107 }
1108
1109 #[allow(clippy::too_many_arguments)]
1110 pub(crate) async fn begin_route_bind_relay_for(
1111 &self,
1112 client_connection_id: ConnectionId,
1113 client_sink: FrameSink,
1114 client_negotiated_ver: u8,
1115 client_corr: u64,
1116 module_id: &str,
1117 principal: Principal,
1118 scope: Option<BoundScope>,
1119 project_root: Option<ProjectRootId>,
1120 deadline: Instant,
1121 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1122 let client_permit =
1126 client_sink
1127 .reserve_owned()
1128 .await
1129 .map_err(|_| ForwardingError::ClientEgressClosed {
1130 connection_id: client_connection_id,
1131 })?;
1132 self.begin_route_bind_relay_inner(
1133 client_connection_id,
1134 client_sink,
1135 client_negotiated_ver,
1136 client_corr,
1137 module_id,
1138 principal,
1139 scope,
1140 project_root,
1141 deadline,
1142 client_permit,
1143 )
1144 }
1145
1146 #[cfg(test)]
1147 pub(crate) fn begin_route_bind_relay_for_test(
1148 &self,
1149 client_connection_id: ConnectionId,
1150 client_sink: FrameSink,
1151 client_corr: u64,
1152 module_id: &str,
1153 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1154 let permit =
1155 client_sink
1156 .try_reserve_owned()
1157 .map_err(|_| ForwardingError::ClientEgressClosed {
1158 connection_id: client_connection_id,
1159 })?;
1160 self.begin_route_bind_relay_inner(
1161 client_connection_id,
1162 client_sink,
1163 subc_protocol::PROTOCOL_VERSION,
1164 client_corr,
1165 module_id,
1166 Principal::Direct,
1167 None,
1168 None,
1169 Instant::now() + std::time::Duration::from_secs(60),
1170 permit,
1171 )
1172 }
1173
1174 pub(crate) fn begin_module_control_rpc_for(
1175 &self,
1176 module_id: &str,
1177 expected_op: &str,
1178 deadline: Instant,
1179 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1180 self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1181 }
1182
1183 pub(crate) fn begin_health_probe_rpc_for(
1184 &self,
1185 module_id: &str,
1186 expected_op: &str,
1187 probe_started_at: Instant,
1188 deadline: Instant,
1189 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1190 self.begin_module_control_rpc_inner(
1191 module_id,
1192 expected_op,
1193 deadline,
1194 Some(probe_started_at),
1195 false,
1196 )
1197 }
1198
1199 pub(crate) fn begin_drain_health_probe_rpc_for(
1200 &self,
1201 module_id: &str,
1202 expected_op: &str,
1203 probe_started_at: Instant,
1204 deadline: Instant,
1205 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1206 self.begin_module_control_rpc_inner(
1207 module_id,
1208 expected_op,
1209 deadline,
1210 Some(probe_started_at),
1211 true,
1212 )
1213 }
1214
1215 pub(crate) fn begin_endpoint_health_probe_rpc_for(
1223 &self,
1224 endpoint: ModuleEndpointId,
1225 expected_op: &str,
1226 probe_started_at: Instant,
1227 deadline: Instant,
1228 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1229 let inner = self.write_inner()?;
1230 let module = module_connection_for_endpoint_locked(&inner, endpoint)
1231 .cloned()
1232 .ok_or(ForwardingError::NoModuleConnection)?;
1233 let module_id = inner
1234 .module_id_by_endpoint
1235 .get(&endpoint)
1236 .cloned()
1237 .unwrap_or_default();
1238 self.begin_control_rpc_locked(
1241 inner,
1242 &module_id,
1243 module,
1244 expected_op,
1245 deadline,
1246 Some(probe_started_at),
1247 true,
1248 )
1249 }
1250
1251 fn begin_module_control_rpc_inner(
1252 &self,
1253 module_id: &str,
1254 expected_op: &str,
1255 deadline: Instant,
1256 health_probe_started_at: Option<Instant>,
1257 allow_draining: bool,
1258 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1259 let inner = self.write_inner()?;
1260 let module = inner
1261 .modules_by_id
1262 .get(module_id)
1263 .cloned()
1264 .ok_or(ForwardingError::NoModuleConnection)?;
1265 self.begin_control_rpc_locked(
1266 inner,
1267 module_id,
1268 module,
1269 expected_op,
1270 deadline,
1271 health_probe_started_at,
1272 allow_draining,
1273 )
1274 }
1275
1276 #[allow(clippy::too_many_arguments)]
1277 fn begin_control_rpc_locked(
1278 &self,
1279 mut inner: RwLockWriteGuard<'_, ForwardingInner>,
1280 module_id: &str,
1281 module: ModuleConnection,
1282 expected_op: &str,
1283 deadline: Instant,
1284 health_probe_started_at: Option<Instant>,
1285 allow_draining: bool,
1286 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1287 if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1288 return Err(ForwardingError::ModuleReloading {
1289 module_id: module_id.to_string(),
1290 });
1291 }
1292 if inner
1293 .closing_connections
1294 .contains(&module.endpoint.connection_id)
1295 {
1296 return Err(ForwardingError::ConnectionClosing {
1297 connection_id: module.endpoint.connection_id,
1298 });
1299 }
1300 if health_probe_started_at.is_some() {
1301 inner
1305 .health_probe_tombstones
1306 .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1307 }
1308 let corr = match inner.allocate_control_corr(module.endpoint) {
1309 Ok(corr) => corr,
1310 Err(err) => {
1311 drop(inner);
1312 self.request_connection_close(
1313 module.endpoint.connection_id,
1314 CloseReason::new(
1315 "control_correlation_exhausted",
1316 "daemon-originated channel-0 correlation space exhausted",
1317 ),
1318 );
1319 return Err(err);
1320 }
1321 };
1322 let (sender, receiver) = oneshot::channel();
1323 inner.pending_control_rpcs.insert(
1324 (module.endpoint, corr),
1325 PendingModuleControlRpcEntry {
1326 expected_op: expected_op.to_string(),
1327 deadline,
1328 health_probe_started_at,
1329 sender,
1330 },
1331 );
1332
1333 Ok(PendingModuleControlRpc {
1334 endpoint: module.endpoint,
1335 module_sink: module.sink,
1336 negotiated_ver: module.negotiated_ver,
1337 corr,
1338 receiver,
1339 })
1340 }
1341
1342 #[allow(clippy::too_many_arguments)]
1343 fn begin_route_bind_relay_inner(
1344 &self,
1345 client_connection_id: ConnectionId,
1346 client_sink: FrameSink,
1347 client_negotiated_ver: u8,
1348 client_corr: u64,
1349 expected_module_id: &str,
1350 principal: Principal,
1351 scope: Option<BoundScope>,
1352 project_root: Option<ProjectRootId>,
1353 deadline: Instant,
1354 client_permit: crate::router::EgressPermit,
1355 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1356 let mut inner = self.write_inner()?;
1357 if inner
1361 .endpoint_by_connection
1362 .contains_key(&client_connection_id)
1363 {
1364 return Err(ForwardingError::ConnectionRoleConflict {
1365 connection_id: client_connection_id,
1366 });
1367 }
1368 if inner.closing_connections.contains(&client_connection_id) {
1369 return Err(ForwardingError::ConnectionClosing {
1370 connection_id: client_connection_id,
1371 });
1372 }
1373 let module = inner
1374 .modules_by_id
1375 .get(expected_module_id)
1376 .cloned()
1377 .ok_or(ForwardingError::NoModuleConnection)?;
1378 if inner.draining_endpoints.contains_key(&module.endpoint) {
1379 return Err(ForwardingError::ModuleReloading {
1380 module_id: expected_module_id.to_string(),
1381 });
1382 }
1383 if inner
1384 .closing_connections
1385 .contains(&module.endpoint.connection_id)
1386 {
1387 return Err(ForwardingError::ConnectionClosing {
1388 connection_id: module.endpoint.connection_id,
1389 });
1390 }
1391
1392 let corr = match inner.allocate_control_corr(module.endpoint) {
1393 Ok(corr) => corr,
1394 Err(err) => {
1395 drop(inner);
1396 self.request_connection_close(
1397 module.endpoint.connection_id,
1398 CloseReason::new(
1399 "control_correlation_exhausted",
1400 "daemon-originated channel-0 correlation space exhausted",
1401 ),
1402 );
1403 return Err(err);
1404 }
1405 };
1406 let (client_channel, client_epoch, module_channel, module_epoch) =
1407 inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1408 let client_key = ClientRouteKey {
1409 connection_id: client_connection_id,
1410 channel: client_channel,
1411 };
1412 let module_key = ModuleRouteKey {
1413 endpoint: module.endpoint,
1414 channel: module_channel,
1415 };
1416 let reservation = RouteReservation {
1417 client_key,
1418 module_key,
1419 client_epoch,
1420 module_epoch,
1421 project_root,
1422 };
1423 let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
1424 route_channel: client_channel,
1425 route_epoch: client_epoch,
1426 })
1427 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1428 let route_open_frame = Frame::build_with_version(
1429 client_negotiated_ver,
1430 FrameType::Response,
1431 Flags::new(false, Priority::Passive, false),
1432 0,
1433 0,
1434 client_corr,
1435 response_body,
1436 )
1437 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1438 let (sender, receiver) = oneshot::channel();
1439 inner.reserved_client.insert(client_key, module_key);
1440 inner.reserved_module.insert(module_key, client_key);
1441 inner.pending_relays.insert(
1442 (module.endpoint, corr),
1443 PendingRouteBindRelayEntry {
1444 reservation,
1445 client_sink,
1446 client_negotiated_ver,
1447 client_permit,
1448 route_open_frame,
1449 principal,
1450 scope,
1451 deadline,
1452 relay_enqueued: false,
1453 sender,
1454 },
1455 );
1456
1457 Ok(PendingRouteBindRelay {
1458 endpoint: module.endpoint,
1459 module_sink: module.sink,
1460 negotiated_ver: module.negotiated_ver,
1461 client_channel,
1462 client_epoch,
1463 module_channel,
1464 module_epoch,
1465 corr,
1466 receiver,
1467 })
1468 }
1469
1470 pub(crate) fn mark_route_bind_relay_enqueued(
1480 &self,
1481 endpoint: ModuleEndpointId,
1482 corr: u64,
1483 ) -> Result<RouteBindRelayMark, ForwardingError> {
1484 let mut inner = self.write_inner()?;
1485 if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
1486 pending.relay_enqueued = true;
1487 return Ok(RouteBindRelayMark::Marked);
1488 }
1489 if inner
1490 .relays_released_before_enqueue
1491 .remove(&(endpoint, corr))
1492 {
1493 return Ok(RouteBindRelayMark::ReleasedWithoutReply);
1494 }
1495 Ok(RouteBindRelayMark::SettledByReply)
1496 }
1497
1498 pub(crate) fn release_client_route(
1499 &self,
1500 client_connection_id: ConnectionId,
1501 client_channel: u16,
1502 expected_epoch: u32,
1503 ) -> Result<RouteRelease, ForwardingError> {
1504 let mut inner = self.write_inner()?;
1505 let release = release_client_route_locked(
1506 &mut inner,
1507 ClientRouteKey {
1508 connection_id: client_connection_id,
1509 channel: client_channel,
1510 },
1511 expected_epoch,
1512 );
1513 self.record_route_release(&release);
1514 Ok(release)
1515 }
1516
1517 pub(crate) fn release_module_route(
1518 &self,
1519 module_connection_id: ConnectionId,
1520 module_channel: u16,
1521 expected_epoch: u32,
1522 ) -> Result<RouteRelease, ForwardingError> {
1523 let mut inner = self.write_inner()?;
1524 let Some(endpoint) = inner
1525 .endpoint_by_connection
1526 .get(&module_connection_id)
1527 .copied()
1528 else {
1529 return Ok(RouteRelease::Absent);
1530 };
1531 let release = release_module_route_locked(
1532 &mut inner,
1533 ModuleRouteKey {
1534 endpoint,
1535 channel: module_channel,
1536 },
1537 expected_epoch,
1538 );
1539 self.record_route_release(&release);
1540 Ok(release)
1541 }
1542
1543 pub(crate) fn abort_pending_relay(
1544 &self,
1545 endpoint: ModuleEndpointId,
1546 corr: u64,
1547 outcome: RouteBindRelayOutcome,
1548 ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
1549 let mut inner = self.write_inner()?;
1550 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1551 inner
1556 .relays_released_before_enqueue
1557 .remove(&(endpoint, corr));
1558 return Ok(None);
1559 };
1560 release_reserved_route_locked(
1561 &mut inner,
1562 pending.reservation.client_key,
1563 pending.reservation.module_key,
1564 );
1565 let target = pending
1566 .relay_enqueued
1567 .then(|| abandoned_route_target(&inner, &pending.reservation));
1568 let _ = pending.sender.send(outcome);
1569 Ok(target.flatten())
1570 }
1571
1572 pub(crate) fn cancel_module_control_rpc(
1573 &self,
1574 endpoint: ModuleEndpointId,
1575 corr: u64,
1576 ) -> Result<(), ForwardingError> {
1577 self.write_inner()?
1578 .pending_control_rpcs
1579 .remove(&(endpoint, corr));
1580 Ok(())
1581 }
1582
1583 pub(crate) fn tombstone_health_probe_rpc(
1584 &self,
1585 endpoint: ModuleEndpointId,
1586 corr: u64,
1587 ) -> Result<bool, ForwardingError> {
1588 let key = (endpoint, corr);
1589 let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
1590 {
1591 let mut inner = self.write_inner()?;
1592 let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
1593 return Ok(false);
1594 };
1595 let Some(probe_started_at) = pending.health_probe_started_at else {
1596 inner.pending_control_rpcs.insert(key, pending);
1597 return Ok(false);
1598 };
1599 let module_id = inner
1600 .module_id_by_endpoint
1601 .get(&endpoint)
1602 .cloned()
1603 .unwrap_or_else(|| "unknown".to_string());
1604 inner.health_probe_tombstones.insert(
1605 key,
1606 HealthProbeTombstone {
1607 expected_op: pending.expected_op,
1608 module_id,
1609 probe_started_at,
1610 expires_at,
1611 },
1612 );
1613 }
1614 self.schedule_health_probe_tombstone_expiration(key, expires_at);
1615 Ok(true)
1616 }
1617
1618 fn schedule_health_probe_tombstone_expiration(
1619 &self,
1620 key: (ModuleEndpointId, u64),
1621 expires_at: Instant,
1622 ) {
1623 let inner = Arc::downgrade(&self.inner);
1624 tokio::spawn(async move {
1625 tokio::time::sleep_until(expires_at).await;
1626 let Some(inner) = inner.upgrade() else {
1627 return;
1628 };
1629 let Ok(mut inner) = inner.write() else {
1630 return;
1631 };
1632 let expired = inner
1633 .health_probe_tombstones
1634 .get(&key)
1635 .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
1636 if expired {
1637 inner.health_probe_tombstones.remove(&key);
1638 }
1639 });
1640 }
1641
1642 pub(crate) fn complete_pending_relay(
1643 &self,
1644 connection_id: ConnectionId,
1645 corr: u64,
1646 outcome: RouteBindRelayOutcome,
1647 ) -> Result<PendingRelayCompletion, ForwardingError> {
1648 let mut inner = self.write_inner()?;
1649 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1650 return Ok(PendingRelayCompletion {
1651 settled: false,
1652 abandoned: None,
1653 });
1654 };
1655 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1656 return Ok(PendingRelayCompletion {
1657 settled: false,
1658 abandoned: None,
1659 });
1660 };
1661
1662 if Instant::now() >= pending.deadline {
1663 release_reserved_route_locked(
1664 &mut inner,
1665 pending.reservation.client_key,
1666 pending.reservation.module_key,
1667 );
1668 let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
1669 .then(|| abandoned_route_target(&inner, &pending.reservation))
1670 .flatten();
1671 let _ = pending
1672 .sender
1673 .send(RouteBindRelayOutcome::Rejected(ErrorBody {
1674 code: "module_timeout".to_string(),
1675 message: "route.bind response arrived after its daemon deadline".to_string(),
1676 detail: None,
1677 }));
1678 return Ok(PendingRelayCompletion {
1679 settled: true,
1680 abandoned,
1681 });
1682 }
1683
1684 match outcome {
1685 RouteBindRelayOutcome::Accepted
1704 if pending.client_sink.is_closed()
1705 || inner
1706 .closing_connections
1707 .contains(&pending.reservation.client_key.connection_id) =>
1708 {
1709 let reason = if pending.client_sink.is_closed() {
1710 "client egress closed before route publication"
1711 } else {
1712 "client connection is closing before route publication"
1713 };
1714 release_reserved_route_locked(
1715 &mut inner,
1716 pending.reservation.client_key,
1717 pending.reservation.module_key,
1718 );
1719 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1723 let _ = pending
1724 .sender
1725 .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
1726 return Ok(PendingRelayCompletion {
1727 settled: true,
1728 abandoned,
1729 });
1730 }
1731 RouteBindRelayOutcome::Accepted
1754 if inner.superseded_endpoints.contains_key(&endpoint)
1755 || inner.draining_endpoints.contains_key(&endpoint) =>
1756 {
1757 release_reserved_route_locked(
1758 &mut inner,
1759 pending.reservation.client_key,
1760 pending.reservation.module_key,
1761 );
1762 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1765 let module_id = inner
1766 .module_id_by_endpoint
1767 .get(&endpoint)
1768 .cloned()
1769 .unwrap_or_else(|| "unknown".to_string());
1770 let _ = pending
1771 .sender
1772 .send(RouteBindRelayOutcome::Rejected(ErrorBody::new(
1773 "module_reloading",
1774 format!("module_id '{module_id}' is reloading"),
1775 )));
1776 return Ok(PendingRelayCompletion {
1777 settled: true,
1778 abandoned,
1779 });
1780 }
1781 RouteBindRelayOutcome::Accepted
1790 if pending
1791 .scope
1792 .as_ref()
1793 .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
1794 {
1795 let refusal = pending
1796 .scope
1797 .as_ref()
1798 .and_then(|scope| scope_refusal_locked(&inner, scope))
1799 .expect("guard matched a refusal under the same lock");
1800 release_reserved_route_locked(
1801 &mut inner,
1802 pending.reservation.client_key,
1803 pending.reservation.module_key,
1804 );
1805 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1806 let _ = pending
1807 .sender
1808 .send(RouteBindRelayOutcome::Rejected(refusal));
1809 return Ok(PendingRelayCompletion {
1810 settled: true,
1811 abandoned,
1812 });
1813 }
1814 RouteBindRelayOutcome::Accepted => {
1815 let abandoned = commit_route_locked(&mut inner, pending)?;
1816 return Ok(PendingRelayCompletion {
1817 settled: true,
1818 abandoned,
1819 });
1820 }
1821 terminal => {
1822 release_reserved_route_locked(
1823 &mut inner,
1824 pending.reservation.client_key,
1825 pending.reservation.module_key,
1826 );
1827 let _ = pending.sender.send(terminal);
1828 }
1829 }
1830 Ok(PendingRelayCompletion {
1831 settled: true,
1832 abandoned: None,
1833 })
1834 }
1835
1836 pub(crate) fn pending_module_control_op(
1837 &self,
1838 connection_id: ConnectionId,
1839 corr: u64,
1840 ) -> Result<Option<String>, ForwardingError> {
1841 let inner = self.read_inner()?;
1842 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1843 return Ok(None);
1844 };
1845 let key = (endpoint, corr);
1846 Ok(inner
1847 .pending_control_rpcs
1848 .get(&key)
1849 .map(|pending| pending.expected_op.clone())
1850 .or_else(|| {
1851 inner
1852 .health_probe_tombstones
1853 .get(&key)
1854 .filter(|tombstone| tombstone.expires_at > Instant::now())
1855 .map(|tombstone| tombstone.expected_op.clone())
1856 }))
1857 }
1858
1859 pub(crate) fn complete_module_control_rpc(
1860 &self,
1861 connection_id: ConnectionId,
1862 corr: u64,
1863 actual_op: Option<&str>,
1864 outcome: ModuleControlRpcOutcome,
1865 ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
1866 let now = Instant::now();
1867 let mut inner = self.write_inner()?;
1868 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1869 return Ok(ModuleControlRpcCompletion::Unknown);
1870 };
1871 let key = (endpoint, corr);
1872 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
1873 if now >= pending.deadline {
1874 let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
1875 ModuleControlRpcCompletion::LateHealthAnswer {
1876 module_id: inner
1877 .module_id_by_endpoint
1878 .get(&endpoint)
1879 .cloned()
1880 .unwrap_or_else(|| "unknown".to_string()),
1881 latency: now.saturating_duration_since(probe_started_at),
1882 }
1883 });
1884 let _ = pending
1885 .sender
1886 .send(ModuleControlRpcOutcome::DeadlineElapsed);
1887 return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
1888 }
1889 let outcome = match actual_op {
1890 Some(actual) if actual != pending.expected_op => {
1891 ModuleControlRpcOutcome::UnexpectedOp {
1892 expected: pending.expected_op,
1893 actual: actual.to_string(),
1894 }
1895 }
1896 _ => outcome,
1897 };
1898 let _ = pending.sender.send(outcome);
1899 return Ok(ModuleControlRpcCompletion::Settled);
1900 }
1901
1902 let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
1903 return Ok(ModuleControlRpcCompletion::Unknown);
1904 };
1905 if tombstone.expires_at <= now {
1906 return Ok(ModuleControlRpcCompletion::Unknown);
1907 }
1908 Ok(ModuleControlRpcCompletion::LateHealthAnswer {
1909 module_id: tombstone.module_id,
1910 latency: now.saturating_duration_since(tombstone.probe_started_at),
1911 })
1912 }
1913
1914 #[cfg(test)]
1915 pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
1916 Ok(self.read_inner()?.health_probe_tombstones.len())
1917 }
1918
1919 #[cfg(test)]
1920 pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
1921 Ok(self.read_inner()?.closing_connections.len())
1922 }
1923
1924 #[cfg(test)]
1927 pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
1928 let inner = self.read_inner()?;
1929 Ok((inner.reserved_client.len(), inner.reserved_module.len()))
1930 }
1931
1932 pub(crate) fn module_endpoint_for_connection(
1933 &self,
1934 connection_id: ConnectionId,
1935 ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
1936 Ok(self
1937 .read_inner()?
1938 .endpoint_by_connection
1939 .get(&connection_id)
1940 .copied())
1941 }
1942
1943 pub(crate) fn module_id_for_connection(
1946 &self,
1947 connection_id: ConnectionId,
1948 ) -> Result<Option<String>, ForwardingError> {
1949 let inner = self.read_inner()?;
1950 Ok(inner
1951 .endpoint_by_connection
1952 .get(&connection_id)
1953 .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
1954 .cloned())
1955 }
1956
1957 pub(crate) fn module_route_epoch_was_allocated(
1964 &self,
1965 connection_id: ConnectionId,
1966 channel: u16,
1967 epoch: u32,
1968 ) -> Result<bool, ForwardingError> {
1969 let inner = self.read_inner()?;
1970 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1971 return Ok(false);
1972 };
1973 Ok(inner
1974 .module_slot_epochs
1975 .get(&ModuleRouteKey { endpoint, channel })
1976 .is_some_and(|last| epoch != 0 && epoch <= *last))
1977 }
1978
1979 pub(crate) fn has_live_module_connection(
1980 &self,
1981 module_id: &str,
1982 ) -> Result<bool, ForwardingError> {
1983 Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
1984 }
1985
1986 pub(crate) fn lookup_data_route(
1987 &self,
1988 connection_id: ConnectionId,
1989 channel: u16,
1990 epoch: u32,
1991 ) -> Result<DataRoute, ForwardingError> {
1992 let inner = self.read_inner()?;
1993 let state = if let Some(endpoint) =
1994 inner.endpoint_by_connection.get(&connection_id).copied()
1995 {
1996 let key = ModuleRouteKey { endpoint, channel };
1997 match inner.module_to_client.get(&key) {
1998 Some(route) if route.module_epoch == epoch => {
1999 DataRouteState::Bound(Arc::clone(route))
2000 }
2001 Some(_) => DataRouteState::EpochMismatch,
2002 None if inner.reserved_module.contains_key(&key)
2003 && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
2004 {
2005 DataRouteState::Reserved
2006 }
2007 None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
2008 None => DataRouteState::Absent,
2009 }
2010 } else {
2011 let key = ClientRouteKey {
2012 connection_id,
2013 channel,
2014 };
2015 match inner.client_to_module.get(&key) {
2016 Some(route) if route.client_epoch == epoch => {
2017 DataRouteState::Bound(Arc::clone(route))
2018 }
2019 Some(_) => DataRouteState::EpochMismatch,
2020 None if inner.reserved_client.contains_key(&key)
2021 && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
2022 {
2023 DataRouteState::Reserved
2024 }
2025 None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
2026 None => DataRouteState::Absent,
2027 }
2028 };
2029 Ok(
2030 if inner.endpoint_by_connection.contains_key(&connection_id) {
2031 DataRoute::Module(state)
2032 } else {
2033 DataRoute::Client(state)
2034 },
2035 )
2036 }
2037
2038 #[cfg(test)]
2039 pub(crate) fn inject_client_slot_epoch(
2040 &self,
2041 connection_id: ConnectionId,
2042 channel: u16,
2043 last_epoch: u32,
2044 ) {
2045 let mut inner = self.write_inner().expect("forwarding lock");
2046 inner.client_slot_epochs.insert(
2047 ClientRouteKey {
2048 connection_id,
2049 channel,
2050 },
2051 last_epoch,
2052 );
2053 inner.next_client_channel.insert(connection_id, channel);
2054 }
2055
2056 #[cfg(test)]
2057 pub(crate) fn inject_module_slot_epoch(
2058 &self,
2059 endpoint: ModuleEndpointId,
2060 channel: u16,
2061 last_epoch: u32,
2062 ) {
2063 let mut inner = self.write_inner().expect("forwarding lock");
2064 inner
2065 .module_slot_epochs
2066 .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
2067 inner.next_module_channel.insert(endpoint, channel);
2068 }
2069
2070 #[cfg(test)]
2071 pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
2072 self.write_inner()
2073 .expect("forwarding lock")
2074 .next_control_corr
2075 .insert(endpoint, next_corr);
2076 }
2077
2078 pub(crate) fn cache_status(
2079 &self,
2080 endpoint: ModuleEndpointId,
2081 module_channel: u16,
2082 module_epoch: u32,
2083 status: String,
2084 ) -> Result<bool, ForwardingError> {
2085 let mut inner = self.write_inner()?;
2086 if !inner.module_id_by_endpoint.contains_key(&endpoint) {
2087 return Err(ForwardingError::StaleModuleEndpoint);
2088 }
2089
2090 let module_key = ModuleRouteKey {
2091 endpoint,
2092 channel: module_channel,
2093 };
2094 let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
2095 (route.module_epoch == module_epoch).then_some((
2096 ClientRouteKey {
2097 connection_id: route.client_connection_id,
2098 channel: route.client_channel,
2099 },
2100 route.client_epoch,
2101 ))
2102 } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
2103 (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
2104 client_key,
2105 inner
2106 .client_slot_epochs
2107 .get(&client_key)
2108 .copied()
2109 .unwrap_or(0),
2110 ))
2111 } else {
2112 None
2113 };
2114
2115 if let Some(handle) = handle {
2116 inner.status.insert(handle, status);
2117 Ok(true)
2118 } else {
2119 debug!(
2120 module_channel,
2121 module_epoch,
2122 generation = endpoint.generation,
2123 connection_id = endpoint.connection_id.get(),
2124 "dropping stale status update for module route handle"
2125 );
2126 Ok(false)
2127 }
2128 }
2129
2130 pub(crate) fn route_poll_snapshot(
2131 &self,
2132 client_connection_id: ConnectionId,
2133 client_channel: u16,
2134 client_epoch: u32,
2135 ) -> Result<RoutePollSnapshot, ForwardingError> {
2136 let inner = self.read_inner()?;
2137 let client_key = ClientRouteKey {
2138 connection_id: client_connection_id,
2139 channel: client_channel,
2140 };
2141 let Some(route) = inner.client_to_module.get(&client_key) else {
2142 return Ok(RoutePollSnapshot::Absent);
2143 };
2144 if route.client_epoch != client_epoch
2145 || !inner
2146 .module_id_by_endpoint
2147 .contains_key(&route.module_endpoint)
2148 {
2149 return Ok(RoutePollSnapshot::Absent);
2150 }
2151 Ok(RoutePollSnapshot::Bound {
2152 module_id: route.module_id.clone(),
2153 status: inner.status.get(&(client_key, client_epoch)).cloned(),
2154 })
2155 }
2156
2157 pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2158 Ok(self.read_inner()?.client_to_module.len())
2159 }
2160
2161 pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2171 let inner = self.read_inner()?;
2172 let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2173 for key in inner.client_to_module.keys() {
2174 *per_connection.entry(key.connection_id).or_insert(0) += 1;
2175 }
2176 let max = per_connection.values().copied().max().unwrap_or(0);
2177 Ok((per_connection.len(), max))
2178 }
2179
2180 pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2181 let inner = self.read_inner()?;
2182 Ok(inner
2183 .client_to_module
2184 .keys()
2185 .any(|key| key.channel == route_channel))
2186 }
2187
2188 pub(crate) fn daemon_drain_result_for_connection(
2192 &self,
2193 connection_id: ConnectionId,
2194 ) -> Result<Option<bool>, ForwardingError> {
2195 let inner = self.read_inner()?;
2196 Ok(inner.daemon_draining.then(|| {
2197 inner
2198 .endpoint_by_connection
2199 .get(&connection_id)
2200 .and_then(|endpoint| inner.daemon_drain_results.get(endpoint))
2201 .copied()
2202 .unwrap_or(false)
2203 }))
2204 }
2205
2206 #[cfg(unix)]
2207 pub(crate) fn record_daemon_drain_result(
2208 &self,
2209 endpoint: ModuleEndpointId,
2210 drained: bool,
2211 ) -> Result<(), ForwardingError> {
2212 let mut inner = self.write_inner()?;
2213 if let Some(result) = inner.daemon_drain_results.get_mut(&endpoint) {
2217 *result = drained;
2218 }
2219 Ok(())
2220 }
2221
2222 #[cfg(unix)]
2225 pub(crate) fn begin_daemon_drain(
2226 &self,
2227 ) -> Result<Vec<(String, ModuleEndpointId)>, ForwardingError> {
2228 let mut inner = self.write_inner()?;
2229 if inner.daemon_draining {
2230 return Ok(Vec::new());
2231 }
2232 inner.daemon_draining = true;
2233 let modules = inner
2234 .modules_by_id
2235 .iter()
2236 .map(|(id, module)| (id.clone(), module.endpoint))
2237 .collect::<Vec<_>>();
2238 for (_, endpoint) in &modules {
2239 inner.daemon_drain_results.insert(*endpoint, false);
2240 inner
2241 .draining_endpoints
2242 .insert(*endpoint, RouteCloseReason::Restart);
2243 }
2244 let off_slot_endpoints = inner
2249 .candidates_by_id
2250 .values()
2251 .map(|module| module.endpoint)
2252 .chain(inner.superseded_endpoints.keys().copied())
2253 .collect::<Vec<_>>();
2254 for endpoint in off_slot_endpoints {
2255 inner
2256 .draining_endpoints
2257 .insert(endpoint, RouteCloseReason::Restart);
2258 }
2259 Ok(modules)
2260 }
2261
2262 pub(crate) fn begin_module_drain(
2269 &self,
2270 module_id: &str,
2271 reason: RouteCloseReason,
2272 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2273 let mut inner = self.write_inner()?;
2274 let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2275 return Ok(None);
2276 };
2277 Ok(Some(begin_drain_locked(
2278 &mut inner, module_id, module, reason,
2279 )))
2280 }
2281
2282 pub(crate) fn begin_endpoint_drain(
2289 &self,
2290 endpoint: ModuleEndpointId,
2291 reason: RouteCloseReason,
2292 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2293 let mut inner = self.write_inner()?;
2294 let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2295 return Ok(None);
2296 };
2297 let module_id = inner
2298 .module_id_by_endpoint
2299 .get(&endpoint)
2300 .cloned()
2301 .expect("an endpoint resolved to a module connection has a module id");
2302 Ok(Some(begin_drain_locked(
2303 &mut inner, &module_id, module, reason,
2304 )))
2305 }
2306}
2307
2308fn begin_drain_locked(
2312 inner: &mut ForwardingInner,
2313 module_id: &str,
2314 module: ModuleConnection,
2315 reason: RouteCloseReason,
2316) -> ModuleDrainTarget {
2317 {
2318 let endpoint = module.endpoint;
2319 inner.draining_endpoints.insert(endpoint, reason);
2320
2321 let flows = inner
2322 .client_to_module
2323 .values()
2324 .filter(|route| route.module_endpoint == endpoint)
2325 .map(|route| Arc::clone(&route.flow))
2326 .collect::<Vec<_>>();
2327 let excluded_subscriptions = flows
2328 .into_iter()
2329 .map(|flow| flow.begin_drain())
2330 .fold(0u32, u32::saturating_add);
2331
2332 let pending_keys = inner
2333 .pending_relays
2334 .keys()
2335 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2336 .copied()
2337 .collect::<Vec<_>>();
2338 let mut abandoned_bindings = Vec::new();
2339 for key in pending_keys {
2340 let Some(pending) = inner.pending_relays.remove(&key) else {
2341 continue;
2342 };
2343 release_reserved_route_locked(
2344 inner,
2345 pending.reservation.client_key,
2346 pending.reservation.module_key,
2347 );
2348 if pending.relay_enqueued {
2349 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2350 abandoned_bindings.push(target);
2351 }
2352 } else {
2353 inner.relays_released_before_enqueue.insert(key);
2357 }
2358 let _ = pending
2359 .sender
2360 .send(RouteBindRelayOutcome::Rejected(ErrorBody::new(
2361 "module_reloading",
2362 format!("module_id '{module_id}' is reloading"),
2363 )));
2364 }
2365
2366 let pending_control_keys = inner
2367 .pending_control_rpcs
2368 .keys()
2369 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2370 .copied()
2371 .collect::<Vec<_>>();
2372 for key in pending_control_keys {
2373 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2374 let _ = pending
2375 .sender
2376 .send(ModuleControlRpcOutcome::ModuleGone(format!(
2377 "module '{module_id}' began draining during module-control RPC"
2378 )));
2379 }
2380 }
2381
2382 ModuleDrainTarget {
2383 endpoint,
2384 sink: module.sink,
2385 negotiated_ver: module.negotiated_ver,
2386 abandoned_bindings,
2387 excluded_subscriptions,
2388 }
2389 }
2390}
2391
2392#[derive(Debug, Default, PartialEq, Eq)]
2395pub(crate) struct DrainHoldouts {
2396 pub(crate) requests: usize,
2399 pub(crate) routes: usize,
2401 pub(crate) total_routes: usize,
2403 pub(crate) top_connections: Vec<(u64, usize)>,
2406 pub(crate) held: Vec<(u16, u64)>,
2415}
2416
2417pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
2419
2420impl ForwardingTable {
2421 pub(crate) fn endpoint_drain_holdouts(
2423 &self,
2424 endpoint: ModuleEndpointId,
2425 ) -> Result<DrainHoldouts, ForwardingError> {
2426 let inner = self.read_inner()?;
2427 let mut holdouts = DrainHoldouts::default();
2428 let mut by_connection: HashMap<u64, usize> = HashMap::new();
2429 for (key, route) in &inner.client_to_module {
2430 if route.module_endpoint != endpoint {
2431 continue;
2432 }
2433 holdouts.total_routes += 1;
2434 let held = route.flow.drain_in_flight();
2435 if held == 0 {
2436 continue;
2437 }
2438 holdouts.requests += held;
2439 holdouts.routes += 1;
2440 *by_connection.entry(key.connection_id.get()).or_default() += held;
2441 holdouts.held.extend(
2442 route
2443 .flow
2444 .drain_held_corrs()
2445 .into_iter()
2446 .map(|corr| (route.module_channel, corr)),
2447 );
2448 }
2449 holdouts.held.sort_unstable();
2450 holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
2451 let mut connections = by_connection.into_iter().collect::<Vec<_>>();
2452 connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
2453 connections.truncate(3);
2454 holdouts.top_connections = connections;
2455 Ok(holdouts)
2456 }
2457
2458 pub(crate) fn endpoint_in_flight_count(
2459 &self,
2460 endpoint: ModuleEndpointId,
2461 ) -> Result<usize, ForwardingError> {
2462 let inner = self.read_inner()?;
2463 Ok(inner
2464 .client_to_module
2465 .values()
2466 .filter(|route| route.module_endpoint == endpoint)
2467 .map(|route| route.flow.drain_in_flight())
2468 .sum())
2469 }
2470
2471 pub(crate) fn endpoint_is_draining(
2472 &self,
2473 endpoint: ModuleEndpointId,
2474 ) -> Result<bool, ForwardingError> {
2475 Ok(self
2476 .read_inner()?
2477 .draining_endpoints
2478 .contains_key(&endpoint))
2479 }
2480
2481 pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
2482 let inner = self.read_inner()?;
2483 Ok(inner
2484 .modules_by_id
2485 .get(module_id)
2486 .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
2487 }
2488
2489 pub(crate) fn release_module_endpoint_routes(
2490 &self,
2491 endpoint: ModuleEndpointId,
2492 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2493 let mut inner = self.write_inner()?;
2494 let routes = inner
2495 .module_to_client
2496 .iter()
2497 .filter(|(module_key, _)| module_key.endpoint == endpoint)
2498 .map(|(module_key, route)| (*module_key, route.module_epoch))
2499 .collect::<Vec<_>>();
2500 let mut released = Vec::with_capacity(routes.len());
2501 for (module_key, epoch) in routes {
2502 if let RouteRelease::Removed(target) =
2503 release_module_route_locked(&mut inner, module_key, epoch)
2504 {
2505 released.push(target);
2506 }
2507 }
2508 Ok(released)
2509 }
2510
2511 pub(crate) fn endpoint_routes(
2517 &self,
2518 endpoint: ModuleEndpointId,
2519 ) -> Result<Vec<EndpointRoute>, ForwardingError> {
2520 let inner = self.read_inner()?;
2521 Ok(endpoint_routes_locked(&inner, endpoint))
2522 }
2523
2524 pub(crate) fn route_census(
2526 &self,
2527 module_id: Option<&str>,
2528 ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
2529 let inner = self.read_inner()?;
2530 let mut endpoints = inner
2531 .modules_by_id
2532 .iter()
2533 .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
2534 .map(|(id, module)| (id.clone(), module.endpoint))
2535 .collect::<Vec<_>>();
2536 endpoints.sort_by(|left, right| left.0.cmp(&right.0));
2537 Ok(endpoints
2538 .into_iter()
2539 .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
2540 .collect())
2541 }
2542
2543 pub(crate) fn live_roots(
2545 &self,
2546 module_id: &str,
2547 ) -> Result<ModuleControlResponseToModule, ForwardingError> {
2548 let inner = self.read_inner()?;
2549 let endpoint = inner
2550 .modules_by_id
2551 .get(module_id)
2552 .map(|module| module.endpoint);
2553 let mut roots = BTreeMap::new();
2554 let mut unknown_root_bindings = 0;
2555 let mut total_bindings = 0;
2556 if let Some(endpoint) = endpoint {
2557 for binding in inner
2558 .module_to_client
2559 .values()
2560 .filter(|binding| binding.module_endpoint == endpoint)
2561 {
2562 total_bindings += 1;
2563 if let Some(root) = &binding.project_root {
2564 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2565 entry.0 += 1;
2566 } else {
2567 unknown_root_bindings += 1;
2568 }
2569 }
2570 for pending in inner
2571 .pending_relays
2572 .values()
2573 .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
2574 {
2575 total_bindings += 1;
2576 if let Some(root) = &pending.reservation.project_root {
2577 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2578 entry.1 += 1;
2579 } else {
2580 unknown_root_bindings += 1;
2581 }
2582 }
2583 }
2584 Ok(ModuleControlResponseToModule::LiveRoots {
2585 roots: roots
2586 .into_iter()
2587 .map(|(project_root, (bound, pending))| LiveRoot {
2588 project_root,
2589 bound,
2590 pending,
2591 })
2592 .collect(),
2593 unknown_root_bindings,
2594 total_bindings,
2595 })
2596 }
2597
2598 pub(crate) fn connection_has_client_routes(
2604 &self,
2605 connection_id: ConnectionId,
2606 ) -> Result<bool, ForwardingError> {
2607 let inner = self.read_inner()?;
2608 Ok(connection_has_client_routes_locked(&inner, connection_id))
2609 }
2610
2611 pub(crate) fn cleanup_connection(
2612 &self,
2613 connection_id: ConnectionId,
2614 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2615 self.cleanup_connection_counted(connection_id)
2616 .map(|cleanup| cleanup.released)
2617 }
2618
2619 pub(crate) fn cleanup_connection_counted(
2624 &self,
2625 connection_id: ConnectionId,
2626 ) -> Result<ConnectionCleanup, ForwardingError> {
2627 let mut inner = self.write_inner()?;
2628 inner.closing_connections.insert(connection_id);
2629 let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
2630 remove_module_connection_locked(&mut inner, endpoint)
2631 } else {
2632 ConnectionCleanup {
2633 released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
2634 abandoned_relays: 0,
2635 }
2636 };
2637 inner.closing_connections.remove(&connection_id);
2646 Ok(cleanup)
2647 }
2648
2649 fn cleanup_client_connection_locked(
2650 inner: &mut ForwardingInner,
2651 connection_id: ConnectionId,
2652 ) -> Vec<GoodbyeTarget> {
2653 let routes = inner
2654 .client_to_module
2655 .iter()
2656 .filter(|(key, _)| key.connection_id == connection_id)
2657 .map(|(key, route)| (*key, route.client_epoch))
2658 .collect::<Vec<_>>();
2659 let mut released = Vec::with_capacity(routes.len());
2660 for (client_key, epoch) in routes {
2661 if let RouteRelease::Removed(target) =
2662 release_client_route_locked(inner, client_key, epoch)
2663 {
2664 released.push(target);
2665 }
2666 }
2667
2668 let pending_keys = inner
2669 .pending_relays
2670 .iter()
2671 .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
2672 .map(|(key, _)| *key)
2673 .collect::<Vec<_>>();
2674 for key in pending_keys {
2675 let Some(pending) = inner.pending_relays.remove(&key) else {
2676 continue;
2677 };
2678 release_reserved_route_locked(
2679 inner,
2680 pending.reservation.client_key,
2681 pending.reservation.module_key,
2682 );
2683 if pending.relay_enqueued {
2684 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2685 released.push(target);
2686 }
2687 } else {
2688 inner.relays_released_before_enqueue.insert(key);
2692 }
2693 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
2694 "client connection closed during route.bind relay".to_string(),
2695 ));
2696 }
2697
2698 let orphaned = inner
2699 .reserved_client
2700 .iter()
2701 .filter(|(key, _)| key.connection_id == connection_id)
2702 .map(|(client, module)| (*client, *module))
2703 .collect::<Vec<_>>();
2704 for (client_key, module_key) in orphaned {
2705 release_reserved_route_locked(inner, client_key, module_key);
2706 }
2707 inner.next_client_channel.remove(&connection_id);
2708 inner
2709 .client_slot_epochs
2710 .retain(|key, _| key.connection_id != connection_id);
2711 inner
2712 .last_published_epoch
2713 .retain(|key, _| key.connection_id != connection_id);
2714 inner
2715 .status
2716 .retain(|(key, _), _| key.connection_id != connection_id);
2717
2718 released
2719 }
2720
2721 pub(crate) fn escalate_client_delivery_failure(
2730 &self,
2731 connection_id: ConnectionId,
2732 channel: u16,
2733 expected_epoch: u32,
2734 reason: CloseReason,
2735 undelivered: UndeliveredFrame<'_>,
2736 ) -> Result<bool, ForwardingError> {
2737 let principals = {
2738 let mut inner = self.write_inner()?;
2739 let key = ClientRouteKey {
2740 connection_id,
2741 channel,
2742 };
2743 if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
2744 None
2745 } else {
2746 inner.closing_connections.insert(connection_id);
2747 Some(connection_principals_locked(&inner, connection_id))
2748 }
2749 };
2750 let Some(principals) = principals else {
2751 return Ok(false);
2752 };
2753 let backlog = undelivered.sink.backlog();
2754 let close_reason = reason.to_string();
2755 if self.request_connection_close(connection_id, reason) {
2756 warn!(
2757 connection_id = connection_id.get(),
2758 principals = %principals,
2759 module_id = undelivered.module_id.unwrap_or("unknown"),
2760 client_channel = channel,
2761 queued_bytes = backlog.queued_bytes,
2762 queued_frames = backlog.queued_frames,
2763 oldest_queued_ms = backlog
2764 .oldest_age
2765 .map(|age| age.as_millis() as u64)
2766 .unwrap_or(0),
2767 close_reason = %close_reason,
2768 "closing client connection: its egress queue could not take a frame"
2769 );
2770 }
2771 Ok(true)
2772 }
2773
2774 pub(crate) fn publish_scope_changes(
2788 &self,
2789 changes: &[ScopeTagChange],
2790 ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
2791 let mut inner = self.write_inner()?;
2792 let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
2793 for change in changes {
2794 let key = (change.owner.clone(), change.scope_ref.clone());
2795 match change.after {
2796 Some(tag) => {
2797 inner.scope_tags.insert(key, tag);
2798 }
2799 None => {
2800 inner.scope_tags.remove(&key);
2801 }
2802 }
2803 by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
2804 }
2805 let mut selected = Vec::new();
2806 for (client_key, route) in &inner.client_to_module {
2807 let Some(scope) = &route.scope else {
2808 continue;
2809 };
2810 let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
2811 else {
2812 continue;
2813 };
2814 if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
2815 continue;
2816 }
2817 let reason = match &change.drain {
2818 ScopeDrain::Nothing => continue,
2819 ScopeDrain::All(reason) => *reason,
2820 ScopeDrain::Carriers(narrowed) => {
2821 let owner = Principal::Reserved {
2822 module_id: scope.owner.clone(),
2823 };
2824 let hit = route.principal != owner
2825 && narrowed.iter().any(|(principal, allowed)| {
2826 *principal == route.principal
2827 && allowed
2828 .as_ref()
2829 .is_none_or(|targets| !targets.contains(&route.module_id))
2830 });
2831 if !hit {
2832 continue;
2833 }
2834 RouteCloseReason::ScopeCarrierRemoved
2835 }
2836 };
2837 selected.push((*client_key, route.client_epoch, reason));
2838 }
2839 let mut drained = Vec::new();
2840 for (client_key, client_epoch, reason) in selected {
2841 let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
2842 continue;
2843 };
2844 let release = release_client_route_locked(&mut inner, client_key, client_epoch);
2845 self.record_route_release(&release);
2846 if let RouteRelease::Removed(module) = release {
2847 drained.push(ScopeDrainedRoute {
2848 reason,
2849 scope: route
2850 .scope
2851 .clone()
2852 .expect("selected scope drain has a bound scope"),
2853 module_id: route.module_id.clone(),
2854 client: GoodbyeTarget {
2855 connection_id: route.client_connection_id,
2856 sink: route.client_sink.clone(),
2857 negotiated_ver: route.client_negotiated_ver,
2858 channel: route.client_channel,
2859 epoch: route.client_epoch,
2860 kind: GoodbyeTargetKind::Client,
2861 module_id: Some(route.module_id.clone()),
2862 },
2863 module,
2864 });
2865 }
2866 }
2867 Ok(drained)
2868 }
2869
2870 #[cfg(test)]
2872 pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
2873 self.read_inner()
2874 .ok()?
2875 .scope_tags
2876 .get(&(owner.to_string(), scope_ref.to_string()))
2877 .copied()
2878 }
2879
2880 fn record_route_release(&self, release: &RouteRelease) {
2881 match release {
2882 RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
2883 RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
2884 RouteRelease::Absent => {}
2885 }
2886 }
2887
2888 fn read_inner(&self) -> Result<RwLockReadGuard<'_, ForwardingInner>, ForwardingError> {
2889 self.inner.read().map_err(|_| ForwardingError::Poisoned)
2890 }
2891
2892 fn write_inner(&self) -> Result<RwLockWriteGuard<'_, ForwardingInner>, ForwardingError> {
2893 self.inner.write().map_err(|_| ForwardingError::Poisoned)
2894 }
2895
2896 fn lock_close_registry(
2897 &self,
2898 ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
2899 self.close_registry
2900 .lock()
2901 .unwrap_or_else(|poisoned| poisoned.into_inner())
2902 }
2903}
2904
2905impl ForwardingInner {
2906 fn allocate_route_slots(
2907 &mut self,
2908 connection_id: ConnectionId,
2909 endpoint: ModuleEndpointId,
2910 ) -> Result<(u16, u32, u16, u32), ForwardingError> {
2911 let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
2912 let mut client_channel = client_start;
2913 let client_channel = loop {
2914 let key = ClientRouteKey {
2915 connection_id,
2916 channel: client_channel,
2917 };
2918 let eligible = !self.client_to_module.contains_key(&key)
2919 && !self.reserved_client.contains_key(&key)
2920 && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2921 if eligible {
2922 break client_channel;
2923 }
2924 client_channel = next_channel(client_channel);
2925 if client_channel == client_start {
2926 return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
2927 }
2928 };
2929
2930 let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
2931 let mut module_channel = module_start;
2932 let module_channel = loop {
2933 let key = ModuleRouteKey {
2934 endpoint,
2935 channel: module_channel,
2936 };
2937 let eligible = !self.module_to_client.contains_key(&key)
2938 && !self.reserved_module.contains_key(&key)
2939 && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2940 if eligible {
2941 break module_channel;
2942 }
2943 module_channel = next_channel(module_channel);
2944 if module_channel == module_start {
2945 return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
2946 }
2947 };
2948
2949 let client_key = ClientRouteKey {
2950 connection_id,
2951 channel: client_channel,
2952 };
2953 let module_key = ModuleRouteKey {
2954 endpoint,
2955 channel: module_channel,
2956 };
2957 let client_epoch = self
2958 .client_slot_epochs
2959 .get(&client_key)
2960 .copied()
2961 .unwrap_or(0)
2962 + 1;
2963 let module_epoch = self
2964 .module_slot_epochs
2965 .get(&module_key)
2966 .copied()
2967 .unwrap_or(0)
2968 + 1;
2969 self.client_slot_epochs.insert(client_key, client_epoch);
2970 self.module_slot_epochs.insert(module_key, module_epoch);
2971 self.next_client_channel
2972 .insert(connection_id, next_channel(client_channel));
2973 self.next_module_channel
2974 .insert(endpoint, next_channel(module_channel));
2975 Ok((client_channel, client_epoch, module_channel, module_epoch))
2976 }
2977
2978 fn allocate_control_corr(
2979 &mut self,
2980 endpoint: ModuleEndpointId,
2981 ) -> Result<u64, ForwardingError> {
2982 let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
2983 if candidate == 0 {
2984 self.closing_connections.insert(endpoint.connection_id);
2985 return Err(ForwardingError::RelayCorrelationExhausted);
2986 }
2987 self.next_control_corr.insert(
2988 endpoint,
2989 if candidate == u64::MAX {
2990 0
2991 } else {
2992 candidate + 1
2993 },
2994 );
2995 Ok(candidate)
2996 }
2997}
2998
2999fn connection_has_client_routes_locked(
3000 inner: &ForwardingInner,
3001 connection_id: ConnectionId,
3002) -> bool {
3003 inner
3004 .client_to_module
3005 .keys()
3006 .any(|key| key.connection_id == connection_id)
3007 || inner
3008 .reserved_client
3009 .keys()
3010 .any(|key| key.connection_id == connection_id)
3011}
3012
3013fn check_module_connection_role_locked(
3014 inner: &ForwardingInner,
3015 connection_id: ConnectionId,
3016) -> Result<(), ForwardingError> {
3017 if inner.endpoint_by_connection.contains_key(&connection_id)
3018 || connection_has_client_routes_locked(inner, connection_id)
3019 {
3020 return Err(ForwardingError::ConnectionRoleConflict { connection_id });
3021 }
3022 Ok(())
3023}
3024
3025fn next_channel(channel: u16) -> u16 {
3026 let next = channel.wrapping_add(1);
3027 if next == 0 {
3028 1
3029 } else {
3030 next
3031 }
3032}
3033
3034fn endpoint_routes_locked(
3035 inner: &ForwardingInner,
3036 endpoint: ModuleEndpointId,
3037) -> Vec<EndpointRoute> {
3038 let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
3039 let draining = drain_reason.is_some();
3040 let mut routes = inner
3041 .module_to_client
3042 .iter()
3043 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3044 .map(|(_, route)| EndpointRoute {
3045 goodbye_target: GoodbyeTarget {
3046 connection_id: route.client_connection_id,
3047 sink: route.client_sink.clone(),
3048 negotiated_ver: route.client_negotiated_ver,
3049 channel: route.client_channel,
3050 epoch: route.client_epoch,
3051 kind: GoodbyeTargetKind::Client,
3052 module_id: Some(route.module_id.clone()),
3053 },
3054 principal: route.principal.clone(),
3055 bound_at: route.bound_at,
3056 draining,
3057 drain_reason,
3058 })
3059 .collect::<Vec<_>>();
3060 routes.sort_by_key(|route| {
3061 (
3062 route.goodbye_target.connection_id.get(),
3063 route.goodbye_target.channel,
3064 route.goodbye_target.epoch,
3065 )
3066 });
3067 routes
3068}
3069
3070fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
3074 let key = (scope.owner.clone(), scope.scope_ref.clone());
3075 match inner.scope_tags.get(&key) {
3076 Some(current) if *current == scope.tag => None,
3077 Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
3078 subc_protocol::error_codes::SCOPE_CHANGED,
3079 format!(
3080 "scope '{}' of {} changed while the route was being bound; re-open it",
3081 scope.scope_ref, scope.owner
3082 ),
3083 )),
3084 _ => Some(ErrorBody::new(
3085 subc_protocol::error_codes::SCOPE_ENDED,
3086 format!(
3087 "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
3088 scope.scope_ref, scope.owner, scope.tag.scope_epoch
3089 ),
3090 )),
3091 }
3092}
3093
3094fn release_reserved_route_locked(
3095 inner: &mut ForwardingInner,
3096 client_key: ClientRouteKey,
3097 module_key: ModuleRouteKey,
3098) {
3099 if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
3100 inner.reserved_client.remove(&client_key);
3101 }
3102 if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
3103 inner.reserved_module.remove(&module_key);
3104 }
3105 inner.status.retain(|(key, _), _| *key != client_key);
3106}
3107
3108fn release_client_route_locked(
3109 inner: &mut ForwardingInner,
3110 client_key: ClientRouteKey,
3111 expected_epoch: u32,
3112) -> RouteRelease {
3113 let Some(route) = inner.client_to_module.get(&client_key) else {
3114 return RouteRelease::Absent;
3115 };
3116 if route.client_epoch != expected_epoch {
3117 return RouteRelease::Stale;
3118 }
3119 let route = inner
3120 .client_to_module
3121 .remove(&client_key)
3122 .expect("route checked under the same forwarding lock");
3123 route.flow.close();
3124 if let Some(scope) = &route.scope {
3125 scope.release_bind();
3126 }
3127 inner.module_to_client.remove(&ModuleRouteKey {
3128 endpoint: route.module_endpoint,
3129 channel: route.module_channel,
3130 });
3131 inner.operator_confirms.route_closed(ModuleRouteKey {
3132 endpoint: route.module_endpoint,
3133 channel: route.module_channel,
3134 });
3135 inner.status.remove(&(client_key, expected_epoch));
3136 RouteRelease::Removed(GoodbyeTarget {
3137 connection_id: route.module_endpoint.connection_id,
3138 sink: route.module_sink.clone(),
3139 negotiated_ver: route.module_negotiated_ver,
3140 channel: route.module_channel,
3141 epoch: route.module_epoch,
3142 kind: GoodbyeTargetKind::Module,
3143 module_id: Some(route.module_id.clone()),
3144 })
3145}
3146
3147fn release_module_route_locked(
3148 inner: &mut ForwardingInner,
3149 module_key: ModuleRouteKey,
3150 expected_epoch: u32,
3151) -> RouteRelease {
3152 let Some(route) = inner.module_to_client.get(&module_key) else {
3153 return RouteRelease::Absent;
3154 };
3155 if route.module_epoch != expected_epoch {
3156 return RouteRelease::Stale;
3157 }
3158 let route = inner
3159 .module_to_client
3160 .remove(&module_key)
3161 .expect("route checked under the same forwarding lock");
3162 inner.operator_confirms.route_closed(module_key);
3163 route.flow.close();
3164 if let Some(scope) = &route.scope {
3165 scope.release_bind();
3166 }
3167 let client_key = ClientRouteKey {
3168 connection_id: route.client_connection_id,
3169 channel: route.client_channel,
3170 };
3171 inner.client_to_module.remove(&client_key);
3172 inner.status.remove(&(client_key, route.client_epoch));
3173 RouteRelease::Removed(GoodbyeTarget {
3174 connection_id: route.client_connection_id,
3175 sink: route.client_sink.clone(),
3176 negotiated_ver: route.client_negotiated_ver,
3177 channel: route.client_channel,
3178 epoch: route.client_epoch,
3179 kind: GoodbyeTargetKind::Client,
3180 module_id: Some(route.module_id.clone()),
3181 })
3182}
3183
3184fn commit_route_locked(
3185 inner: &mut ForwardingInner,
3186 pending: PendingRouteBindRelayEntry,
3187) -> Result<Option<GoodbyeTarget>, ForwardingError> {
3188 let reservation = pending.reservation;
3189 if inner
3190 .closing_connections
3191 .contains(&reservation.client_key.connection_id)
3192 {
3193 return Err(ForwardingError::ConnectionClosing {
3194 connection_id: reservation.client_key.connection_id,
3195 });
3196 }
3197 let module_id = inner
3198 .module_id_by_endpoint
3199 .get(&reservation.module_key.endpoint)
3200 .cloned()
3201 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3202 if inner
3203 .draining_endpoints
3204 .contains_key(&reservation.module_key.endpoint)
3205 {
3206 return Err(ForwardingError::ModuleReloading { module_id });
3207 }
3208 if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
3209 || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
3210 {
3211 return Err(ForwardingError::UnknownReservation {
3212 client_channel: reservation.client_key.channel,
3213 module_channel: reservation.module_key.channel,
3214 });
3215 }
3216 let module = inner
3217 .modules_by_id
3218 .get(&module_id)
3219 .filter(|module| module.endpoint == reservation.module_key.endpoint)
3220 .cloned()
3221 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3222 let binding = Arc::new(RouteBinding {
3223 client_connection_id: reservation.client_key.connection_id,
3224 client_sink: pending.client_sink,
3225 client_negotiated_ver: pending.client_negotiated_ver,
3226 client_channel: reservation.client_key.channel,
3227 client_epoch: reservation.client_epoch,
3228 module_id,
3229 module_endpoint: reservation.module_key.endpoint,
3230 module_sink: module.sink,
3231 module_negotiated_ver: module.negotiated_ver,
3232 module_channel: reservation.module_key.channel,
3233 module_epoch: reservation.module_epoch,
3234 principal: pending.principal,
3235 project_root: reservation.project_root.clone(),
3236 bound_at: Instant::now(),
3237 flow: Arc::new(ChannelFlow::new(window_for(&module.concurrency))),
3238 scope: pending.scope,
3239 });
3240 inner
3241 .client_to_module
3242 .insert(reservation.client_key, Arc::clone(&binding));
3243 inner
3244 .module_to_client
3245 .insert(reservation.module_key, Arc::clone(&binding));
3246 let previous_published = inner
3247 .last_published_epoch
3248 .insert(reservation.client_key, reservation.client_epoch);
3249
3250 let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
3255 if client_writer_closed {
3256 let abandoned = abandoned_route_target(inner, &reservation);
3260 if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
3261 route.flow.close();
3262 }
3263 inner.module_to_client.remove(&reservation.module_key);
3264 inner
3265 .status
3266 .remove(&(reservation.client_key, reservation.client_epoch));
3267 match previous_published {
3268 Some(epoch) => {
3269 inner
3270 .last_published_epoch
3271 .insert(reservation.client_key, epoch);
3272 }
3273 None => {
3274 inner.last_published_epoch.remove(&reservation.client_key);
3275 }
3276 }
3277 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3278 "client egress closed during route publication".to_string(),
3279 ));
3280 return Ok(abandoned);
3281 }
3282
3283 if let Some(scope) = &binding.scope {
3284 scope.commit_presence();
3285 }
3286 let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
3287 Ok(None)
3288}
3289
3290fn module_connection_for_endpoint_locked(
3298 inner: &ForwardingInner,
3299 endpoint: ModuleEndpointId,
3300) -> Option<&ModuleConnection> {
3301 let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
3302 inner
3303 .modules_by_id
3304 .get(module_id)
3305 .filter(|module| module.endpoint == endpoint)
3306 .or_else(|| {
3307 inner
3308 .candidates_by_id
3309 .get(module_id)
3310 .filter(|module| module.endpoint == endpoint)
3311 })
3312 .or_else(|| inner.superseded_endpoints.get(&endpoint))
3313}
3314
3315fn abandoned_route_target(
3316 inner: &ForwardingInner,
3317 reservation: &RouteReservation,
3318) -> Option<GoodbyeTarget> {
3319 let module_id = inner
3320 .module_id_by_endpoint
3321 .get(&reservation.module_key.endpoint)?;
3322 let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
3323 (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
3324 connection_id: module.endpoint.connection_id,
3325 sink: module.sink.clone(),
3326 negotiated_ver: module.negotiated_ver,
3327 channel: reservation.module_key.channel,
3328 epoch: reservation.module_epoch,
3329 kind: GoodbyeTargetKind::Module,
3330 module_id: Some(module_id.clone()),
3331 })
3332}
3333
3334fn enqueue_hello_ack_locked(
3339 sink: &FrameSink,
3340 connection_id: ConnectionId,
3341 hello_ack: Option<Frame>,
3342) -> Result<(), ForwardingError> {
3343 let Some(hello_ack) = hello_ack else {
3344 return Ok(());
3345 };
3346 sink.try_send(hello_ack)
3347 .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
3348}
3349
3350fn remove_module_connection_locked(
3351 inner: &mut ForwardingInner,
3352 endpoint: ModuleEndpointId,
3353) -> ConnectionCleanup {
3354 inner.operator_confirms.module_closed(endpoint);
3357 inner.draining_endpoints.remove(&endpoint);
3358 inner.daemon_drain_results.remove(&endpoint);
3359 let module_id = inner.module_id_by_endpoint.remove(&endpoint);
3360 if let Some(module_id) = module_id.as_ref() {
3361 if inner
3362 .modules_by_id
3363 .get(module_id)
3364 .is_some_and(|module| module.endpoint == endpoint)
3365 {
3366 inner.modules_by_id.remove(module_id);
3367 }
3368 if inner
3369 .candidates_by_id
3370 .get(module_id)
3371 .is_some_and(|module| module.endpoint == endpoint)
3372 {
3373 inner.candidates_by_id.remove(module_id);
3374 }
3375 }
3376 inner.superseded_endpoints.remove(&endpoint);
3377 inner.endpoint_by_connection.remove(&endpoint.connection_id);
3378 inner.next_module_channel.remove(&endpoint);
3379 inner.next_control_corr.remove(&endpoint);
3380 inner
3381 .health_probe_tombstones
3382 .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
3383 inner
3384 .module_slot_epochs
3385 .retain(|key, _| key.endpoint != endpoint);
3386 let reserved_module_keys: Vec<ModuleRouteKey> = inner
3387 .reserved_module
3388 .keys()
3389 .filter(|module_key| module_key.endpoint == endpoint)
3390 .copied()
3391 .collect();
3392 for module_key in reserved_module_keys {
3393 if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
3394 release_reserved_route_locked(inner, client_key, module_key);
3395 }
3396 }
3397
3398 let pending_keys: Vec<_> = inner
3399 .pending_relays
3400 .keys()
3401 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3402 .copied()
3403 .collect();
3404 let pending: Vec<_> = pending_keys
3405 .into_iter()
3406 .filter_map(|key| {
3407 inner
3408 .pending_relays
3409 .remove(&key)
3410 .map(|pending| (key, pending))
3411 })
3412 .collect();
3413 let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
3414 for (key, pending) in pending {
3415 if !pending.relay_enqueued {
3419 inner.relays_released_before_enqueue.insert(key);
3420 }
3421 let module_label = module_id.as_deref().unwrap_or("unknown");
3422 let _ = pending
3423 .sender
3424 .send(RouteBindRelayOutcome::ModuleGone(format!(
3425 "module '{module_label}' connection closed during route.bind relay"
3426 )));
3427 }
3428
3429 let pending_control_keys: Vec<_> = inner
3430 .pending_control_rpcs
3431 .keys()
3432 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3433 .copied()
3434 .collect();
3435 let pending_control: Vec<_> = pending_control_keys
3436 .into_iter()
3437 .filter_map(|key| inner.pending_control_rpcs.remove(&key))
3438 .collect();
3439 for pending in pending_control {
3440 let module_label = module_id.as_deref().unwrap_or("unknown");
3441 let _ = pending
3442 .sender
3443 .send(ModuleControlRpcOutcome::ModuleGone(format!(
3444 "module '{module_label}' connection closed during module-control RPC"
3445 )));
3446 }
3447
3448 let module_routes = inner
3449 .module_to_client
3450 .iter()
3451 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3452 .map(|(module_key, route)| (*module_key, route.module_epoch))
3453 .collect::<Vec<_>>();
3454 let mut released = Vec::with_capacity(module_routes.len());
3455 for (module_key, epoch) in module_routes {
3456 if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
3457 {
3458 released.push(target);
3459 }
3460 }
3461 ConnectionCleanup {
3462 released,
3463 abandoned_relays,
3464 }
3465}
3466
3467#[derive(Debug, Clone, Copy)]
3468struct RequestCredit {
3469 subscription: bool,
3470 excluded_from_drain: bool,
3471}
3472
3473#[derive(Debug, Default)]
3474struct CreditLedger {
3475 by_corr: HashMap<u64, Vec<RequestCredit>>,
3476}
3477
3478impl CreditLedger {
3479 fn acquire(&mut self, corr: u64, subscription: bool) {
3480 self.by_corr.entry(corr).or_default().push(RequestCredit {
3481 subscription,
3482 excluded_from_drain: false,
3483 });
3484 }
3485
3486 fn release(&mut self, corr: u64) -> bool {
3487 let Some(credits) = self.by_corr.get_mut(&corr) else {
3488 return false;
3489 };
3490 let released = credits.pop().is_some();
3491 if credits.is_empty() {
3492 self.by_corr.remove(&corr);
3493 }
3494 released
3495 }
3496
3497 fn capture_subscription_exclusions(&mut self) -> u32 {
3498 let mut excluded = 0u32;
3499 for credit in self.by_corr.values_mut().flatten() {
3500 if credit.subscription && !credit.excluded_from_drain {
3501 credit.excluded_from_drain = true;
3502 excluded = excluded.saturating_add(1);
3503 }
3504 }
3505 excluded
3506 }
3507
3508 #[cfg(test)]
3509 fn in_flight(&self) -> usize {
3510 self.by_corr.values().map(Vec::len).sum()
3511 }
3512
3513 fn drain_in_flight(&self) -> usize {
3514 self.by_corr
3515 .values()
3516 .flatten()
3517 .filter(|credit| !credit.excluded_from_drain)
3518 .count()
3519 }
3520
3521 fn drain_held_corrs(&self) -> Vec<u64> {
3524 let mut corrs = self
3525 .by_corr
3526 .iter()
3527 .flat_map(|(corr, credits)| {
3528 credits
3529 .iter()
3530 .filter(|credit| !credit.excluded_from_drain)
3531 .map(move |_| *corr)
3532 })
3533 .collect::<Vec<_>>();
3534 corrs.sort_unstable();
3535 corrs
3536 }
3537}
3538
3539#[derive(Debug, Default)]
3540struct ChannelFlowState {
3541 closed: bool,
3542 credits: CreditLedger,
3543}
3544
3545#[derive(Debug)]
3547pub(crate) struct ChannelFlow {
3548 sem: Semaphore,
3549 window: usize,
3550 state: Mutex<ChannelFlowState>,
3551}
3552
3553impl ChannelFlow {
3554 pub(crate) fn new(window: usize) -> Self {
3555 debug_assert!(window > 0, "flow-control window must be non-zero");
3556 Self {
3557 sem: Semaphore::new(window),
3558 window,
3559 state: Mutex::new(ChannelFlowState::default()),
3560 }
3561 }
3562
3563 #[cfg(test)]
3564 pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
3565 self.acquire_tagged(0, false).await
3566 }
3567
3568 pub(crate) async fn acquire_tagged(
3569 &self,
3570 corr: u64,
3571 subscription: bool,
3572 ) -> Result<(), ChannelFlowClosed> {
3573 let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
3574 let mut state = self
3575 .state
3576 .lock()
3577 .unwrap_or_else(|poisoned| poisoned.into_inner());
3578 if state.closed {
3579 return Err(ChannelFlowClosed);
3580 }
3581 state.credits.acquire(corr, subscription);
3582 permit.forget();
3583 Ok(())
3584 }
3585
3586 #[cfg(test)]
3587 pub(crate) fn release(&self) {
3588 self.release_corr(0);
3589 }
3590
3591 pub(crate) fn release_corr(&self, corr: u64) {
3592 let released = self
3593 .state
3594 .lock()
3595 .unwrap_or_else(|poisoned| poisoned.into_inner())
3596 .credits
3597 .release(corr);
3598 if !released {
3599 warn!(
3603 window = self.window,
3604 available = self.sem.available_permits(),
3605 "flow-control over-release ignored"
3606 );
3607 return;
3608 }
3609 if !self.sem.is_closed() {
3610 self.sem.add_permits(1);
3611 }
3612 }
3613
3614 #[cfg(test)]
3615 pub(crate) fn in_flight(&self) -> usize {
3616 self.state
3617 .lock()
3618 .unwrap_or_else(|poisoned| poisoned.into_inner())
3619 .credits
3620 .in_flight()
3621 }
3622
3623 pub(crate) fn drain_in_flight(&self) -> usize {
3624 self.state
3625 .lock()
3626 .unwrap_or_else(|poisoned| poisoned.into_inner())
3627 .credits
3628 .drain_in_flight()
3629 }
3630
3631 pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
3632 self.state
3633 .lock()
3634 .unwrap_or_else(|poisoned| poisoned.into_inner())
3635 .credits
3636 .drain_held_corrs()
3637 }
3638
3639 #[cfg(test)]
3640 pub(crate) fn available_permits(&self) -> usize {
3641 self.sem.available_permits()
3642 }
3643
3644 pub(crate) fn begin_drain(&self) -> u32 {
3645 let mut state = self
3646 .state
3647 .lock()
3648 .unwrap_or_else(|poisoned| poisoned.into_inner());
3649 state.closed = true;
3650 self.sem.close();
3651 state.credits.capture_subscription_exclusions()
3652 }
3653
3654 pub(crate) fn close(&self) {
3655 self.state
3656 .lock()
3657 .unwrap_or_else(|poisoned| poisoned.into_inner())
3658 .closed = true;
3659 self.sem.close();
3660 }
3661}
3662
3663#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3664pub(crate) struct ChannelFlowClosed;
3665
3666impl fmt::Display for ChannelFlowClosed {
3667 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3668 write!(f, "flow-control window closed")
3669 }
3670}
3671
3672impl Error for ChannelFlowClosed {}
3673
3674fn window_for(concurrency: &Concurrency) -> usize {
3675 match concurrency {
3676 Concurrency::Serial => 1,
3677 Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
3678 Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
3679 }
3680}
3681
3682#[derive(Debug, Clone, PartialEq, Eq)]
3683pub enum ForwardingError {
3684 ConnectionRoleConflict {
3687 connection_id: ConnectionId,
3688 },
3689 NoModuleConnection,
3690 ModuleReloading {
3691 module_id: String,
3692 },
3693 StaleModuleEndpoint,
3694 UnknownReservation {
3695 client_channel: u16,
3696 module_channel: u16,
3697 },
3698 ClientRouteChannelExhausted {
3699 connection_id: ConnectionId,
3700 },
3701 ModuleRouteChannelExhausted {
3702 endpoint: ModuleEndpointId,
3703 },
3704 RelayCorrelationExhausted,
3705 ConnectionClosing {
3706 connection_id: ConnectionId,
3707 },
3708 ClientEgressClosed {
3709 connection_id: ConnectionId,
3710 },
3711 RouteOpenBuild(String),
3712 CandidateSlotOccupied {
3714 module_id: String,
3715 },
3716 ModuleEgressUnavailable {
3719 connection_id: ConnectionId,
3720 },
3721 Poisoned,
3722}
3723
3724impl fmt::Display for ForwardingError {
3725 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3726 match self {
3727 Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
3728 Self::NoModuleConnection => write!(f, "no module connection is registered"),
3729 Self::ModuleReloading { module_id } => {
3730 write!(f, "module_id '{module_id}' is reloading")
3731 }
3732 Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
3733 Self::UnknownReservation {
3734 client_channel,
3735 module_channel,
3736 } => write!(
3737 f,
3738 "route reservation client channel {client_channel} / module channel {module_channel} was not found"
3739 ),
3740 Self::ClientRouteChannelExhausted { connection_id } => write!(
3741 f,
3742 "no client route channels are available for connection {}",
3743 connection_id.get()
3744 ),
3745 Self::ModuleRouteChannelExhausted { endpoint } => write!(
3746 f,
3747 "no module route channels are available for endpoint generation {} on connection {}",
3748 endpoint.generation,
3749 endpoint.connection_id.get()
3750 ),
3751 Self::RelayCorrelationExhausted => {
3752 write!(f, "module control correlation ids are exhausted")
3753 }
3754 Self::ConnectionClosing { connection_id } => write!(
3755 f,
3756 "connection {} is closing and cannot accept route allocation",
3757 connection_id.get()
3758 ),
3759 Self::ClientEgressClosed { connection_id } => write!(
3760 f,
3761 "client connection {} egress is closed",
3762 connection_id.get()
3763 ),
3764 Self::RouteOpenBuild(message) => {
3765 write!(f, "failed to prebuild route.open response: {message}")
3766 }
3767 Self::CandidateSlotOccupied { module_id } => write!(
3768 f,
3769 "module_id '{module_id}' already has a swap candidate registered"
3770 ),
3771 Self::ModuleEgressUnavailable { connection_id } => write!(
3772 f,
3773 "module connection {} egress is unavailable; HELLO_ACK could not be queued",
3774 connection_id.get()
3775 ),
3776 Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
3777 }
3778 }
3779}
3780
3781impl Error for ForwardingError {}
3782
3783#[cfg(test)]
3784mod tests {
3785 use std::time::Duration;
3786
3787 use super::*;
3788 use tokio::sync::mpsc;
3789
3790 #[test]
3791 fn ordinary_long_running_request_is_not_excluded_from_drain() {
3792 let mut ledger = CreditLedger::default();
3793 ledger.acquire(1, false);
3794
3795 assert_eq!(ledger.capture_subscription_exclusions(), 0);
3796 assert_eq!(ledger.drain_in_flight(), 1);
3797 }
3798
3799 #[test]
3800 fn bit_set_subscription_is_excluded_and_counted() {
3801 let mut ledger = CreditLedger::default();
3802 ledger.acquire(1, true);
3803
3804 assert_eq!(ledger.capture_subscription_exclusions(), 1);
3805 assert_eq!(ledger.drain_in_flight(), 0);
3806 }
3807
3808 #[test]
3809 fn subscription_opened_after_drain_snapshot_is_not_excluded() {
3810 let mut ledger = CreditLedger::default();
3811 ledger.acquire(1, true);
3812 assert_eq!(ledger.capture_subscription_exclusions(), 1);
3813
3814 ledger.acquire(2, true);
3815
3816 assert_eq!(ledger.drain_in_flight(), 1);
3817 }
3818
3819 #[test]
3820 fn drain_with_no_subscriptions_reports_zero_excluded() {
3821 let mut ledger = CreditLedger::default();
3822 assert_eq!(ledger.capture_subscription_exclusions(), 0);
3823 }
3824
3825 fn test_hello_ack(corr: u64) -> Frame {
3826 Frame::build(
3827 FrameType::HelloAck,
3828 Flags::new(false, Priority::Passive, false),
3829 0,
3830 0,
3831 corr,
3832 Vec::new(),
3833 )
3834 .unwrap()
3835 }
3836
3837 #[test]
3841 fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
3842 let forwarding = ForwardingTable::default();
3843
3844 let (closed_tx, closed_rx) = mpsc::channel(8);
3845 drop(closed_rx);
3846 let closed = ConnectionId::new(1);
3847 assert_eq!(
3848 forwarding.register_module_connection_acked(
3849 closed,
3850 "closed".to_string(),
3851 2,
3852 Concurrency::ModuleManaged,
3853 FrameSink::new(closed_tx),
3854 test_hello_ack(1),
3855 ),
3856 Err(ForwardingError::ModuleEgressUnavailable {
3857 connection_id: closed
3858 })
3859 );
3860
3861 let (full_tx, _full_rx) = mpsc::channel(1);
3862 let full_sink = FrameSink::new(full_tx);
3863 full_sink.try_send(test_hello_ack(99)).unwrap();
3864 let full = ConnectionId::new(2);
3865 assert_eq!(
3866 forwarding.register_module_connection_acked(
3867 full,
3868 "full".to_string(),
3869 2,
3870 Concurrency::ModuleManaged,
3871 full_sink.clone(),
3872 test_hello_ack(2),
3873 ),
3874 Err(ForwardingError::ModuleEgressUnavailable {
3875 connection_id: full
3876 })
3877 );
3878 assert_eq!(
3879 forwarding.register_candidate_module_connection_acked(
3880 full,
3881 "full".to_string(),
3882 2,
3883 Concurrency::ModuleManaged,
3884 full_sink,
3885 test_hello_ack(3),
3886 ),
3887 Err(ForwardingError::ModuleEgressUnavailable {
3888 connection_id: full
3889 })
3890 );
3891
3892 for (connection, module_id) in [(closed, "closed"), (full, "full")] {
3893 assert_eq!(
3894 forwarding
3895 .module_endpoint_for_connection(connection)
3896 .unwrap(),
3897 None
3898 );
3899 let (client_tx, _client_rx) = mpsc::channel(8);
3900 assert_eq!(
3901 forwarding
3902 .begin_route_bind_relay_for_test(
3903 ConnectionId::new(50),
3904 FrameSink::new(client_tx),
3905 1,
3906 module_id,
3907 )
3908 .err(),
3909 Some(ForwardingError::NoModuleConnection)
3910 );
3911 }
3912 assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
3913 }
3914
3915 #[test]
3918 fn acked_registration_queues_the_hello_ack_first() {
3919 let forwarding = ForwardingTable::default();
3920 let (active_tx, mut active_rx) = mpsc::channel(8);
3921 forwarding
3922 .register_module_connection_acked(
3923 ConnectionId::new(1),
3924 "acked".to_string(),
3925 2,
3926 Concurrency::ModuleManaged,
3927 FrameSink::new(active_tx),
3928 test_hello_ack(11),
3929 )
3930 .unwrap();
3931 let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
3932 forwarding
3933 .register_candidate_module_connection_acked(
3934 ConnectionId::new(2),
3935 "acked".to_string(),
3936 2,
3937 Concurrency::ModuleManaged,
3938 FrameSink::new(candidate_tx),
3939 test_hello_ack(12),
3940 )
3941 .unwrap();
3942
3943 let active_first = active_rx.try_recv().unwrap().frame;
3944 assert_eq!(active_first.header.ty, FrameType::HelloAck);
3945 assert_eq!(active_first.header.corr, 11);
3946 let candidate_first = candidate_rx.try_recv().unwrap().frame;
3947 assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
3948 assert_eq!(candidate_first.header.corr, 12);
3949 }
3950
3951 #[test]
3952 fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
3953 let forwarding = ForwardingTable::default();
3954 let module_connection = ConnectionId::new(10);
3955 let exhausted_client = ConnectionId::new(20);
3956 let second_client = ConnectionId::new(30);
3957 let (module_tx, _module_rx) = mpsc::channel(1);
3958 let endpoint = forwarding
3959 .register_module_connection(
3960 module_connection,
3961 "route-limit-provider".to_string(),
3962 1,
3963 Concurrency::ModuleManaged,
3964 FrameSink::new(module_tx),
3965 )
3966 .unwrap();
3967
3968 {
3969 let mut inner = forwarding.inner.write().unwrap();
3970 for channel in 1..=u16::MAX {
3971 inner.reserved_client.insert(
3972 ClientRouteKey {
3973 connection_id: exhausted_client,
3974 channel,
3975 },
3976 ModuleRouteKey {
3977 endpoint,
3978 channel: 1,
3979 },
3980 );
3981 }
3982 }
3983
3984 let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
3985 let err = forwarding
3986 .begin_route_bind_relay_for_test(
3987 exhausted_client,
3988 FrameSink::new(exhausted_tx),
3989 1,
3990 "route-limit-provider",
3991 )
3992 .unwrap_err();
3993 assert!(matches!(
3994 err,
3995 ForwardingError::ClientRouteChannelExhausted { connection_id }
3996 if connection_id == exhausted_client
3997 ));
3998
3999 let (second_tx, _second_rx) = mpsc::channel(1);
4000 let pending = forwarding
4001 .begin_route_bind_relay_for_test(
4002 second_client,
4003 FrameSink::new(second_tx),
4004 2,
4005 "route-limit-provider",
4006 )
4007 .unwrap();
4008 assert_eq!(pending.client_channel, 1);
4009 }
4010
4011 #[test]
4012 fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
4013 let forwarding = ForwardingTable::default();
4014 let module_connection = ConnectionId::new(40);
4015 let client = ConnectionId::new(50);
4016 let (module_tx, _module_rx) = mpsc::channel(1);
4017 forwarding
4018 .register_module_connection(
4019 module_connection,
4020 "slot-reuse-provider".to_string(),
4021 1,
4022 Concurrency::ModuleManaged,
4023 FrameSink::new(module_tx),
4024 )
4025 .unwrap();
4026
4027 let (client_tx, _client_rx) = mpsc::channel(1);
4028 let client_sink = FrameSink::new(client_tx);
4029 let mut wrapped_channel = None;
4030 for index in 0..=usize::from(u16::MAX) {
4031 let pending = forwarding
4032 .begin_route_bind_relay_for_test(
4033 client,
4034 client_sink.clone(),
4035 index as u64 + 1,
4036 "slot-reuse-provider",
4037 )
4038 .unwrap();
4039 if index == usize::from(u16::MAX) {
4040 wrapped_channel = Some(pending.module_channel);
4041 }
4042 forwarding
4043 .abort_pending_relay(
4044 pending.endpoint,
4045 pending.corr,
4046 RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
4047 )
4048 .unwrap();
4049 }
4050
4051 assert_eq!(wrapped_channel, Some(1));
4052 }
4053
4054 #[test]
4055 fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
4056 let forwarding = ForwardingTable::default();
4057 let client = ConnectionId::new(60);
4058 forwarding
4059 .inner
4060 .write()
4061 .unwrap()
4062 .next_client_channel
4063 .insert(client, 41);
4064
4065 let released = forwarding.cleanup_connection(client).unwrap();
4066
4067 assert!(released.is_empty());
4068 assert!(!forwarding
4069 .inner
4070 .read()
4071 .unwrap()
4072 .next_client_channel
4073 .contains_key(&client));
4074 }
4075
4076 #[test]
4077 fn stale_module_cleanup_preserves_fast_reconnect_successor() {
4078 let forwarding = ForwardingTable::default();
4079 let module_id = "fast-reconnect-provider";
4080 let first_connection = ConnectionId::new(70);
4081 let second_connection = ConnectionId::new(80);
4082 let (first_tx, _first_rx) = mpsc::channel(1);
4083 let first_endpoint = forwarding
4084 .register_module_connection(
4085 first_connection,
4086 module_id.to_string(),
4087 1,
4088 Concurrency::ModuleManaged,
4089 FrameSink::new(first_tx),
4090 )
4091 .unwrap();
4092 let (second_tx, _second_rx) = mpsc::channel(1);
4093 let second_endpoint = forwarding
4094 .register_module_connection(
4095 second_connection,
4096 module_id.to_string(),
4097 1,
4098 Concurrency::ModuleManaged,
4099 FrameSink::new(second_tx),
4100 )
4101 .unwrap();
4102 assert_ne!(first_endpoint, second_endpoint);
4103
4104 let released = forwarding.cleanup_connection(first_connection).unwrap();
4105
4106 assert!(released.is_empty());
4107 assert_eq!(
4108 forwarding
4109 .inner
4110 .read()
4111 .unwrap()
4112 .modules_by_id
4113 .get(module_id)
4114 .map(|module| module.endpoint),
4115 Some(second_endpoint)
4116 );
4117 assert!(forwarding.has_live_module_connection(module_id).unwrap());
4118 let control_rpc = forwarding
4119 .begin_module_control_rpc_for(
4120 module_id,
4121 "health.check",
4122 Instant::now() + Duration::from_secs(1),
4123 )
4124 .unwrap();
4125 assert_eq!(control_rpc.endpoint, second_endpoint);
4126 }
4127
4128 fn route_fixture(
4129 module_id: &str,
4130 ) -> (
4131 ForwardingTable,
4132 ConnectionId,
4133 ModuleEndpointId,
4134 ConnectionId,
4135 FrameSink,
4136 mpsc::Receiver<crate::router::OutboundFrame>,
4137 ) {
4138 let forwarding = ForwardingTable::default();
4139 let module_connection = ConnectionId::new(100);
4140 let client_connection = ConnectionId::new(200);
4141 let (module_tx, _module_rx) = mpsc::channel(8);
4142 let endpoint = forwarding
4143 .register_module_connection(
4144 module_connection,
4145 module_id.to_string(),
4146 2,
4147 Concurrency::ModuleManaged,
4148 FrameSink::new(module_tx),
4149 )
4150 .unwrap();
4151 let (client_tx, client_rx) = mpsc::channel(8);
4152 (
4153 forwarding,
4154 module_connection,
4155 endpoint,
4156 client_connection,
4157 FrameSink::new(client_tx),
4158 client_rx,
4159 )
4160 }
4161
4162 #[test]
4163 #[cfg(unix)]
4164 fn daemon_drain_gates_current_and_racing_provider_registrations() {
4165 let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
4166 assert_eq!(
4167 forwarding.begin_daemon_drain().unwrap(),
4168 [("provider".to_string(), endpoint)]
4169 );
4170 assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
4171 assert!(matches!(
4172 forwarding.register_module_connection(
4173 ConnectionId::new(300),
4174 "late-provider".into(),
4175 2,
4176 Concurrency::ModuleManaged,
4177 sink,
4178 ),
4179 Err(ForwardingError::ConnectionClosing { .. })
4180 ));
4181 }
4182
4183 #[test]
4184 #[cfg(unix)]
4185 fn daemon_drain_results_are_endpoint_fenced_and_removed_on_cleanup() {
4186 let (forwarding, connection, endpoint, _, _, _) = route_fixture("provider");
4187 assert_eq!(
4188 forwarding
4189 .daemon_drain_result_for_connection(connection)
4190 .unwrap(),
4191 None
4192 );
4193 forwarding
4194 .record_daemon_drain_result(endpoint, true)
4195 .unwrap();
4196 forwarding.begin_daemon_drain().unwrap();
4197 assert_eq!(
4198 forwarding
4199 .daemon_drain_result_for_connection(connection)
4200 .unwrap(),
4201 Some(false)
4202 );
4203 let replacement = ModuleEndpointId {
4204 generation: endpoint.generation + 1,
4205 ..endpoint
4206 };
4207 forwarding
4208 .record_daemon_drain_result(replacement, true)
4209 .unwrap();
4210 assert_eq!(
4211 forwarding
4212 .daemon_drain_result_for_connection(connection)
4213 .unwrap(),
4214 Some(false)
4215 );
4216 forwarding
4217 .record_daemon_drain_result(endpoint, true)
4218 .unwrap();
4219 assert_eq!(
4220 forwarding
4221 .daemon_drain_result_for_connection(connection)
4222 .unwrap(),
4223 Some(true)
4224 );
4225 assert_eq!(
4226 forwarding
4227 .daemon_drain_result_for_connection(ConnectionId::new(300))
4228 .unwrap(),
4229 Some(false)
4230 );
4231 forwarding.cleanup_connection(connection).unwrap();
4232 forwarding
4233 .record_daemon_drain_result(endpoint, true)
4234 .unwrap();
4235 assert_eq!(
4236 forwarding
4237 .daemon_drain_result_for_connection(connection)
4238 .unwrap(),
4239 Some(false)
4240 );
4241 assert!(forwarding
4242 .read_inner()
4243 .unwrap()
4244 .daemon_drain_results
4245 .is_empty());
4246 }
4247
4248 #[test]
4249 #[cfg(unix)]
4250 fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
4251 let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
4252 route_fixture("provider");
4253 let mut pending =
4254 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
4255 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
4256 forwarding.begin_daemon_drain().unwrap();
4257 let completion = forwarding
4258 .complete_pending_relay(
4259 module_connection,
4260 pending.corr,
4261 RouteBindRelayOutcome::Accepted,
4262 )
4263 .expect("draining admission is not a fatal module error");
4264 assert!(completion.settled);
4265 let abandoned = completion
4266 .abandoned
4267 .expect("late accepted bind needs a route GOODBYE");
4268 assert_eq!(abandoned.channel, pending.module_channel);
4269 assert!(matches!(pending.receiver.try_recv().unwrap(),
4270 RouteBindRelayOutcome::Rejected(error) if error.code == "module_reloading"));
4271 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4272 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4273 assert!(
4274 rx.try_recv().is_err(),
4275 "no route.open may be published during drain"
4276 );
4277 assert_eq!(
4278 forwarding
4279 .module_endpoint_for_connection(module_connection)
4280 .unwrap(),
4281 Some(endpoint)
4282 );
4283 assert!(
4284 !forwarding
4285 .complete_pending_relay(
4286 module_connection,
4287 pending.corr,
4288 RouteBindRelayOutcome::Accepted,
4289 )
4290 .unwrap()
4291 .settled
4292 );
4293 }
4294
4295 fn test_ping(corr: u64) -> Frame {
4296 Frame::build(
4297 FrameType::Ping,
4298 Flags::new(false, Priority::Passive, false),
4299 0,
4300 0,
4301 corr,
4302 Vec::new(),
4303 )
4304 .unwrap()
4305 }
4306
4307 fn begin_test_route(
4308 forwarding: &ForwardingTable,
4309 client_connection: ConnectionId,
4310 client_sink: FrameSink,
4311 corr: u64,
4312 module_id: &str,
4313 ) -> PendingRouteBindRelay {
4314 forwarding
4315 .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
4316 .unwrap()
4317 }
4318
4319 #[test]
4320 fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
4321 for candidate in [false, true] {
4322 for register_first in [false, true] {
4323 let (forwarding, module_connection, _, client_connection, sink, _rx) =
4324 route_fixture("target");
4325 let register = || {
4326 if candidate {
4327 forwarding.register_candidate_module_connection(
4328 client_connection,
4329 "source".into(),
4330 2,
4331 Concurrency::ModuleManaged,
4332 sink.clone(),
4333 )
4334 } else {
4335 forwarding.register_module_connection(
4336 client_connection,
4337 "source".into(),
4338 2,
4339 Concurrency::ModuleManaged,
4340 sink.clone(),
4341 )
4342 }
4343 };
4344 if register_first {
4345 register().unwrap();
4346 assert!(
4347 forwarding
4348 .begin_route_bind_relay_for_test(
4349 client_connection,
4350 sink.clone(),
4351 1,
4352 "target",
4353 )
4354 .is_err(),
4355 "a deferred route.open cannot reserve after HELLO"
4356 );
4357 assert!(forwarding
4358 .cleanup_connection(client_connection)
4359 .unwrap()
4360 .is_empty());
4361 } else {
4362 let pending =
4363 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
4364 assert!(
4365 register().is_err(),
4366 "HELLO cannot register after a route reservation"
4367 );
4368 forwarding
4369 .complete_pending_relay(
4370 module_connection,
4371 pending.corr,
4372 RouteBindRelayOutcome::Accepted,
4373 )
4374 .unwrap();
4375 assert_eq!(
4376 forwarding
4377 .cleanup_connection(client_connection)
4378 .unwrap()
4379 .len(),
4380 1
4381 );
4382 }
4383 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4384 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4385 }
4386 }
4387 }
4388
4389 #[tokio::test]
4395 async fn pending_route_open_completes_behind_queued_data_frames() {
4396 assert_eq!(
4397 crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
4398 8,
4399 "the per-connection pending route.open limit is its own constant"
4400 );
4401 let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
4402 route_fixture("open-behind-data");
4403 let (sink, mut client_rx) = crate::server::connection_egress();
4404 const DATA_FRAMES: usize = 1_000;
4405 let data = |corr: u64| {
4406 Frame::build(
4407 FrameType::StreamData,
4408 Flags::new(false, Priority::Interactive, false),
4409 9,
4410 1,
4411 corr,
4412 vec![b'x'; 200],
4413 )
4414 .unwrap()
4415 };
4416 for corr in 0..DATA_FRAMES as u64 {
4417 sink.try_send(data(corr)).unwrap();
4418 }
4419 let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
4420 assert_eq!(sink.backlog().queued_bytes, data_bytes);
4421
4422 let pending = tokio::time::timeout(
4423 Duration::from_secs(5),
4424 forwarding.begin_route_bind_relay_for(
4425 client,
4426 sink.clone(),
4427 subc_protocol::PROTOCOL_VERSION,
4428 4_242,
4429 "open-behind-data",
4430 Principal::Direct,
4431 None,
4432 None,
4433 Instant::now() + Duration::from_secs(60),
4434 ),
4435 )
4436 .await
4437 .expect("reserving the route.open slot must not wait behind data frames")
4438 .unwrap();
4439 forwarding
4440 .complete_pending_relay(
4441 module_connection,
4442 pending.corr,
4443 RouteBindRelayOutcome::Accepted,
4444 )
4445 .unwrap();
4446
4447 let backlog = sink.backlog();
4448 assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
4449 assert!(
4450 backlog.queued_bytes > data_bytes,
4451 "the route.open response must be counted in queued bytes: {backlog:?}"
4452 );
4453 for corr in 0..DATA_FRAMES as u64 {
4454 assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
4455 }
4456 let open = client_rx.recv().await.unwrap();
4457 assert_eq!(open.header.corr, 4_242);
4458 assert_eq!(open.header.ty, FrameType::Response);
4459 drop(open);
4460 assert_eq!(sink.backlog().queued_bytes, 0);
4461 assert_eq!(sink.backlog().queued_frames, 0);
4462 }
4463
4464 #[tokio::test]
4469 async fn drain_holdouts_count_held_requests_and_name_the_connection() {
4470 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4471 route_fixture("holdouts");
4472 let mut bound = |corr| {
4473 let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
4474 forwarding
4475 .complete_pending_relay(
4476 module_connection,
4477 route.corr,
4478 RouteBindRelayOutcome::Accepted,
4479 )
4480 .unwrap();
4481 client_rx.try_recv().unwrap();
4482 match forwarding
4483 .lookup_data_route(client, route.client_channel, route.client_epoch)
4484 .unwrap()
4485 {
4486 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4487 other => panic!("expected live route, got {other:?}"),
4488 }
4489 };
4490 let holding = bound(61);
4491 let _idle = bound(62);
4492 holding.flow.acquire_tagged(7, false).await.unwrap();
4493 holding.flow.acquire_tagged(2, false).await.unwrap();
4494 holding.flow.acquire_tagged(3, true).await.unwrap();
4495 forwarding
4496 .begin_module_drain("holdouts", RouteCloseReason::Restart)
4497 .unwrap();
4498
4499 let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
4500 assert_eq!(
4501 holdouts,
4502 DrainHoldouts {
4503 requests: 2,
4504 routes: 1,
4505 total_routes: 2,
4506 top_connections: vec![(client.get(), 2)],
4507 held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
4510 }
4511 );
4512 }
4513
4514 #[test]
4515 fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
4516 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4517 route_fixture("census");
4518 let pending = begin_test_route(&forwarding, client, sink, 1, "census");
4519 forwarding
4520 .complete_pending_relay(
4521 module_connection,
4522 pending.corr,
4523 RouteBindRelayOutcome::Accepted,
4524 )
4525 .unwrap();
4526
4527 let routes = forwarding.endpoint_routes(endpoint).unwrap();
4528 assert_eq!(routes.len(), 1);
4529 assert!(matches!(routes[0].principal, Principal::Direct));
4530 assert_eq!(routes[0].goodbye_target.connection_id, client);
4531 assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
4532 assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
4533 assert!(!routes[0].draining);
4534
4535 forwarding
4536 .begin_module_drain("census", RouteCloseReason::Restart)
4537 .unwrap();
4538 let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
4539 assert_eq!(draining_routes.len(), 1);
4540 assert!(draining_routes[0].draining);
4541 }
4542
4543 #[test]
4544 fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
4545 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
4546 let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
4547 assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
4548 forwarding
4549 .abort_pending_relay(
4550 first.endpoint,
4551 first.corr,
4552 RouteBindRelayOutcome::ModuleGone("abort".into()),
4553 )
4554 .unwrap();
4555 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4556 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4557
4558 let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
4559 assert_eq!(second.client_channel, first.client_channel);
4560 assert_eq!(second.module_channel, first.module_channel);
4561 assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
4562 }
4563
4564 #[test]
4565 fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
4566 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4567 route_fixture("epoch-release");
4568 let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
4569 forwarding
4570 .complete_pending_relay(
4571 module_connection,
4572 first.corr,
4573 RouteBindRelayOutcome::Accepted,
4574 )
4575 .unwrap();
4576 assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
4577 assert!(matches!(
4578 forwarding
4579 .release_client_route(client, first.client_channel, first.client_epoch)
4580 .unwrap(),
4581 RouteRelease::Removed(_)
4582 ));
4583 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4584 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4585
4586 let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
4587 forwarding
4588 .complete_pending_relay(
4589 module_connection,
4590 second.corr,
4591 RouteBindRelayOutcome::Accepted,
4592 )
4593 .unwrap();
4594 assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
4595 assert!(matches!(
4596 forwarding
4597 .release_client_route(client, second.client_channel, first.client_epoch)
4598 .unwrap(),
4599 RouteRelease::Stale
4600 ));
4601 assert!(!forwarding
4602 .cache_status(
4603 endpoint,
4604 second.module_channel,
4605 first.module_epoch,
4606 "stale".into(),
4607 )
4608 .unwrap());
4609 assert!(forwarding
4610 .cache_status(
4611 endpoint,
4612 second.module_channel,
4613 second.module_epoch,
4614 "current".into(),
4615 )
4616 .unwrap());
4617 match forwarding
4618 .route_poll_snapshot(client, second.client_channel, second.client_epoch)
4619 .unwrap()
4620 {
4621 RoutePollSnapshot::Bound { status, .. } => {
4622 assert_eq!(status.as_deref(), Some("current"));
4623 }
4624 RoutePollSnapshot::Absent => panic!("successor binding was removed"),
4625 }
4626 let counters = forwarding.counters().snapshot();
4627 assert_eq!(counters["route_released_epoch_fenced"], 1);
4628 assert_eq!(counters["route_release_stale_skipped"], 1);
4629 }
4630
4631 #[test]
4632 fn max_epoch_reservation_retires_only_that_slot() {
4633 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
4634 forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
4635 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
4636 let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
4637 assert_eq!(
4638 (final_use.client_channel, final_use.client_epoch),
4639 (7, u32::MAX)
4640 );
4641 assert_eq!(
4642 (final_use.module_channel, final_use.module_epoch),
4643 (9, u32::MAX)
4644 );
4645 forwarding
4646 .abort_pending_relay(
4647 endpoint,
4648 final_use.corr,
4649 RouteBindRelayOutcome::ModuleGone("abort".into()),
4650 )
4651 .unwrap();
4652 forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
4653 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
4654 let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
4655 assert_ne!(next.client_channel, 7);
4656 assert_ne!(next.module_channel, 9);
4657 assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
4658 }
4659
4660 #[test]
4661 fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
4662 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4663 route_fixture("corr-shared");
4664 let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
4665 assert_eq!(bind.corr, 1);
4666 forwarding
4667 .abort_pending_relay(
4668 endpoint,
4669 bind.corr,
4670 RouteBindRelayOutcome::ModuleGone("abort".into()),
4671 )
4672 .unwrap();
4673 let rpc = forwarding
4674 .begin_module_control_rpc_for(
4675 "corr-shared",
4676 "health.check",
4677 Instant::now() - Duration::from_millis(1),
4678 )
4679 .unwrap();
4680 assert_eq!(rpc.corr, 2);
4681 assert_eq!(
4682 forwarding
4683 .complete_module_control_rpc(
4684 module_connection,
4685 rpc.corr,
4686 Some("health.check"),
4687 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
4688 status: subc_protocol::session::HealthStatus::Ok,
4689 detail: None,
4690 metrics: None,
4691 }),
4692 )
4693 .unwrap(),
4694 ModuleControlRpcCompletion::Settled
4695 );
4696 assert!(matches!(
4697 rpc.receiver.blocking_recv().unwrap(),
4698 ModuleControlRpcOutcome::DeadlineElapsed
4699 ));
4700 }
4701
4702 #[tokio::test(start_paused = true)]
4703 async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
4704 let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
4705 let probe_started_at = Instant::now();
4706 let rpc = forwarding
4707 .begin_health_probe_rpc_for(
4708 "tombstone-ttl",
4709 "health.check",
4710 probe_started_at,
4711 probe_started_at + Duration::from_secs(5),
4712 )
4713 .unwrap();
4714 assert!(forwarding
4715 .tombstone_health_probe_rpc(endpoint, rpc.corr)
4716 .unwrap());
4717 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
4718
4719 tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
4720 tokio::task::yield_now().await;
4721
4722 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
4723 }
4724
4725 #[test]
4726 fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
4727 let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
4728 let mut close = forwarding.register_connection_close(endpoint.connection_id);
4729 forwarding.inject_control_corr(endpoint, u64::MAX);
4730 let final_rpc = forwarding
4731 .begin_module_control_rpc_for(
4732 "corr-max",
4733 "health.check",
4734 Instant::now() + Duration::from_secs(1),
4735 )
4736 .unwrap();
4737 assert_eq!(final_rpc.corr, u64::MAX);
4738 forwarding
4739 .cancel_module_control_rpc(endpoint, final_rpc.corr)
4740 .unwrap();
4741 assert!(matches!(
4742 forwarding.begin_module_control_rpc_for(
4743 "corr-max",
4744 "health.check",
4745 Instant::now() + Duration::from_secs(1),
4746 ),
4747 Err(ForwardingError::RelayCorrelationExhausted)
4748 ));
4749 assert!(close.try_recv().is_ok());
4750 }
4751
4752 #[test]
4753 fn publication_epoch_controls_delivery_failure_escalation() {
4754 fn setup_successor(
4755 commit_successor: Option<bool>,
4756 ) -> (ForwardingTable, ConnectionId, u16, u32) {
4757 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4758 route_fixture("escalation");
4759 let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
4760 forwarding
4761 .complete_pending_relay(
4762 module_connection,
4763 first.corr,
4764 RouteBindRelayOutcome::Accepted,
4765 )
4766 .unwrap();
4767 client_rx.try_recv().unwrap();
4768 assert!(matches!(
4769 forwarding
4770 .release_client_route(client, first.client_channel, first.client_epoch)
4771 .unwrap(),
4772 RouteRelease::Removed(_)
4773 ));
4774 if let Some(commit_successor) = commit_successor {
4775 forwarding.inject_client_slot_epoch(
4776 client,
4777 first.client_channel,
4778 first.client_epoch,
4779 );
4780 forwarding.inject_module_slot_epoch(
4781 endpoint,
4782 first.module_channel,
4783 first.module_epoch,
4784 );
4785 let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
4786 if commit_successor {
4787 forwarding
4788 .complete_pending_relay(
4789 module_connection,
4790 successor.corr,
4791 RouteBindRelayOutcome::Accepted,
4792 )
4793 .unwrap();
4794 client_rx.try_recv().unwrap();
4795 } else {
4796 forwarding
4797 .abort_pending_relay(
4798 endpoint,
4799 successor.corr,
4800 RouteBindRelayOutcome::ModuleGone("abort".into()),
4801 )
4802 .unwrap();
4803 }
4804 }
4805 (forwarding, client, first.client_channel, first.client_epoch)
4806 }
4807
4808 let probe_sink = FrameSink::new(mpsc::channel(1).0);
4809 let (no_successor, client, channel, epoch) = setup_successor(None);
4810 let mut close = no_successor.register_connection_close(client);
4811 assert!(no_successor
4812 .escalate_client_delivery_failure(
4813 client,
4814 channel,
4815 epoch,
4816 CloseReason::new("delivery", "failed"),
4817 UndeliveredFrame {
4818 module_id: None,
4819 sink: &probe_sink,
4820 },
4821 )
4822 .unwrap());
4823 assert!(close.try_recv().is_ok());
4824
4825 let (aborted, client, channel, epoch) = setup_successor(Some(false));
4826 let mut close = aborted.register_connection_close(client);
4827 assert!(aborted
4828 .escalate_client_delivery_failure(
4829 client,
4830 channel,
4831 epoch,
4832 CloseReason::new("delivery", "failed"),
4833 UndeliveredFrame {
4834 module_id: None,
4835 sink: &probe_sink,
4836 },
4837 )
4838 .unwrap());
4839 assert!(close.try_recv().is_ok());
4840
4841 let (published, client, channel, epoch) = setup_successor(Some(true));
4842 let mut close = published.register_connection_close(client);
4843 assert!(!published
4844 .escalate_client_delivery_failure(
4845 client,
4846 channel,
4847 epoch,
4848 CloseReason::new("delivery", "stale failure"),
4849 UndeliveredFrame {
4850 module_id: None,
4851 sink: &probe_sink,
4852 },
4853 )
4854 .unwrap());
4855 assert!(close.try_recv().is_err());
4856 }
4857
4858 #[test]
4859 fn route_concentration_separates_client_count_from_routes_per_client() {
4860 let (forwarding, module_connection, _, client, sink, _client_rx) =
4864 route_fixture("concentration");
4865 assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
4866
4867 for corr in [70_u64, 71] {
4868 let pending =
4869 begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
4870 forwarding
4871 .complete_pending_relay(
4872 module_connection,
4873 pending.corr,
4874 RouteBindRelayOutcome::Accepted,
4875 )
4876 .unwrap();
4877 }
4878
4879 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
4881 assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
4882 }
4883
4884 #[test]
4885 fn cleanup_and_accepted_resolution_have_one_lock_winner() {
4886 let (forwarding, module_connection, _, client, sink, mut client_rx) =
4887 route_fixture("cleanup-race");
4888 let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
4889 forwarding
4890 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4891 .unwrap();
4892 let released = forwarding.cleanup_connection(client).unwrap();
4893 assert_eq!(released.len(), 1);
4894 let completion = forwarding
4895 .complete_pending_relay(
4896 module_connection,
4897 pending.corr,
4898 RouteBindRelayOutcome::Accepted,
4899 )
4900 .unwrap();
4901 assert!(!completion.settled);
4902 assert!(client_rx.try_recv().is_err());
4903 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4904
4905 let (forwarding, module_connection, _, client, sink, mut client_rx) =
4906 route_fixture("accepted-race");
4907 let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
4908 forwarding
4909 .complete_pending_relay(
4910 module_connection,
4911 pending.corr,
4912 RouteBindRelayOutcome::Accepted,
4913 )
4914 .unwrap();
4915 assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
4916 let released = forwarding.cleanup_connection(client).unwrap();
4917 assert_eq!(released.len(), 1);
4918 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4919 }
4920
4921 #[test]
4922 fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
4923 let (forwarding, module_connection, _, client, sink, mut client_rx) =
4924 route_fixture("drain-gap");
4925 let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
4926 forwarding
4927 .complete_pending_relay(
4928 module_connection,
4929 live.corr,
4930 RouteBindRelayOutcome::Accepted,
4931 )
4932 .unwrap();
4933 client_rx.try_recv().unwrap();
4934 let binding = match forwarding
4935 .lookup_data_route(client, live.client_channel, live.client_epoch)
4936 .unwrap()
4937 {
4938 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4939 other => panic!("expected live route, got {other:?}"),
4940 };
4941
4942 let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
4943 forwarding
4944 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4945 .unwrap();
4946 let control_rpc = forwarding
4947 .begin_module_control_rpc_for(
4948 "drain-gap",
4949 "health.check",
4950 Instant::now() + Duration::from_secs(1),
4951 )
4952 .unwrap();
4953 let target = forwarding
4954 .begin_module_drain("drain-gap", RouteCloseReason::Reload)
4955 .unwrap()
4956 .unwrap();
4957 assert!(matches!(
4958 control_rpc.receiver.blocking_recv().unwrap(),
4959 ModuleControlRpcOutcome::ModuleGone(_)
4960 ));
4961 assert_eq!(target.abandoned_bindings.len(), 1);
4962 assert!(binding.flow.sem.is_closed());
4963 assert!(
4964 !forwarding
4965 .complete_pending_relay(
4966 module_connection,
4967 pending.corr,
4968 RouteBindRelayOutcome::Accepted,
4969 )
4970 .unwrap()
4971 .settled
4972 );
4973 assert!(matches!(
4974 forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
4975 Err(ForwardingError::ModuleReloading { .. })
4976 ));
4977 let released = forwarding
4978 .release_module_endpoint_routes(target.endpoint)
4979 .unwrap();
4980 assert_eq!(released.len(), 1);
4981 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4982 }
4983
4984 #[test]
4992 fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
4993 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4994 route_fixture("closing-client");
4995
4996 let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
4999 forwarding
5000 .complete_pending_relay(
5001 module_connection,
5002 live.corr,
5003 RouteBindRelayOutcome::Accepted,
5004 )
5005 .unwrap();
5006 client_rx.try_recv().unwrap();
5007
5008 let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
5010 forwarding
5011 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5012 .unwrap();
5013
5014 assert!(forwarding
5016 .escalate_client_delivery_failure(
5017 client,
5018 live.client_channel,
5019 live.client_epoch,
5020 CloseReason::new(
5021 "module_to_client_delivery_failed",
5022 "client egress refused a module frame",
5023 ),
5024 UndeliveredFrame {
5025 module_id: None,
5026 sink: &sink,
5027 },
5028 )
5029 .unwrap());
5030 assert!(!sink.is_closed());
5031
5032 let completion = forwarding
5033 .complete_pending_relay(
5034 module_connection,
5035 pending.corr,
5036 RouteBindRelayOutcome::Accepted,
5037 )
5038 .expect("a closing client must not turn a module's ack into an error");
5039
5040 assert!(completion.settled);
5041 let abandoned = completion
5042 .abandoned
5043 .expect("the module must be told to drop the binding it just created");
5044 assert_eq!(abandoned.connection_id, module_connection);
5045 assert_eq!(abandoned.channel, pending.module_channel);
5046 assert_eq!(abandoned.epoch, pending.module_epoch);
5047 assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
5048 assert!(matches!(
5049 pending.receiver.blocking_recv().unwrap(),
5050 RouteBindRelayOutcome::ModuleGone(_)
5051 ));
5052 assert!(client_rx.try_recv().is_err());
5055 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
5056
5057 assert!(forwarding
5060 .has_live_module_connection("closing-client")
5061 .unwrap());
5062 let cotenant = ConnectionId::new(201);
5063 let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
5064 let cotenant_route = begin_test_route(
5065 &forwarding,
5066 cotenant,
5067 FrameSink::new(cotenant_tx),
5068 62,
5069 "closing-client",
5070 );
5071 assert_eq!(cotenant_route.endpoint, endpoint);
5072 forwarding
5073 .complete_pending_relay(
5074 module_connection,
5075 cotenant_route.corr,
5076 RouteBindRelayOutcome::Accepted,
5077 )
5078 .unwrap();
5079 assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
5080 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5081 }
5082
5083 #[test]
5084 fn pending_route_permit_is_released_on_rejection_and_abort() {
5085 let forwarding = ForwardingTable::default();
5086 let module_connection = ConnectionId::new(300);
5087 let client = ConnectionId::new(301);
5088 let (module_tx, _module_rx) = mpsc::channel(1);
5089 let endpoint = forwarding
5090 .register_module_connection(
5091 module_connection,
5092 "permit".into(),
5093 2,
5094 Concurrency::ModuleManaged,
5095 FrameSink::new(module_tx),
5096 )
5097 .unwrap();
5098 let (client_tx, mut client_rx) = mpsc::channel(1);
5099 let sink = FrameSink::new(client_tx);
5100 let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
5101 assert!(sink.try_send(test_ping(999)).is_err());
5102 forwarding
5103 .complete_pending_relay(
5104 module_connection,
5105 rejected.corr,
5106 RouteBindRelayOutcome::Rejected(ErrorBody {
5107 code: "no".into(),
5108 message: "rejected".into(),
5109 detail: None,
5110 }),
5111 )
5112 .unwrap();
5113 sink.try_send(test_ping(1000)).unwrap();
5114 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
5115
5116 let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
5117 assert!(sink.try_send(test_ping(1001)).is_err());
5118 forwarding
5119 .abort_pending_relay(
5120 endpoint,
5121 aborted.corr,
5122 RouteBindRelayOutcome::ModuleGone("abort".into()),
5123 )
5124 .unwrap();
5125 sink.try_send(test_ping(1002)).unwrap();
5126 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
5127
5128 let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
5129 forwarding
5130 .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
5131 .unwrap();
5132 drop(client_rx);
5133 let completion = forwarding
5134 .complete_pending_relay(
5135 module_connection,
5136 receiver_closed.corr,
5137 RouteBindRelayOutcome::Accepted,
5138 )
5139 .unwrap();
5140 assert!(completion.abandoned.is_some());
5141 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5142 }
5143
5144 #[test]
5150 fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
5151 let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
5152 route_fixture("closing-set-leak");
5153
5154 const CONNECTIONS: u64 = 32;
5155 for index in 0..CONNECTIONS {
5156 let client = ConnectionId::new(1000 + index);
5157 let (client_tx, _client_rx) = mpsc::channel(8);
5158 let route = begin_test_route(
5159 &forwarding,
5160 client,
5161 FrameSink::new(client_tx),
5162 index + 1,
5163 "closing-set-leak",
5164 );
5165 forwarding
5166 .complete_pending_relay(
5167 module_connection,
5168 route.corr,
5169 RouteBindRelayOutcome::Accepted,
5170 )
5171 .unwrap();
5172 forwarding.cleanup_connection(client).unwrap();
5173 }
5174 forwarding.cleanup_connection(module_connection).unwrap();
5175
5176 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5177 }
5178
5179 #[test]
5186 fn closing_connection_is_refused_new_work_until_cleanup_completes() {
5187 let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
5188 route_fixture("closing-gate");
5189
5190 let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
5193 forwarding
5194 .complete_pending_relay(
5195 module_connection,
5196 live.corr,
5197 RouteBindRelayOutcome::Accepted,
5198 )
5199 .unwrap();
5200 client_rx.try_recv().unwrap();
5201
5202 assert!(forwarding
5205 .escalate_client_delivery_failure(
5206 client,
5207 live.client_channel,
5208 live.client_epoch,
5209 CloseReason::new(
5210 "module_to_client_delivery_failed",
5211 "client egress refused a module frame",
5212 ),
5213 UndeliveredFrame {
5214 module_id: None,
5215 sink: &sink,
5216 },
5217 )
5218 .unwrap());
5219 assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
5220
5221 assert!(matches!(
5223 forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
5224 Err(ForwardingError::ConnectionClosing { connection_id })
5225 if connection_id == client
5226 ));
5227 let (late_tx, _late_rx) = mpsc::channel(1);
5229 assert!(matches!(
5230 forwarding.register_module_connection(
5231 client,
5232 "late-module".into(),
5233 2,
5234 Concurrency::ModuleManaged,
5235 FrameSink::new(late_tx),
5236 ),
5237 Err(ForwardingError::ConnectionClosing { connection_id })
5238 if connection_id == client
5239 ));
5240
5241 forwarding.cleanup_connection(client).unwrap();
5245 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5246 }
5247}
5248
5249#[cfg(test)]
5252mod swap_slot_tests {
5253 use std::time::Duration;
5254
5255 use super::*;
5256 use tokio::sync::mpsc;
5257
5258 const MODULE_ID: &str = "swapped";
5259
5260 struct SwapFixture {
5261 forwarding: ForwardingTable,
5262 incumbent_connection: ConnectionId,
5263 incumbent: ModuleEndpointId,
5264 candidate_connection: ConnectionId,
5265 candidate: ModuleEndpointId,
5266 _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
5267 }
5268
5269 fn swap_fixture() -> SwapFixture {
5270 let forwarding = ForwardingTable::default();
5271 let incumbent_connection = ConnectionId::new(100);
5272 let candidate_connection = ConnectionId::new(110);
5273 let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
5274 let incumbent = forwarding
5275 .register_module_connection(
5276 incumbent_connection,
5277 MODULE_ID.to_string(),
5278 2,
5279 Concurrency::ModuleManaged,
5280 FrameSink::new(incumbent_tx),
5281 )
5282 .unwrap();
5283 let (candidate_tx, candidate_rx) = mpsc::channel(8);
5284 let candidate = forwarding
5285 .register_candidate_module_connection(
5286 candidate_connection,
5287 MODULE_ID.to_string(),
5288 2,
5289 Concurrency::ModuleManaged,
5290 FrameSink::new(candidate_tx),
5291 )
5292 .unwrap();
5293 SwapFixture {
5294 forwarding,
5295 incumbent_connection,
5296 incumbent,
5297 candidate_connection,
5298 candidate,
5299 _module_rxs: vec![incumbent_rx, candidate_rx],
5300 }
5301 }
5302
5303 fn client(
5304 raw: u64,
5305 ) -> (
5306 ConnectionId,
5307 FrameSink,
5308 mpsc::Receiver<crate::router::OutboundFrame>,
5309 ) {
5310 let (tx, rx) = mpsc::channel(8);
5311 (ConnectionId::new(raw), FrameSink::new(tx), rx)
5312 }
5313
5314 fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
5315 forwarding
5316 .read_inner()
5317 .unwrap()
5318 .client_to_module
5319 .values()
5320 .map(|route| route.module_endpoint)
5321 .collect()
5322 }
5323
5324 #[test]
5325 fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
5326 let fixture = swap_fixture();
5327 let forwarding = &fixture.forwarding;
5328 assert_ne!(fixture.incumbent, fixture.candidate);
5329
5330 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5332 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5333 let (client_connection, client_sink, _client_rx) = client(200);
5334 let pending = forwarding
5335 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5336 .unwrap();
5337 assert_eq!(pending.endpoint, fixture.incumbent);
5338 let rpc = forwarding
5339 .begin_module_control_rpc_for(
5340 MODULE_ID,
5341 "health.check",
5342 Instant::now() + Duration::from_secs(1),
5343 )
5344 .unwrap();
5345 assert_eq!(rpc.endpoint, fixture.incumbent);
5346 let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
5347 assert_eq!(census.len(), 1, "the census lists one endpoint per id");
5348
5349 assert_eq!(
5351 forwarding
5352 .module_endpoint_for_connection(fixture.candidate_connection)
5353 .unwrap(),
5354 Some(fixture.candidate)
5355 );
5356 assert_eq!(
5357 forwarding
5358 .module_id_for_connection(fixture.candidate_connection)
5359 .unwrap()
5360 .as_deref(),
5361 Some(MODULE_ID)
5362 );
5363
5364 let (other_tx, _other_rx) = mpsc::channel(1);
5366 assert_eq!(
5367 forwarding.register_candidate_module_connection(
5368 ConnectionId::new(120),
5369 MODULE_ID.to_string(),
5370 2,
5371 Concurrency::ModuleManaged,
5372 FrameSink::new(other_tx),
5373 ),
5374 Err(ForwardingError::CandidateSlotOccupied {
5375 module_id: MODULE_ID.to_string()
5376 })
5377 );
5378 }
5379
5380 #[test]
5384 fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
5385 let fixture = swap_fixture();
5386 let forwarding = &fixture.forwarding;
5387 let (early_client, early_sink, _early_rx) = client(200);
5388 let mut early = forwarding
5389 .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
5390 .unwrap();
5391 assert_eq!(early.endpoint, fixture.incumbent);
5392 assert_eq!(
5393 forwarding
5394 .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
5395 .unwrap(),
5396 RouteBindRelayMark::Marked
5397 );
5398
5399 let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5400 assert_eq!(
5401 cutover,
5402 ForwardingCutover {
5403 promoted: fixture.candidate,
5404 incumbent: Some(fixture.incumbent),
5405 }
5406 );
5407
5408 let (late_client, late_sink, _late_rx) = client(201);
5410 let late = forwarding
5411 .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
5412 .unwrap();
5413 assert_eq!(
5414 late.endpoint, fixture.candidate,
5415 "a route.open after cutover was reserved on the incumbent"
5416 );
5417
5418 let completion = forwarding
5420 .complete_pending_relay(
5421 fixture.incumbent_connection,
5422 early.corr,
5423 RouteBindRelayOutcome::Accepted,
5424 )
5425 .expect("a superseded endpoint's ack is not an error on its connection");
5426 assert!(completion.settled);
5427 assert!(
5428 !committed_endpoints(forwarding).contains(&fixture.incumbent),
5429 "a relay reserved before cutover committed a route on the incumbent"
5430 );
5431 let goodbye = completion
5432 .abandoned
5433 .expect("the incumbent is told to drop the binding it just created");
5434 assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
5435 assert_eq!(goodbye.channel, early.module_channel);
5436 assert_eq!(goodbye.epoch, early.module_epoch);
5437 assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
5438 match early.receiver.try_recv() {
5439 Ok(RouteBindRelayOutcome::Rejected(body)) => assert_eq!(body.code, "module_reloading"),
5440 other => panic!("expected a retryable module_reloading answer, got {other:?}"),
5441 }
5442 assert!(matches!(
5443 forwarding
5444 .lookup_data_route(early_client, early.client_channel, early.client_epoch)
5445 .unwrap(),
5446 DataRoute::Client(DataRouteState::Absent)
5447 ));
5448
5449 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
5452 forwarding
5453 .complete_pending_relay(
5454 fixture.candidate_connection,
5455 late.corr,
5456 RouteBindRelayOutcome::Accepted,
5457 )
5458 .unwrap();
5459 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5460 assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
5461 }
5462
5463 #[test]
5464 fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
5465 let fixture = swap_fixture();
5466 let forwarding = &fixture.forwarding;
5467 let (bound_client, bound_sink, _bound_rx) = client(200);
5469 let bound = forwarding
5470 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5471 .unwrap();
5472 forwarding
5473 .complete_pending_relay(
5474 fixture.incumbent_connection,
5475 bound.corr,
5476 RouteBindRelayOutcome::Accepted,
5477 )
5478 .unwrap();
5479 let (pending_client, pending_sink, _pending_rx) = client(201);
5480 let mut in_flight = forwarding
5481 .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
5482 .unwrap();
5483 forwarding
5484 .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
5485 .unwrap();
5486
5487 let incumbent = forwarding
5488 .cutover_candidate(MODULE_ID)
5489 .unwrap()
5490 .unwrap()
5491 .incumbent
5492 .unwrap();
5493 let target = forwarding
5494 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
5495 .unwrap()
5496 .expect("the superseded incumbent is still registered");
5497
5498 assert_eq!(target.endpoint, fixture.incumbent);
5499 assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
5500 assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
5501 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5502 assert_eq!(target.abandoned_bindings.len(), 1);
5503 assert_eq!(
5504 target.abandoned_bindings[0].channel,
5505 in_flight.module_channel
5506 );
5507 assert!(matches!(
5508 in_flight.receiver.try_recv(),
5509 Ok(RouteBindRelayOutcome::Rejected(body)) if body.code == "module_reloading"
5510 ));
5511 assert_eq!(
5512 forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
5513 1,
5514 "the incumbent's bound route stays until its drain finishes"
5515 );
5516
5517 let (next_client, next_sink, _next_rx) = client(202);
5518 let next = forwarding
5519 .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
5520 .expect("the promoted candidate keeps accepting routes");
5521 assert_eq!(next.endpoint, fixture.candidate);
5522 }
5523
5524 #[test]
5530 fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
5531 let forwarding = ForwardingTable::default();
5532 let first_connection = ConnectionId::new(70);
5533 let (first_tx, _first_rx) = mpsc::channel(8);
5534 forwarding
5535 .register_module_connection(
5536 first_connection,
5537 MODULE_ID.to_string(),
5538 2,
5539 Concurrency::ModuleManaged,
5540 FrameSink::new(first_tx),
5541 )
5542 .unwrap();
5543 let (client_connection, client_sink, _client_rx) = client(200);
5544 let mut pending = forwarding
5545 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5546 .unwrap();
5547 let (second_tx, _second_rx) = mpsc::channel(8);
5548 forwarding
5549 .register_module_connection(
5550 ConnectionId::new(80),
5551 MODULE_ID.to_string(),
5552 2,
5553 Concurrency::ModuleManaged,
5554 FrameSink::new(second_tx),
5555 )
5556 .unwrap();
5557
5558 assert_eq!(
5559 forwarding
5560 .complete_pending_relay(
5561 first_connection,
5562 pending.corr,
5563 RouteBindRelayOutcome::Accepted
5564 )
5565 .unwrap_err(),
5566 ForwardingError::StaleModuleEndpoint
5567 );
5568 assert!(committed_endpoints(&forwarding).is_empty());
5569 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5570 assert!(matches!(
5571 pending.receiver.try_recv(),
5572 Err(oneshot::error::TryRecvError::Closed)
5573 ));
5574 }
5575
5576 #[test]
5577 fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
5578 let fixture = swap_fixture();
5580 let forwarding = &fixture.forwarding;
5581 assert!(forwarding
5582 .cleanup_connection(fixture.candidate_connection)
5583 .unwrap()
5584 .is_empty());
5585 assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
5586 let (client_connection, client_sink, _client_rx) = client(200);
5587 assert_eq!(
5588 forwarding
5589 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5590 .unwrap()
5591 .endpoint,
5592 fixture.incumbent
5593 );
5594
5595 let fixture = swap_fixture();
5598 let forwarding = &fixture.forwarding;
5599 let (bound_client, bound_sink, _bound_rx) = client(200);
5600 let bound = forwarding
5601 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5602 .unwrap();
5603 forwarding
5604 .complete_pending_relay(
5605 fixture.incumbent_connection,
5606 bound.corr,
5607 RouteBindRelayOutcome::Accepted,
5608 )
5609 .unwrap();
5610 forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5611 let released = forwarding
5612 .cleanup_connection(fixture.incumbent_connection)
5613 .unwrap();
5614 assert_eq!(released.len(), 1);
5615 assert_eq!(released[0].connection_id, bound_client);
5616 assert!(forwarding
5617 .read_inner()
5618 .unwrap()
5619 .superseded_endpoints
5620 .is_empty());
5621 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5622 let (next_client, next_sink, _next_rx) = client(201);
5623 assert_eq!(
5624 forwarding
5625 .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
5626 .unwrap()
5627 .endpoint,
5628 fixture.candidate
5629 );
5630 }
5631}