1use std::{
2 collections::{BTreeMap, HashMap, HashSet},
3 error::Error,
4 fmt,
5 sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
6 time::Duration,
7};
8
9use subc_control::{ClientControlResponse, RouteCloseReason};
10use subc_protocol::{
11 manifest::Concurrency,
12 session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
13 ErrorBody, Flags, FrameType, Principal, Priority,
14};
15use tokio::sync::{oneshot, Semaphore};
16use tokio::time::Instant;
17use tracing::{debug, info, warn};
18
19use crate::{
20 control::{RouteBindBreakers, RouteBindConcurrency},
21 observability::DaemonCounters,
22 registry::ConnectionId,
23 router::FrameSink,
24 scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
25 Frame, ProjectRootId,
26};
27
28const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
30
31const STATELESS_PARALLEL_WINDOW: usize = 1024;
33
34const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
38
39#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
44pub struct ModuleEndpointId {
45 pub connection_id: ConnectionId,
46 pub generation: u64,
47}
48
49#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
51pub(crate) struct ClientRouteKey {
52 pub connection_id: ConnectionId,
53 pub channel: u16,
54}
55
56#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
58pub(crate) struct ModuleRouteKey {
59 pub endpoint: ModuleEndpointId,
60 pub channel: u16,
61}
62
63#[derive(Debug)]
64pub(crate) struct RouteBinding {
65 pub client_connection_id: ConnectionId,
66 pub client_sink: FrameSink,
67 pub client_negotiated_ver: u8,
68 pub client_channel: u16,
69 pub client_epoch: u32,
70 pub module_id: String,
71 pub module_endpoint: ModuleEndpointId,
72 pub module_sink: FrameSink,
73 pub module_negotiated_ver: u8,
74 pub module_channel: u16,
75 pub module_epoch: u32,
76 pub principal: Principal,
77 pub project_root: Option<ProjectRootId>,
78 pub bound_at: Instant,
79 pub flow: Arc<ChannelFlow>,
80 pub scope: Option<BoundScope>,
83}
84
85#[derive(Debug, Clone)]
86pub(crate) enum DataRoute {
87 Client(DataRouteState),
88 Module(DataRouteState),
89}
90
91#[derive(Debug, Clone)]
92pub(crate) enum DataRouteState {
93 Bound(Arc<RouteBinding>),
94 Reserved,
95 EpochMismatch,
96 Absent,
97}
98
99#[derive(Debug, Clone, Copy, PartialEq, Eq)]
128pub(crate) enum GoodbyeTargetKind {
129 Client,
130 Module,
131}
132
133#[derive(Debug, Clone)]
134pub(crate) struct GoodbyeTarget {
135 pub connection_id: ConnectionId,
136 pub sink: FrameSink,
137 pub negotiated_ver: u8,
138 pub channel: u16,
139 pub epoch: u32,
140 pub kind: GoodbyeTargetKind,
141 pub module_id: Option<String>,
146}
147
148#[derive(Debug, Clone, Copy)]
151pub(crate) struct UndeliveredFrame<'a> {
152 pub module_id: Option<&'a str>,
154 pub sink: &'a FrameSink,
156}
157
158pub(crate) fn principal_label(principal: &Principal) -> String {
160 match principal {
161 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
162 Principal::Direct => "direct".to_string(),
163 other => format!("{other:?}"),
164 }
165}
166
167fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
170 let labels = inner
171 .client_to_module
172 .iter()
173 .filter(|(key, _)| key.connection_id == connection_id)
174 .map(|(_, route)| principal_label(&route.principal))
175 .collect::<std::collections::BTreeSet<_>>();
176 if labels.is_empty() {
177 "none".to_string()
178 } else {
179 labels.into_iter().collect::<Vec<_>>().join(",")
180 }
181}
182
183impl GoodbyeTarget {
184 pub(crate) fn close_on_delivery_failure(&self) -> bool {
187 matches!(self.kind, GoodbyeTargetKind::Client)
188 }
189}
190
191pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
204
205pub(crate) fn send_module_route_goodbye(
217 counters: &DaemonCounters,
218 sink: &FrameSink,
219 frame: Frame,
220 module_id: Option<&str>,
221 context: &'static str,
222) {
223 let channel = frame.header.channel;
224 let epoch = frame.header.epoch;
225 let Err(err) = sink.try_send(frame.clone()) else {
226 return;
227 };
228 let runtime = match tokio::runtime::Handle::try_current() {
231 Ok(runtime) if !sink.is_closed() => runtime,
232 _ => {
233 counters.increment_goodbye_relay_module_dropped(module_id);
234 warn!(
235 module_id = module_id.unwrap_or("unknown"),
236 route_channel = channel,
237 route_epoch = epoch,
238 error = %err,
239 context,
240 "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
241 );
242 return;
243 }
244 };
245 debug!(
246 module_id = module_id.unwrap_or("unknown"),
247 route_channel = channel,
248 route_epoch = epoch,
249 error = %err,
250 context,
251 "module egress queue refused route GOODBYE; delivering it once the module frees room"
252 );
253 let counters = counters.clone();
254 let sink = sink.clone();
255 let module_id = module_id.map(str::to_string);
256 runtime.spawn(async move {
257 let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
258 let why = match outcome {
259 Ok(Ok(())) => {
260 debug!(
261 module_id = module_id.as_deref().unwrap_or("unknown"),
262 route_channel = channel,
263 route_epoch = epoch,
264 context,
265 "late route GOODBYE delivered to module"
266 );
267 return;
268 }
269 Ok(Err(err)) => err.to_string(),
270 Err(_) => format!(
271 "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
272 ),
273 };
274 counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
275 warn!(
276 module_id = module_id.as_deref().unwrap_or("unknown"),
277 route_channel = channel,
278 route_epoch = epoch,
279 error = %why,
280 context,
281 "route GOODBYE to module dropped under backpressure; not closing shared module connection"
282 );
283 });
284}
285
286#[derive(Debug, Clone)]
292pub(crate) struct EndpointRoute {
293 pub goodbye_target: GoodbyeTarget,
294 pub principal: Principal,
295 pub bound_at: Instant,
296 pub draining: bool,
297 pub drain_reason: Option<RouteCloseReason>,
301}
302
303#[derive(Debug, Clone)]
306pub(crate) struct ScopeDrainedRoute {
307 pub reason: RouteCloseReason,
308 pub scope: BoundScope,
309 pub module_id: String,
310 pub client: GoodbyeTarget,
311 pub module: GoodbyeTarget,
312}
313
314#[derive(Debug)]
315pub(crate) struct PendingRouteBindRelay {
316 pub endpoint: ModuleEndpointId,
317 pub module_sink: FrameSink,
318 pub negotiated_ver: u8,
319 pub client_channel: u16,
320 pub client_epoch: u32,
321 pub module_channel: u16,
322 pub module_epoch: u32,
323 pub corr: u64,
324 pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
325}
326
327#[derive(Debug, Clone)]
328pub(crate) struct ModuleDrainTarget {
329 pub endpoint: ModuleEndpointId,
330 pub sink: FrameSink,
331 pub negotiated_ver: u8,
332 pub abandoned_bindings: Vec<GoodbyeTarget>,
333 pub excluded_subscriptions: u32,
334}
335
336#[cfg(unix)]
339#[derive(Debug, Clone)]
340pub(crate) struct ModuleConnectionTarget {
341 pub module_id: String,
342 pub endpoint: ModuleEndpointId,
343 pub sink: FrameSink,
344 pub negotiated_ver: u8,
345}
346
347#[derive(Debug, Clone)]
348pub(crate) enum RouteBindRelayOutcome {
349 Accepted,
350 Rejected(ErrorBody),
355 DaemonRejected(ErrorBody),
363 ModuleGone(String),
364}
365
366#[derive(Debug, Clone, Copy, PartialEq, Eq)]
368pub(crate) struct ForwardingCutover {
369 pub promoted: ModuleEndpointId,
371 pub incumbent: Option<ModuleEndpointId>,
374}
375
376#[derive(Debug)]
378pub(crate) struct ConnectionCleanup {
379 pub released: Vec<GoodbyeTarget>,
381 pub abandoned_relays: u32,
384}
385
386#[derive(Debug, Clone)]
387pub(crate) struct PendingRelayCompletion {
388 pub settled: bool,
389 pub abandoned: Option<GoodbyeTarget>,
390}
391
392#[derive(Debug)]
393pub(crate) struct PendingModuleControlRpc {
394 pub endpoint: ModuleEndpointId,
395 pub module_sink: FrameSink,
396 pub negotiated_ver: u8,
397 pub corr: u64,
398 pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
399}
400
401#[derive(Debug, Clone)]
402pub(crate) enum ModuleControlRpcOutcome {
403 Response(ModuleControlResponse),
404 Rejected(ErrorBody),
405 ModuleGone(String),
406 MalformedResponse(String),
407 UnexpectedOp { expected: String, actual: String },
408 DeadlineElapsed,
409}
410
411#[derive(Debug, Clone, PartialEq, Eq)]
412pub(crate) enum ModuleControlRpcCompletion {
413 Unknown,
414 Settled,
415 LateHealthAnswer {
416 module_id: String,
417 latency: Duration,
418 },
419}
420
421#[derive(Debug)]
422struct PendingModuleControlRpcEntry {
423 expected_op: String,
424 deadline: Instant,
425 health_probe_started_at: Option<Instant>,
426 sender: oneshot::Sender<ModuleControlRpcOutcome>,
427}
428
429#[derive(Debug)]
430struct HealthProbeTombstone {
431 expected_op: String,
432 module_id: String,
433 probe_started_at: Instant,
434 expires_at: Instant,
435}
436
437#[derive(Debug, Clone)]
438struct RouteReservation {
439 client_key: ClientRouteKey,
440 module_key: ModuleRouteKey,
441 client_epoch: u32,
442 module_epoch: u32,
443 project_root: Option<ProjectRootId>,
444}
445
446#[derive(Debug)]
447struct PendingRouteBindRelayEntry {
448 reservation: RouteReservation,
449 client_sink: FrameSink,
450 client_negotiated_ver: u8,
451 client_permit: crate::router::EgressPermit,
452 route_open_frame: Frame,
453 principal: Principal,
454 scope: Option<BoundScope>,
457 deadline: Instant,
458 relay_enqueued: bool,
469 sender: oneshot::Sender<RouteBindRelayOutcome>,
470}
471
472#[derive(Debug, Clone, Copy, PartialEq, Eq)]
475pub(crate) enum RouteBindRelayMark {
476 Marked,
478 SettledByReply,
482 ReleasedWithoutReply,
486}
487
488#[derive(Debug, Clone)]
489pub(crate) enum RouteRelease {
490 Removed(GoodbyeTarget),
491 Stale,
492 Absent,
493}
494
495#[derive(Debug, Clone)]
496pub(crate) enum RoutePollSnapshot {
497 Bound {
498 module_id: String,
499 status: Option<String>,
500 },
501 Absent,
502}
503
504#[derive(Debug, Clone)]
505struct ModuleConnection {
506 endpoint: ModuleEndpointId,
507 sink: FrameSink,
508 negotiated_ver: u8,
509 concurrency: Concurrency,
510}
511
512#[derive(Debug, Default)]
513struct ForwardingInner {
514 operator_confirms: Arc<crate::operator_confirm::OperatorConfirms>,
515 daemon_draining: bool,
516 daemon_drain_results: HashMap<ModuleEndpointId, bool>,
521 modules_by_id: HashMap<String, ModuleConnection>,
525 candidates_by_id: HashMap<String, ModuleConnection>,
531 superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
537 endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
538 module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
539 draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
543 closing_connections: HashSet<ConnectionId>,
544 next_generation: u64,
545 reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
546 reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
547 next_client_channel: HashMap<ConnectionId, u16>,
548 next_module_channel: HashMap<ModuleEndpointId, u16>,
549 client_slot_epochs: HashMap<ClientRouteKey, u32>,
550 module_slot_epochs: HashMap<ModuleRouteKey, u32>,
551 last_published_epoch: HashMap<ClientRouteKey, u32>,
552 client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
553 module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
554 status: HashMap<(ClientRouteKey, u32), String>,
555 pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
556 relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
561 scope_tags: HashMap<(String, String), ScopeTag>,
566 next_control_corr: HashMap<ModuleEndpointId, u64>,
567 pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
568 health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
569}
570
571#[derive(Debug, Clone)]
572pub(crate) struct CloseReason {
573 code: &'static str,
574 message: String,
575}
576
577impl CloseReason {
578 pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
579 Self {
580 code,
581 message: message.into(),
582 }
583 }
584}
585
586impl fmt::Display for CloseReason {
587 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
588 write!(f, "{}: {}", self.code, self.message)
589 }
590}
591
592pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
593
594#[derive(Debug, Default)]
596pub struct ForwardingTable {
597 inner: Arc<RwLock<ForwardingInner>>,
598 #[cfg(feature = "test-support")]
599 route_closing_hold: Mutex<Option<oneshot::Receiver<()>>>,
600 close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
601 counters: DaemonCounters,
602 route_bind_breakers: RouteBindBreakers,
607 route_bind_concurrency: RouteBindConcurrency,
610 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
614}
615
616impl ForwardingTable {
617 #[cfg(feature = "test-support")]
659 pub fn configure_operator_script(
660 &self,
661 script: &std::path::Path,
662 events: Option<&std::path::Path>,
663 limits: Option<crate::operator_confirm::OperatorLimits>,
664 ) {
665 self.operator_confirms()
666 .configure_script(script, events, limits);
667 }
668
669 #[cfg(feature = "test-support")]
673 pub fn hold_route_closing_for_test(&self) -> oneshot::Sender<()> {
674 let (release, hold) = oneshot::channel();
675 *self.route_closing_hold.lock().unwrap() = Some(hold);
676 release
677 }
678
679 #[cfg(feature = "test-support")]
680 pub(crate) async fn wait_before_route_closing_for_test(&self) {
681 let hold = self.route_closing_hold.lock().unwrap().take();
682 if let Some(hold) = hold {
683 let _ = hold.await;
684 }
685 }
686
687 pub(crate) fn operator_confirms(&self) -> Arc<crate::operator_confirm::OperatorConfirms> {
688 Arc::clone(
689 &self
690 .inner
691 .read()
692 .unwrap_or_else(|p| p.into_inner())
693 .operator_confirms,
694 )
695 }
696
697 #[cfg(test)]
698 pub(crate) fn inject_operator_principal(&self, key: ModuleRouteKey, principal: Principal) {
699 let mut inner = self.write_inner().unwrap();
700 let old = inner.module_to_client.remove(&key).unwrap();
701 let client = ClientRouteKey {
702 connection_id: old.client_connection_id,
703 channel: old.client_channel,
704 };
705 inner.client_to_module.remove(&client);
706 let mut binding = Arc::try_unwrap(old).expect("test binding has no outstanding readers");
707 binding.principal = principal;
708 let binding = Arc::new(binding);
709 inner.client_to_module.insert(client, Arc::clone(&binding));
710 inner.module_to_client.insert(key, binding);
711 }
712
713 pub(crate) fn with_operator_route<T>(
717 &self,
718 connection_id: ConnectionId,
719 channel: u16,
720 epoch: u32,
721 admit: impl FnOnce(Option<&RouteBinding>) -> T,
722 ) -> Result<T, ForwardingError> {
723 let inner = self.read_inner()?;
724 let binding = inner
725 .endpoint_by_connection
726 .get(&connection_id)
727 .and_then(|endpoint| {
728 inner.module_to_client.get(&ModuleRouteKey {
729 endpoint: *endpoint,
730 channel,
731 })
732 })
733 .filter(|binding| binding.module_epoch == epoch);
734 Ok(admit(binding.map(Arc::as_ref)))
735 }
736
737 pub(crate) fn counters(&self) -> DaemonCounters {
738 self.counters.clone()
739 }
740
741 pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
742 self.route_bind_breakers.clone()
743 }
744
745 pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
746 self.route_bind_concurrency.clone()
747 }
748
749 pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
750 Arc::clone(&self.route_outages)
751 }
752
753 pub(crate) fn register_connection_close(
754 &self,
755 connection_id: ConnectionId,
756 ) -> ConnectionCloseReceiver {
757 let (sender, receiver) = oneshot::channel();
758 let replaced = self
759 .lock_close_registry()
760 .insert(connection_id, sender)
761 .is_some();
762 if replaced {
763 warn!(
764 connection_id = connection_id.get(),
765 "replaced existing connection close registration"
766 );
767 }
768 receiver
769 }
770
771 pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
772 self.lock_close_registry().remove(&connection_id);
773 }
774
775 #[cfg(unix)]
784 pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
785 let senders: Vec<_> = self.lock_close_registry().drain().collect();
786 let count = senders.len();
787 for (_, sender) in senders {
788 let _ = sender.send(reason.clone());
789 }
790 count
791 }
792
793 #[cfg(unix)]
797 pub(crate) fn module_connections(
798 &self,
799 ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
800 let inner = self.read_inner()?;
801 let mut seen = HashSet::new();
802 Ok(inner
803 .modules_by_id
804 .values()
805 .chain(inner.candidates_by_id.values())
806 .chain(inner.superseded_endpoints.values())
807 .filter(|module| seen.insert(module.endpoint))
808 .map(|module| ModuleConnectionTarget {
809 module_id: inner
810 .module_id_by_endpoint
811 .get(&module.endpoint)
812 .cloned()
813 .unwrap_or_default(),
814 endpoint: module.endpoint,
815 sink: module.sink.clone(),
816 negotiated_ver: module.negotiated_ver,
817 })
818 .collect())
819 }
820
821 pub(crate) fn request_connection_close(
825 &self,
826 connection_id: ConnectionId,
827 reason: CloseReason,
828 ) -> bool {
829 let sender = self.lock_close_registry().remove(&connection_id);
830 if let Some(sender) = sender {
831 debug!(
832 connection_id = connection_id.get(),
833 close_reason = %reason,
834 "requesting connection close"
835 );
836 let _ = sender.send(reason);
837 true
838 } else {
839 debug!(
840 connection_id = connection_id.get(),
841 close_reason = %reason,
842 "connection close request ignored for inactive connection"
843 );
844 false
845 }
846 }
847
848 pub fn register_module_connection(
849 &self,
850 connection_id: ConnectionId,
851 module_id: String,
852 negotiated_ver: u8,
853 concurrency: Concurrency,
854 sink: FrameSink,
855 ) -> Result<ModuleEndpointId, ForwardingError> {
856 self.register_module_connection_inner(
857 connection_id,
858 module_id,
859 negotiated_ver,
860 concurrency,
861 sink,
862 None,
863 )
864 }
865
866 pub(crate) fn register_module_connection_acked(
879 &self,
880 connection_id: ConnectionId,
881 module_id: String,
882 negotiated_ver: u8,
883 concurrency: Concurrency,
884 sink: FrameSink,
885 hello_ack: Frame,
886 ) -> Result<ModuleEndpointId, ForwardingError> {
887 self.register_module_connection_inner(
888 connection_id,
889 module_id,
890 negotiated_ver,
891 concurrency,
892 sink,
893 Some(hello_ack),
894 )
895 }
896
897 fn register_module_connection_inner(
898 &self,
899 connection_id: ConnectionId,
900 module_id: String,
901 negotiated_ver: u8,
902 concurrency: Concurrency,
903 sink: FrameSink,
904 hello_ack: Option<Frame>,
905 ) -> Result<ModuleEndpointId, ForwardingError> {
906 let mut inner = self.write_inner()?;
907 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
908 return Err(ForwardingError::ConnectionClosing { connection_id });
909 }
910 check_module_connection_role_locked(&inner, connection_id)?;
911 enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
914
915 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
916 let endpoint = ModuleEndpointId {
917 connection_id,
918 generation: inner.next_generation,
919 };
920 inner.endpoint_by_connection.insert(connection_id, endpoint);
921 inner
922 .module_id_by_endpoint
923 .insert(endpoint, module_id.clone());
924 inner.next_module_channel.insert(endpoint, 1);
925 inner.next_control_corr.insert(endpoint, 1);
926 inner.modules_by_id.insert(
927 module_id.clone(),
928 ModuleConnection {
929 endpoint,
930 sink,
931 negotiated_ver,
932 concurrency,
933 },
934 );
935 drop(inner);
936
937 if let Some(discarded) = self
950 .route_bind_breakers
951 .reset_for_new_module_connection(&module_id)
952 {
953 info!(
954 module_id = %module_id,
955 discarded_consecutive_timeouts = discarded,
956 "route.bind breaker state discarded: a new module connection replaced the process it described"
957 );
958 }
959 Ok(endpoint)
960 }
961
962 #[cfg(test)]
976 pub(crate) fn register_candidate_module_connection(
977 &self,
978 connection_id: ConnectionId,
979 module_id: String,
980 negotiated_ver: u8,
981 concurrency: Concurrency,
982 sink: FrameSink,
983 ) -> Result<ModuleEndpointId, ForwardingError> {
984 self.register_candidate_module_connection_inner(
985 connection_id,
986 module_id,
987 negotiated_ver,
988 concurrency,
989 sink,
990 None,
991 )
992 }
993
994 pub(crate) fn register_candidate_module_connection_acked(
1001 &self,
1002 connection_id: ConnectionId,
1003 module_id: String,
1004 negotiated_ver: u8,
1005 concurrency: Concurrency,
1006 sink: FrameSink,
1007 hello_ack: Frame,
1008 ) -> Result<ModuleEndpointId, ForwardingError> {
1009 self.register_candidate_module_connection_inner(
1010 connection_id,
1011 module_id,
1012 negotiated_ver,
1013 concurrency,
1014 sink,
1015 Some(hello_ack),
1016 )
1017 }
1018
1019 fn register_candidate_module_connection_inner(
1020 &self,
1021 connection_id: ConnectionId,
1022 module_id: String,
1023 negotiated_ver: u8,
1024 concurrency: Concurrency,
1025 sink: FrameSink,
1026 hello_ack: Option<Frame>,
1027 ) -> Result<ModuleEndpointId, ForwardingError> {
1028 let mut inner = self.write_inner()?;
1029 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1030 return Err(ForwardingError::ConnectionClosing { connection_id });
1031 }
1032 check_module_connection_role_locked(&inner, connection_id)?;
1033 if inner.candidates_by_id.contains_key(&module_id) {
1034 return Err(ForwardingError::CandidateSlotOccupied { module_id });
1035 }
1036 enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1039
1040 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1041 let endpoint = ModuleEndpointId {
1042 connection_id,
1043 generation: inner.next_generation,
1044 };
1045 inner.endpoint_by_connection.insert(connection_id, endpoint);
1046 inner
1047 .module_id_by_endpoint
1048 .insert(endpoint, module_id.clone());
1049 inner.next_module_channel.insert(endpoint, 1);
1050 inner.next_control_corr.insert(endpoint, 1);
1051 inner.candidates_by_id.insert(
1052 module_id,
1053 ModuleConnection {
1054 endpoint,
1055 sink,
1056 negotiated_ver,
1057 concurrency,
1058 },
1059 );
1060 Ok(endpoint)
1061 }
1062
1063 pub(crate) fn cutover_candidate(
1081 &self,
1082 module_id: &str,
1083 ) -> Result<Option<ForwardingCutover>, ForwardingError> {
1084 let mut inner = self.write_inner()?;
1085 if inner.daemon_draining {
1086 return Err(ForwardingError::ModuleReloading {
1087 module_id: module_id.to_string(),
1088 });
1089 }
1090 let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
1091 return Ok(None);
1092 };
1093 let promoted = candidate.endpoint;
1094 let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
1095 let incumbent = incumbent.map(|incumbent| {
1096 let endpoint = incumbent.endpoint;
1097 inner.superseded_endpoints.insert(endpoint, incumbent);
1098 endpoint
1099 });
1100 drop(inner);
1101
1102 if let Some(discarded) = self
1106 .route_bind_breakers
1107 .reset_for_new_module_connection(module_id)
1108 {
1109 info!(
1110 module_id = %module_id,
1111 discarded_consecutive_timeouts = discarded,
1112 "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
1113 );
1114 }
1115 Ok(Some(ForwardingCutover {
1116 promoted,
1117 incumbent,
1118 }))
1119 }
1120
1121 #[allow(clippy::too_many_arguments)]
1122 pub(crate) async fn begin_route_bind_relay_for(
1123 &self,
1124 client_connection_id: ConnectionId,
1125 client_sink: FrameSink,
1126 client_negotiated_ver: u8,
1127 client_corr: u64,
1128 module_id: &str,
1129 principal: Principal,
1130 scope: Option<BoundScope>,
1131 project_root: Option<ProjectRootId>,
1132 deadline: Instant,
1133 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1134 let client_permit =
1138 client_sink
1139 .reserve_owned()
1140 .await
1141 .map_err(|_| ForwardingError::ClientEgressClosed {
1142 connection_id: client_connection_id,
1143 })?;
1144 self.begin_route_bind_relay_inner(
1145 client_connection_id,
1146 client_sink,
1147 client_negotiated_ver,
1148 client_corr,
1149 module_id,
1150 principal,
1151 scope,
1152 project_root,
1153 deadline,
1154 client_permit,
1155 )
1156 }
1157
1158 #[cfg(test)]
1159 pub(crate) fn begin_route_bind_relay_for_test(
1160 &self,
1161 client_connection_id: ConnectionId,
1162 client_sink: FrameSink,
1163 client_corr: u64,
1164 module_id: &str,
1165 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1166 let permit =
1167 client_sink
1168 .try_reserve_owned()
1169 .map_err(|_| ForwardingError::ClientEgressClosed {
1170 connection_id: client_connection_id,
1171 })?;
1172 self.begin_route_bind_relay_inner(
1173 client_connection_id,
1174 client_sink,
1175 subc_protocol::PROTOCOL_VERSION,
1176 client_corr,
1177 module_id,
1178 Principal::Direct,
1179 None,
1180 None,
1181 Instant::now() + std::time::Duration::from_secs(60),
1182 permit,
1183 )
1184 }
1185
1186 pub(crate) fn begin_module_control_rpc_for(
1187 &self,
1188 module_id: &str,
1189 expected_op: &str,
1190 deadline: Instant,
1191 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1192 self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1193 }
1194
1195 pub(crate) fn begin_health_probe_rpc_for(
1196 &self,
1197 module_id: &str,
1198 expected_op: &str,
1199 probe_started_at: Instant,
1200 deadline: Instant,
1201 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1202 self.begin_module_control_rpc_inner(
1203 module_id,
1204 expected_op,
1205 deadline,
1206 Some(probe_started_at),
1207 false,
1208 )
1209 }
1210
1211 pub(crate) fn begin_drain_health_probe_rpc_for(
1212 &self,
1213 module_id: &str,
1214 expected_op: &str,
1215 probe_started_at: Instant,
1216 deadline: Instant,
1217 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1218 self.begin_module_control_rpc_inner(
1219 module_id,
1220 expected_op,
1221 deadline,
1222 Some(probe_started_at),
1223 true,
1224 )
1225 }
1226
1227 pub(crate) fn begin_endpoint_health_probe_rpc_for(
1235 &self,
1236 endpoint: ModuleEndpointId,
1237 expected_op: &str,
1238 probe_started_at: Instant,
1239 deadline: Instant,
1240 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1241 let inner = self.write_inner()?;
1242 let module = module_connection_for_endpoint_locked(&inner, endpoint)
1243 .cloned()
1244 .ok_or(ForwardingError::NoModuleConnection)?;
1245 let module_id = inner
1246 .module_id_by_endpoint
1247 .get(&endpoint)
1248 .cloned()
1249 .unwrap_or_default();
1250 self.begin_control_rpc_locked(
1253 inner,
1254 &module_id,
1255 module,
1256 expected_op,
1257 deadline,
1258 Some(probe_started_at),
1259 true,
1260 )
1261 }
1262
1263 fn begin_module_control_rpc_inner(
1264 &self,
1265 module_id: &str,
1266 expected_op: &str,
1267 deadline: Instant,
1268 health_probe_started_at: Option<Instant>,
1269 allow_draining: bool,
1270 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1271 let inner = self.write_inner()?;
1272 let module = inner
1273 .modules_by_id
1274 .get(module_id)
1275 .cloned()
1276 .ok_or(ForwardingError::NoModuleConnection)?;
1277 self.begin_control_rpc_locked(
1278 inner,
1279 module_id,
1280 module,
1281 expected_op,
1282 deadline,
1283 health_probe_started_at,
1284 allow_draining,
1285 )
1286 }
1287
1288 #[allow(clippy::too_many_arguments)]
1289 fn begin_control_rpc_locked(
1290 &self,
1291 mut inner: RwLockWriteGuard<'_, ForwardingInner>,
1292 module_id: &str,
1293 module: ModuleConnection,
1294 expected_op: &str,
1295 deadline: Instant,
1296 health_probe_started_at: Option<Instant>,
1297 allow_draining: bool,
1298 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1299 if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1300 return Err(ForwardingError::ModuleReloading {
1301 module_id: module_id.to_string(),
1302 });
1303 }
1304 if inner
1305 .closing_connections
1306 .contains(&module.endpoint.connection_id)
1307 {
1308 return Err(ForwardingError::ConnectionClosing {
1309 connection_id: module.endpoint.connection_id,
1310 });
1311 }
1312 if health_probe_started_at.is_some() {
1313 inner
1317 .health_probe_tombstones
1318 .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1319 }
1320 let corr = match inner.allocate_control_corr(module.endpoint) {
1321 Ok(corr) => corr,
1322 Err(err) => {
1323 drop(inner);
1324 self.request_connection_close(
1325 module.endpoint.connection_id,
1326 CloseReason::new(
1327 "control_correlation_exhausted",
1328 "daemon-originated channel-0 correlation space exhausted",
1329 ),
1330 );
1331 return Err(err);
1332 }
1333 };
1334 let (sender, receiver) = oneshot::channel();
1335 inner.pending_control_rpcs.insert(
1336 (module.endpoint, corr),
1337 PendingModuleControlRpcEntry {
1338 expected_op: expected_op.to_string(),
1339 deadline,
1340 health_probe_started_at,
1341 sender,
1342 },
1343 );
1344
1345 Ok(PendingModuleControlRpc {
1346 endpoint: module.endpoint,
1347 module_sink: module.sink,
1348 negotiated_ver: module.negotiated_ver,
1349 corr,
1350 receiver,
1351 })
1352 }
1353
1354 #[allow(clippy::too_many_arguments)]
1355 fn begin_route_bind_relay_inner(
1356 &self,
1357 client_connection_id: ConnectionId,
1358 client_sink: FrameSink,
1359 client_negotiated_ver: u8,
1360 client_corr: u64,
1361 expected_module_id: &str,
1362 principal: Principal,
1363 scope: Option<BoundScope>,
1364 project_root: Option<ProjectRootId>,
1365 deadline: Instant,
1366 client_permit: crate::router::EgressPermit,
1367 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1368 let mut inner = self.write_inner()?;
1369 if inner
1373 .endpoint_by_connection
1374 .contains_key(&client_connection_id)
1375 {
1376 return Err(ForwardingError::ConnectionRoleConflict {
1377 connection_id: client_connection_id,
1378 });
1379 }
1380 if inner.closing_connections.contains(&client_connection_id) {
1381 return Err(ForwardingError::ConnectionClosing {
1382 connection_id: client_connection_id,
1383 });
1384 }
1385 let module = inner
1386 .modules_by_id
1387 .get(expected_module_id)
1388 .cloned()
1389 .ok_or(ForwardingError::NoModuleConnection)?;
1390 if inner.draining_endpoints.contains_key(&module.endpoint) {
1391 return Err(ForwardingError::ModuleReloading {
1392 module_id: expected_module_id.to_string(),
1393 });
1394 }
1395 if inner
1396 .closing_connections
1397 .contains(&module.endpoint.connection_id)
1398 {
1399 return Err(ForwardingError::ConnectionClosing {
1400 connection_id: module.endpoint.connection_id,
1401 });
1402 }
1403
1404 let corr = match inner.allocate_control_corr(module.endpoint) {
1405 Ok(corr) => corr,
1406 Err(err) => {
1407 drop(inner);
1408 self.request_connection_close(
1409 module.endpoint.connection_id,
1410 CloseReason::new(
1411 "control_correlation_exhausted",
1412 "daemon-originated channel-0 correlation space exhausted",
1413 ),
1414 );
1415 return Err(err);
1416 }
1417 };
1418 let (client_channel, client_epoch, module_channel, module_epoch) =
1419 inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1420 let client_key = ClientRouteKey {
1421 connection_id: client_connection_id,
1422 channel: client_channel,
1423 };
1424 let module_key = ModuleRouteKey {
1425 endpoint: module.endpoint,
1426 channel: module_channel,
1427 };
1428 let reservation = RouteReservation {
1429 client_key,
1430 module_key,
1431 client_epoch,
1432 module_epoch,
1433 project_root,
1434 };
1435 let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
1436 route_channel: client_channel,
1437 route_epoch: client_epoch,
1438 })
1439 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1440 let route_open_frame = Frame::build_with_version(
1441 client_negotiated_ver,
1442 FrameType::Response,
1443 Flags::new(false, Priority::Passive, false),
1444 0,
1445 0,
1446 client_corr,
1447 response_body,
1448 )
1449 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1450 let (sender, receiver) = oneshot::channel();
1451 inner.reserved_client.insert(client_key, module_key);
1452 inner.reserved_module.insert(module_key, client_key);
1453 inner.pending_relays.insert(
1454 (module.endpoint, corr),
1455 PendingRouteBindRelayEntry {
1456 reservation,
1457 client_sink,
1458 client_negotiated_ver,
1459 client_permit,
1460 route_open_frame,
1461 principal,
1462 scope,
1463 deadline,
1464 relay_enqueued: false,
1465 sender,
1466 },
1467 );
1468
1469 Ok(PendingRouteBindRelay {
1470 endpoint: module.endpoint,
1471 module_sink: module.sink,
1472 negotiated_ver: module.negotiated_ver,
1473 client_channel,
1474 client_epoch,
1475 module_channel,
1476 module_epoch,
1477 corr,
1478 receiver,
1479 })
1480 }
1481
1482 pub(crate) fn mark_route_bind_relay_enqueued(
1492 &self,
1493 endpoint: ModuleEndpointId,
1494 corr: u64,
1495 ) -> Result<RouteBindRelayMark, ForwardingError> {
1496 let mut inner = self.write_inner()?;
1497 if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
1498 pending.relay_enqueued = true;
1499 return Ok(RouteBindRelayMark::Marked);
1500 }
1501 if inner
1502 .relays_released_before_enqueue
1503 .remove(&(endpoint, corr))
1504 {
1505 return Ok(RouteBindRelayMark::ReleasedWithoutReply);
1506 }
1507 Ok(RouteBindRelayMark::SettledByReply)
1508 }
1509
1510 pub(crate) fn release_client_route(
1511 &self,
1512 client_connection_id: ConnectionId,
1513 client_channel: u16,
1514 expected_epoch: u32,
1515 ) -> Result<RouteRelease, ForwardingError> {
1516 let mut inner = self.write_inner()?;
1517 let release = release_client_route_locked(
1518 &mut inner,
1519 ClientRouteKey {
1520 connection_id: client_connection_id,
1521 channel: client_channel,
1522 },
1523 expected_epoch,
1524 );
1525 self.record_route_release(&release);
1526 Ok(release)
1527 }
1528
1529 pub(crate) fn release_module_route(
1530 &self,
1531 module_connection_id: ConnectionId,
1532 module_channel: u16,
1533 expected_epoch: u32,
1534 ) -> Result<RouteRelease, ForwardingError> {
1535 let mut inner = self.write_inner()?;
1536 let Some(endpoint) = inner
1537 .endpoint_by_connection
1538 .get(&module_connection_id)
1539 .copied()
1540 else {
1541 return Ok(RouteRelease::Absent);
1542 };
1543 let release = release_module_route_locked(
1544 &mut inner,
1545 ModuleRouteKey {
1546 endpoint,
1547 channel: module_channel,
1548 },
1549 expected_epoch,
1550 );
1551 self.record_route_release(&release);
1552 Ok(release)
1553 }
1554
1555 pub(crate) fn abort_pending_relay(
1556 &self,
1557 endpoint: ModuleEndpointId,
1558 corr: u64,
1559 outcome: RouteBindRelayOutcome,
1560 ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
1561 let mut inner = self.write_inner()?;
1562 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1563 inner
1568 .relays_released_before_enqueue
1569 .remove(&(endpoint, corr));
1570 return Ok(None);
1571 };
1572 release_reserved_route_locked(
1573 &mut inner,
1574 pending.reservation.client_key,
1575 pending.reservation.module_key,
1576 );
1577 let target = pending
1578 .relay_enqueued
1579 .then(|| abandoned_route_target(&inner, &pending.reservation));
1580 let _ = pending.sender.send(outcome);
1581 Ok(target.flatten())
1582 }
1583
1584 pub(crate) fn cancel_module_control_rpc(
1585 &self,
1586 endpoint: ModuleEndpointId,
1587 corr: u64,
1588 ) -> Result<(), ForwardingError> {
1589 self.write_inner()?
1590 .pending_control_rpcs
1591 .remove(&(endpoint, corr));
1592 Ok(())
1593 }
1594
1595 pub(crate) fn tombstone_health_probe_rpc(
1596 &self,
1597 endpoint: ModuleEndpointId,
1598 corr: u64,
1599 ) -> Result<bool, ForwardingError> {
1600 let key = (endpoint, corr);
1601 let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
1602 {
1603 let mut inner = self.write_inner()?;
1604 let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
1605 return Ok(false);
1606 };
1607 let Some(probe_started_at) = pending.health_probe_started_at else {
1608 inner.pending_control_rpcs.insert(key, pending);
1609 return Ok(false);
1610 };
1611 let module_id = inner
1612 .module_id_by_endpoint
1613 .get(&endpoint)
1614 .cloned()
1615 .unwrap_or_else(|| "unknown".to_string());
1616 inner.health_probe_tombstones.insert(
1617 key,
1618 HealthProbeTombstone {
1619 expected_op: pending.expected_op,
1620 module_id,
1621 probe_started_at,
1622 expires_at,
1623 },
1624 );
1625 }
1626 self.schedule_health_probe_tombstone_expiration(key, expires_at);
1627 Ok(true)
1628 }
1629
1630 fn schedule_health_probe_tombstone_expiration(
1631 &self,
1632 key: (ModuleEndpointId, u64),
1633 expires_at: Instant,
1634 ) {
1635 let inner = Arc::downgrade(&self.inner);
1636 tokio::spawn(async move {
1637 tokio::time::sleep_until(expires_at).await;
1638 let Some(inner) = inner.upgrade() else {
1639 return;
1640 };
1641 let Ok(mut inner) = inner.write() else {
1642 return;
1643 };
1644 let expired = inner
1645 .health_probe_tombstones
1646 .get(&key)
1647 .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
1648 if expired {
1649 inner.health_probe_tombstones.remove(&key);
1650 }
1651 });
1652 }
1653
1654 pub(crate) fn complete_pending_relay(
1655 &self,
1656 connection_id: ConnectionId,
1657 corr: u64,
1658 outcome: RouteBindRelayOutcome,
1659 ) -> Result<PendingRelayCompletion, ForwardingError> {
1660 let mut inner = self.write_inner()?;
1661 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1662 return Ok(PendingRelayCompletion {
1663 settled: false,
1664 abandoned: None,
1665 });
1666 };
1667 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1668 return Ok(PendingRelayCompletion {
1669 settled: false,
1670 abandoned: None,
1671 });
1672 };
1673
1674 if Instant::now() >= pending.deadline {
1675 release_reserved_route_locked(
1676 &mut inner,
1677 pending.reservation.client_key,
1678 pending.reservation.module_key,
1679 );
1680 let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
1681 .then(|| abandoned_route_target(&inner, &pending.reservation))
1682 .flatten();
1683 let _ = pending
1684 .sender
1685 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody {
1686 code: "module_timeout".to_string(),
1687 message: "route.bind response arrived after its daemon deadline".to_string(),
1688 detail: None,
1689 }));
1690 return Ok(PendingRelayCompletion {
1691 settled: true,
1692 abandoned,
1693 });
1694 }
1695
1696 match outcome {
1697 RouteBindRelayOutcome::Accepted
1716 if pending.client_sink.is_closed()
1717 || inner
1718 .closing_connections
1719 .contains(&pending.reservation.client_key.connection_id) =>
1720 {
1721 let reason = if pending.client_sink.is_closed() {
1722 "client egress closed before route publication"
1723 } else {
1724 "client connection is closing before route publication"
1725 };
1726 release_reserved_route_locked(
1727 &mut inner,
1728 pending.reservation.client_key,
1729 pending.reservation.module_key,
1730 );
1731 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1735 let _ = pending
1736 .sender
1737 .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
1738 return Ok(PendingRelayCompletion {
1739 settled: true,
1740 abandoned,
1741 });
1742 }
1743 RouteBindRelayOutcome::Accepted
1766 if inner.superseded_endpoints.contains_key(&endpoint)
1767 || inner.draining_endpoints.contains_key(&endpoint) =>
1768 {
1769 release_reserved_route_locked(
1770 &mut inner,
1771 pending.reservation.client_key,
1772 pending.reservation.module_key,
1773 );
1774 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1777 let module_id = inner
1778 .module_id_by_endpoint
1779 .get(&endpoint)
1780 .cloned()
1781 .unwrap_or_else(|| "unknown".to_string());
1782 let _ = pending
1783 .sender
1784 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
1785 "module_reloading",
1786 format!("module_id '{module_id}' is reloading"),
1787 )));
1788 return Ok(PendingRelayCompletion {
1789 settled: true,
1790 abandoned,
1791 });
1792 }
1793 RouteBindRelayOutcome::Accepted
1802 if pending
1803 .scope
1804 .as_ref()
1805 .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
1806 {
1807 let refusal = pending
1808 .scope
1809 .as_ref()
1810 .and_then(|scope| scope_refusal_locked(&inner, scope))
1811 .expect("guard matched a refusal under the same lock");
1812 release_reserved_route_locked(
1813 &mut inner,
1814 pending.reservation.client_key,
1815 pending.reservation.module_key,
1816 );
1817 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1818 let _ = pending
1819 .sender
1820 .send(RouteBindRelayOutcome::DaemonRejected(refusal));
1821 return Ok(PendingRelayCompletion {
1822 settled: true,
1823 abandoned,
1824 });
1825 }
1826 RouteBindRelayOutcome::Accepted => {
1827 let abandoned = commit_route_locked(&mut inner, pending)?;
1828 return Ok(PendingRelayCompletion {
1829 settled: true,
1830 abandoned,
1831 });
1832 }
1833 terminal => {
1834 release_reserved_route_locked(
1835 &mut inner,
1836 pending.reservation.client_key,
1837 pending.reservation.module_key,
1838 );
1839 let _ = pending.sender.send(terminal);
1840 }
1841 }
1842 Ok(PendingRelayCompletion {
1843 settled: true,
1844 abandoned: None,
1845 })
1846 }
1847
1848 pub(crate) fn pending_module_control_op(
1849 &self,
1850 connection_id: ConnectionId,
1851 corr: u64,
1852 ) -> Result<Option<String>, ForwardingError> {
1853 let inner = self.read_inner()?;
1854 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1855 return Ok(None);
1856 };
1857 let key = (endpoint, corr);
1858 Ok(inner
1859 .pending_control_rpcs
1860 .get(&key)
1861 .map(|pending| pending.expected_op.clone())
1862 .or_else(|| {
1863 inner
1864 .health_probe_tombstones
1865 .get(&key)
1866 .filter(|tombstone| tombstone.expires_at > Instant::now())
1867 .map(|tombstone| tombstone.expected_op.clone())
1868 }))
1869 }
1870
1871 pub(crate) fn complete_module_control_rpc(
1872 &self,
1873 connection_id: ConnectionId,
1874 corr: u64,
1875 actual_op: Option<&str>,
1876 outcome: ModuleControlRpcOutcome,
1877 ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
1878 let now = Instant::now();
1879 let mut inner = self.write_inner()?;
1880 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1881 return Ok(ModuleControlRpcCompletion::Unknown);
1882 };
1883 let key = (endpoint, corr);
1884 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
1885 if now >= pending.deadline {
1886 let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
1887 ModuleControlRpcCompletion::LateHealthAnswer {
1888 module_id: inner
1889 .module_id_by_endpoint
1890 .get(&endpoint)
1891 .cloned()
1892 .unwrap_or_else(|| "unknown".to_string()),
1893 latency: now.saturating_duration_since(probe_started_at),
1894 }
1895 });
1896 let _ = pending
1897 .sender
1898 .send(ModuleControlRpcOutcome::DeadlineElapsed);
1899 return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
1900 }
1901 let outcome = match actual_op {
1902 Some(actual) if actual != pending.expected_op => {
1903 ModuleControlRpcOutcome::UnexpectedOp {
1904 expected: pending.expected_op,
1905 actual: actual.to_string(),
1906 }
1907 }
1908 _ => outcome,
1909 };
1910 let _ = pending.sender.send(outcome);
1911 return Ok(ModuleControlRpcCompletion::Settled);
1912 }
1913
1914 let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
1915 return Ok(ModuleControlRpcCompletion::Unknown);
1916 };
1917 if tombstone.expires_at <= now {
1918 return Ok(ModuleControlRpcCompletion::Unknown);
1919 }
1920 Ok(ModuleControlRpcCompletion::LateHealthAnswer {
1921 module_id: tombstone.module_id,
1922 latency: now.saturating_duration_since(tombstone.probe_started_at),
1923 })
1924 }
1925
1926 #[cfg(test)]
1927 pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
1928 Ok(self.read_inner()?.health_probe_tombstones.len())
1929 }
1930
1931 #[cfg(test)]
1932 pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
1933 Ok(self.read_inner()?.closing_connections.len())
1934 }
1935
1936 #[cfg(test)]
1939 pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
1940 let inner = self.read_inner()?;
1941 Ok((inner.reserved_client.len(), inner.reserved_module.len()))
1942 }
1943
1944 pub(crate) fn module_endpoint_for_connection(
1945 &self,
1946 connection_id: ConnectionId,
1947 ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
1948 Ok(self
1949 .read_inner()?
1950 .endpoint_by_connection
1951 .get(&connection_id)
1952 .copied())
1953 }
1954
1955 pub(crate) fn module_id_for_connection(
1958 &self,
1959 connection_id: ConnectionId,
1960 ) -> Result<Option<String>, ForwardingError> {
1961 let inner = self.read_inner()?;
1962 Ok(inner
1963 .endpoint_by_connection
1964 .get(&connection_id)
1965 .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
1966 .cloned())
1967 }
1968
1969 pub(crate) fn module_route_epoch_was_allocated(
1976 &self,
1977 connection_id: ConnectionId,
1978 channel: u16,
1979 epoch: u32,
1980 ) -> Result<bool, ForwardingError> {
1981 let inner = self.read_inner()?;
1982 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1983 return Ok(false);
1984 };
1985 Ok(inner
1986 .module_slot_epochs
1987 .get(&ModuleRouteKey { endpoint, channel })
1988 .is_some_and(|last| epoch != 0 && epoch <= *last))
1989 }
1990
1991 pub(crate) fn has_live_module_connection(
1992 &self,
1993 module_id: &str,
1994 ) -> Result<bool, ForwardingError> {
1995 Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
1996 }
1997
1998 pub(crate) fn lookup_data_route(
1999 &self,
2000 connection_id: ConnectionId,
2001 channel: u16,
2002 epoch: u32,
2003 ) -> Result<DataRoute, ForwardingError> {
2004 let inner = self.read_inner()?;
2005 let state = if let Some(endpoint) =
2006 inner.endpoint_by_connection.get(&connection_id).copied()
2007 {
2008 let key = ModuleRouteKey { endpoint, channel };
2009 match inner.module_to_client.get(&key) {
2010 Some(route) if route.module_epoch == epoch => {
2011 DataRouteState::Bound(Arc::clone(route))
2012 }
2013 Some(_) => DataRouteState::EpochMismatch,
2014 None if inner.reserved_module.contains_key(&key)
2015 && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
2016 {
2017 DataRouteState::Reserved
2018 }
2019 None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
2020 None => DataRouteState::Absent,
2021 }
2022 } else {
2023 let key = ClientRouteKey {
2024 connection_id,
2025 channel,
2026 };
2027 match inner.client_to_module.get(&key) {
2028 Some(route) if route.client_epoch == epoch => {
2029 DataRouteState::Bound(Arc::clone(route))
2030 }
2031 Some(_) => DataRouteState::EpochMismatch,
2032 None if inner.reserved_client.contains_key(&key)
2033 && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
2034 {
2035 DataRouteState::Reserved
2036 }
2037 None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
2038 None => DataRouteState::Absent,
2039 }
2040 };
2041 Ok(
2042 if inner.endpoint_by_connection.contains_key(&connection_id) {
2043 DataRoute::Module(state)
2044 } else {
2045 DataRoute::Client(state)
2046 },
2047 )
2048 }
2049
2050 #[cfg(test)]
2051 pub(crate) fn inject_client_slot_epoch(
2052 &self,
2053 connection_id: ConnectionId,
2054 channel: u16,
2055 last_epoch: u32,
2056 ) {
2057 let mut inner = self.write_inner().expect("forwarding lock");
2058 inner.client_slot_epochs.insert(
2059 ClientRouteKey {
2060 connection_id,
2061 channel,
2062 },
2063 last_epoch,
2064 );
2065 inner.next_client_channel.insert(connection_id, channel);
2066 }
2067
2068 #[cfg(test)]
2069 pub(crate) fn inject_module_slot_epoch(
2070 &self,
2071 endpoint: ModuleEndpointId,
2072 channel: u16,
2073 last_epoch: u32,
2074 ) {
2075 let mut inner = self.write_inner().expect("forwarding lock");
2076 inner
2077 .module_slot_epochs
2078 .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
2079 inner.next_module_channel.insert(endpoint, channel);
2080 }
2081
2082 #[cfg(test)]
2083 pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
2084 self.write_inner()
2085 .expect("forwarding lock")
2086 .next_control_corr
2087 .insert(endpoint, next_corr);
2088 }
2089
2090 pub(crate) fn cache_status(
2091 &self,
2092 endpoint: ModuleEndpointId,
2093 module_channel: u16,
2094 module_epoch: u32,
2095 status: String,
2096 ) -> Result<bool, ForwardingError> {
2097 let mut inner = self.write_inner()?;
2098 if !inner.module_id_by_endpoint.contains_key(&endpoint) {
2099 return Err(ForwardingError::StaleModuleEndpoint);
2100 }
2101
2102 let module_key = ModuleRouteKey {
2103 endpoint,
2104 channel: module_channel,
2105 };
2106 let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
2107 (route.module_epoch == module_epoch).then_some((
2108 ClientRouteKey {
2109 connection_id: route.client_connection_id,
2110 channel: route.client_channel,
2111 },
2112 route.client_epoch,
2113 ))
2114 } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
2115 (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
2116 client_key,
2117 inner
2118 .client_slot_epochs
2119 .get(&client_key)
2120 .copied()
2121 .unwrap_or(0),
2122 ))
2123 } else {
2124 None
2125 };
2126
2127 if let Some(handle) = handle {
2128 inner.status.insert(handle, status);
2129 Ok(true)
2130 } else {
2131 debug!(
2132 module_channel,
2133 module_epoch,
2134 generation = endpoint.generation,
2135 connection_id = endpoint.connection_id.get(),
2136 "dropping stale status update for module route handle"
2137 );
2138 Ok(false)
2139 }
2140 }
2141
2142 pub(crate) fn route_poll_snapshot(
2143 &self,
2144 client_connection_id: ConnectionId,
2145 client_channel: u16,
2146 client_epoch: u32,
2147 ) -> Result<RoutePollSnapshot, ForwardingError> {
2148 let inner = self.read_inner()?;
2149 let client_key = ClientRouteKey {
2150 connection_id: client_connection_id,
2151 channel: client_channel,
2152 };
2153 let Some(route) = inner.client_to_module.get(&client_key) else {
2154 return Ok(RoutePollSnapshot::Absent);
2155 };
2156 if route.client_epoch != client_epoch
2157 || !inner
2158 .module_id_by_endpoint
2159 .contains_key(&route.module_endpoint)
2160 {
2161 return Ok(RoutePollSnapshot::Absent);
2162 }
2163 Ok(RoutePollSnapshot::Bound {
2164 module_id: route.module_id.clone(),
2165 status: inner.status.get(&(client_key, client_epoch)).cloned(),
2166 })
2167 }
2168
2169 pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2170 Ok(self.read_inner()?.client_to_module.len())
2171 }
2172
2173 pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2183 let inner = self.read_inner()?;
2184 let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2185 for key in inner.client_to_module.keys() {
2186 *per_connection.entry(key.connection_id).or_insert(0) += 1;
2187 }
2188 let max = per_connection.values().copied().max().unwrap_or(0);
2189 Ok((per_connection.len(), max))
2190 }
2191
2192 pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2193 let inner = self.read_inner()?;
2194 Ok(inner
2195 .client_to_module
2196 .keys()
2197 .any(|key| key.channel == route_channel))
2198 }
2199
2200 pub(crate) fn daemon_drain_result_for_connection(
2204 &self,
2205 connection_id: ConnectionId,
2206 ) -> Result<Option<bool>, ForwardingError> {
2207 let inner = self.read_inner()?;
2208 Ok(inner.daemon_draining.then(|| {
2209 inner
2210 .endpoint_by_connection
2211 .get(&connection_id)
2212 .and_then(|endpoint| inner.daemon_drain_results.get(endpoint))
2213 .copied()
2214 .unwrap_or(false)
2215 }))
2216 }
2217
2218 #[cfg(unix)]
2219 pub(crate) fn record_daemon_drain_result(
2220 &self,
2221 endpoint: ModuleEndpointId,
2222 drained: bool,
2223 ) -> Result<(), ForwardingError> {
2224 let mut inner = self.write_inner()?;
2225 if let Some(result) = inner.daemon_drain_results.get_mut(&endpoint) {
2229 *result = drained;
2230 }
2231 Ok(())
2232 }
2233
2234 #[cfg(unix)]
2237 pub(crate) fn begin_daemon_drain(
2238 &self,
2239 ) -> Result<Vec<(String, ModuleEndpointId)>, ForwardingError> {
2240 let mut inner = self.write_inner()?;
2241 if inner.daemon_draining {
2242 return Ok(Vec::new());
2243 }
2244 inner.daemon_draining = true;
2245 let modules = inner
2246 .modules_by_id
2247 .iter()
2248 .map(|(id, module)| (id.clone(), module.endpoint))
2249 .collect::<Vec<_>>();
2250 for (_, endpoint) in &modules {
2251 inner.daemon_drain_results.insert(*endpoint, false);
2252 inner
2253 .draining_endpoints
2254 .insert(*endpoint, RouteCloseReason::Restart);
2255 }
2256 let off_slot_endpoints = inner
2261 .candidates_by_id
2262 .values()
2263 .map(|module| module.endpoint)
2264 .chain(inner.superseded_endpoints.keys().copied())
2265 .collect::<Vec<_>>();
2266 for endpoint in off_slot_endpoints {
2267 inner
2268 .draining_endpoints
2269 .insert(endpoint, RouteCloseReason::Restart);
2270 }
2271 Ok(modules)
2272 }
2273
2274 pub(crate) fn begin_module_drain(
2281 &self,
2282 module_id: &str,
2283 reason: RouteCloseReason,
2284 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2285 let mut inner = self.write_inner()?;
2286 let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2287 return Ok(None);
2288 };
2289 Ok(Some(begin_drain_locked(
2290 &mut inner, module_id, module, reason,
2291 )))
2292 }
2293
2294 pub(crate) fn begin_endpoint_drain(
2301 &self,
2302 endpoint: ModuleEndpointId,
2303 reason: RouteCloseReason,
2304 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2305 let mut inner = self.write_inner()?;
2306 let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2307 return Ok(None);
2308 };
2309 let module_id = inner
2310 .module_id_by_endpoint
2311 .get(&endpoint)
2312 .cloned()
2313 .expect("an endpoint resolved to a module connection has a module id");
2314 Ok(Some(begin_drain_locked(
2315 &mut inner, &module_id, module, reason,
2316 )))
2317 }
2318}
2319
2320fn begin_drain_locked(
2324 inner: &mut ForwardingInner,
2325 module_id: &str,
2326 module: ModuleConnection,
2327 reason: RouteCloseReason,
2328) -> ModuleDrainTarget {
2329 {
2330 let endpoint = module.endpoint;
2331 inner.draining_endpoints.insert(endpoint, reason);
2332
2333 let flows = inner
2334 .client_to_module
2335 .values()
2336 .filter(|route| route.module_endpoint == endpoint)
2337 .map(|route| Arc::clone(&route.flow))
2338 .collect::<Vec<_>>();
2339 let excluded_subscriptions = flows
2340 .into_iter()
2341 .map(|flow| flow.begin_drain())
2342 .fold(0u32, u32::saturating_add);
2343
2344 let pending_keys = inner
2345 .pending_relays
2346 .keys()
2347 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2348 .copied()
2349 .collect::<Vec<_>>();
2350 let mut abandoned_bindings = Vec::new();
2351 for key in pending_keys {
2352 let Some(pending) = inner.pending_relays.remove(&key) else {
2353 continue;
2354 };
2355 release_reserved_route_locked(
2356 inner,
2357 pending.reservation.client_key,
2358 pending.reservation.module_key,
2359 );
2360 if pending.relay_enqueued {
2361 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2362 abandoned_bindings.push(target);
2363 }
2364 } else {
2365 inner.relays_released_before_enqueue.insert(key);
2369 }
2370 let _ = pending
2371 .sender
2372 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
2373 "module_reloading",
2374 format!("module_id '{module_id}' is reloading"),
2375 )));
2376 }
2377
2378 let pending_control_keys = inner
2379 .pending_control_rpcs
2380 .keys()
2381 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2382 .copied()
2383 .collect::<Vec<_>>();
2384 for key in pending_control_keys {
2385 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2386 let _ = pending
2387 .sender
2388 .send(ModuleControlRpcOutcome::ModuleGone(format!(
2389 "module '{module_id}' began draining during module-control RPC"
2390 )));
2391 }
2392 }
2393
2394 ModuleDrainTarget {
2395 endpoint,
2396 sink: module.sink,
2397 negotiated_ver: module.negotiated_ver,
2398 abandoned_bindings,
2399 excluded_subscriptions,
2400 }
2401 }
2402}
2403
2404#[derive(Debug, Default, PartialEq, Eq)]
2407pub(crate) struct DrainHoldouts {
2408 pub(crate) requests: usize,
2411 pub(crate) routes: usize,
2413 pub(crate) total_routes: usize,
2415 pub(crate) top_connections: Vec<(u64, usize)>,
2418 pub(crate) held: Vec<(u16, u64)>,
2427}
2428
2429pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
2431
2432impl ForwardingTable {
2433 pub(crate) fn endpoint_drain_holdouts(
2435 &self,
2436 endpoint: ModuleEndpointId,
2437 ) -> Result<DrainHoldouts, ForwardingError> {
2438 let inner = self.read_inner()?;
2439 let mut holdouts = DrainHoldouts::default();
2440 let mut by_connection: HashMap<u64, usize> = HashMap::new();
2441 for (key, route) in &inner.client_to_module {
2442 if route.module_endpoint != endpoint {
2443 continue;
2444 }
2445 holdouts.total_routes += 1;
2446 let held = route.flow.drain_in_flight();
2447 if held == 0 {
2448 continue;
2449 }
2450 holdouts.requests += held;
2451 holdouts.routes += 1;
2452 *by_connection.entry(key.connection_id.get()).or_default() += held;
2453 holdouts.held.extend(
2454 route
2455 .flow
2456 .drain_held_corrs()
2457 .into_iter()
2458 .map(|corr| (route.module_channel, corr)),
2459 );
2460 }
2461 holdouts.held.sort_unstable();
2462 holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
2463 let mut connections = by_connection.into_iter().collect::<Vec<_>>();
2464 connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
2465 connections.truncate(3);
2466 holdouts.top_connections = connections;
2467 Ok(holdouts)
2468 }
2469
2470 pub(crate) fn endpoint_in_flight_count(
2471 &self,
2472 endpoint: ModuleEndpointId,
2473 ) -> Result<usize, ForwardingError> {
2474 let inner = self.read_inner()?;
2475 Ok(inner
2476 .client_to_module
2477 .values()
2478 .filter(|route| route.module_endpoint == endpoint)
2479 .map(|route| route.flow.drain_in_flight())
2480 .sum())
2481 }
2482
2483 pub(crate) fn endpoint_is_draining(
2484 &self,
2485 endpoint: ModuleEndpointId,
2486 ) -> Result<bool, ForwardingError> {
2487 Ok(self
2488 .read_inner()?
2489 .draining_endpoints
2490 .contains_key(&endpoint))
2491 }
2492
2493 pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
2494 let inner = self.read_inner()?;
2495 Ok(inner
2496 .modules_by_id
2497 .get(module_id)
2498 .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
2499 }
2500
2501 pub(crate) fn release_module_endpoint_routes(
2502 &self,
2503 endpoint: ModuleEndpointId,
2504 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2505 let mut inner = self.write_inner()?;
2506 let routes = inner
2507 .module_to_client
2508 .iter()
2509 .filter(|(module_key, _)| module_key.endpoint == endpoint)
2510 .map(|(module_key, route)| (*module_key, route.module_epoch))
2511 .collect::<Vec<_>>();
2512 let mut released = Vec::with_capacity(routes.len());
2513 for (module_key, epoch) in routes {
2514 if let RouteRelease::Removed(target) =
2515 release_module_route_locked(&mut inner, module_key, epoch)
2516 {
2517 released.push(target);
2518 }
2519 }
2520 Ok(released)
2521 }
2522
2523 pub(crate) fn endpoint_routes(
2529 &self,
2530 endpoint: ModuleEndpointId,
2531 ) -> Result<Vec<EndpointRoute>, ForwardingError> {
2532 let inner = self.read_inner()?;
2533 Ok(endpoint_routes_locked(&inner, endpoint))
2534 }
2535
2536 pub(crate) fn route_census(
2538 &self,
2539 module_id: Option<&str>,
2540 ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
2541 let inner = self.read_inner()?;
2542 let mut endpoints = inner
2543 .modules_by_id
2544 .iter()
2545 .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
2546 .map(|(id, module)| (id.clone(), module.endpoint))
2547 .collect::<Vec<_>>();
2548 endpoints.sort_by(|left, right| left.0.cmp(&right.0));
2549 Ok(endpoints
2550 .into_iter()
2551 .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
2552 .collect())
2553 }
2554
2555 pub(crate) fn live_roots(
2557 &self,
2558 module_id: &str,
2559 ) -> Result<ModuleControlResponseToModule, ForwardingError> {
2560 let inner = self.read_inner()?;
2561 let endpoint = inner
2562 .modules_by_id
2563 .get(module_id)
2564 .map(|module| module.endpoint);
2565 let mut roots = BTreeMap::new();
2566 let mut unknown_root_bindings = 0;
2567 let mut total_bindings = 0;
2568 if let Some(endpoint) = endpoint {
2569 for binding in inner
2570 .module_to_client
2571 .values()
2572 .filter(|binding| binding.module_endpoint == endpoint)
2573 {
2574 total_bindings += 1;
2575 let workdir = binding
2576 .scope
2577 .as_ref()
2578 .and_then(|scope| scope.workdir.as_deref())
2579 .map(std::path::Path::new);
2580 if let Some(root) = &binding.project_root {
2581 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2582 entry.0 += 1;
2583 } else if workdir.is_none() {
2584 unknown_root_bindings += 1;
2585 }
2586 if let Some(workdir) = workdir.filter(|workdir| {
2587 binding.project_root.as_ref().map(ProjectRootId::as_path) != Some(*workdir)
2588 }) {
2589 roots.entry(workdir.to_path_buf()).or_insert((0, 0)).0 += 1;
2590 }
2591 }
2592 for pending in inner
2593 .pending_relays
2594 .values()
2595 .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
2596 {
2597 total_bindings += 1;
2598 if let Some(root) = &pending.reservation.project_root {
2599 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2600 entry.1 += 1;
2601 } else {
2602 unknown_root_bindings += 1;
2603 }
2604 }
2605 }
2606 Ok(ModuleControlResponseToModule::LiveRoots {
2607 roots: roots
2608 .into_iter()
2609 .map(|(project_root, (bound, pending))| LiveRoot {
2610 project_root,
2611 bound,
2612 pending,
2613 })
2614 .collect(),
2615 unknown_root_bindings,
2616 total_bindings,
2617 })
2618 }
2619
2620 pub(crate) fn connection_has_client_routes(
2626 &self,
2627 connection_id: ConnectionId,
2628 ) -> Result<bool, ForwardingError> {
2629 let inner = self.read_inner()?;
2630 Ok(connection_has_client_routes_locked(&inner, connection_id))
2631 }
2632
2633 pub(crate) fn cleanup_connection(
2634 &self,
2635 connection_id: ConnectionId,
2636 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2637 self.cleanup_connection_counted(connection_id)
2638 .map(|cleanup| cleanup.released)
2639 }
2640
2641 pub(crate) fn cleanup_connection_counted(
2646 &self,
2647 connection_id: ConnectionId,
2648 ) -> Result<ConnectionCleanup, ForwardingError> {
2649 let mut inner = self.write_inner()?;
2650 inner.closing_connections.insert(connection_id);
2651 let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
2652 remove_module_connection_locked(&mut inner, endpoint)
2653 } else {
2654 ConnectionCleanup {
2655 released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
2656 abandoned_relays: 0,
2657 }
2658 };
2659 inner.closing_connections.remove(&connection_id);
2668 Ok(cleanup)
2669 }
2670
2671 fn cleanup_client_connection_locked(
2672 inner: &mut ForwardingInner,
2673 connection_id: ConnectionId,
2674 ) -> Vec<GoodbyeTarget> {
2675 let routes = inner
2676 .client_to_module
2677 .iter()
2678 .filter(|(key, _)| key.connection_id == connection_id)
2679 .map(|(key, route)| (*key, route.client_epoch))
2680 .collect::<Vec<_>>();
2681 let mut released = Vec::with_capacity(routes.len());
2682 for (client_key, epoch) in routes {
2683 if let RouteRelease::Removed(target) =
2684 release_client_route_locked(inner, client_key, epoch)
2685 {
2686 released.push(target);
2687 }
2688 }
2689
2690 let pending_keys = inner
2691 .pending_relays
2692 .iter()
2693 .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
2694 .map(|(key, _)| *key)
2695 .collect::<Vec<_>>();
2696 for key in pending_keys {
2697 let Some(pending) = inner.pending_relays.remove(&key) else {
2698 continue;
2699 };
2700 release_reserved_route_locked(
2701 inner,
2702 pending.reservation.client_key,
2703 pending.reservation.module_key,
2704 );
2705 if pending.relay_enqueued {
2706 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2707 released.push(target);
2708 }
2709 } else {
2710 inner.relays_released_before_enqueue.insert(key);
2714 }
2715 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
2716 "client connection closed during route.bind relay".to_string(),
2717 ));
2718 }
2719
2720 let orphaned = inner
2721 .reserved_client
2722 .iter()
2723 .filter(|(key, _)| key.connection_id == connection_id)
2724 .map(|(client, module)| (*client, *module))
2725 .collect::<Vec<_>>();
2726 for (client_key, module_key) in orphaned {
2727 release_reserved_route_locked(inner, client_key, module_key);
2728 }
2729 inner.next_client_channel.remove(&connection_id);
2730 inner
2731 .client_slot_epochs
2732 .retain(|key, _| key.connection_id != connection_id);
2733 inner
2734 .last_published_epoch
2735 .retain(|key, _| key.connection_id != connection_id);
2736 inner
2737 .status
2738 .retain(|(key, _), _| key.connection_id != connection_id);
2739
2740 released
2741 }
2742
2743 pub(crate) fn escalate_client_delivery_failure(
2752 &self,
2753 connection_id: ConnectionId,
2754 channel: u16,
2755 expected_epoch: u32,
2756 reason: CloseReason,
2757 undelivered: UndeliveredFrame<'_>,
2758 ) -> Result<bool, ForwardingError> {
2759 self.escalate_client_delivery_failure_diagnosed(
2760 connection_id,
2761 channel,
2762 expected_epoch,
2763 reason,
2764 undelivered,
2765 None,
2766 )
2767 }
2768
2769 pub(crate) fn escalate_client_delivery_failure_diagnosed(
2770 &self,
2771 connection_id: ConnectionId,
2772 channel: u16,
2773 expected_epoch: u32,
2774 reason: CloseReason,
2775 undelivered: UndeliveredFrame<'_>,
2776 diagnosis: Option<&crate::router::ClientDeliveryDiagnosis>,
2777 ) -> Result<bool, ForwardingError> {
2778 let principals = {
2779 let mut inner = self.write_inner()?;
2780 let key = ClientRouteKey {
2781 connection_id,
2782 channel,
2783 };
2784 if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
2785 None
2786 } else {
2787 inner.closing_connections.insert(connection_id);
2788 Some(connection_principals_locked(&inner, connection_id))
2789 }
2790 };
2791 let Some(principals) = principals else {
2792 return Ok(false);
2793 };
2794 let backlog = undelivered.sink.backlog();
2795 let close_reason = reason.to_string();
2796 if self.request_connection_close(connection_id, reason) {
2797 warn!(
2798 connection_id = connection_id.get(),
2799 principals = %principals,
2800 module_id = undelivered.module_id.unwrap_or("unknown"),
2801 client_channel = channel,
2802 queued_bytes = backlog.queued_bytes,
2803 queued_frames = backlog.queued_frames,
2804 frame_type = ?diagnosis.map(|failure| failure.frame_type),
2805 frame_flags = ?diagnosis.map(|failure| failure.flags.0),
2806 close_cause = diagnosis.map(|failure| failure.cause).unwrap_or("delivery_failed"),
2807 oldest_queued_ms = backlog
2808 .oldest_age
2809 .map(|age| age.as_millis() as u64)
2810 .unwrap_or(0),
2811 close_reason = %close_reason,
2812 "closing client connection: its egress queue could not take a frame"
2813 );
2814 }
2815 Ok(true)
2816 }
2817
2818 pub(crate) fn publish_scope_changes(
2832 &self,
2833 changes: &[ScopeTagChange],
2834 ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
2835 let mut inner = self.write_inner()?;
2836 let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
2837 for change in changes {
2838 let key = (change.owner.clone(), change.scope_ref.clone());
2839 match change.after {
2840 Some(tag) => {
2841 inner.scope_tags.insert(key, tag);
2842 }
2843 None => {
2844 inner.scope_tags.remove(&key);
2845 }
2846 }
2847 by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
2848 }
2849 let mut selected = Vec::new();
2850 for (client_key, route) in &inner.client_to_module {
2851 let Some(scope) = &route.scope else {
2852 continue;
2853 };
2854 let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
2855 else {
2856 continue;
2857 };
2858 if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
2859 continue;
2860 }
2861 let reason = match &change.drain {
2862 ScopeDrain::Nothing => continue,
2863 ScopeDrain::All(reason) => *reason,
2864 ScopeDrain::Carriers(narrowed) => {
2865 let owner = Principal::Reserved {
2866 module_id: scope.owner.clone(),
2867 };
2868 let hit = route.principal != owner
2869 && narrowed.iter().any(|(principal, allowed)| {
2870 *principal == route.principal
2871 && allowed
2872 .as_ref()
2873 .is_none_or(|targets| !targets.contains(&route.module_id))
2874 });
2875 if !hit {
2876 continue;
2877 }
2878 RouteCloseReason::ScopeCarrierRemoved
2879 }
2880 };
2881 selected.push((*client_key, route.client_epoch, reason));
2882 }
2883 let mut drained = Vec::new();
2884 for (client_key, client_epoch, reason) in selected {
2885 let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
2886 continue;
2887 };
2888 let release = release_client_route_locked(&mut inner, client_key, client_epoch);
2889 self.record_route_release(&release);
2890 if let RouteRelease::Removed(module) = release {
2891 drained.push(ScopeDrainedRoute {
2892 reason,
2893 scope: route
2894 .scope
2895 .clone()
2896 .expect("selected scope drain has a bound scope"),
2897 module_id: route.module_id.clone(),
2898 client: GoodbyeTarget {
2899 connection_id: route.client_connection_id,
2900 sink: route.client_sink.clone(),
2901 negotiated_ver: route.client_negotiated_ver,
2902 channel: route.client_channel,
2903 epoch: route.client_epoch,
2904 kind: GoodbyeTargetKind::Client,
2905 module_id: Some(route.module_id.clone()),
2906 },
2907 module,
2908 });
2909 }
2910 }
2911 Ok(drained)
2912 }
2913
2914 #[cfg(test)]
2916 pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
2917 self.read_inner()
2918 .ok()?
2919 .scope_tags
2920 .get(&(owner.to_string(), scope_ref.to_string()))
2921 .copied()
2922 }
2923
2924 fn record_route_release(&self, release: &RouteRelease) {
2925 match release {
2926 RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
2927 RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
2928 RouteRelease::Absent => {}
2929 }
2930 }
2931
2932 fn read_inner(&self) -> Result<RwLockReadGuard<'_, ForwardingInner>, ForwardingError> {
2933 self.inner.read().map_err(|_| ForwardingError::Poisoned)
2934 }
2935
2936 fn write_inner(&self) -> Result<RwLockWriteGuard<'_, ForwardingInner>, ForwardingError> {
2937 self.inner.write().map_err(|_| ForwardingError::Poisoned)
2938 }
2939
2940 fn lock_close_registry(
2941 &self,
2942 ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
2943 self.close_registry
2944 .lock()
2945 .unwrap_or_else(|poisoned| poisoned.into_inner())
2946 }
2947}
2948
2949impl ForwardingInner {
2950 fn allocate_route_slots(
2951 &mut self,
2952 connection_id: ConnectionId,
2953 endpoint: ModuleEndpointId,
2954 ) -> Result<(u16, u32, u16, u32), ForwardingError> {
2955 let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
2956 let mut client_channel = client_start;
2957 let client_channel = loop {
2958 let key = ClientRouteKey {
2959 connection_id,
2960 channel: client_channel,
2961 };
2962 let eligible = !self.client_to_module.contains_key(&key)
2963 && !self.reserved_client.contains_key(&key)
2964 && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2965 if eligible {
2966 break client_channel;
2967 }
2968 client_channel = next_channel(client_channel);
2969 if client_channel == client_start {
2970 return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
2971 }
2972 };
2973
2974 let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
2975 let mut module_channel = module_start;
2976 let module_channel = loop {
2977 let key = ModuleRouteKey {
2978 endpoint,
2979 channel: module_channel,
2980 };
2981 let eligible = !self.module_to_client.contains_key(&key)
2982 && !self.reserved_module.contains_key(&key)
2983 && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2984 if eligible {
2985 break module_channel;
2986 }
2987 module_channel = next_channel(module_channel);
2988 if module_channel == module_start {
2989 return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
2990 }
2991 };
2992
2993 let client_key = ClientRouteKey {
2994 connection_id,
2995 channel: client_channel,
2996 };
2997 let module_key = ModuleRouteKey {
2998 endpoint,
2999 channel: module_channel,
3000 };
3001 let client_epoch = self
3002 .client_slot_epochs
3003 .get(&client_key)
3004 .copied()
3005 .unwrap_or(0)
3006 + 1;
3007 let module_epoch = self
3008 .module_slot_epochs
3009 .get(&module_key)
3010 .copied()
3011 .unwrap_or(0)
3012 + 1;
3013 self.client_slot_epochs.insert(client_key, client_epoch);
3014 self.module_slot_epochs.insert(module_key, module_epoch);
3015 self.next_client_channel
3016 .insert(connection_id, next_channel(client_channel));
3017 self.next_module_channel
3018 .insert(endpoint, next_channel(module_channel));
3019 Ok((client_channel, client_epoch, module_channel, module_epoch))
3020 }
3021
3022 fn allocate_control_corr(
3023 &mut self,
3024 endpoint: ModuleEndpointId,
3025 ) -> Result<u64, ForwardingError> {
3026 let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
3027 if candidate == 0 {
3028 self.closing_connections.insert(endpoint.connection_id);
3029 return Err(ForwardingError::RelayCorrelationExhausted);
3030 }
3031 self.next_control_corr.insert(
3032 endpoint,
3033 if candidate == u64::MAX {
3034 0
3035 } else {
3036 candidate + 1
3037 },
3038 );
3039 Ok(candidate)
3040 }
3041}
3042
3043fn connection_has_client_routes_locked(
3044 inner: &ForwardingInner,
3045 connection_id: ConnectionId,
3046) -> bool {
3047 inner
3048 .client_to_module
3049 .keys()
3050 .any(|key| key.connection_id == connection_id)
3051 || inner
3052 .reserved_client
3053 .keys()
3054 .any(|key| key.connection_id == connection_id)
3055}
3056
3057fn check_module_connection_role_locked(
3058 inner: &ForwardingInner,
3059 connection_id: ConnectionId,
3060) -> Result<(), ForwardingError> {
3061 if inner.endpoint_by_connection.contains_key(&connection_id)
3062 || connection_has_client_routes_locked(inner, connection_id)
3063 {
3064 return Err(ForwardingError::ConnectionRoleConflict { connection_id });
3065 }
3066 Ok(())
3067}
3068
3069fn next_channel(channel: u16) -> u16 {
3070 let next = channel.wrapping_add(1);
3071 if next == 0 {
3072 1
3073 } else {
3074 next
3075 }
3076}
3077
3078fn endpoint_routes_locked(
3079 inner: &ForwardingInner,
3080 endpoint: ModuleEndpointId,
3081) -> Vec<EndpointRoute> {
3082 let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
3083 let draining = drain_reason.is_some();
3084 let mut routes = inner
3085 .module_to_client
3086 .iter()
3087 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3088 .map(|(_, route)| EndpointRoute {
3089 goodbye_target: GoodbyeTarget {
3090 connection_id: route.client_connection_id,
3091 sink: route.client_sink.clone(),
3092 negotiated_ver: route.client_negotiated_ver,
3093 channel: route.client_channel,
3094 epoch: route.client_epoch,
3095 kind: GoodbyeTargetKind::Client,
3096 module_id: Some(route.module_id.clone()),
3097 },
3098 principal: route.principal.clone(),
3099 bound_at: route.bound_at,
3100 draining,
3101 drain_reason,
3102 })
3103 .collect::<Vec<_>>();
3104 routes.sort_by_key(|route| {
3105 (
3106 route.goodbye_target.connection_id.get(),
3107 route.goodbye_target.channel,
3108 route.goodbye_target.epoch,
3109 )
3110 });
3111 routes
3112}
3113
3114fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
3118 let key = (scope.owner.clone(), scope.scope_ref.clone());
3119 match inner.scope_tags.get(&key) {
3120 Some(current) if *current == scope.tag => None,
3121 Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
3122 subc_protocol::error_codes::SCOPE_CHANGED,
3123 format!(
3124 "scope '{}' of {} changed while the route was being bound; re-open it",
3125 scope.scope_ref, scope.owner
3126 ),
3127 )),
3128 _ => Some(ErrorBody::new(
3129 subc_protocol::error_codes::SCOPE_ENDED,
3130 format!(
3131 "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
3132 scope.scope_ref, scope.owner, scope.tag.scope_epoch
3133 ),
3134 )),
3135 }
3136}
3137
3138fn release_reserved_route_locked(
3139 inner: &mut ForwardingInner,
3140 client_key: ClientRouteKey,
3141 module_key: ModuleRouteKey,
3142) {
3143 if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
3144 inner.reserved_client.remove(&client_key);
3145 }
3146 if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
3147 inner.reserved_module.remove(&module_key);
3148 }
3149 inner.status.retain(|(key, _), _| *key != client_key);
3150}
3151
3152fn release_client_route_locked(
3153 inner: &mut ForwardingInner,
3154 client_key: ClientRouteKey,
3155 expected_epoch: u32,
3156) -> RouteRelease {
3157 let Some(route) = inner.client_to_module.get(&client_key) else {
3158 return RouteRelease::Absent;
3159 };
3160 if route.client_epoch != expected_epoch {
3161 return RouteRelease::Stale;
3162 }
3163 let route = inner
3164 .client_to_module
3165 .remove(&client_key)
3166 .expect("route checked under the same forwarding lock");
3167 route.flow.close();
3168 if let Some(scope) = &route.scope {
3169 scope.release_bind();
3170 }
3171 inner.module_to_client.remove(&ModuleRouteKey {
3172 endpoint: route.module_endpoint,
3173 channel: route.module_channel,
3174 });
3175 inner.operator_confirms.route_closed(ModuleRouteKey {
3176 endpoint: route.module_endpoint,
3177 channel: route.module_channel,
3178 });
3179 inner.status.remove(&(client_key, expected_epoch));
3180 RouteRelease::Removed(GoodbyeTarget {
3181 connection_id: route.module_endpoint.connection_id,
3182 sink: route.module_sink.clone(),
3183 negotiated_ver: route.module_negotiated_ver,
3184 channel: route.module_channel,
3185 epoch: route.module_epoch,
3186 kind: GoodbyeTargetKind::Module,
3187 module_id: Some(route.module_id.clone()),
3188 })
3189}
3190
3191fn release_module_route_locked(
3192 inner: &mut ForwardingInner,
3193 module_key: ModuleRouteKey,
3194 expected_epoch: u32,
3195) -> RouteRelease {
3196 let Some(route) = inner.module_to_client.get(&module_key) else {
3197 return RouteRelease::Absent;
3198 };
3199 if route.module_epoch != expected_epoch {
3200 return RouteRelease::Stale;
3201 }
3202 let route = inner
3203 .module_to_client
3204 .remove(&module_key)
3205 .expect("route checked under the same forwarding lock");
3206 inner.operator_confirms.route_closed(module_key);
3207 route.flow.close();
3208 if let Some(scope) = &route.scope {
3209 scope.release_bind();
3210 }
3211 let client_key = ClientRouteKey {
3212 connection_id: route.client_connection_id,
3213 channel: route.client_channel,
3214 };
3215 inner.client_to_module.remove(&client_key);
3216 inner.status.remove(&(client_key, route.client_epoch));
3217 RouteRelease::Removed(GoodbyeTarget {
3218 connection_id: route.client_connection_id,
3219 sink: route.client_sink.clone(),
3220 negotiated_ver: route.client_negotiated_ver,
3221 channel: route.client_channel,
3222 epoch: route.client_epoch,
3223 kind: GoodbyeTargetKind::Client,
3224 module_id: Some(route.module_id.clone()),
3225 })
3226}
3227
3228fn commit_route_locked(
3229 inner: &mut ForwardingInner,
3230 pending: PendingRouteBindRelayEntry,
3231) -> Result<Option<GoodbyeTarget>, ForwardingError> {
3232 let reservation = pending.reservation;
3233 if inner
3234 .closing_connections
3235 .contains(&reservation.client_key.connection_id)
3236 {
3237 return Err(ForwardingError::ConnectionClosing {
3238 connection_id: reservation.client_key.connection_id,
3239 });
3240 }
3241 let module_id = inner
3242 .module_id_by_endpoint
3243 .get(&reservation.module_key.endpoint)
3244 .cloned()
3245 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3246 if inner
3247 .draining_endpoints
3248 .contains_key(&reservation.module_key.endpoint)
3249 {
3250 return Err(ForwardingError::ModuleReloading { module_id });
3251 }
3252 if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
3253 || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
3254 {
3255 return Err(ForwardingError::UnknownReservation {
3256 client_channel: reservation.client_key.channel,
3257 module_channel: reservation.module_key.channel,
3258 });
3259 }
3260 let module = inner
3261 .modules_by_id
3262 .get(&module_id)
3263 .filter(|module| module.endpoint == reservation.module_key.endpoint)
3264 .cloned()
3265 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3266 let binding = Arc::new(RouteBinding {
3267 client_connection_id: reservation.client_key.connection_id,
3268 client_sink: pending.client_sink,
3269 client_negotiated_ver: pending.client_negotiated_ver,
3270 client_channel: reservation.client_key.channel,
3271 client_epoch: reservation.client_epoch,
3272 module_id,
3273 module_endpoint: reservation.module_key.endpoint,
3274 module_sink: module.sink,
3275 module_negotiated_ver: module.negotiated_ver,
3276 module_channel: reservation.module_key.channel,
3277 module_epoch: reservation.module_epoch,
3278 principal: pending.principal,
3279 project_root: reservation.project_root.clone(),
3280 bound_at: Instant::now(),
3281 flow: Arc::new(ChannelFlow::new(window_for(&module.concurrency))),
3282 scope: pending.scope,
3283 });
3284 inner
3285 .client_to_module
3286 .insert(reservation.client_key, Arc::clone(&binding));
3287 inner
3288 .module_to_client
3289 .insert(reservation.module_key, Arc::clone(&binding));
3290 let previous_published = inner
3291 .last_published_epoch
3292 .insert(reservation.client_key, reservation.client_epoch);
3293
3294 if let Some(scope) = &binding.scope {
3298 scope.record_served(&binding.module_id);
3299 }
3300
3301 let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
3306 if client_writer_closed {
3307 let abandoned = abandoned_route_target(inner, &reservation);
3311 if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
3312 route.flow.close();
3313 }
3314 inner.module_to_client.remove(&reservation.module_key);
3315 inner
3316 .status
3317 .remove(&(reservation.client_key, reservation.client_epoch));
3318 match previous_published {
3319 Some(epoch) => {
3320 inner
3321 .last_published_epoch
3322 .insert(reservation.client_key, epoch);
3323 }
3324 None => {
3325 inner.last_published_epoch.remove(&reservation.client_key);
3326 }
3327 }
3328 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3329 "client egress closed during route publication".to_string(),
3330 ));
3331 return Ok(abandoned);
3332 }
3333
3334 if let Some(scope) = &binding.scope {
3335 scope.commit_presence();
3336 }
3337 let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
3338 Ok(None)
3339}
3340
3341fn module_connection_for_endpoint_locked(
3349 inner: &ForwardingInner,
3350 endpoint: ModuleEndpointId,
3351) -> Option<&ModuleConnection> {
3352 let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
3353 inner
3354 .modules_by_id
3355 .get(module_id)
3356 .filter(|module| module.endpoint == endpoint)
3357 .or_else(|| {
3358 inner
3359 .candidates_by_id
3360 .get(module_id)
3361 .filter(|module| module.endpoint == endpoint)
3362 })
3363 .or_else(|| inner.superseded_endpoints.get(&endpoint))
3364}
3365
3366fn abandoned_route_target(
3367 inner: &ForwardingInner,
3368 reservation: &RouteReservation,
3369) -> Option<GoodbyeTarget> {
3370 let module_id = inner
3371 .module_id_by_endpoint
3372 .get(&reservation.module_key.endpoint)?;
3373 let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
3374 (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
3375 connection_id: module.endpoint.connection_id,
3376 sink: module.sink.clone(),
3377 negotiated_ver: module.negotiated_ver,
3378 channel: reservation.module_key.channel,
3379 epoch: reservation.module_epoch,
3380 kind: GoodbyeTargetKind::Module,
3381 module_id: Some(module_id.clone()),
3382 })
3383}
3384
3385fn enqueue_hello_ack_locked(
3390 sink: &FrameSink,
3391 connection_id: ConnectionId,
3392 hello_ack: Option<Frame>,
3393) -> Result<(), ForwardingError> {
3394 let Some(hello_ack) = hello_ack else {
3395 return Ok(());
3396 };
3397 sink.try_send(hello_ack)
3398 .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
3399}
3400
3401fn remove_module_connection_locked(
3402 inner: &mut ForwardingInner,
3403 endpoint: ModuleEndpointId,
3404) -> ConnectionCleanup {
3405 inner.operator_confirms.module_closed(endpoint);
3408 inner.draining_endpoints.remove(&endpoint);
3409 inner.daemon_drain_results.remove(&endpoint);
3410 let module_id = inner.module_id_by_endpoint.remove(&endpoint);
3411 if let Some(module_id) = module_id.as_ref() {
3412 if inner
3413 .modules_by_id
3414 .get(module_id)
3415 .is_some_and(|module| module.endpoint == endpoint)
3416 {
3417 inner.modules_by_id.remove(module_id);
3418 }
3419 if inner
3420 .candidates_by_id
3421 .get(module_id)
3422 .is_some_and(|module| module.endpoint == endpoint)
3423 {
3424 inner.candidates_by_id.remove(module_id);
3425 }
3426 }
3427 inner.superseded_endpoints.remove(&endpoint);
3428 inner.endpoint_by_connection.remove(&endpoint.connection_id);
3429 inner.next_module_channel.remove(&endpoint);
3430 inner.next_control_corr.remove(&endpoint);
3431 inner
3432 .health_probe_tombstones
3433 .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
3434 inner
3435 .module_slot_epochs
3436 .retain(|key, _| key.endpoint != endpoint);
3437 let reserved_module_keys: Vec<ModuleRouteKey> = inner
3438 .reserved_module
3439 .keys()
3440 .filter(|module_key| module_key.endpoint == endpoint)
3441 .copied()
3442 .collect();
3443 for module_key in reserved_module_keys {
3444 if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
3445 release_reserved_route_locked(inner, client_key, module_key);
3446 }
3447 }
3448
3449 let pending_keys: Vec<_> = inner
3450 .pending_relays
3451 .keys()
3452 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3453 .copied()
3454 .collect();
3455 let pending: Vec<_> = pending_keys
3456 .into_iter()
3457 .filter_map(|key| {
3458 inner
3459 .pending_relays
3460 .remove(&key)
3461 .map(|pending| (key, pending))
3462 })
3463 .collect();
3464 let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
3465 for (key, pending) in pending {
3466 if !pending.relay_enqueued {
3470 inner.relays_released_before_enqueue.insert(key);
3471 }
3472 let module_label = module_id.as_deref().unwrap_or("unknown");
3473 let _ = pending
3474 .sender
3475 .send(RouteBindRelayOutcome::ModuleGone(format!(
3476 "module '{module_label}' connection closed during route.bind relay"
3477 )));
3478 }
3479
3480 let pending_control_keys: Vec<_> = inner
3481 .pending_control_rpcs
3482 .keys()
3483 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3484 .copied()
3485 .collect();
3486 let pending_control: Vec<_> = pending_control_keys
3487 .into_iter()
3488 .filter_map(|key| inner.pending_control_rpcs.remove(&key))
3489 .collect();
3490 for pending in pending_control {
3491 let module_label = module_id.as_deref().unwrap_or("unknown");
3492 let _ = pending
3493 .sender
3494 .send(ModuleControlRpcOutcome::ModuleGone(format!(
3495 "module '{module_label}' connection closed during module-control RPC"
3496 )));
3497 }
3498
3499 let module_routes = inner
3500 .module_to_client
3501 .iter()
3502 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3503 .map(|(module_key, route)| (*module_key, route.module_epoch))
3504 .collect::<Vec<_>>();
3505 let mut released = Vec::with_capacity(module_routes.len());
3506 for (module_key, epoch) in module_routes {
3507 if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
3508 {
3509 released.push(target);
3510 }
3511 }
3512 ConnectionCleanup {
3513 released,
3514 abandoned_relays,
3515 }
3516}
3517
3518#[derive(Debug, Clone, Copy)]
3519struct RequestCredit {
3520 subscription: bool,
3521 excluded_from_drain: bool,
3522}
3523
3524#[derive(Debug, Default)]
3525struct CreditLedger {
3526 by_corr: HashMap<u64, Vec<RequestCredit>>,
3527}
3528
3529impl CreditLedger {
3530 fn acquire(&mut self, corr: u64, subscription: bool) {
3531 self.by_corr.entry(corr).or_default().push(RequestCredit {
3532 subscription,
3533 excluded_from_drain: false,
3534 });
3535 }
3536
3537 fn release(&mut self, corr: u64) -> bool {
3538 let Some(credits) = self.by_corr.get_mut(&corr) else {
3539 return false;
3540 };
3541 let released = credits.pop().is_some();
3542 if credits.is_empty() {
3543 self.by_corr.remove(&corr);
3544 }
3545 released
3546 }
3547
3548 fn capture_subscription_exclusions(&mut self) -> u32 {
3549 let mut excluded = 0u32;
3550 for credit in self.by_corr.values_mut().flatten() {
3551 if credit.subscription && !credit.excluded_from_drain {
3552 credit.excluded_from_drain = true;
3553 excluded = excluded.saturating_add(1);
3554 }
3555 }
3556 excluded
3557 }
3558
3559 #[cfg(test)]
3560 fn in_flight(&self) -> usize {
3561 self.by_corr.values().map(Vec::len).sum()
3562 }
3563
3564 fn drain_in_flight(&self) -> usize {
3565 self.by_corr
3566 .values()
3567 .flatten()
3568 .filter(|credit| !credit.excluded_from_drain)
3569 .count()
3570 }
3571
3572 fn drain_held_corrs(&self) -> Vec<u64> {
3575 let mut corrs = self
3576 .by_corr
3577 .iter()
3578 .flat_map(|(corr, credits)| {
3579 credits
3580 .iter()
3581 .filter(|credit| !credit.excluded_from_drain)
3582 .map(move |_| *corr)
3583 })
3584 .collect::<Vec<_>>();
3585 corrs.sort_unstable();
3586 corrs
3587 }
3588}
3589
3590#[derive(Debug, Default)]
3591struct ChannelFlowState {
3592 closed: bool,
3593 credits: CreditLedger,
3594}
3595
3596#[derive(Debug)]
3598pub(crate) struct ChannelFlow {
3599 sem: Semaphore,
3600 window: usize,
3601 state: Mutex<ChannelFlowState>,
3602}
3603
3604impl ChannelFlow {
3605 pub(crate) fn new(window: usize) -> Self {
3606 debug_assert!(window > 0, "flow-control window must be non-zero");
3607 Self {
3608 sem: Semaphore::new(window),
3609 window,
3610 state: Mutex::new(ChannelFlowState::default()),
3611 }
3612 }
3613
3614 #[cfg(test)]
3615 pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
3616 self.acquire_tagged(0, false).await
3617 }
3618
3619 pub(crate) async fn acquire_tagged(
3620 &self,
3621 corr: u64,
3622 subscription: bool,
3623 ) -> Result<(), ChannelFlowClosed> {
3624 let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
3625 let mut state = self
3626 .state
3627 .lock()
3628 .unwrap_or_else(|poisoned| poisoned.into_inner());
3629 if state.closed {
3630 return Err(ChannelFlowClosed);
3631 }
3632 state.credits.acquire(corr, subscription);
3633 permit.forget();
3634 Ok(())
3635 }
3636
3637 #[cfg(test)]
3638 pub(crate) fn release(&self) {
3639 self.release_corr(0);
3640 }
3641
3642 pub(crate) fn release_corr(&self, corr: u64) {
3643 let released = self
3644 .state
3645 .lock()
3646 .unwrap_or_else(|poisoned| poisoned.into_inner())
3647 .credits
3648 .release(corr);
3649 if !released {
3650 warn!(
3654 window = self.window,
3655 available = self.sem.available_permits(),
3656 "flow-control over-release ignored"
3657 );
3658 return;
3659 }
3660 if !self.sem.is_closed() {
3661 self.sem.add_permits(1);
3662 }
3663 }
3664
3665 #[cfg(test)]
3666 pub(crate) fn in_flight(&self) -> usize {
3667 self.state
3668 .lock()
3669 .unwrap_or_else(|poisoned| poisoned.into_inner())
3670 .credits
3671 .in_flight()
3672 }
3673
3674 pub(crate) fn drain_in_flight(&self) -> usize {
3675 self.state
3676 .lock()
3677 .unwrap_or_else(|poisoned| poisoned.into_inner())
3678 .credits
3679 .drain_in_flight()
3680 }
3681
3682 pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
3683 self.state
3684 .lock()
3685 .unwrap_or_else(|poisoned| poisoned.into_inner())
3686 .credits
3687 .drain_held_corrs()
3688 }
3689
3690 #[cfg(test)]
3691 pub(crate) fn available_permits(&self) -> usize {
3692 self.sem.available_permits()
3693 }
3694
3695 pub(crate) fn begin_drain(&self) -> u32 {
3696 let mut state = self
3697 .state
3698 .lock()
3699 .unwrap_or_else(|poisoned| poisoned.into_inner());
3700 state.closed = true;
3701 self.sem.close();
3702 state.credits.capture_subscription_exclusions()
3703 }
3704
3705 pub(crate) fn close(&self) {
3706 self.state
3707 .lock()
3708 .unwrap_or_else(|poisoned| poisoned.into_inner())
3709 .closed = true;
3710 self.sem.close();
3711 }
3712}
3713
3714#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3715pub(crate) struct ChannelFlowClosed;
3716
3717impl fmt::Display for ChannelFlowClosed {
3718 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3719 write!(f, "flow-control window closed")
3720 }
3721}
3722
3723impl Error for ChannelFlowClosed {}
3724
3725fn window_for(concurrency: &Concurrency) -> usize {
3726 match concurrency {
3727 Concurrency::Serial => 1,
3728 Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
3729 Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
3730 }
3731}
3732
3733#[derive(Debug, Clone, PartialEq, Eq)]
3734pub enum ForwardingError {
3735 ConnectionRoleConflict {
3738 connection_id: ConnectionId,
3739 },
3740 NoModuleConnection,
3741 ModuleReloading {
3742 module_id: String,
3743 },
3744 StaleModuleEndpoint,
3745 UnknownReservation {
3746 client_channel: u16,
3747 module_channel: u16,
3748 },
3749 ClientRouteChannelExhausted {
3750 connection_id: ConnectionId,
3751 },
3752 ModuleRouteChannelExhausted {
3753 endpoint: ModuleEndpointId,
3754 },
3755 RelayCorrelationExhausted,
3756 ConnectionClosing {
3757 connection_id: ConnectionId,
3758 },
3759 ClientEgressClosed {
3760 connection_id: ConnectionId,
3761 },
3762 RouteOpenBuild(String),
3763 CandidateSlotOccupied {
3765 module_id: String,
3766 },
3767 ModuleEgressUnavailable {
3770 connection_id: ConnectionId,
3771 },
3772 Poisoned,
3773}
3774
3775impl fmt::Display for ForwardingError {
3776 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3777 match self {
3778 Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
3779 Self::NoModuleConnection => write!(f, "no module connection is registered"),
3780 Self::ModuleReloading { module_id } => {
3781 write!(f, "module_id '{module_id}' is reloading")
3782 }
3783 Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
3784 Self::UnknownReservation {
3785 client_channel,
3786 module_channel,
3787 } => write!(
3788 f,
3789 "route reservation client channel {client_channel} / module channel {module_channel} was not found"
3790 ),
3791 Self::ClientRouteChannelExhausted { connection_id } => write!(
3792 f,
3793 "no client route channels are available for connection {}",
3794 connection_id.get()
3795 ),
3796 Self::ModuleRouteChannelExhausted { endpoint } => write!(
3797 f,
3798 "no module route channels are available for endpoint generation {} on connection {}",
3799 endpoint.generation,
3800 endpoint.connection_id.get()
3801 ),
3802 Self::RelayCorrelationExhausted => {
3803 write!(f, "module control correlation ids are exhausted")
3804 }
3805 Self::ConnectionClosing { connection_id } => write!(
3806 f,
3807 "connection {} is closing and cannot accept route allocation",
3808 connection_id.get()
3809 ),
3810 Self::ClientEgressClosed { connection_id } => write!(
3811 f,
3812 "client connection {} egress is closed",
3813 connection_id.get()
3814 ),
3815 Self::RouteOpenBuild(message) => {
3816 write!(f, "failed to prebuild route.open response: {message}")
3817 }
3818 Self::CandidateSlotOccupied { module_id } => write!(
3819 f,
3820 "module_id '{module_id}' already has a swap candidate registered"
3821 ),
3822 Self::ModuleEgressUnavailable { connection_id } => write!(
3823 f,
3824 "module connection {} egress is unavailable; HELLO_ACK could not be queued",
3825 connection_id.get()
3826 ),
3827 Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
3828 }
3829 }
3830}
3831
3832impl Error for ForwardingError {}
3833
3834#[cfg(test)]
3835mod tests {
3836 use std::time::Duration;
3837
3838 use super::*;
3839 use tokio::sync::mpsc;
3840
3841 #[test]
3842 fn ordinary_long_running_request_is_not_excluded_from_drain() {
3843 let mut ledger = CreditLedger::default();
3844 ledger.acquire(1, false);
3845
3846 assert_eq!(ledger.capture_subscription_exclusions(), 0);
3847 assert_eq!(ledger.drain_in_flight(), 1);
3848 }
3849
3850 #[test]
3851 fn bit_set_subscription_is_excluded_and_counted() {
3852 let mut ledger = CreditLedger::default();
3853 ledger.acquire(1, true);
3854
3855 assert_eq!(ledger.capture_subscription_exclusions(), 1);
3856 assert_eq!(ledger.drain_in_flight(), 0);
3857 }
3858
3859 #[test]
3860 fn subscription_opened_after_drain_snapshot_is_not_excluded() {
3861 let mut ledger = CreditLedger::default();
3862 ledger.acquire(1, true);
3863 assert_eq!(ledger.capture_subscription_exclusions(), 1);
3864
3865 ledger.acquire(2, true);
3866
3867 assert_eq!(ledger.drain_in_flight(), 1);
3868 }
3869
3870 #[test]
3871 fn drain_with_no_subscriptions_reports_zero_excluded() {
3872 let mut ledger = CreditLedger::default();
3873 assert_eq!(ledger.capture_subscription_exclusions(), 0);
3874 }
3875
3876 fn test_hello_ack(corr: u64) -> Frame {
3877 Frame::build(
3878 FrameType::HelloAck,
3879 Flags::new(false, Priority::Passive, false),
3880 0,
3881 0,
3882 corr,
3883 Vec::new(),
3884 )
3885 .unwrap()
3886 }
3887
3888 #[test]
3892 fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
3893 let forwarding = ForwardingTable::default();
3894
3895 let (closed_tx, closed_rx) = mpsc::channel(8);
3896 drop(closed_rx);
3897 let closed = ConnectionId::new(1);
3898 assert_eq!(
3899 forwarding.register_module_connection_acked(
3900 closed,
3901 "closed".to_string(),
3902 2,
3903 Concurrency::ModuleManaged,
3904 FrameSink::new(closed_tx),
3905 test_hello_ack(1),
3906 ),
3907 Err(ForwardingError::ModuleEgressUnavailable {
3908 connection_id: closed
3909 })
3910 );
3911
3912 let (full_tx, _full_rx) = mpsc::channel(1);
3913 let full_sink = FrameSink::new(full_tx);
3914 full_sink.try_send(test_hello_ack(99)).unwrap();
3915 let full = ConnectionId::new(2);
3916 assert_eq!(
3917 forwarding.register_module_connection_acked(
3918 full,
3919 "full".to_string(),
3920 2,
3921 Concurrency::ModuleManaged,
3922 full_sink.clone(),
3923 test_hello_ack(2),
3924 ),
3925 Err(ForwardingError::ModuleEgressUnavailable {
3926 connection_id: full
3927 })
3928 );
3929 assert_eq!(
3930 forwarding.register_candidate_module_connection_acked(
3931 full,
3932 "full".to_string(),
3933 2,
3934 Concurrency::ModuleManaged,
3935 full_sink,
3936 test_hello_ack(3),
3937 ),
3938 Err(ForwardingError::ModuleEgressUnavailable {
3939 connection_id: full
3940 })
3941 );
3942
3943 for (connection, module_id) in [(closed, "closed"), (full, "full")] {
3944 assert_eq!(
3945 forwarding
3946 .module_endpoint_for_connection(connection)
3947 .unwrap(),
3948 None
3949 );
3950 let (client_tx, _client_rx) = mpsc::channel(8);
3951 assert_eq!(
3952 forwarding
3953 .begin_route_bind_relay_for_test(
3954 ConnectionId::new(50),
3955 FrameSink::new(client_tx),
3956 1,
3957 module_id,
3958 )
3959 .err(),
3960 Some(ForwardingError::NoModuleConnection)
3961 );
3962 }
3963 assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
3964 }
3965
3966 #[test]
3969 fn acked_registration_queues_the_hello_ack_first() {
3970 let forwarding = ForwardingTable::default();
3971 let (active_tx, mut active_rx) = mpsc::channel(8);
3972 forwarding
3973 .register_module_connection_acked(
3974 ConnectionId::new(1),
3975 "acked".to_string(),
3976 2,
3977 Concurrency::ModuleManaged,
3978 FrameSink::new(active_tx),
3979 test_hello_ack(11),
3980 )
3981 .unwrap();
3982 let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
3983 forwarding
3984 .register_candidate_module_connection_acked(
3985 ConnectionId::new(2),
3986 "acked".to_string(),
3987 2,
3988 Concurrency::ModuleManaged,
3989 FrameSink::new(candidate_tx),
3990 test_hello_ack(12),
3991 )
3992 .unwrap();
3993
3994 let active_first = active_rx.try_recv().unwrap().frame;
3995 assert_eq!(active_first.header.ty, FrameType::HelloAck);
3996 assert_eq!(active_first.header.corr, 11);
3997 let candidate_first = candidate_rx.try_recv().unwrap().frame;
3998 assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
3999 assert_eq!(candidate_first.header.corr, 12);
4000 }
4001
4002 #[test]
4003 fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
4004 let forwarding = ForwardingTable::default();
4005 let module_connection = ConnectionId::new(10);
4006 let exhausted_client = ConnectionId::new(20);
4007 let second_client = ConnectionId::new(30);
4008 let (module_tx, _module_rx) = mpsc::channel(1);
4009 let endpoint = forwarding
4010 .register_module_connection(
4011 module_connection,
4012 "route-limit-provider".to_string(),
4013 1,
4014 Concurrency::ModuleManaged,
4015 FrameSink::new(module_tx),
4016 )
4017 .unwrap();
4018
4019 {
4020 let mut inner = forwarding.inner.write().unwrap();
4021 for channel in 1..=u16::MAX {
4022 inner.reserved_client.insert(
4023 ClientRouteKey {
4024 connection_id: exhausted_client,
4025 channel,
4026 },
4027 ModuleRouteKey {
4028 endpoint,
4029 channel: 1,
4030 },
4031 );
4032 }
4033 }
4034
4035 let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
4036 let err = forwarding
4037 .begin_route_bind_relay_for_test(
4038 exhausted_client,
4039 FrameSink::new(exhausted_tx),
4040 1,
4041 "route-limit-provider",
4042 )
4043 .unwrap_err();
4044 assert!(matches!(
4045 err,
4046 ForwardingError::ClientRouteChannelExhausted { connection_id }
4047 if connection_id == exhausted_client
4048 ));
4049
4050 let (second_tx, _second_rx) = mpsc::channel(1);
4051 let pending = forwarding
4052 .begin_route_bind_relay_for_test(
4053 second_client,
4054 FrameSink::new(second_tx),
4055 2,
4056 "route-limit-provider",
4057 )
4058 .unwrap();
4059 assert_eq!(pending.client_channel, 1);
4060 }
4061
4062 #[test]
4063 fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
4064 let forwarding = ForwardingTable::default();
4065 let module_connection = ConnectionId::new(40);
4066 let client = ConnectionId::new(50);
4067 let (module_tx, _module_rx) = mpsc::channel(1);
4068 forwarding
4069 .register_module_connection(
4070 module_connection,
4071 "slot-reuse-provider".to_string(),
4072 1,
4073 Concurrency::ModuleManaged,
4074 FrameSink::new(module_tx),
4075 )
4076 .unwrap();
4077
4078 let (client_tx, _client_rx) = mpsc::channel(1);
4079 let client_sink = FrameSink::new(client_tx);
4080 let mut wrapped_channel = None;
4081 for index in 0..=usize::from(u16::MAX) {
4082 let pending = forwarding
4083 .begin_route_bind_relay_for_test(
4084 client,
4085 client_sink.clone(),
4086 index as u64 + 1,
4087 "slot-reuse-provider",
4088 )
4089 .unwrap();
4090 if index == usize::from(u16::MAX) {
4091 wrapped_channel = Some(pending.module_channel);
4092 }
4093 forwarding
4094 .abort_pending_relay(
4095 pending.endpoint,
4096 pending.corr,
4097 RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
4098 )
4099 .unwrap();
4100 }
4101
4102 assert_eq!(wrapped_channel, Some(1));
4103 }
4104
4105 #[test]
4106 fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
4107 let forwarding = ForwardingTable::default();
4108 let client = ConnectionId::new(60);
4109 forwarding
4110 .inner
4111 .write()
4112 .unwrap()
4113 .next_client_channel
4114 .insert(client, 41);
4115
4116 let released = forwarding.cleanup_connection(client).unwrap();
4117
4118 assert!(released.is_empty());
4119 assert!(!forwarding
4120 .inner
4121 .read()
4122 .unwrap()
4123 .next_client_channel
4124 .contains_key(&client));
4125 }
4126
4127 #[test]
4128 fn stale_module_cleanup_preserves_fast_reconnect_successor() {
4129 let forwarding = ForwardingTable::default();
4130 let module_id = "fast-reconnect-provider";
4131 let first_connection = ConnectionId::new(70);
4132 let second_connection = ConnectionId::new(80);
4133 let (first_tx, _first_rx) = mpsc::channel(1);
4134 let first_endpoint = forwarding
4135 .register_module_connection(
4136 first_connection,
4137 module_id.to_string(),
4138 1,
4139 Concurrency::ModuleManaged,
4140 FrameSink::new(first_tx),
4141 )
4142 .unwrap();
4143 let (second_tx, _second_rx) = mpsc::channel(1);
4144 let second_endpoint = forwarding
4145 .register_module_connection(
4146 second_connection,
4147 module_id.to_string(),
4148 1,
4149 Concurrency::ModuleManaged,
4150 FrameSink::new(second_tx),
4151 )
4152 .unwrap();
4153 assert_ne!(first_endpoint, second_endpoint);
4154
4155 let released = forwarding.cleanup_connection(first_connection).unwrap();
4156
4157 assert!(released.is_empty());
4158 assert_eq!(
4159 forwarding
4160 .inner
4161 .read()
4162 .unwrap()
4163 .modules_by_id
4164 .get(module_id)
4165 .map(|module| module.endpoint),
4166 Some(second_endpoint)
4167 );
4168 assert!(forwarding.has_live_module_connection(module_id).unwrap());
4169 let control_rpc = forwarding
4170 .begin_module_control_rpc_for(
4171 module_id,
4172 "health.check",
4173 Instant::now() + Duration::from_secs(1),
4174 )
4175 .unwrap();
4176 assert_eq!(control_rpc.endpoint, second_endpoint);
4177 }
4178
4179 fn route_fixture(
4180 module_id: &str,
4181 ) -> (
4182 ForwardingTable,
4183 ConnectionId,
4184 ModuleEndpointId,
4185 ConnectionId,
4186 FrameSink,
4187 mpsc::Receiver<crate::router::OutboundFrame>,
4188 ) {
4189 let forwarding = ForwardingTable::default();
4190 let module_connection = ConnectionId::new(100);
4191 let client_connection = ConnectionId::new(200);
4192 let (module_tx, _module_rx) = mpsc::channel(8);
4193 let endpoint = forwarding
4194 .register_module_connection(
4195 module_connection,
4196 module_id.to_string(),
4197 2,
4198 Concurrency::ModuleManaged,
4199 FrameSink::new(module_tx),
4200 )
4201 .unwrap();
4202 let (client_tx, client_rx) = mpsc::channel(8);
4203 (
4204 forwarding,
4205 module_connection,
4206 endpoint,
4207 client_connection,
4208 FrameSink::new(client_tx),
4209 client_rx,
4210 )
4211 }
4212
4213 #[test]
4214 #[cfg(unix)]
4215 fn daemon_drain_gates_current_and_racing_provider_registrations() {
4216 let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
4217 assert_eq!(
4218 forwarding.begin_daemon_drain().unwrap(),
4219 [("provider".to_string(), endpoint)]
4220 );
4221 assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
4222 assert!(matches!(
4223 forwarding.register_module_connection(
4224 ConnectionId::new(300),
4225 "late-provider".into(),
4226 2,
4227 Concurrency::ModuleManaged,
4228 sink,
4229 ),
4230 Err(ForwardingError::ConnectionClosing { .. })
4231 ));
4232 }
4233
4234 #[test]
4235 #[cfg(unix)]
4236 fn daemon_drain_results_are_endpoint_fenced_and_removed_on_cleanup() {
4237 let (forwarding, connection, endpoint, _, _, _) = route_fixture("provider");
4238 assert_eq!(
4239 forwarding
4240 .daemon_drain_result_for_connection(connection)
4241 .unwrap(),
4242 None
4243 );
4244 forwarding
4245 .record_daemon_drain_result(endpoint, true)
4246 .unwrap();
4247 forwarding.begin_daemon_drain().unwrap();
4248 assert_eq!(
4249 forwarding
4250 .daemon_drain_result_for_connection(connection)
4251 .unwrap(),
4252 Some(false)
4253 );
4254 let replacement = ModuleEndpointId {
4255 generation: endpoint.generation + 1,
4256 ..endpoint
4257 };
4258 forwarding
4259 .record_daemon_drain_result(replacement, true)
4260 .unwrap();
4261 assert_eq!(
4262 forwarding
4263 .daemon_drain_result_for_connection(connection)
4264 .unwrap(),
4265 Some(false)
4266 );
4267 forwarding
4268 .record_daemon_drain_result(endpoint, true)
4269 .unwrap();
4270 assert_eq!(
4271 forwarding
4272 .daemon_drain_result_for_connection(connection)
4273 .unwrap(),
4274 Some(true)
4275 );
4276 assert_eq!(
4277 forwarding
4278 .daemon_drain_result_for_connection(ConnectionId::new(300))
4279 .unwrap(),
4280 Some(false)
4281 );
4282 forwarding.cleanup_connection(connection).unwrap();
4283 forwarding
4284 .record_daemon_drain_result(endpoint, true)
4285 .unwrap();
4286 assert_eq!(
4287 forwarding
4288 .daemon_drain_result_for_connection(connection)
4289 .unwrap(),
4290 Some(false)
4291 );
4292 assert!(forwarding
4293 .read_inner()
4294 .unwrap()
4295 .daemon_drain_results
4296 .is_empty());
4297 }
4298
4299 #[test]
4300 #[cfg(unix)]
4301 fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
4302 let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
4303 route_fixture("provider");
4304 let mut pending =
4305 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
4306 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
4307 forwarding.begin_daemon_drain().unwrap();
4308 let completion = forwarding
4309 .complete_pending_relay(
4310 module_connection,
4311 pending.corr,
4312 RouteBindRelayOutcome::Accepted,
4313 )
4314 .expect("draining admission is not a fatal module error");
4315 assert!(completion.settled);
4316 let abandoned = completion
4317 .abandoned
4318 .expect("late accepted bind needs a route GOODBYE");
4319 assert_eq!(abandoned.channel, pending.module_channel);
4320 assert!(matches!(pending.receiver.try_recv().unwrap(),
4321 RouteBindRelayOutcome::DaemonRejected(error) if error.code == "module_reloading"));
4322 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4323 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4324 assert!(
4325 rx.try_recv().is_err(),
4326 "no route.open may be published during drain"
4327 );
4328 assert_eq!(
4329 forwarding
4330 .module_endpoint_for_connection(module_connection)
4331 .unwrap(),
4332 Some(endpoint)
4333 );
4334 assert!(
4335 !forwarding
4336 .complete_pending_relay(
4337 module_connection,
4338 pending.corr,
4339 RouteBindRelayOutcome::Accepted,
4340 )
4341 .unwrap()
4342 .settled
4343 );
4344 }
4345
4346 fn test_ping(corr: u64) -> Frame {
4347 Frame::build(
4348 FrameType::Ping,
4349 Flags::new(false, Priority::Passive, false),
4350 0,
4351 0,
4352 corr,
4353 Vec::new(),
4354 )
4355 .unwrap()
4356 }
4357
4358 fn begin_test_route(
4359 forwarding: &ForwardingTable,
4360 client_connection: ConnectionId,
4361 client_sink: FrameSink,
4362 corr: u64,
4363 module_id: &str,
4364 ) -> PendingRouteBindRelay {
4365 forwarding
4366 .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
4367 .unwrap()
4368 }
4369
4370 #[test]
4371 fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
4372 for candidate in [false, true] {
4373 for register_first in [false, true] {
4374 let (forwarding, module_connection, _, client_connection, sink, _rx) =
4375 route_fixture("target");
4376 let register = || {
4377 if candidate {
4378 forwarding.register_candidate_module_connection(
4379 client_connection,
4380 "source".into(),
4381 2,
4382 Concurrency::ModuleManaged,
4383 sink.clone(),
4384 )
4385 } else {
4386 forwarding.register_module_connection(
4387 client_connection,
4388 "source".into(),
4389 2,
4390 Concurrency::ModuleManaged,
4391 sink.clone(),
4392 )
4393 }
4394 };
4395 if register_first {
4396 register().unwrap();
4397 assert!(
4398 forwarding
4399 .begin_route_bind_relay_for_test(
4400 client_connection,
4401 sink.clone(),
4402 1,
4403 "target",
4404 )
4405 .is_err(),
4406 "a deferred route.open cannot reserve after HELLO"
4407 );
4408 assert!(forwarding
4409 .cleanup_connection(client_connection)
4410 .unwrap()
4411 .is_empty());
4412 } else {
4413 let pending =
4414 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
4415 assert!(
4416 register().is_err(),
4417 "HELLO cannot register after a route reservation"
4418 );
4419 forwarding
4420 .complete_pending_relay(
4421 module_connection,
4422 pending.corr,
4423 RouteBindRelayOutcome::Accepted,
4424 )
4425 .unwrap();
4426 assert_eq!(
4427 forwarding
4428 .cleanup_connection(client_connection)
4429 .unwrap()
4430 .len(),
4431 1
4432 );
4433 }
4434 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4435 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4436 }
4437 }
4438 }
4439
4440 #[tokio::test]
4446 async fn pending_route_open_completes_behind_queued_data_frames() {
4447 assert_eq!(
4448 crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
4449 8,
4450 "the per-connection pending route.open limit is its own constant"
4451 );
4452 let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
4453 route_fixture("open-behind-data");
4454 let (sink, mut client_rx) = crate::server::connection_egress();
4455 const DATA_FRAMES: usize = 1_000;
4456 let data = |corr: u64| {
4457 Frame::build(
4458 FrameType::StreamData,
4459 Flags::new(false, Priority::Interactive, false),
4460 9,
4461 1,
4462 corr,
4463 vec![b'x'; 200],
4464 )
4465 .unwrap()
4466 };
4467 for corr in 0..DATA_FRAMES as u64 {
4468 sink.try_send(data(corr)).unwrap();
4469 }
4470 let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
4471 assert_eq!(sink.backlog().queued_bytes, data_bytes);
4472
4473 let pending = tokio::time::timeout(
4474 Duration::from_secs(5),
4475 forwarding.begin_route_bind_relay_for(
4476 client,
4477 sink.clone(),
4478 subc_protocol::PROTOCOL_VERSION,
4479 4_242,
4480 "open-behind-data",
4481 Principal::Direct,
4482 None,
4483 None,
4484 Instant::now() + Duration::from_secs(60),
4485 ),
4486 )
4487 .await
4488 .expect("reserving the route.open slot must not wait behind data frames")
4489 .unwrap();
4490 forwarding
4491 .complete_pending_relay(
4492 module_connection,
4493 pending.corr,
4494 RouteBindRelayOutcome::Accepted,
4495 )
4496 .unwrap();
4497
4498 let backlog = sink.backlog();
4499 assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
4500 assert!(
4501 backlog.queued_bytes > data_bytes,
4502 "the route.open response must be counted in queued bytes: {backlog:?}"
4503 );
4504 for corr in 0..DATA_FRAMES as u64 {
4505 assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
4506 }
4507 let open = client_rx.recv().await.unwrap();
4508 assert_eq!(open.header.corr, 4_242);
4509 assert_eq!(open.header.ty, FrameType::Response);
4510 drop(open);
4511 assert_eq!(sink.backlog().queued_bytes, 0);
4512 assert_eq!(sink.backlog().queued_frames, 0);
4513 }
4514
4515 #[tokio::test]
4520 async fn drain_holdouts_count_held_requests_and_name_the_connection() {
4521 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4522 route_fixture("holdouts");
4523 let mut bound = |corr| {
4524 let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
4525 forwarding
4526 .complete_pending_relay(
4527 module_connection,
4528 route.corr,
4529 RouteBindRelayOutcome::Accepted,
4530 )
4531 .unwrap();
4532 client_rx.try_recv().unwrap();
4533 match forwarding
4534 .lookup_data_route(client, route.client_channel, route.client_epoch)
4535 .unwrap()
4536 {
4537 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4538 other => panic!("expected live route, got {other:?}"),
4539 }
4540 };
4541 let holding = bound(61);
4542 let _idle = bound(62);
4543 holding.flow.acquire_tagged(7, false).await.unwrap();
4544 holding.flow.acquire_tagged(2, false).await.unwrap();
4545 holding.flow.acquire_tagged(3, true).await.unwrap();
4546 forwarding
4547 .begin_module_drain("holdouts", RouteCloseReason::Restart)
4548 .unwrap();
4549
4550 let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
4551 assert_eq!(
4552 holdouts,
4553 DrainHoldouts {
4554 requests: 2,
4555 routes: 1,
4556 total_routes: 2,
4557 top_connections: vec![(client.get(), 2)],
4558 held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
4561 }
4562 );
4563 }
4564
4565 #[test]
4566 fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
4567 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4568 route_fixture("census");
4569 let pending = begin_test_route(&forwarding, client, sink, 1, "census");
4570 forwarding
4571 .complete_pending_relay(
4572 module_connection,
4573 pending.corr,
4574 RouteBindRelayOutcome::Accepted,
4575 )
4576 .unwrap();
4577
4578 let routes = forwarding.endpoint_routes(endpoint).unwrap();
4579 assert_eq!(routes.len(), 1);
4580 assert!(matches!(routes[0].principal, Principal::Direct));
4581 assert_eq!(routes[0].goodbye_target.connection_id, client);
4582 assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
4583 assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
4584 assert!(!routes[0].draining);
4585
4586 forwarding
4587 .begin_module_drain("census", RouteCloseReason::Restart)
4588 .unwrap();
4589 let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
4590 assert_eq!(draining_routes.len(), 1);
4591 assert!(draining_routes[0].draining);
4592 }
4593
4594 #[test]
4595 fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
4596 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
4597 let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
4598 assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
4599 forwarding
4600 .abort_pending_relay(
4601 first.endpoint,
4602 first.corr,
4603 RouteBindRelayOutcome::ModuleGone("abort".into()),
4604 )
4605 .unwrap();
4606 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4607 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4608
4609 let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
4610 assert_eq!(second.client_channel, first.client_channel);
4611 assert_eq!(second.module_channel, first.module_channel);
4612 assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
4613 }
4614
4615 #[test]
4616 fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
4617 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4618 route_fixture("epoch-release");
4619 let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
4620 forwarding
4621 .complete_pending_relay(
4622 module_connection,
4623 first.corr,
4624 RouteBindRelayOutcome::Accepted,
4625 )
4626 .unwrap();
4627 assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
4628 assert!(matches!(
4629 forwarding
4630 .release_client_route(client, first.client_channel, first.client_epoch)
4631 .unwrap(),
4632 RouteRelease::Removed(_)
4633 ));
4634 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4635 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4636
4637 let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
4638 forwarding
4639 .complete_pending_relay(
4640 module_connection,
4641 second.corr,
4642 RouteBindRelayOutcome::Accepted,
4643 )
4644 .unwrap();
4645 assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
4646 assert!(matches!(
4647 forwarding
4648 .release_client_route(client, second.client_channel, first.client_epoch)
4649 .unwrap(),
4650 RouteRelease::Stale
4651 ));
4652 assert!(!forwarding
4653 .cache_status(
4654 endpoint,
4655 second.module_channel,
4656 first.module_epoch,
4657 "stale".into(),
4658 )
4659 .unwrap());
4660 assert!(forwarding
4661 .cache_status(
4662 endpoint,
4663 second.module_channel,
4664 second.module_epoch,
4665 "current".into(),
4666 )
4667 .unwrap());
4668 match forwarding
4669 .route_poll_snapshot(client, second.client_channel, second.client_epoch)
4670 .unwrap()
4671 {
4672 RoutePollSnapshot::Bound { status, .. } => {
4673 assert_eq!(status.as_deref(), Some("current"));
4674 }
4675 RoutePollSnapshot::Absent => panic!("successor binding was removed"),
4676 }
4677 let counters = forwarding.counters().snapshot();
4678 assert_eq!(counters["route_released_epoch_fenced"], 1);
4679 assert_eq!(counters["route_release_stale_skipped"], 1);
4680 }
4681
4682 #[test]
4683 fn max_epoch_reservation_retires_only_that_slot() {
4684 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
4685 forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
4686 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
4687 let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
4688 assert_eq!(
4689 (final_use.client_channel, final_use.client_epoch),
4690 (7, u32::MAX)
4691 );
4692 assert_eq!(
4693 (final_use.module_channel, final_use.module_epoch),
4694 (9, u32::MAX)
4695 );
4696 forwarding
4697 .abort_pending_relay(
4698 endpoint,
4699 final_use.corr,
4700 RouteBindRelayOutcome::ModuleGone("abort".into()),
4701 )
4702 .unwrap();
4703 forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
4704 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
4705 let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
4706 assert_ne!(next.client_channel, 7);
4707 assert_ne!(next.module_channel, 9);
4708 assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
4709 }
4710
4711 #[test]
4712 fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
4713 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4714 route_fixture("corr-shared");
4715 let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
4716 assert_eq!(bind.corr, 1);
4717 forwarding
4718 .abort_pending_relay(
4719 endpoint,
4720 bind.corr,
4721 RouteBindRelayOutcome::ModuleGone("abort".into()),
4722 )
4723 .unwrap();
4724 let rpc = forwarding
4725 .begin_module_control_rpc_for(
4726 "corr-shared",
4727 "health.check",
4728 Instant::now() - Duration::from_millis(1),
4729 )
4730 .unwrap();
4731 assert_eq!(rpc.corr, 2);
4732 assert_eq!(
4733 forwarding
4734 .complete_module_control_rpc(
4735 module_connection,
4736 rpc.corr,
4737 Some("health.check"),
4738 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
4739 status: subc_protocol::session::HealthStatus::Ok,
4740 detail: None,
4741 metrics: None,
4742 }),
4743 )
4744 .unwrap(),
4745 ModuleControlRpcCompletion::Settled
4746 );
4747 assert!(matches!(
4748 rpc.receiver.blocking_recv().unwrap(),
4749 ModuleControlRpcOutcome::DeadlineElapsed
4750 ));
4751 }
4752
4753 #[tokio::test(start_paused = true)]
4754 async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
4755 let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
4756 let probe_started_at = Instant::now();
4757 let rpc = forwarding
4758 .begin_health_probe_rpc_for(
4759 "tombstone-ttl",
4760 "health.check",
4761 probe_started_at,
4762 probe_started_at + Duration::from_secs(5),
4763 )
4764 .unwrap();
4765 assert!(forwarding
4766 .tombstone_health_probe_rpc(endpoint, rpc.corr)
4767 .unwrap());
4768 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
4769
4770 tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
4771 tokio::task::yield_now().await;
4772
4773 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
4774 }
4775
4776 #[test]
4777 fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
4778 let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
4779 let mut close = forwarding.register_connection_close(endpoint.connection_id);
4780 forwarding.inject_control_corr(endpoint, u64::MAX);
4781 let final_rpc = forwarding
4782 .begin_module_control_rpc_for(
4783 "corr-max",
4784 "health.check",
4785 Instant::now() + Duration::from_secs(1),
4786 )
4787 .unwrap();
4788 assert_eq!(final_rpc.corr, u64::MAX);
4789 forwarding
4790 .cancel_module_control_rpc(endpoint, final_rpc.corr)
4791 .unwrap();
4792 assert!(matches!(
4793 forwarding.begin_module_control_rpc_for(
4794 "corr-max",
4795 "health.check",
4796 Instant::now() + Duration::from_secs(1),
4797 ),
4798 Err(ForwardingError::RelayCorrelationExhausted)
4799 ));
4800 assert!(close.try_recv().is_ok());
4801 }
4802
4803 #[test]
4804 fn publication_epoch_controls_delivery_failure_escalation() {
4805 fn setup_successor(
4806 commit_successor: Option<bool>,
4807 ) -> (ForwardingTable, ConnectionId, u16, u32) {
4808 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4809 route_fixture("escalation");
4810 let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
4811 forwarding
4812 .complete_pending_relay(
4813 module_connection,
4814 first.corr,
4815 RouteBindRelayOutcome::Accepted,
4816 )
4817 .unwrap();
4818 client_rx.try_recv().unwrap();
4819 assert!(matches!(
4820 forwarding
4821 .release_client_route(client, first.client_channel, first.client_epoch)
4822 .unwrap(),
4823 RouteRelease::Removed(_)
4824 ));
4825 if let Some(commit_successor) = commit_successor {
4826 forwarding.inject_client_slot_epoch(
4827 client,
4828 first.client_channel,
4829 first.client_epoch,
4830 );
4831 forwarding.inject_module_slot_epoch(
4832 endpoint,
4833 first.module_channel,
4834 first.module_epoch,
4835 );
4836 let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
4837 if commit_successor {
4838 forwarding
4839 .complete_pending_relay(
4840 module_connection,
4841 successor.corr,
4842 RouteBindRelayOutcome::Accepted,
4843 )
4844 .unwrap();
4845 client_rx.try_recv().unwrap();
4846 } else {
4847 forwarding
4848 .abort_pending_relay(
4849 endpoint,
4850 successor.corr,
4851 RouteBindRelayOutcome::ModuleGone("abort".into()),
4852 )
4853 .unwrap();
4854 }
4855 }
4856 (forwarding, client, first.client_channel, first.client_epoch)
4857 }
4858
4859 let probe_sink = FrameSink::new(mpsc::channel(1).0);
4860 let (no_successor, client, channel, epoch) = setup_successor(None);
4861 let mut close = no_successor.register_connection_close(client);
4862 assert!(no_successor
4863 .escalate_client_delivery_failure(
4864 client,
4865 channel,
4866 epoch,
4867 CloseReason::new("delivery", "failed"),
4868 UndeliveredFrame {
4869 module_id: None,
4870 sink: &probe_sink,
4871 },
4872 )
4873 .unwrap());
4874 assert!(close.try_recv().is_ok());
4875
4876 let (aborted, client, channel, epoch) = setup_successor(Some(false));
4877 let mut close = aborted.register_connection_close(client);
4878 assert!(aborted
4879 .escalate_client_delivery_failure(
4880 client,
4881 channel,
4882 epoch,
4883 CloseReason::new("delivery", "failed"),
4884 UndeliveredFrame {
4885 module_id: None,
4886 sink: &probe_sink,
4887 },
4888 )
4889 .unwrap());
4890 assert!(close.try_recv().is_ok());
4891
4892 let (published, client, channel, epoch) = setup_successor(Some(true));
4893 let mut close = published.register_connection_close(client);
4894 assert!(!published
4895 .escalate_client_delivery_failure(
4896 client,
4897 channel,
4898 epoch,
4899 CloseReason::new("delivery", "stale failure"),
4900 UndeliveredFrame {
4901 module_id: None,
4902 sink: &probe_sink,
4903 },
4904 )
4905 .unwrap());
4906 assert!(close.try_recv().is_err());
4907 }
4908
4909 #[test]
4910 fn route_concentration_separates_client_count_from_routes_per_client() {
4911 let (forwarding, module_connection, _, client, sink, _client_rx) =
4915 route_fixture("concentration");
4916 assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
4917
4918 for corr in [70_u64, 71] {
4919 let pending =
4920 begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
4921 forwarding
4922 .complete_pending_relay(
4923 module_connection,
4924 pending.corr,
4925 RouteBindRelayOutcome::Accepted,
4926 )
4927 .unwrap();
4928 }
4929
4930 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
4932 assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
4933 }
4934
4935 #[test]
4936 fn cleanup_and_accepted_resolution_have_one_lock_winner() {
4937 let (forwarding, module_connection, _, client, sink, mut client_rx) =
4938 route_fixture("cleanup-race");
4939 let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
4940 forwarding
4941 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4942 .unwrap();
4943 let released = forwarding.cleanup_connection(client).unwrap();
4944 assert_eq!(released.len(), 1);
4945 let completion = forwarding
4946 .complete_pending_relay(
4947 module_connection,
4948 pending.corr,
4949 RouteBindRelayOutcome::Accepted,
4950 )
4951 .unwrap();
4952 assert!(!completion.settled);
4953 assert!(client_rx.try_recv().is_err());
4954 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4955
4956 let (forwarding, module_connection, _, client, sink, mut client_rx) =
4957 route_fixture("accepted-race");
4958 let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
4959 forwarding
4960 .complete_pending_relay(
4961 module_connection,
4962 pending.corr,
4963 RouteBindRelayOutcome::Accepted,
4964 )
4965 .unwrap();
4966 assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
4967 let released = forwarding.cleanup_connection(client).unwrap();
4968 assert_eq!(released.len(), 1);
4969 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4970 }
4971
4972 #[test]
4973 fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
4974 let (forwarding, module_connection, _, client, sink, mut client_rx) =
4975 route_fixture("drain-gap");
4976 let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
4977 forwarding
4978 .complete_pending_relay(
4979 module_connection,
4980 live.corr,
4981 RouteBindRelayOutcome::Accepted,
4982 )
4983 .unwrap();
4984 client_rx.try_recv().unwrap();
4985 let binding = match forwarding
4986 .lookup_data_route(client, live.client_channel, live.client_epoch)
4987 .unwrap()
4988 {
4989 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4990 other => panic!("expected live route, got {other:?}"),
4991 };
4992
4993 let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
4994 forwarding
4995 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4996 .unwrap();
4997 let control_rpc = forwarding
4998 .begin_module_control_rpc_for(
4999 "drain-gap",
5000 "health.check",
5001 Instant::now() + Duration::from_secs(1),
5002 )
5003 .unwrap();
5004 let target = forwarding
5005 .begin_module_drain("drain-gap", RouteCloseReason::Reload)
5006 .unwrap()
5007 .unwrap();
5008 assert!(matches!(
5009 control_rpc.receiver.blocking_recv().unwrap(),
5010 ModuleControlRpcOutcome::ModuleGone(_)
5011 ));
5012 assert_eq!(target.abandoned_bindings.len(), 1);
5013 assert!(binding.flow.sem.is_closed());
5014 assert!(
5015 !forwarding
5016 .complete_pending_relay(
5017 module_connection,
5018 pending.corr,
5019 RouteBindRelayOutcome::Accepted,
5020 )
5021 .unwrap()
5022 .settled
5023 );
5024 assert!(matches!(
5025 forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
5026 Err(ForwardingError::ModuleReloading { .. })
5027 ));
5028 let released = forwarding
5029 .release_module_endpoint_routes(target.endpoint)
5030 .unwrap();
5031 assert_eq!(released.len(), 1);
5032 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5033 }
5034
5035 #[test]
5043 fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
5044 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5045 route_fixture("closing-client");
5046
5047 let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
5050 forwarding
5051 .complete_pending_relay(
5052 module_connection,
5053 live.corr,
5054 RouteBindRelayOutcome::Accepted,
5055 )
5056 .unwrap();
5057 client_rx.try_recv().unwrap();
5058
5059 let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
5061 forwarding
5062 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5063 .unwrap();
5064
5065 assert!(forwarding
5067 .escalate_client_delivery_failure(
5068 client,
5069 live.client_channel,
5070 live.client_epoch,
5071 CloseReason::new(
5072 "module_to_client_delivery_failed",
5073 "client egress refused a module frame",
5074 ),
5075 UndeliveredFrame {
5076 module_id: None,
5077 sink: &sink,
5078 },
5079 )
5080 .unwrap());
5081 assert!(!sink.is_closed());
5082
5083 let completion = forwarding
5084 .complete_pending_relay(
5085 module_connection,
5086 pending.corr,
5087 RouteBindRelayOutcome::Accepted,
5088 )
5089 .expect("a closing client must not turn a module's ack into an error");
5090
5091 assert!(completion.settled);
5092 let abandoned = completion
5093 .abandoned
5094 .expect("the module must be told to drop the binding it just created");
5095 assert_eq!(abandoned.connection_id, module_connection);
5096 assert_eq!(abandoned.channel, pending.module_channel);
5097 assert_eq!(abandoned.epoch, pending.module_epoch);
5098 assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
5099 assert!(matches!(
5100 pending.receiver.blocking_recv().unwrap(),
5101 RouteBindRelayOutcome::ModuleGone(_)
5102 ));
5103 assert!(client_rx.try_recv().is_err());
5106 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
5107
5108 assert!(forwarding
5111 .has_live_module_connection("closing-client")
5112 .unwrap());
5113 let cotenant = ConnectionId::new(201);
5114 let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
5115 let cotenant_route = begin_test_route(
5116 &forwarding,
5117 cotenant,
5118 FrameSink::new(cotenant_tx),
5119 62,
5120 "closing-client",
5121 );
5122 assert_eq!(cotenant_route.endpoint, endpoint);
5123 forwarding
5124 .complete_pending_relay(
5125 module_connection,
5126 cotenant_route.corr,
5127 RouteBindRelayOutcome::Accepted,
5128 )
5129 .unwrap();
5130 assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
5131 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5132 }
5133
5134 #[test]
5135 fn pending_route_permit_is_released_on_rejection_and_abort() {
5136 let forwarding = ForwardingTable::default();
5137 let module_connection = ConnectionId::new(300);
5138 let client = ConnectionId::new(301);
5139 let (module_tx, _module_rx) = mpsc::channel(1);
5140 let endpoint = forwarding
5141 .register_module_connection(
5142 module_connection,
5143 "permit".into(),
5144 2,
5145 Concurrency::ModuleManaged,
5146 FrameSink::new(module_tx),
5147 )
5148 .unwrap();
5149 let (client_tx, mut client_rx) = mpsc::channel(1);
5150 let sink = FrameSink::new(client_tx);
5151 let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
5152 assert!(sink.try_send(test_ping(999)).is_err());
5153 forwarding
5154 .complete_pending_relay(
5155 module_connection,
5156 rejected.corr,
5157 RouteBindRelayOutcome::Rejected(ErrorBody {
5158 code: "no".into(),
5159 message: "rejected".into(),
5160 detail: None,
5161 }),
5162 )
5163 .unwrap();
5164 sink.try_send(test_ping(1000)).unwrap();
5165 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
5166
5167 let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
5168 assert!(sink.try_send(test_ping(1001)).is_err());
5169 forwarding
5170 .abort_pending_relay(
5171 endpoint,
5172 aborted.corr,
5173 RouteBindRelayOutcome::ModuleGone("abort".into()),
5174 )
5175 .unwrap();
5176 sink.try_send(test_ping(1002)).unwrap();
5177 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
5178
5179 let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
5180 forwarding
5181 .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
5182 .unwrap();
5183 drop(client_rx);
5184 let completion = forwarding
5185 .complete_pending_relay(
5186 module_connection,
5187 receiver_closed.corr,
5188 RouteBindRelayOutcome::Accepted,
5189 )
5190 .unwrap();
5191 assert!(completion.abandoned.is_some());
5192 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5193 }
5194
5195 #[test]
5201 fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
5202 let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
5203 route_fixture("closing-set-leak");
5204
5205 const CONNECTIONS: u64 = 32;
5206 for index in 0..CONNECTIONS {
5207 let client = ConnectionId::new(1000 + index);
5208 let (client_tx, _client_rx) = mpsc::channel(8);
5209 let route = begin_test_route(
5210 &forwarding,
5211 client,
5212 FrameSink::new(client_tx),
5213 index + 1,
5214 "closing-set-leak",
5215 );
5216 forwarding
5217 .complete_pending_relay(
5218 module_connection,
5219 route.corr,
5220 RouteBindRelayOutcome::Accepted,
5221 )
5222 .unwrap();
5223 forwarding.cleanup_connection(client).unwrap();
5224 }
5225 forwarding.cleanup_connection(module_connection).unwrap();
5226
5227 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5228 }
5229
5230 #[test]
5237 fn closing_connection_is_refused_new_work_until_cleanup_completes() {
5238 let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
5239 route_fixture("closing-gate");
5240
5241 let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
5244 forwarding
5245 .complete_pending_relay(
5246 module_connection,
5247 live.corr,
5248 RouteBindRelayOutcome::Accepted,
5249 )
5250 .unwrap();
5251 client_rx.try_recv().unwrap();
5252
5253 assert!(forwarding
5256 .escalate_client_delivery_failure(
5257 client,
5258 live.client_channel,
5259 live.client_epoch,
5260 CloseReason::new(
5261 "module_to_client_delivery_failed",
5262 "client egress refused a module frame",
5263 ),
5264 UndeliveredFrame {
5265 module_id: None,
5266 sink: &sink,
5267 },
5268 )
5269 .unwrap());
5270 assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
5271
5272 assert!(matches!(
5274 forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
5275 Err(ForwardingError::ConnectionClosing { connection_id })
5276 if connection_id == client
5277 ));
5278 let (late_tx, _late_rx) = mpsc::channel(1);
5280 assert!(matches!(
5281 forwarding.register_module_connection(
5282 client,
5283 "late-module".into(),
5284 2,
5285 Concurrency::ModuleManaged,
5286 FrameSink::new(late_tx),
5287 ),
5288 Err(ForwardingError::ConnectionClosing { connection_id })
5289 if connection_id == client
5290 ));
5291
5292 forwarding.cleanup_connection(client).unwrap();
5296 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5297 }
5298}
5299
5300#[cfg(test)]
5303mod swap_slot_tests {
5304 use std::time::Duration;
5305
5306 use super::*;
5307 use tokio::sync::mpsc;
5308
5309 const MODULE_ID: &str = "swapped";
5310
5311 struct SwapFixture {
5312 forwarding: ForwardingTable,
5313 incumbent_connection: ConnectionId,
5314 incumbent: ModuleEndpointId,
5315 candidate_connection: ConnectionId,
5316 candidate: ModuleEndpointId,
5317 _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
5318 }
5319
5320 fn swap_fixture() -> SwapFixture {
5321 let forwarding = ForwardingTable::default();
5322 let incumbent_connection = ConnectionId::new(100);
5323 let candidate_connection = ConnectionId::new(110);
5324 let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
5325 let incumbent = forwarding
5326 .register_module_connection(
5327 incumbent_connection,
5328 MODULE_ID.to_string(),
5329 2,
5330 Concurrency::ModuleManaged,
5331 FrameSink::new(incumbent_tx),
5332 )
5333 .unwrap();
5334 let (candidate_tx, candidate_rx) = mpsc::channel(8);
5335 let candidate = forwarding
5336 .register_candidate_module_connection(
5337 candidate_connection,
5338 MODULE_ID.to_string(),
5339 2,
5340 Concurrency::ModuleManaged,
5341 FrameSink::new(candidate_tx),
5342 )
5343 .unwrap();
5344 SwapFixture {
5345 forwarding,
5346 incumbent_connection,
5347 incumbent,
5348 candidate_connection,
5349 candidate,
5350 _module_rxs: vec![incumbent_rx, candidate_rx],
5351 }
5352 }
5353
5354 fn client(
5355 raw: u64,
5356 ) -> (
5357 ConnectionId,
5358 FrameSink,
5359 mpsc::Receiver<crate::router::OutboundFrame>,
5360 ) {
5361 let (tx, rx) = mpsc::channel(8);
5362 (ConnectionId::new(raw), FrameSink::new(tx), rx)
5363 }
5364
5365 fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
5366 forwarding
5367 .read_inner()
5368 .unwrap()
5369 .client_to_module
5370 .values()
5371 .map(|route| route.module_endpoint)
5372 .collect()
5373 }
5374
5375 #[test]
5376 fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
5377 let fixture = swap_fixture();
5378 let forwarding = &fixture.forwarding;
5379 assert_ne!(fixture.incumbent, fixture.candidate);
5380
5381 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5383 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5384 let (client_connection, client_sink, _client_rx) = client(200);
5385 let pending = forwarding
5386 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5387 .unwrap();
5388 assert_eq!(pending.endpoint, fixture.incumbent);
5389 let rpc = forwarding
5390 .begin_module_control_rpc_for(
5391 MODULE_ID,
5392 "health.check",
5393 Instant::now() + Duration::from_secs(1),
5394 )
5395 .unwrap();
5396 assert_eq!(rpc.endpoint, fixture.incumbent);
5397 let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
5398 assert_eq!(census.len(), 1, "the census lists one endpoint per id");
5399
5400 assert_eq!(
5402 forwarding
5403 .module_endpoint_for_connection(fixture.candidate_connection)
5404 .unwrap(),
5405 Some(fixture.candidate)
5406 );
5407 assert_eq!(
5408 forwarding
5409 .module_id_for_connection(fixture.candidate_connection)
5410 .unwrap()
5411 .as_deref(),
5412 Some(MODULE_ID)
5413 );
5414
5415 let (other_tx, _other_rx) = mpsc::channel(1);
5417 assert_eq!(
5418 forwarding.register_candidate_module_connection(
5419 ConnectionId::new(120),
5420 MODULE_ID.to_string(),
5421 2,
5422 Concurrency::ModuleManaged,
5423 FrameSink::new(other_tx),
5424 ),
5425 Err(ForwardingError::CandidateSlotOccupied {
5426 module_id: MODULE_ID.to_string()
5427 })
5428 );
5429 }
5430
5431 #[test]
5435 fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
5436 let fixture = swap_fixture();
5437 let forwarding = &fixture.forwarding;
5438 let (early_client, early_sink, _early_rx) = client(200);
5439 let mut early = forwarding
5440 .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
5441 .unwrap();
5442 assert_eq!(early.endpoint, fixture.incumbent);
5443 assert_eq!(
5444 forwarding
5445 .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
5446 .unwrap(),
5447 RouteBindRelayMark::Marked
5448 );
5449
5450 let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5451 assert_eq!(
5452 cutover,
5453 ForwardingCutover {
5454 promoted: fixture.candidate,
5455 incumbent: Some(fixture.incumbent),
5456 }
5457 );
5458
5459 let (late_client, late_sink, _late_rx) = client(201);
5461 let late = forwarding
5462 .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
5463 .unwrap();
5464 assert_eq!(
5465 late.endpoint, fixture.candidate,
5466 "a route.open after cutover was reserved on the incumbent"
5467 );
5468
5469 let completion = forwarding
5471 .complete_pending_relay(
5472 fixture.incumbent_connection,
5473 early.corr,
5474 RouteBindRelayOutcome::Accepted,
5475 )
5476 .expect("a superseded endpoint's ack is not an error on its connection");
5477 assert!(completion.settled);
5478 assert!(
5479 !committed_endpoints(forwarding).contains(&fixture.incumbent),
5480 "a relay reserved before cutover committed a route on the incumbent"
5481 );
5482 let goodbye = completion
5483 .abandoned
5484 .expect("the incumbent is told to drop the binding it just created");
5485 assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
5486 assert_eq!(goodbye.channel, early.module_channel);
5487 assert_eq!(goodbye.epoch, early.module_epoch);
5488 assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
5489 match early.receiver.try_recv() {
5490 Ok(RouteBindRelayOutcome::DaemonRejected(body)) => {
5491 assert_eq!(body.code, "module_reloading")
5492 }
5493 other => panic!("expected a retryable module_reloading answer, got {other:?}"),
5494 }
5495 assert!(matches!(
5496 forwarding
5497 .lookup_data_route(early_client, early.client_channel, early.client_epoch)
5498 .unwrap(),
5499 DataRoute::Client(DataRouteState::Absent)
5500 ));
5501
5502 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
5505 forwarding
5506 .complete_pending_relay(
5507 fixture.candidate_connection,
5508 late.corr,
5509 RouteBindRelayOutcome::Accepted,
5510 )
5511 .unwrap();
5512 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5513 assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
5514 }
5515
5516 #[test]
5517 fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
5518 let fixture = swap_fixture();
5519 let forwarding = &fixture.forwarding;
5520 let (bound_client, bound_sink, _bound_rx) = client(200);
5522 let bound = forwarding
5523 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5524 .unwrap();
5525 forwarding
5526 .complete_pending_relay(
5527 fixture.incumbent_connection,
5528 bound.corr,
5529 RouteBindRelayOutcome::Accepted,
5530 )
5531 .unwrap();
5532 let (pending_client, pending_sink, _pending_rx) = client(201);
5533 let mut in_flight = forwarding
5534 .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
5535 .unwrap();
5536 forwarding
5537 .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
5538 .unwrap();
5539
5540 let incumbent = forwarding
5541 .cutover_candidate(MODULE_ID)
5542 .unwrap()
5543 .unwrap()
5544 .incumbent
5545 .unwrap();
5546 let target = forwarding
5547 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
5548 .unwrap()
5549 .expect("the superseded incumbent is still registered");
5550
5551 assert_eq!(target.endpoint, fixture.incumbent);
5552 assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
5553 assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
5554 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5555 assert_eq!(target.abandoned_bindings.len(), 1);
5556 assert_eq!(
5557 target.abandoned_bindings[0].channel,
5558 in_flight.module_channel
5559 );
5560 assert!(matches!(
5561 in_flight.receiver.try_recv(),
5562 Ok(RouteBindRelayOutcome::DaemonRejected(body)) if body.code == "module_reloading"
5563 ));
5564 assert_eq!(
5565 forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
5566 1,
5567 "the incumbent's bound route stays until its drain finishes"
5568 );
5569
5570 let (next_client, next_sink, _next_rx) = client(202);
5571 let next = forwarding
5572 .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
5573 .expect("the promoted candidate keeps accepting routes");
5574 assert_eq!(next.endpoint, fixture.candidate);
5575 }
5576
5577 #[test]
5583 fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
5584 let forwarding = ForwardingTable::default();
5585 let first_connection = ConnectionId::new(70);
5586 let (first_tx, _first_rx) = mpsc::channel(8);
5587 forwarding
5588 .register_module_connection(
5589 first_connection,
5590 MODULE_ID.to_string(),
5591 2,
5592 Concurrency::ModuleManaged,
5593 FrameSink::new(first_tx),
5594 )
5595 .unwrap();
5596 let (client_connection, client_sink, _client_rx) = client(200);
5597 let mut pending = forwarding
5598 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5599 .unwrap();
5600 let (second_tx, _second_rx) = mpsc::channel(8);
5601 forwarding
5602 .register_module_connection(
5603 ConnectionId::new(80),
5604 MODULE_ID.to_string(),
5605 2,
5606 Concurrency::ModuleManaged,
5607 FrameSink::new(second_tx),
5608 )
5609 .unwrap();
5610
5611 assert_eq!(
5612 forwarding
5613 .complete_pending_relay(
5614 first_connection,
5615 pending.corr,
5616 RouteBindRelayOutcome::Accepted
5617 )
5618 .unwrap_err(),
5619 ForwardingError::StaleModuleEndpoint
5620 );
5621 assert!(committed_endpoints(&forwarding).is_empty());
5622 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5623 assert!(matches!(
5624 pending.receiver.try_recv(),
5625 Err(oneshot::error::TryRecvError::Closed)
5626 ));
5627 }
5628
5629 #[test]
5630 fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
5631 let fixture = swap_fixture();
5633 let forwarding = &fixture.forwarding;
5634 assert!(forwarding
5635 .cleanup_connection(fixture.candidate_connection)
5636 .unwrap()
5637 .is_empty());
5638 assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
5639 let (client_connection, client_sink, _client_rx) = client(200);
5640 assert_eq!(
5641 forwarding
5642 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5643 .unwrap()
5644 .endpoint,
5645 fixture.incumbent
5646 );
5647
5648 let fixture = swap_fixture();
5651 let forwarding = &fixture.forwarding;
5652 let (bound_client, bound_sink, _bound_rx) = client(200);
5653 let bound = forwarding
5654 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5655 .unwrap();
5656 forwarding
5657 .complete_pending_relay(
5658 fixture.incumbent_connection,
5659 bound.corr,
5660 RouteBindRelayOutcome::Accepted,
5661 )
5662 .unwrap();
5663 forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5664 let released = forwarding
5665 .cleanup_connection(fixture.incumbent_connection)
5666 .unwrap();
5667 assert_eq!(released.len(), 1);
5668 assert_eq!(released[0].connection_id, bound_client);
5669 assert!(forwarding
5670 .read_inner()
5671 .unwrap()
5672 .superseded_endpoints
5673 .is_empty());
5674 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5675 let (next_client, next_sink, _next_rx) = client(201);
5676 assert_eq!(
5677 forwarding
5678 .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
5679 .unwrap()
5680 .endpoint,
5681 fixture.candidate
5682 );
5683 }
5684}