1use std::{
2 collections::{BTreeMap, HashMap, HashSet},
3 error::Error,
4 fmt,
5 sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
6 time::Duration,
7};
8
9use subc_control::{ClientControlResponse, RouteCloseReason};
10use subc_protocol::{
11 manifest::Concurrency,
12 session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
13 ErrorBody, Flags, FrameType, Principal, Priority,
14};
15use tokio::sync::{oneshot, Semaphore};
16use tokio::time::Instant;
17use tracing::{debug, info, warn};
18
19use crate::{
20 control::{RouteBindBreakers, RouteBindConcurrency},
21 observability::DaemonCounters,
22 registry::ConnectionId,
23 router::FrameSink,
24 scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
25 Frame, ProjectRootId,
26};
27
28const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
30
31const STATELESS_PARALLEL_WINDOW: usize = 1024;
33
34const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
38
39#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
44pub struct ModuleEndpointId {
45 pub connection_id: ConnectionId,
46 pub generation: u64,
47}
48
49#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
51pub(crate) struct ClientRouteKey {
52 pub connection_id: ConnectionId,
53 pub channel: u16,
54}
55
56#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
58pub(crate) struct ModuleRouteKey {
59 pub endpoint: ModuleEndpointId,
60 pub channel: u16,
61}
62
63#[derive(Debug)]
64pub(crate) struct RouteBinding {
65 pub client_connection_id: ConnectionId,
66 pub client_sink: FrameSink,
67 pub client_negotiated_ver: u8,
68 pub client_channel: u16,
69 pub client_epoch: u32,
70 pub module_id: String,
71 pub module_endpoint: ModuleEndpointId,
72 pub module_sink: FrameSink,
73 pub module_negotiated_ver: u8,
74 pub module_channel: u16,
75 pub module_epoch: u32,
76 pub principal: Principal,
77 pub project_root: Option<ProjectRootId>,
78 pub bound_at: Instant,
79 pub flow: Arc<ChannelFlow>,
80 pub scope: Option<BoundScope>,
83}
84
85#[derive(Debug, Clone)]
86pub(crate) enum DataRoute {
87 Client(DataRouteState),
88 Module(DataRouteState),
89}
90
91#[derive(Debug, Clone)]
92pub(crate) enum DataRouteState {
93 Bound(Arc<RouteBinding>),
94 Reserved,
95 EpochMismatch,
96 Absent,
97}
98
99#[derive(Debug, Clone, Copy, PartialEq, Eq)]
128pub(crate) enum GoodbyeTargetKind {
129 Client,
130 Module,
131}
132
133#[derive(Debug, Clone)]
134pub(crate) struct GoodbyeTarget {
135 pub connection_id: ConnectionId,
136 pub sink: FrameSink,
137 pub negotiated_ver: u8,
138 pub channel: u16,
139 pub epoch: u32,
140 pub kind: GoodbyeTargetKind,
141 pub module_id: Option<String>,
146}
147
148#[derive(Debug, Clone, Copy)]
151pub(crate) struct UndeliveredFrame<'a> {
152 pub module_id: Option<&'a str>,
154 pub sink: &'a FrameSink,
156}
157
158pub(crate) fn principal_label(principal: &Principal) -> String {
160 match principal {
161 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
162 Principal::Direct => "direct".to_string(),
163 other => format!("{other:?}"),
164 }
165}
166
167fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
170 let labels = inner
171 .client_to_module
172 .iter()
173 .filter(|(key, _)| key.connection_id == connection_id)
174 .map(|(_, route)| principal_label(&route.principal))
175 .collect::<std::collections::BTreeSet<_>>();
176 if labels.is_empty() {
177 "none".to_string()
178 } else {
179 labels.into_iter().collect::<Vec<_>>().join(",")
180 }
181}
182
183impl GoodbyeTarget {
184 pub(crate) fn close_on_delivery_failure(&self) -> bool {
187 matches!(self.kind, GoodbyeTargetKind::Client)
188 }
189}
190
191pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
204
205pub(crate) fn send_module_route_goodbye(
217 counters: &DaemonCounters,
218 sink: &FrameSink,
219 frame: Frame,
220 module_id: Option<&str>,
221 context: &'static str,
222) {
223 let channel = frame.header.channel;
224 let epoch = frame.header.epoch;
225 let Err(err) = sink.try_send(frame.clone()) else {
226 return;
227 };
228 let runtime = match tokio::runtime::Handle::try_current() {
231 Ok(runtime) if !sink.is_closed() => runtime,
232 _ => {
233 counters.increment_goodbye_relay_module_dropped(module_id);
234 warn!(
235 module_id = module_id.unwrap_or("unknown"),
236 route_channel = channel,
237 route_epoch = epoch,
238 error = %err,
239 context,
240 "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
241 );
242 return;
243 }
244 };
245 debug!(
246 module_id = module_id.unwrap_or("unknown"),
247 route_channel = channel,
248 route_epoch = epoch,
249 error = %err,
250 context,
251 "module egress queue refused route GOODBYE; delivering it once the module frees room"
252 );
253 let counters = counters.clone();
254 let sink = sink.clone();
255 let module_id = module_id.map(str::to_string);
256 runtime.spawn(async move {
257 let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
258 let why = match outcome {
259 Ok(Ok(())) => {
260 debug!(
261 module_id = module_id.as_deref().unwrap_or("unknown"),
262 route_channel = channel,
263 route_epoch = epoch,
264 context,
265 "late route GOODBYE delivered to module"
266 );
267 return;
268 }
269 Ok(Err(err)) => err.to_string(),
270 Err(_) => format!(
271 "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
272 ),
273 };
274 counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
275 warn!(
276 module_id = module_id.as_deref().unwrap_or("unknown"),
277 route_channel = channel,
278 route_epoch = epoch,
279 error = %why,
280 context,
281 "route GOODBYE to module dropped under backpressure; not closing shared module connection"
282 );
283 });
284}
285
286#[derive(Debug, Clone)]
292pub(crate) struct EndpointRoute {
293 pub goodbye_target: GoodbyeTarget,
294 pub principal: Principal,
295 pub bound_at: Instant,
296 pub draining: bool,
297 pub drain_reason: Option<RouteCloseReason>,
301}
302
303#[derive(Debug, Clone)]
306pub(crate) struct ScopeDrainedRoute {
307 pub reason: RouteCloseReason,
308 pub scope: BoundScope,
309 pub module_id: String,
310 pub client: GoodbyeTarget,
311 pub module: GoodbyeTarget,
312}
313
314#[derive(Debug)]
315pub(crate) struct PendingRouteBindRelay {
316 pub endpoint: ModuleEndpointId,
317 pub module_sink: FrameSink,
318 pub negotiated_ver: u8,
319 pub client_channel: u16,
320 pub client_epoch: u32,
321 pub module_channel: u16,
322 pub module_epoch: u32,
323 pub corr: u64,
324 pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
325}
326
327#[derive(Debug, Clone)]
328pub(crate) struct ModuleDrainTarget {
329 pub endpoint: ModuleEndpointId,
330 pub sink: FrameSink,
331 pub negotiated_ver: u8,
332 pub abandoned_bindings: Vec<GoodbyeTarget>,
333 pub excluded_subscriptions: u32,
334}
335
336#[cfg(unix)]
339#[derive(Debug, Clone)]
340pub(crate) struct ModuleConnectionTarget {
341 pub module_id: String,
342 pub endpoint: ModuleEndpointId,
343 pub sink: FrameSink,
344 pub negotiated_ver: u8,
345}
346
347#[derive(Debug, Clone)]
348pub(crate) enum RouteBindRelayOutcome {
349 Accepted,
350 Rejected(ErrorBody),
351 ModuleGone(String),
352}
353
354#[derive(Debug, Clone, Copy, PartialEq, Eq)]
356pub(crate) struct ForwardingCutover {
357 pub promoted: ModuleEndpointId,
359 pub incumbent: Option<ModuleEndpointId>,
362}
363
364#[derive(Debug)]
366pub(crate) struct ConnectionCleanup {
367 pub released: Vec<GoodbyeTarget>,
369 pub abandoned_relays: u32,
372}
373
374#[derive(Debug, Clone)]
375pub(crate) struct PendingRelayCompletion {
376 pub settled: bool,
377 pub abandoned: Option<GoodbyeTarget>,
378}
379
380#[derive(Debug)]
381pub(crate) struct PendingModuleControlRpc {
382 pub endpoint: ModuleEndpointId,
383 pub module_sink: FrameSink,
384 pub negotiated_ver: u8,
385 pub corr: u64,
386 pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
387}
388
389#[derive(Debug, Clone)]
390pub(crate) enum ModuleControlRpcOutcome {
391 Response(ModuleControlResponse),
392 Rejected(ErrorBody),
393 ModuleGone(String),
394 MalformedResponse(String),
395 UnexpectedOp { expected: String, actual: String },
396 DeadlineElapsed,
397}
398
399#[derive(Debug, Clone, PartialEq, Eq)]
400pub(crate) enum ModuleControlRpcCompletion {
401 Unknown,
402 Settled,
403 LateHealthAnswer {
404 module_id: String,
405 latency: Duration,
406 },
407}
408
409#[derive(Debug)]
410struct PendingModuleControlRpcEntry {
411 expected_op: String,
412 deadline: Instant,
413 health_probe_started_at: Option<Instant>,
414 sender: oneshot::Sender<ModuleControlRpcOutcome>,
415}
416
417#[derive(Debug)]
418struct HealthProbeTombstone {
419 expected_op: String,
420 module_id: String,
421 probe_started_at: Instant,
422 expires_at: Instant,
423}
424
425#[derive(Debug, Clone)]
426struct RouteReservation {
427 client_key: ClientRouteKey,
428 module_key: ModuleRouteKey,
429 client_epoch: u32,
430 module_epoch: u32,
431 project_root: Option<ProjectRootId>,
432}
433
434#[derive(Debug)]
435struct PendingRouteBindRelayEntry {
436 reservation: RouteReservation,
437 client_sink: FrameSink,
438 client_negotiated_ver: u8,
439 client_permit: crate::router::EgressPermit,
440 route_open_frame: Frame,
441 principal: Principal,
442 scope: Option<BoundScope>,
445 deadline: Instant,
446 relay_enqueued: bool,
457 sender: oneshot::Sender<RouteBindRelayOutcome>,
458}
459
460#[derive(Debug, Clone, Copy, PartialEq, Eq)]
463pub(crate) enum RouteBindRelayMark {
464 Marked,
466 SettledByReply,
470 ReleasedWithoutReply,
474}
475
476#[derive(Debug, Clone)]
477pub(crate) enum RouteRelease {
478 Removed(GoodbyeTarget),
479 Stale,
480 Absent,
481}
482
483#[derive(Debug, Clone)]
484pub(crate) enum RoutePollSnapshot {
485 Bound {
486 module_id: String,
487 status: Option<String>,
488 },
489 Absent,
490}
491
492#[derive(Debug, Clone)]
493struct ModuleConnection {
494 endpoint: ModuleEndpointId,
495 sink: FrameSink,
496 negotiated_ver: u8,
497 concurrency: Concurrency,
498}
499
500#[derive(Debug, Default)]
501struct ForwardingInner {
502 operator_confirms: Arc<crate::operator_confirm::OperatorConfirms>,
503 daemon_draining: bool,
504 daemon_drain_results: HashMap<ModuleEndpointId, bool>,
509 modules_by_id: HashMap<String, ModuleConnection>,
513 candidates_by_id: HashMap<String, ModuleConnection>,
519 superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
525 endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
526 module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
527 draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
531 closing_connections: HashSet<ConnectionId>,
532 next_generation: u64,
533 reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
534 reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
535 next_client_channel: HashMap<ConnectionId, u16>,
536 next_module_channel: HashMap<ModuleEndpointId, u16>,
537 client_slot_epochs: HashMap<ClientRouteKey, u32>,
538 module_slot_epochs: HashMap<ModuleRouteKey, u32>,
539 last_published_epoch: HashMap<ClientRouteKey, u32>,
540 client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
541 module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
542 status: HashMap<(ClientRouteKey, u32), String>,
543 pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
544 relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
549 scope_tags: HashMap<(String, String), ScopeTag>,
554 next_control_corr: HashMap<ModuleEndpointId, u64>,
555 pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
556 health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
557}
558
559#[derive(Debug, Clone)]
560pub(crate) struct CloseReason {
561 code: &'static str,
562 message: String,
563}
564
565impl CloseReason {
566 pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
567 Self {
568 code,
569 message: message.into(),
570 }
571 }
572}
573
574impl fmt::Display for CloseReason {
575 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
576 write!(f, "{}: {}", self.code, self.message)
577 }
578}
579
580pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
581
582#[derive(Debug, Default)]
584pub struct ForwardingTable {
585 inner: Arc<RwLock<ForwardingInner>>,
586 #[cfg(feature = "test-support")]
587 route_closing_hold: Mutex<Option<oneshot::Receiver<()>>>,
588 close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
589 counters: DaemonCounters,
590 route_bind_breakers: RouteBindBreakers,
595 route_bind_concurrency: RouteBindConcurrency,
598 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
602}
603
604impl ForwardingTable {
605 #[cfg(feature = "test-support")]
647 pub fn configure_operator_script(
648 &self,
649 script: &std::path::Path,
650 events: Option<&std::path::Path>,
651 limits: Option<crate::operator_confirm::OperatorLimits>,
652 ) {
653 self.operator_confirms()
654 .configure_script(script, events, limits);
655 }
656
657 #[cfg(feature = "test-support")]
661 pub fn hold_route_closing_for_test(&self) -> oneshot::Sender<()> {
662 let (release, hold) = oneshot::channel();
663 *self.route_closing_hold.lock().unwrap() = Some(hold);
664 release
665 }
666
667 #[cfg(feature = "test-support")]
668 pub(crate) async fn wait_before_route_closing_for_test(&self) {
669 let hold = self.route_closing_hold.lock().unwrap().take();
670 if let Some(hold) = hold {
671 let _ = hold.await;
672 }
673 }
674
675 pub(crate) fn operator_confirms(&self) -> Arc<crate::operator_confirm::OperatorConfirms> {
676 Arc::clone(
677 &self
678 .inner
679 .read()
680 .unwrap_or_else(|p| p.into_inner())
681 .operator_confirms,
682 )
683 }
684
685 #[cfg(test)]
686 pub(crate) fn inject_operator_principal(&self, key: ModuleRouteKey, principal: Principal) {
687 let mut inner = self.write_inner().unwrap();
688 let old = inner.module_to_client.remove(&key).unwrap();
689 let client = ClientRouteKey {
690 connection_id: old.client_connection_id,
691 channel: old.client_channel,
692 };
693 inner.client_to_module.remove(&client);
694 let mut binding = Arc::try_unwrap(old).expect("test binding has no outstanding readers");
695 binding.principal = principal;
696 let binding = Arc::new(binding);
697 inner.client_to_module.insert(client, Arc::clone(&binding));
698 inner.module_to_client.insert(key, binding);
699 }
700
701 pub(crate) fn with_operator_route<T>(
705 &self,
706 connection_id: ConnectionId,
707 channel: u16,
708 epoch: u32,
709 admit: impl FnOnce(Option<&RouteBinding>) -> T,
710 ) -> Result<T, ForwardingError> {
711 let inner = self.read_inner()?;
712 let binding = inner
713 .endpoint_by_connection
714 .get(&connection_id)
715 .and_then(|endpoint| {
716 inner.module_to_client.get(&ModuleRouteKey {
717 endpoint: *endpoint,
718 channel,
719 })
720 })
721 .filter(|binding| binding.module_epoch == epoch);
722 Ok(admit(binding.map(Arc::as_ref)))
723 }
724
725 pub(crate) fn counters(&self) -> DaemonCounters {
726 self.counters.clone()
727 }
728
729 pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
730 self.route_bind_breakers.clone()
731 }
732
733 pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
734 self.route_bind_concurrency.clone()
735 }
736
737 pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
738 Arc::clone(&self.route_outages)
739 }
740
741 pub(crate) fn register_connection_close(
742 &self,
743 connection_id: ConnectionId,
744 ) -> ConnectionCloseReceiver {
745 let (sender, receiver) = oneshot::channel();
746 let replaced = self
747 .lock_close_registry()
748 .insert(connection_id, sender)
749 .is_some();
750 if replaced {
751 warn!(
752 connection_id = connection_id.get(),
753 "replaced existing connection close registration"
754 );
755 }
756 receiver
757 }
758
759 pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
760 self.lock_close_registry().remove(&connection_id);
761 }
762
763 #[cfg(unix)]
772 pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
773 let senders: Vec<_> = self.lock_close_registry().drain().collect();
774 let count = senders.len();
775 for (_, sender) in senders {
776 let _ = sender.send(reason.clone());
777 }
778 count
779 }
780
781 #[cfg(unix)]
785 pub(crate) fn module_connections(
786 &self,
787 ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
788 let inner = self.read_inner()?;
789 let mut seen = HashSet::new();
790 Ok(inner
791 .modules_by_id
792 .values()
793 .chain(inner.candidates_by_id.values())
794 .chain(inner.superseded_endpoints.values())
795 .filter(|module| seen.insert(module.endpoint))
796 .map(|module| ModuleConnectionTarget {
797 module_id: inner
798 .module_id_by_endpoint
799 .get(&module.endpoint)
800 .cloned()
801 .unwrap_or_default(),
802 endpoint: module.endpoint,
803 sink: module.sink.clone(),
804 negotiated_ver: module.negotiated_ver,
805 })
806 .collect())
807 }
808
809 pub(crate) fn request_connection_close(
813 &self,
814 connection_id: ConnectionId,
815 reason: CloseReason,
816 ) -> bool {
817 let sender = self.lock_close_registry().remove(&connection_id);
818 if let Some(sender) = sender {
819 debug!(
820 connection_id = connection_id.get(),
821 close_reason = %reason,
822 "requesting connection close"
823 );
824 let _ = sender.send(reason);
825 true
826 } else {
827 debug!(
828 connection_id = connection_id.get(),
829 close_reason = %reason,
830 "connection close request ignored for inactive connection"
831 );
832 false
833 }
834 }
835
836 pub fn register_module_connection(
837 &self,
838 connection_id: ConnectionId,
839 module_id: String,
840 negotiated_ver: u8,
841 concurrency: Concurrency,
842 sink: FrameSink,
843 ) -> Result<ModuleEndpointId, ForwardingError> {
844 self.register_module_connection_inner(
845 connection_id,
846 module_id,
847 negotiated_ver,
848 concurrency,
849 sink,
850 None,
851 )
852 }
853
854 pub(crate) fn register_module_connection_acked(
867 &self,
868 connection_id: ConnectionId,
869 module_id: String,
870 negotiated_ver: u8,
871 concurrency: Concurrency,
872 sink: FrameSink,
873 hello_ack: Frame,
874 ) -> Result<ModuleEndpointId, ForwardingError> {
875 self.register_module_connection_inner(
876 connection_id,
877 module_id,
878 negotiated_ver,
879 concurrency,
880 sink,
881 Some(hello_ack),
882 )
883 }
884
885 fn register_module_connection_inner(
886 &self,
887 connection_id: ConnectionId,
888 module_id: String,
889 negotiated_ver: u8,
890 concurrency: Concurrency,
891 sink: FrameSink,
892 hello_ack: Option<Frame>,
893 ) -> Result<ModuleEndpointId, ForwardingError> {
894 let mut inner = self.write_inner()?;
895 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
896 return Err(ForwardingError::ConnectionClosing { connection_id });
897 }
898 check_module_connection_role_locked(&inner, connection_id)?;
899 enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
902
903 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
904 let endpoint = ModuleEndpointId {
905 connection_id,
906 generation: inner.next_generation,
907 };
908 inner.endpoint_by_connection.insert(connection_id, endpoint);
909 inner
910 .module_id_by_endpoint
911 .insert(endpoint, module_id.clone());
912 inner.next_module_channel.insert(endpoint, 1);
913 inner.next_control_corr.insert(endpoint, 1);
914 inner.modules_by_id.insert(
915 module_id.clone(),
916 ModuleConnection {
917 endpoint,
918 sink,
919 negotiated_ver,
920 concurrency,
921 },
922 );
923 drop(inner);
924
925 if let Some(discarded) = self
938 .route_bind_breakers
939 .reset_for_new_module_connection(&module_id)
940 {
941 info!(
942 module_id = %module_id,
943 discarded_consecutive_timeouts = discarded,
944 "route.bind breaker state discarded: a new module connection replaced the process it described"
945 );
946 }
947 Ok(endpoint)
948 }
949
950 #[cfg(test)]
964 pub(crate) fn register_candidate_module_connection(
965 &self,
966 connection_id: ConnectionId,
967 module_id: String,
968 negotiated_ver: u8,
969 concurrency: Concurrency,
970 sink: FrameSink,
971 ) -> Result<ModuleEndpointId, ForwardingError> {
972 self.register_candidate_module_connection_inner(
973 connection_id,
974 module_id,
975 negotiated_ver,
976 concurrency,
977 sink,
978 None,
979 )
980 }
981
982 pub(crate) fn register_candidate_module_connection_acked(
989 &self,
990 connection_id: ConnectionId,
991 module_id: String,
992 negotiated_ver: u8,
993 concurrency: Concurrency,
994 sink: FrameSink,
995 hello_ack: Frame,
996 ) -> Result<ModuleEndpointId, ForwardingError> {
997 self.register_candidate_module_connection_inner(
998 connection_id,
999 module_id,
1000 negotiated_ver,
1001 concurrency,
1002 sink,
1003 Some(hello_ack),
1004 )
1005 }
1006
1007 fn register_candidate_module_connection_inner(
1008 &self,
1009 connection_id: ConnectionId,
1010 module_id: String,
1011 negotiated_ver: u8,
1012 concurrency: Concurrency,
1013 sink: FrameSink,
1014 hello_ack: Option<Frame>,
1015 ) -> Result<ModuleEndpointId, ForwardingError> {
1016 let mut inner = self.write_inner()?;
1017 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1018 return Err(ForwardingError::ConnectionClosing { connection_id });
1019 }
1020 check_module_connection_role_locked(&inner, connection_id)?;
1021 if inner.candidates_by_id.contains_key(&module_id) {
1022 return Err(ForwardingError::CandidateSlotOccupied { module_id });
1023 }
1024 enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1027
1028 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1029 let endpoint = ModuleEndpointId {
1030 connection_id,
1031 generation: inner.next_generation,
1032 };
1033 inner.endpoint_by_connection.insert(connection_id, endpoint);
1034 inner
1035 .module_id_by_endpoint
1036 .insert(endpoint, module_id.clone());
1037 inner.next_module_channel.insert(endpoint, 1);
1038 inner.next_control_corr.insert(endpoint, 1);
1039 inner.candidates_by_id.insert(
1040 module_id,
1041 ModuleConnection {
1042 endpoint,
1043 sink,
1044 negotiated_ver,
1045 concurrency,
1046 },
1047 );
1048 Ok(endpoint)
1049 }
1050
1051 pub(crate) fn cutover_candidate(
1069 &self,
1070 module_id: &str,
1071 ) -> Result<Option<ForwardingCutover>, ForwardingError> {
1072 let mut inner = self.write_inner()?;
1073 if inner.daemon_draining {
1074 return Err(ForwardingError::ModuleReloading {
1075 module_id: module_id.to_string(),
1076 });
1077 }
1078 let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
1079 return Ok(None);
1080 };
1081 let promoted = candidate.endpoint;
1082 let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
1083 let incumbent = incumbent.map(|incumbent| {
1084 let endpoint = incumbent.endpoint;
1085 inner.superseded_endpoints.insert(endpoint, incumbent);
1086 endpoint
1087 });
1088 drop(inner);
1089
1090 if let Some(discarded) = self
1094 .route_bind_breakers
1095 .reset_for_new_module_connection(module_id)
1096 {
1097 info!(
1098 module_id = %module_id,
1099 discarded_consecutive_timeouts = discarded,
1100 "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
1101 );
1102 }
1103 Ok(Some(ForwardingCutover {
1104 promoted,
1105 incumbent,
1106 }))
1107 }
1108
1109 #[allow(clippy::too_many_arguments)]
1110 pub(crate) async fn begin_route_bind_relay_for(
1111 &self,
1112 client_connection_id: ConnectionId,
1113 client_sink: FrameSink,
1114 client_negotiated_ver: u8,
1115 client_corr: u64,
1116 module_id: &str,
1117 principal: Principal,
1118 scope: Option<BoundScope>,
1119 project_root: Option<ProjectRootId>,
1120 deadline: Instant,
1121 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1122 let client_permit =
1126 client_sink
1127 .reserve_owned()
1128 .await
1129 .map_err(|_| ForwardingError::ClientEgressClosed {
1130 connection_id: client_connection_id,
1131 })?;
1132 self.begin_route_bind_relay_inner(
1133 client_connection_id,
1134 client_sink,
1135 client_negotiated_ver,
1136 client_corr,
1137 module_id,
1138 principal,
1139 scope,
1140 project_root,
1141 deadline,
1142 client_permit,
1143 )
1144 }
1145
1146 #[cfg(test)]
1147 pub(crate) fn begin_route_bind_relay_for_test(
1148 &self,
1149 client_connection_id: ConnectionId,
1150 client_sink: FrameSink,
1151 client_corr: u64,
1152 module_id: &str,
1153 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1154 let permit =
1155 client_sink
1156 .try_reserve_owned()
1157 .map_err(|_| ForwardingError::ClientEgressClosed {
1158 connection_id: client_connection_id,
1159 })?;
1160 self.begin_route_bind_relay_inner(
1161 client_connection_id,
1162 client_sink,
1163 subc_protocol::PROTOCOL_VERSION,
1164 client_corr,
1165 module_id,
1166 Principal::Direct,
1167 None,
1168 None,
1169 Instant::now() + std::time::Duration::from_secs(60),
1170 permit,
1171 )
1172 }
1173
1174 pub(crate) fn begin_module_control_rpc_for(
1175 &self,
1176 module_id: &str,
1177 expected_op: &str,
1178 deadline: Instant,
1179 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1180 self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1181 }
1182
1183 pub(crate) fn begin_health_probe_rpc_for(
1184 &self,
1185 module_id: &str,
1186 expected_op: &str,
1187 probe_started_at: Instant,
1188 deadline: Instant,
1189 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1190 self.begin_module_control_rpc_inner(
1191 module_id,
1192 expected_op,
1193 deadline,
1194 Some(probe_started_at),
1195 false,
1196 )
1197 }
1198
1199 pub(crate) fn begin_drain_health_probe_rpc_for(
1200 &self,
1201 module_id: &str,
1202 expected_op: &str,
1203 probe_started_at: Instant,
1204 deadline: Instant,
1205 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1206 self.begin_module_control_rpc_inner(
1207 module_id,
1208 expected_op,
1209 deadline,
1210 Some(probe_started_at),
1211 true,
1212 )
1213 }
1214
1215 pub(crate) fn begin_endpoint_health_probe_rpc_for(
1223 &self,
1224 endpoint: ModuleEndpointId,
1225 expected_op: &str,
1226 probe_started_at: Instant,
1227 deadline: Instant,
1228 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1229 let inner = self.write_inner()?;
1230 let module = module_connection_for_endpoint_locked(&inner, endpoint)
1231 .cloned()
1232 .ok_or(ForwardingError::NoModuleConnection)?;
1233 let module_id = inner
1234 .module_id_by_endpoint
1235 .get(&endpoint)
1236 .cloned()
1237 .unwrap_or_default();
1238 self.begin_control_rpc_locked(
1241 inner,
1242 &module_id,
1243 module,
1244 expected_op,
1245 deadline,
1246 Some(probe_started_at),
1247 true,
1248 )
1249 }
1250
1251 fn begin_module_control_rpc_inner(
1252 &self,
1253 module_id: &str,
1254 expected_op: &str,
1255 deadline: Instant,
1256 health_probe_started_at: Option<Instant>,
1257 allow_draining: bool,
1258 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1259 let inner = self.write_inner()?;
1260 let module = inner
1261 .modules_by_id
1262 .get(module_id)
1263 .cloned()
1264 .ok_or(ForwardingError::NoModuleConnection)?;
1265 self.begin_control_rpc_locked(
1266 inner,
1267 module_id,
1268 module,
1269 expected_op,
1270 deadline,
1271 health_probe_started_at,
1272 allow_draining,
1273 )
1274 }
1275
1276 #[allow(clippy::too_many_arguments)]
1277 fn begin_control_rpc_locked(
1278 &self,
1279 mut inner: RwLockWriteGuard<'_, ForwardingInner>,
1280 module_id: &str,
1281 module: ModuleConnection,
1282 expected_op: &str,
1283 deadline: Instant,
1284 health_probe_started_at: Option<Instant>,
1285 allow_draining: bool,
1286 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1287 if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1288 return Err(ForwardingError::ModuleReloading {
1289 module_id: module_id.to_string(),
1290 });
1291 }
1292 if inner
1293 .closing_connections
1294 .contains(&module.endpoint.connection_id)
1295 {
1296 return Err(ForwardingError::ConnectionClosing {
1297 connection_id: module.endpoint.connection_id,
1298 });
1299 }
1300 if health_probe_started_at.is_some() {
1301 inner
1305 .health_probe_tombstones
1306 .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1307 }
1308 let corr = match inner.allocate_control_corr(module.endpoint) {
1309 Ok(corr) => corr,
1310 Err(err) => {
1311 drop(inner);
1312 self.request_connection_close(
1313 module.endpoint.connection_id,
1314 CloseReason::new(
1315 "control_correlation_exhausted",
1316 "daemon-originated channel-0 correlation space exhausted",
1317 ),
1318 );
1319 return Err(err);
1320 }
1321 };
1322 let (sender, receiver) = oneshot::channel();
1323 inner.pending_control_rpcs.insert(
1324 (module.endpoint, corr),
1325 PendingModuleControlRpcEntry {
1326 expected_op: expected_op.to_string(),
1327 deadline,
1328 health_probe_started_at,
1329 sender,
1330 },
1331 );
1332
1333 Ok(PendingModuleControlRpc {
1334 endpoint: module.endpoint,
1335 module_sink: module.sink,
1336 negotiated_ver: module.negotiated_ver,
1337 corr,
1338 receiver,
1339 })
1340 }
1341
1342 #[allow(clippy::too_many_arguments)]
1343 fn begin_route_bind_relay_inner(
1344 &self,
1345 client_connection_id: ConnectionId,
1346 client_sink: FrameSink,
1347 client_negotiated_ver: u8,
1348 client_corr: u64,
1349 expected_module_id: &str,
1350 principal: Principal,
1351 scope: Option<BoundScope>,
1352 project_root: Option<ProjectRootId>,
1353 deadline: Instant,
1354 client_permit: crate::router::EgressPermit,
1355 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1356 let mut inner = self.write_inner()?;
1357 if inner
1361 .endpoint_by_connection
1362 .contains_key(&client_connection_id)
1363 {
1364 return Err(ForwardingError::ConnectionRoleConflict {
1365 connection_id: client_connection_id,
1366 });
1367 }
1368 if inner.closing_connections.contains(&client_connection_id) {
1369 return Err(ForwardingError::ConnectionClosing {
1370 connection_id: client_connection_id,
1371 });
1372 }
1373 let module = inner
1374 .modules_by_id
1375 .get(expected_module_id)
1376 .cloned()
1377 .ok_or(ForwardingError::NoModuleConnection)?;
1378 if inner.draining_endpoints.contains_key(&module.endpoint) {
1379 return Err(ForwardingError::ModuleReloading {
1380 module_id: expected_module_id.to_string(),
1381 });
1382 }
1383 if inner
1384 .closing_connections
1385 .contains(&module.endpoint.connection_id)
1386 {
1387 return Err(ForwardingError::ConnectionClosing {
1388 connection_id: module.endpoint.connection_id,
1389 });
1390 }
1391
1392 let corr = match inner.allocate_control_corr(module.endpoint) {
1393 Ok(corr) => corr,
1394 Err(err) => {
1395 drop(inner);
1396 self.request_connection_close(
1397 module.endpoint.connection_id,
1398 CloseReason::new(
1399 "control_correlation_exhausted",
1400 "daemon-originated channel-0 correlation space exhausted",
1401 ),
1402 );
1403 return Err(err);
1404 }
1405 };
1406 let (client_channel, client_epoch, module_channel, module_epoch) =
1407 inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1408 let client_key = ClientRouteKey {
1409 connection_id: client_connection_id,
1410 channel: client_channel,
1411 };
1412 let module_key = ModuleRouteKey {
1413 endpoint: module.endpoint,
1414 channel: module_channel,
1415 };
1416 let reservation = RouteReservation {
1417 client_key,
1418 module_key,
1419 client_epoch,
1420 module_epoch,
1421 project_root,
1422 };
1423 let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
1424 route_channel: client_channel,
1425 route_epoch: client_epoch,
1426 })
1427 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1428 let route_open_frame = Frame::build_with_version(
1429 client_negotiated_ver,
1430 FrameType::Response,
1431 Flags::new(false, Priority::Passive, false),
1432 0,
1433 0,
1434 client_corr,
1435 response_body,
1436 )
1437 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1438 let (sender, receiver) = oneshot::channel();
1439 inner.reserved_client.insert(client_key, module_key);
1440 inner.reserved_module.insert(module_key, client_key);
1441 inner.pending_relays.insert(
1442 (module.endpoint, corr),
1443 PendingRouteBindRelayEntry {
1444 reservation,
1445 client_sink,
1446 client_negotiated_ver,
1447 client_permit,
1448 route_open_frame,
1449 principal,
1450 scope,
1451 deadline,
1452 relay_enqueued: false,
1453 sender,
1454 },
1455 );
1456
1457 Ok(PendingRouteBindRelay {
1458 endpoint: module.endpoint,
1459 module_sink: module.sink,
1460 negotiated_ver: module.negotiated_ver,
1461 client_channel,
1462 client_epoch,
1463 module_channel,
1464 module_epoch,
1465 corr,
1466 receiver,
1467 })
1468 }
1469
1470 pub(crate) fn mark_route_bind_relay_enqueued(
1480 &self,
1481 endpoint: ModuleEndpointId,
1482 corr: u64,
1483 ) -> Result<RouteBindRelayMark, ForwardingError> {
1484 let mut inner = self.write_inner()?;
1485 if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
1486 pending.relay_enqueued = true;
1487 return Ok(RouteBindRelayMark::Marked);
1488 }
1489 if inner
1490 .relays_released_before_enqueue
1491 .remove(&(endpoint, corr))
1492 {
1493 return Ok(RouteBindRelayMark::ReleasedWithoutReply);
1494 }
1495 Ok(RouteBindRelayMark::SettledByReply)
1496 }
1497
1498 pub(crate) fn release_client_route(
1499 &self,
1500 client_connection_id: ConnectionId,
1501 client_channel: u16,
1502 expected_epoch: u32,
1503 ) -> Result<RouteRelease, ForwardingError> {
1504 let mut inner = self.write_inner()?;
1505 let release = release_client_route_locked(
1506 &mut inner,
1507 ClientRouteKey {
1508 connection_id: client_connection_id,
1509 channel: client_channel,
1510 },
1511 expected_epoch,
1512 );
1513 self.record_route_release(&release);
1514 Ok(release)
1515 }
1516
1517 pub(crate) fn release_module_route(
1518 &self,
1519 module_connection_id: ConnectionId,
1520 module_channel: u16,
1521 expected_epoch: u32,
1522 ) -> Result<RouteRelease, ForwardingError> {
1523 let mut inner = self.write_inner()?;
1524 let Some(endpoint) = inner
1525 .endpoint_by_connection
1526 .get(&module_connection_id)
1527 .copied()
1528 else {
1529 return Ok(RouteRelease::Absent);
1530 };
1531 let release = release_module_route_locked(
1532 &mut inner,
1533 ModuleRouteKey {
1534 endpoint,
1535 channel: module_channel,
1536 },
1537 expected_epoch,
1538 );
1539 self.record_route_release(&release);
1540 Ok(release)
1541 }
1542
1543 pub(crate) fn abort_pending_relay(
1544 &self,
1545 endpoint: ModuleEndpointId,
1546 corr: u64,
1547 outcome: RouteBindRelayOutcome,
1548 ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
1549 let mut inner = self.write_inner()?;
1550 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1551 inner
1556 .relays_released_before_enqueue
1557 .remove(&(endpoint, corr));
1558 return Ok(None);
1559 };
1560 release_reserved_route_locked(
1561 &mut inner,
1562 pending.reservation.client_key,
1563 pending.reservation.module_key,
1564 );
1565 let target = pending
1566 .relay_enqueued
1567 .then(|| abandoned_route_target(&inner, &pending.reservation));
1568 let _ = pending.sender.send(outcome);
1569 Ok(target.flatten())
1570 }
1571
1572 pub(crate) fn cancel_module_control_rpc(
1573 &self,
1574 endpoint: ModuleEndpointId,
1575 corr: u64,
1576 ) -> Result<(), ForwardingError> {
1577 self.write_inner()?
1578 .pending_control_rpcs
1579 .remove(&(endpoint, corr));
1580 Ok(())
1581 }
1582
1583 pub(crate) fn tombstone_health_probe_rpc(
1584 &self,
1585 endpoint: ModuleEndpointId,
1586 corr: u64,
1587 ) -> Result<bool, ForwardingError> {
1588 let key = (endpoint, corr);
1589 let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
1590 {
1591 let mut inner = self.write_inner()?;
1592 let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
1593 return Ok(false);
1594 };
1595 let Some(probe_started_at) = pending.health_probe_started_at else {
1596 inner.pending_control_rpcs.insert(key, pending);
1597 return Ok(false);
1598 };
1599 let module_id = inner
1600 .module_id_by_endpoint
1601 .get(&endpoint)
1602 .cloned()
1603 .unwrap_or_else(|| "unknown".to_string());
1604 inner.health_probe_tombstones.insert(
1605 key,
1606 HealthProbeTombstone {
1607 expected_op: pending.expected_op,
1608 module_id,
1609 probe_started_at,
1610 expires_at,
1611 },
1612 );
1613 }
1614 self.schedule_health_probe_tombstone_expiration(key, expires_at);
1615 Ok(true)
1616 }
1617
1618 fn schedule_health_probe_tombstone_expiration(
1619 &self,
1620 key: (ModuleEndpointId, u64),
1621 expires_at: Instant,
1622 ) {
1623 let inner = Arc::downgrade(&self.inner);
1624 tokio::spawn(async move {
1625 tokio::time::sleep_until(expires_at).await;
1626 let Some(inner) = inner.upgrade() else {
1627 return;
1628 };
1629 let Ok(mut inner) = inner.write() else {
1630 return;
1631 };
1632 let expired = inner
1633 .health_probe_tombstones
1634 .get(&key)
1635 .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
1636 if expired {
1637 inner.health_probe_tombstones.remove(&key);
1638 }
1639 });
1640 }
1641
1642 pub(crate) fn complete_pending_relay(
1643 &self,
1644 connection_id: ConnectionId,
1645 corr: u64,
1646 outcome: RouteBindRelayOutcome,
1647 ) -> Result<PendingRelayCompletion, ForwardingError> {
1648 let mut inner = self.write_inner()?;
1649 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1650 return Ok(PendingRelayCompletion {
1651 settled: false,
1652 abandoned: None,
1653 });
1654 };
1655 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1656 return Ok(PendingRelayCompletion {
1657 settled: false,
1658 abandoned: None,
1659 });
1660 };
1661
1662 if Instant::now() >= pending.deadline {
1663 release_reserved_route_locked(
1664 &mut inner,
1665 pending.reservation.client_key,
1666 pending.reservation.module_key,
1667 );
1668 let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
1669 .then(|| abandoned_route_target(&inner, &pending.reservation))
1670 .flatten();
1671 let _ = pending
1672 .sender
1673 .send(RouteBindRelayOutcome::Rejected(ErrorBody {
1674 code: "module_timeout".to_string(),
1675 message: "route.bind response arrived after its daemon deadline".to_string(),
1676 detail: None,
1677 }));
1678 return Ok(PendingRelayCompletion {
1679 settled: true,
1680 abandoned,
1681 });
1682 }
1683
1684 match outcome {
1685 RouteBindRelayOutcome::Accepted
1704 if pending.client_sink.is_closed()
1705 || inner
1706 .closing_connections
1707 .contains(&pending.reservation.client_key.connection_id) =>
1708 {
1709 let reason = if pending.client_sink.is_closed() {
1710 "client egress closed before route publication"
1711 } else {
1712 "client connection is closing before route publication"
1713 };
1714 release_reserved_route_locked(
1715 &mut inner,
1716 pending.reservation.client_key,
1717 pending.reservation.module_key,
1718 );
1719 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1723 let _ = pending
1724 .sender
1725 .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
1726 return Ok(PendingRelayCompletion {
1727 settled: true,
1728 abandoned,
1729 });
1730 }
1731 RouteBindRelayOutcome::Accepted
1754 if inner.superseded_endpoints.contains_key(&endpoint)
1755 || inner.draining_endpoints.contains_key(&endpoint) =>
1756 {
1757 release_reserved_route_locked(
1758 &mut inner,
1759 pending.reservation.client_key,
1760 pending.reservation.module_key,
1761 );
1762 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1765 let module_id = inner
1766 .module_id_by_endpoint
1767 .get(&endpoint)
1768 .cloned()
1769 .unwrap_or_else(|| "unknown".to_string());
1770 let _ = pending
1771 .sender
1772 .send(RouteBindRelayOutcome::Rejected(ErrorBody::new(
1773 "module_reloading",
1774 format!("module_id '{module_id}' is reloading"),
1775 )));
1776 return Ok(PendingRelayCompletion {
1777 settled: true,
1778 abandoned,
1779 });
1780 }
1781 RouteBindRelayOutcome::Accepted
1790 if pending
1791 .scope
1792 .as_ref()
1793 .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
1794 {
1795 let refusal = pending
1796 .scope
1797 .as_ref()
1798 .and_then(|scope| scope_refusal_locked(&inner, scope))
1799 .expect("guard matched a refusal under the same lock");
1800 release_reserved_route_locked(
1801 &mut inner,
1802 pending.reservation.client_key,
1803 pending.reservation.module_key,
1804 );
1805 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1806 let _ = pending
1807 .sender
1808 .send(RouteBindRelayOutcome::Rejected(refusal));
1809 return Ok(PendingRelayCompletion {
1810 settled: true,
1811 abandoned,
1812 });
1813 }
1814 RouteBindRelayOutcome::Accepted => {
1815 let abandoned = commit_route_locked(&mut inner, pending)?;
1816 return Ok(PendingRelayCompletion {
1817 settled: true,
1818 abandoned,
1819 });
1820 }
1821 terminal => {
1822 release_reserved_route_locked(
1823 &mut inner,
1824 pending.reservation.client_key,
1825 pending.reservation.module_key,
1826 );
1827 let _ = pending.sender.send(terminal);
1828 }
1829 }
1830 Ok(PendingRelayCompletion {
1831 settled: true,
1832 abandoned: None,
1833 })
1834 }
1835
1836 pub(crate) fn pending_module_control_op(
1837 &self,
1838 connection_id: ConnectionId,
1839 corr: u64,
1840 ) -> Result<Option<String>, ForwardingError> {
1841 let inner = self.read_inner()?;
1842 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1843 return Ok(None);
1844 };
1845 let key = (endpoint, corr);
1846 Ok(inner
1847 .pending_control_rpcs
1848 .get(&key)
1849 .map(|pending| pending.expected_op.clone())
1850 .or_else(|| {
1851 inner
1852 .health_probe_tombstones
1853 .get(&key)
1854 .filter(|tombstone| tombstone.expires_at > Instant::now())
1855 .map(|tombstone| tombstone.expected_op.clone())
1856 }))
1857 }
1858
1859 pub(crate) fn complete_module_control_rpc(
1860 &self,
1861 connection_id: ConnectionId,
1862 corr: u64,
1863 actual_op: Option<&str>,
1864 outcome: ModuleControlRpcOutcome,
1865 ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
1866 let now = Instant::now();
1867 let mut inner = self.write_inner()?;
1868 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1869 return Ok(ModuleControlRpcCompletion::Unknown);
1870 };
1871 let key = (endpoint, corr);
1872 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
1873 if now >= pending.deadline {
1874 let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
1875 ModuleControlRpcCompletion::LateHealthAnswer {
1876 module_id: inner
1877 .module_id_by_endpoint
1878 .get(&endpoint)
1879 .cloned()
1880 .unwrap_or_else(|| "unknown".to_string()),
1881 latency: now.saturating_duration_since(probe_started_at),
1882 }
1883 });
1884 let _ = pending
1885 .sender
1886 .send(ModuleControlRpcOutcome::DeadlineElapsed);
1887 return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
1888 }
1889 let outcome = match actual_op {
1890 Some(actual) if actual != pending.expected_op => {
1891 ModuleControlRpcOutcome::UnexpectedOp {
1892 expected: pending.expected_op,
1893 actual: actual.to_string(),
1894 }
1895 }
1896 _ => outcome,
1897 };
1898 let _ = pending.sender.send(outcome);
1899 return Ok(ModuleControlRpcCompletion::Settled);
1900 }
1901
1902 let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
1903 return Ok(ModuleControlRpcCompletion::Unknown);
1904 };
1905 if tombstone.expires_at <= now {
1906 return Ok(ModuleControlRpcCompletion::Unknown);
1907 }
1908 Ok(ModuleControlRpcCompletion::LateHealthAnswer {
1909 module_id: tombstone.module_id,
1910 latency: now.saturating_duration_since(tombstone.probe_started_at),
1911 })
1912 }
1913
1914 #[cfg(test)]
1915 pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
1916 Ok(self.read_inner()?.health_probe_tombstones.len())
1917 }
1918
1919 #[cfg(test)]
1920 pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
1921 Ok(self.read_inner()?.closing_connections.len())
1922 }
1923
1924 #[cfg(test)]
1927 pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
1928 let inner = self.read_inner()?;
1929 Ok((inner.reserved_client.len(), inner.reserved_module.len()))
1930 }
1931
1932 pub(crate) fn module_endpoint_for_connection(
1933 &self,
1934 connection_id: ConnectionId,
1935 ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
1936 Ok(self
1937 .read_inner()?
1938 .endpoint_by_connection
1939 .get(&connection_id)
1940 .copied())
1941 }
1942
1943 pub(crate) fn module_id_for_connection(
1946 &self,
1947 connection_id: ConnectionId,
1948 ) -> Result<Option<String>, ForwardingError> {
1949 let inner = self.read_inner()?;
1950 Ok(inner
1951 .endpoint_by_connection
1952 .get(&connection_id)
1953 .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
1954 .cloned())
1955 }
1956
1957 pub(crate) fn module_route_epoch_was_allocated(
1964 &self,
1965 connection_id: ConnectionId,
1966 channel: u16,
1967 epoch: u32,
1968 ) -> Result<bool, ForwardingError> {
1969 let inner = self.read_inner()?;
1970 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1971 return Ok(false);
1972 };
1973 Ok(inner
1974 .module_slot_epochs
1975 .get(&ModuleRouteKey { endpoint, channel })
1976 .is_some_and(|last| epoch != 0 && epoch <= *last))
1977 }
1978
1979 pub(crate) fn has_live_module_connection(
1980 &self,
1981 module_id: &str,
1982 ) -> Result<bool, ForwardingError> {
1983 Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
1984 }
1985
1986 pub(crate) fn lookup_data_route(
1987 &self,
1988 connection_id: ConnectionId,
1989 channel: u16,
1990 epoch: u32,
1991 ) -> Result<DataRoute, ForwardingError> {
1992 let inner = self.read_inner()?;
1993 let state = if let Some(endpoint) =
1994 inner.endpoint_by_connection.get(&connection_id).copied()
1995 {
1996 let key = ModuleRouteKey { endpoint, channel };
1997 match inner.module_to_client.get(&key) {
1998 Some(route) if route.module_epoch == epoch => {
1999 DataRouteState::Bound(Arc::clone(route))
2000 }
2001 Some(_) => DataRouteState::EpochMismatch,
2002 None if inner.reserved_module.contains_key(&key)
2003 && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
2004 {
2005 DataRouteState::Reserved
2006 }
2007 None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
2008 None => DataRouteState::Absent,
2009 }
2010 } else {
2011 let key = ClientRouteKey {
2012 connection_id,
2013 channel,
2014 };
2015 match inner.client_to_module.get(&key) {
2016 Some(route) if route.client_epoch == epoch => {
2017 DataRouteState::Bound(Arc::clone(route))
2018 }
2019 Some(_) => DataRouteState::EpochMismatch,
2020 None if inner.reserved_client.contains_key(&key)
2021 && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
2022 {
2023 DataRouteState::Reserved
2024 }
2025 None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
2026 None => DataRouteState::Absent,
2027 }
2028 };
2029 Ok(
2030 if inner.endpoint_by_connection.contains_key(&connection_id) {
2031 DataRoute::Module(state)
2032 } else {
2033 DataRoute::Client(state)
2034 },
2035 )
2036 }
2037
2038 #[cfg(test)]
2039 pub(crate) fn inject_client_slot_epoch(
2040 &self,
2041 connection_id: ConnectionId,
2042 channel: u16,
2043 last_epoch: u32,
2044 ) {
2045 let mut inner = self.write_inner().expect("forwarding lock");
2046 inner.client_slot_epochs.insert(
2047 ClientRouteKey {
2048 connection_id,
2049 channel,
2050 },
2051 last_epoch,
2052 );
2053 inner.next_client_channel.insert(connection_id, channel);
2054 }
2055
2056 #[cfg(test)]
2057 pub(crate) fn inject_module_slot_epoch(
2058 &self,
2059 endpoint: ModuleEndpointId,
2060 channel: u16,
2061 last_epoch: u32,
2062 ) {
2063 let mut inner = self.write_inner().expect("forwarding lock");
2064 inner
2065 .module_slot_epochs
2066 .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
2067 inner.next_module_channel.insert(endpoint, channel);
2068 }
2069
2070 #[cfg(test)]
2071 pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
2072 self.write_inner()
2073 .expect("forwarding lock")
2074 .next_control_corr
2075 .insert(endpoint, next_corr);
2076 }
2077
2078 pub(crate) fn cache_status(
2079 &self,
2080 endpoint: ModuleEndpointId,
2081 module_channel: u16,
2082 module_epoch: u32,
2083 status: String,
2084 ) -> Result<bool, ForwardingError> {
2085 let mut inner = self.write_inner()?;
2086 if !inner.module_id_by_endpoint.contains_key(&endpoint) {
2087 return Err(ForwardingError::StaleModuleEndpoint);
2088 }
2089
2090 let module_key = ModuleRouteKey {
2091 endpoint,
2092 channel: module_channel,
2093 };
2094 let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
2095 (route.module_epoch == module_epoch).then_some((
2096 ClientRouteKey {
2097 connection_id: route.client_connection_id,
2098 channel: route.client_channel,
2099 },
2100 route.client_epoch,
2101 ))
2102 } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
2103 (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
2104 client_key,
2105 inner
2106 .client_slot_epochs
2107 .get(&client_key)
2108 .copied()
2109 .unwrap_or(0),
2110 ))
2111 } else {
2112 None
2113 };
2114
2115 if let Some(handle) = handle {
2116 inner.status.insert(handle, status);
2117 Ok(true)
2118 } else {
2119 debug!(
2120 module_channel,
2121 module_epoch,
2122 generation = endpoint.generation,
2123 connection_id = endpoint.connection_id.get(),
2124 "dropping stale status update for module route handle"
2125 );
2126 Ok(false)
2127 }
2128 }
2129
2130 pub(crate) fn route_poll_snapshot(
2131 &self,
2132 client_connection_id: ConnectionId,
2133 client_channel: u16,
2134 client_epoch: u32,
2135 ) -> Result<RoutePollSnapshot, ForwardingError> {
2136 let inner = self.read_inner()?;
2137 let client_key = ClientRouteKey {
2138 connection_id: client_connection_id,
2139 channel: client_channel,
2140 };
2141 let Some(route) = inner.client_to_module.get(&client_key) else {
2142 return Ok(RoutePollSnapshot::Absent);
2143 };
2144 if route.client_epoch != client_epoch
2145 || !inner
2146 .module_id_by_endpoint
2147 .contains_key(&route.module_endpoint)
2148 {
2149 return Ok(RoutePollSnapshot::Absent);
2150 }
2151 Ok(RoutePollSnapshot::Bound {
2152 module_id: route.module_id.clone(),
2153 status: inner.status.get(&(client_key, client_epoch)).cloned(),
2154 })
2155 }
2156
2157 pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2158 Ok(self.read_inner()?.client_to_module.len())
2159 }
2160
2161 pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2171 let inner = self.read_inner()?;
2172 let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2173 for key in inner.client_to_module.keys() {
2174 *per_connection.entry(key.connection_id).or_insert(0) += 1;
2175 }
2176 let max = per_connection.values().copied().max().unwrap_or(0);
2177 Ok((per_connection.len(), max))
2178 }
2179
2180 pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2181 let inner = self.read_inner()?;
2182 Ok(inner
2183 .client_to_module
2184 .keys()
2185 .any(|key| key.channel == route_channel))
2186 }
2187
2188 pub(crate) fn daemon_drain_result_for_connection(
2192 &self,
2193 connection_id: ConnectionId,
2194 ) -> Result<Option<bool>, ForwardingError> {
2195 let inner = self.read_inner()?;
2196 Ok(inner.daemon_draining.then(|| {
2197 inner
2198 .endpoint_by_connection
2199 .get(&connection_id)
2200 .and_then(|endpoint| inner.daemon_drain_results.get(endpoint))
2201 .copied()
2202 .unwrap_or(false)
2203 }))
2204 }
2205
2206 #[cfg(unix)]
2207 pub(crate) fn record_daemon_drain_result(
2208 &self,
2209 endpoint: ModuleEndpointId,
2210 drained: bool,
2211 ) -> Result<(), ForwardingError> {
2212 let mut inner = self.write_inner()?;
2213 if let Some(result) = inner.daemon_drain_results.get_mut(&endpoint) {
2217 *result = drained;
2218 }
2219 Ok(())
2220 }
2221
2222 #[cfg(unix)]
2225 pub(crate) fn begin_daemon_drain(
2226 &self,
2227 ) -> Result<Vec<(String, ModuleEndpointId)>, ForwardingError> {
2228 let mut inner = self.write_inner()?;
2229 if inner.daemon_draining {
2230 return Ok(Vec::new());
2231 }
2232 inner.daemon_draining = true;
2233 let modules = inner
2234 .modules_by_id
2235 .iter()
2236 .map(|(id, module)| (id.clone(), module.endpoint))
2237 .collect::<Vec<_>>();
2238 for (_, endpoint) in &modules {
2239 inner.daemon_drain_results.insert(*endpoint, false);
2240 inner
2241 .draining_endpoints
2242 .insert(*endpoint, RouteCloseReason::Restart);
2243 }
2244 let off_slot_endpoints = inner
2249 .candidates_by_id
2250 .values()
2251 .map(|module| module.endpoint)
2252 .chain(inner.superseded_endpoints.keys().copied())
2253 .collect::<Vec<_>>();
2254 for endpoint in off_slot_endpoints {
2255 inner
2256 .draining_endpoints
2257 .insert(endpoint, RouteCloseReason::Restart);
2258 }
2259 Ok(modules)
2260 }
2261
2262 pub(crate) fn begin_module_drain(
2269 &self,
2270 module_id: &str,
2271 reason: RouteCloseReason,
2272 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2273 let mut inner = self.write_inner()?;
2274 let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2275 return Ok(None);
2276 };
2277 Ok(Some(begin_drain_locked(
2278 &mut inner, module_id, module, reason,
2279 )))
2280 }
2281
2282 pub(crate) fn begin_endpoint_drain(
2289 &self,
2290 endpoint: ModuleEndpointId,
2291 reason: RouteCloseReason,
2292 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2293 let mut inner = self.write_inner()?;
2294 let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2295 return Ok(None);
2296 };
2297 let module_id = inner
2298 .module_id_by_endpoint
2299 .get(&endpoint)
2300 .cloned()
2301 .expect("an endpoint resolved to a module connection has a module id");
2302 Ok(Some(begin_drain_locked(
2303 &mut inner, &module_id, module, reason,
2304 )))
2305 }
2306}
2307
2308fn begin_drain_locked(
2312 inner: &mut ForwardingInner,
2313 module_id: &str,
2314 module: ModuleConnection,
2315 reason: RouteCloseReason,
2316) -> ModuleDrainTarget {
2317 {
2318 let endpoint = module.endpoint;
2319 inner.draining_endpoints.insert(endpoint, reason);
2320
2321 let flows = inner
2322 .client_to_module
2323 .values()
2324 .filter(|route| route.module_endpoint == endpoint)
2325 .map(|route| Arc::clone(&route.flow))
2326 .collect::<Vec<_>>();
2327 let excluded_subscriptions = flows
2328 .into_iter()
2329 .map(|flow| flow.begin_drain())
2330 .fold(0u32, u32::saturating_add);
2331
2332 let pending_keys = inner
2333 .pending_relays
2334 .keys()
2335 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2336 .copied()
2337 .collect::<Vec<_>>();
2338 let mut abandoned_bindings = Vec::new();
2339 for key in pending_keys {
2340 let Some(pending) = inner.pending_relays.remove(&key) else {
2341 continue;
2342 };
2343 release_reserved_route_locked(
2344 inner,
2345 pending.reservation.client_key,
2346 pending.reservation.module_key,
2347 );
2348 if pending.relay_enqueued {
2349 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2350 abandoned_bindings.push(target);
2351 }
2352 } else {
2353 inner.relays_released_before_enqueue.insert(key);
2357 }
2358 let _ = pending
2359 .sender
2360 .send(RouteBindRelayOutcome::Rejected(ErrorBody::new(
2361 "module_reloading",
2362 format!("module_id '{module_id}' is reloading"),
2363 )));
2364 }
2365
2366 let pending_control_keys = inner
2367 .pending_control_rpcs
2368 .keys()
2369 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2370 .copied()
2371 .collect::<Vec<_>>();
2372 for key in pending_control_keys {
2373 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2374 let _ = pending
2375 .sender
2376 .send(ModuleControlRpcOutcome::ModuleGone(format!(
2377 "module '{module_id}' began draining during module-control RPC"
2378 )));
2379 }
2380 }
2381
2382 ModuleDrainTarget {
2383 endpoint,
2384 sink: module.sink,
2385 negotiated_ver: module.negotiated_ver,
2386 abandoned_bindings,
2387 excluded_subscriptions,
2388 }
2389 }
2390}
2391
2392#[derive(Debug, Default, PartialEq, Eq)]
2395pub(crate) struct DrainHoldouts {
2396 pub(crate) requests: usize,
2399 pub(crate) routes: usize,
2401 pub(crate) total_routes: usize,
2403 pub(crate) top_connections: Vec<(u64, usize)>,
2406 pub(crate) held: Vec<(u16, u64)>,
2415}
2416
2417pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
2419
2420impl ForwardingTable {
2421 pub(crate) fn endpoint_drain_holdouts(
2423 &self,
2424 endpoint: ModuleEndpointId,
2425 ) -> Result<DrainHoldouts, ForwardingError> {
2426 let inner = self.read_inner()?;
2427 let mut holdouts = DrainHoldouts::default();
2428 let mut by_connection: HashMap<u64, usize> = HashMap::new();
2429 for (key, route) in &inner.client_to_module {
2430 if route.module_endpoint != endpoint {
2431 continue;
2432 }
2433 holdouts.total_routes += 1;
2434 let held = route.flow.drain_in_flight();
2435 if held == 0 {
2436 continue;
2437 }
2438 holdouts.requests += held;
2439 holdouts.routes += 1;
2440 *by_connection.entry(key.connection_id.get()).or_default() += held;
2441 holdouts.held.extend(
2442 route
2443 .flow
2444 .drain_held_corrs()
2445 .into_iter()
2446 .map(|corr| (route.module_channel, corr)),
2447 );
2448 }
2449 holdouts.held.sort_unstable();
2450 holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
2451 let mut connections = by_connection.into_iter().collect::<Vec<_>>();
2452 connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
2453 connections.truncate(3);
2454 holdouts.top_connections = connections;
2455 Ok(holdouts)
2456 }
2457
2458 pub(crate) fn endpoint_in_flight_count(
2459 &self,
2460 endpoint: ModuleEndpointId,
2461 ) -> Result<usize, ForwardingError> {
2462 let inner = self.read_inner()?;
2463 Ok(inner
2464 .client_to_module
2465 .values()
2466 .filter(|route| route.module_endpoint == endpoint)
2467 .map(|route| route.flow.drain_in_flight())
2468 .sum())
2469 }
2470
2471 pub(crate) fn endpoint_is_draining(
2472 &self,
2473 endpoint: ModuleEndpointId,
2474 ) -> Result<bool, ForwardingError> {
2475 Ok(self
2476 .read_inner()?
2477 .draining_endpoints
2478 .contains_key(&endpoint))
2479 }
2480
2481 pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
2482 let inner = self.read_inner()?;
2483 Ok(inner
2484 .modules_by_id
2485 .get(module_id)
2486 .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
2487 }
2488
2489 pub(crate) fn release_module_endpoint_routes(
2490 &self,
2491 endpoint: ModuleEndpointId,
2492 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2493 let mut inner = self.write_inner()?;
2494 let routes = inner
2495 .module_to_client
2496 .iter()
2497 .filter(|(module_key, _)| module_key.endpoint == endpoint)
2498 .map(|(module_key, route)| (*module_key, route.module_epoch))
2499 .collect::<Vec<_>>();
2500 let mut released = Vec::with_capacity(routes.len());
2501 for (module_key, epoch) in routes {
2502 if let RouteRelease::Removed(target) =
2503 release_module_route_locked(&mut inner, module_key, epoch)
2504 {
2505 released.push(target);
2506 }
2507 }
2508 Ok(released)
2509 }
2510
2511 pub(crate) fn endpoint_routes(
2517 &self,
2518 endpoint: ModuleEndpointId,
2519 ) -> Result<Vec<EndpointRoute>, ForwardingError> {
2520 let inner = self.read_inner()?;
2521 Ok(endpoint_routes_locked(&inner, endpoint))
2522 }
2523
2524 pub(crate) fn route_census(
2526 &self,
2527 module_id: Option<&str>,
2528 ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
2529 let inner = self.read_inner()?;
2530 let mut endpoints = inner
2531 .modules_by_id
2532 .iter()
2533 .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
2534 .map(|(id, module)| (id.clone(), module.endpoint))
2535 .collect::<Vec<_>>();
2536 endpoints.sort_by(|left, right| left.0.cmp(&right.0));
2537 Ok(endpoints
2538 .into_iter()
2539 .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
2540 .collect())
2541 }
2542
2543 pub(crate) fn live_roots(
2545 &self,
2546 module_id: &str,
2547 ) -> Result<ModuleControlResponseToModule, ForwardingError> {
2548 let inner = self.read_inner()?;
2549 let endpoint = inner
2550 .modules_by_id
2551 .get(module_id)
2552 .map(|module| module.endpoint);
2553 let mut roots = BTreeMap::new();
2554 let mut unknown_root_bindings = 0;
2555 let mut total_bindings = 0;
2556 if let Some(endpoint) = endpoint {
2557 for binding in inner
2558 .module_to_client
2559 .values()
2560 .filter(|binding| binding.module_endpoint == endpoint)
2561 {
2562 total_bindings += 1;
2563 if let Some(root) = &binding.project_root {
2564 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2565 entry.0 += 1;
2566 } else {
2567 unknown_root_bindings += 1;
2568 }
2569 }
2570 for pending in inner
2571 .pending_relays
2572 .values()
2573 .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
2574 {
2575 total_bindings += 1;
2576 if let Some(root) = &pending.reservation.project_root {
2577 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2578 entry.1 += 1;
2579 } else {
2580 unknown_root_bindings += 1;
2581 }
2582 }
2583 }
2584 Ok(ModuleControlResponseToModule::LiveRoots {
2585 roots: roots
2586 .into_iter()
2587 .map(|(project_root, (bound, pending))| LiveRoot {
2588 project_root,
2589 bound,
2590 pending,
2591 })
2592 .collect(),
2593 unknown_root_bindings,
2594 total_bindings,
2595 })
2596 }
2597
2598 pub(crate) fn connection_has_client_routes(
2604 &self,
2605 connection_id: ConnectionId,
2606 ) -> Result<bool, ForwardingError> {
2607 let inner = self.read_inner()?;
2608 Ok(connection_has_client_routes_locked(&inner, connection_id))
2609 }
2610
2611 pub(crate) fn cleanup_connection(
2612 &self,
2613 connection_id: ConnectionId,
2614 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2615 self.cleanup_connection_counted(connection_id)
2616 .map(|cleanup| cleanup.released)
2617 }
2618
2619 pub(crate) fn cleanup_connection_counted(
2624 &self,
2625 connection_id: ConnectionId,
2626 ) -> Result<ConnectionCleanup, ForwardingError> {
2627 let mut inner = self.write_inner()?;
2628 inner.closing_connections.insert(connection_id);
2629 let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
2630 remove_module_connection_locked(&mut inner, endpoint)
2631 } else {
2632 ConnectionCleanup {
2633 released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
2634 abandoned_relays: 0,
2635 }
2636 };
2637 inner.closing_connections.remove(&connection_id);
2646 Ok(cleanup)
2647 }
2648
2649 fn cleanup_client_connection_locked(
2650 inner: &mut ForwardingInner,
2651 connection_id: ConnectionId,
2652 ) -> Vec<GoodbyeTarget> {
2653 let routes = inner
2654 .client_to_module
2655 .iter()
2656 .filter(|(key, _)| key.connection_id == connection_id)
2657 .map(|(key, route)| (*key, route.client_epoch))
2658 .collect::<Vec<_>>();
2659 let mut released = Vec::with_capacity(routes.len());
2660 for (client_key, epoch) in routes {
2661 if let RouteRelease::Removed(target) =
2662 release_client_route_locked(inner, client_key, epoch)
2663 {
2664 released.push(target);
2665 }
2666 }
2667
2668 let pending_keys = inner
2669 .pending_relays
2670 .iter()
2671 .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
2672 .map(|(key, _)| *key)
2673 .collect::<Vec<_>>();
2674 for key in pending_keys {
2675 let Some(pending) = inner.pending_relays.remove(&key) else {
2676 continue;
2677 };
2678 release_reserved_route_locked(
2679 inner,
2680 pending.reservation.client_key,
2681 pending.reservation.module_key,
2682 );
2683 if pending.relay_enqueued {
2684 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2685 released.push(target);
2686 }
2687 } else {
2688 inner.relays_released_before_enqueue.insert(key);
2692 }
2693 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
2694 "client connection closed during route.bind relay".to_string(),
2695 ));
2696 }
2697
2698 let orphaned = inner
2699 .reserved_client
2700 .iter()
2701 .filter(|(key, _)| key.connection_id == connection_id)
2702 .map(|(client, module)| (*client, *module))
2703 .collect::<Vec<_>>();
2704 for (client_key, module_key) in orphaned {
2705 release_reserved_route_locked(inner, client_key, module_key);
2706 }
2707 inner.next_client_channel.remove(&connection_id);
2708 inner
2709 .client_slot_epochs
2710 .retain(|key, _| key.connection_id != connection_id);
2711 inner
2712 .last_published_epoch
2713 .retain(|key, _| key.connection_id != connection_id);
2714 inner
2715 .status
2716 .retain(|(key, _), _| key.connection_id != connection_id);
2717
2718 released
2719 }
2720
2721 pub(crate) fn escalate_client_delivery_failure(
2730 &self,
2731 connection_id: ConnectionId,
2732 channel: u16,
2733 expected_epoch: u32,
2734 reason: CloseReason,
2735 undelivered: UndeliveredFrame<'_>,
2736 ) -> Result<bool, ForwardingError> {
2737 let principals = {
2738 let mut inner = self.write_inner()?;
2739 let key = ClientRouteKey {
2740 connection_id,
2741 channel,
2742 };
2743 if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
2744 None
2745 } else {
2746 inner.closing_connections.insert(connection_id);
2747 Some(connection_principals_locked(&inner, connection_id))
2748 }
2749 };
2750 let Some(principals) = principals else {
2751 return Ok(false);
2752 };
2753 let backlog = undelivered.sink.backlog();
2754 let close_reason = reason.to_string();
2755 if self.request_connection_close(connection_id, reason) {
2756 warn!(
2757 connection_id = connection_id.get(),
2758 principals = %principals,
2759 module_id = undelivered.module_id.unwrap_or("unknown"),
2760 client_channel = channel,
2761 queued_bytes = backlog.queued_bytes,
2762 queued_frames = backlog.queued_frames,
2763 oldest_queued_ms = backlog
2764 .oldest_age
2765 .map(|age| age.as_millis() as u64)
2766 .unwrap_or(0),
2767 close_reason = %close_reason,
2768 "closing client connection: its egress queue could not take a frame"
2769 );
2770 }
2771 Ok(true)
2772 }
2773
2774 pub(crate) fn publish_scope_changes(
2788 &self,
2789 changes: &[ScopeTagChange],
2790 ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
2791 let mut inner = self.write_inner()?;
2792 let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
2793 for change in changes {
2794 let key = (change.owner.clone(), change.scope_ref.clone());
2795 match change.after {
2796 Some(tag) => {
2797 inner.scope_tags.insert(key, tag);
2798 }
2799 None => {
2800 inner.scope_tags.remove(&key);
2801 }
2802 }
2803 by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
2804 }
2805 let mut selected = Vec::new();
2806 for (client_key, route) in &inner.client_to_module {
2807 let Some(scope) = &route.scope else {
2808 continue;
2809 };
2810 let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
2811 else {
2812 continue;
2813 };
2814 if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
2815 continue;
2816 }
2817 let reason = match &change.drain {
2818 ScopeDrain::Nothing => continue,
2819 ScopeDrain::All(reason) => *reason,
2820 ScopeDrain::Carriers(narrowed) => {
2821 let owner = Principal::Reserved {
2822 module_id: scope.owner.clone(),
2823 };
2824 let hit = route.principal != owner
2825 && narrowed.iter().any(|(principal, allowed)| {
2826 *principal == route.principal
2827 && allowed
2828 .as_ref()
2829 .is_none_or(|targets| !targets.contains(&route.module_id))
2830 });
2831 if !hit {
2832 continue;
2833 }
2834 RouteCloseReason::ScopeCarrierRemoved
2835 }
2836 };
2837 selected.push((*client_key, route.client_epoch, reason));
2838 }
2839 let mut drained = Vec::new();
2840 for (client_key, client_epoch, reason) in selected {
2841 let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
2842 continue;
2843 };
2844 let release = release_client_route_locked(&mut inner, client_key, client_epoch);
2845 self.record_route_release(&release);
2846 if let RouteRelease::Removed(module) = release {
2847 drained.push(ScopeDrainedRoute {
2848 reason,
2849 scope: route
2850 .scope
2851 .clone()
2852 .expect("selected scope drain has a bound scope"),
2853 module_id: route.module_id.clone(),
2854 client: GoodbyeTarget {
2855 connection_id: route.client_connection_id,
2856 sink: route.client_sink.clone(),
2857 negotiated_ver: route.client_negotiated_ver,
2858 channel: route.client_channel,
2859 epoch: route.client_epoch,
2860 kind: GoodbyeTargetKind::Client,
2861 module_id: Some(route.module_id.clone()),
2862 },
2863 module,
2864 });
2865 }
2866 }
2867 Ok(drained)
2868 }
2869
2870 #[cfg(test)]
2872 pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
2873 self.read_inner()
2874 .ok()?
2875 .scope_tags
2876 .get(&(owner.to_string(), scope_ref.to_string()))
2877 .copied()
2878 }
2879
2880 fn record_route_release(&self, release: &RouteRelease) {
2881 match release {
2882 RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
2883 RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
2884 RouteRelease::Absent => {}
2885 }
2886 }
2887
2888 fn read_inner(&self) -> Result<RwLockReadGuard<'_, ForwardingInner>, ForwardingError> {
2889 self.inner.read().map_err(|_| ForwardingError::Poisoned)
2890 }
2891
2892 fn write_inner(&self) -> Result<RwLockWriteGuard<'_, ForwardingInner>, ForwardingError> {
2893 self.inner.write().map_err(|_| ForwardingError::Poisoned)
2894 }
2895
2896 fn lock_close_registry(
2897 &self,
2898 ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
2899 self.close_registry
2900 .lock()
2901 .unwrap_or_else(|poisoned| poisoned.into_inner())
2902 }
2903}
2904
2905impl ForwardingInner {
2906 fn allocate_route_slots(
2907 &mut self,
2908 connection_id: ConnectionId,
2909 endpoint: ModuleEndpointId,
2910 ) -> Result<(u16, u32, u16, u32), ForwardingError> {
2911 let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
2912 let mut client_channel = client_start;
2913 let client_channel = loop {
2914 let key = ClientRouteKey {
2915 connection_id,
2916 channel: client_channel,
2917 };
2918 let eligible = !self.client_to_module.contains_key(&key)
2919 && !self.reserved_client.contains_key(&key)
2920 && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2921 if eligible {
2922 break client_channel;
2923 }
2924 client_channel = next_channel(client_channel);
2925 if client_channel == client_start {
2926 return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
2927 }
2928 };
2929
2930 let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
2931 let mut module_channel = module_start;
2932 let module_channel = loop {
2933 let key = ModuleRouteKey {
2934 endpoint,
2935 channel: module_channel,
2936 };
2937 let eligible = !self.module_to_client.contains_key(&key)
2938 && !self.reserved_module.contains_key(&key)
2939 && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2940 if eligible {
2941 break module_channel;
2942 }
2943 module_channel = next_channel(module_channel);
2944 if module_channel == module_start {
2945 return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
2946 }
2947 };
2948
2949 let client_key = ClientRouteKey {
2950 connection_id,
2951 channel: client_channel,
2952 };
2953 let module_key = ModuleRouteKey {
2954 endpoint,
2955 channel: module_channel,
2956 };
2957 let client_epoch = self
2958 .client_slot_epochs
2959 .get(&client_key)
2960 .copied()
2961 .unwrap_or(0)
2962 + 1;
2963 let module_epoch = self
2964 .module_slot_epochs
2965 .get(&module_key)
2966 .copied()
2967 .unwrap_or(0)
2968 + 1;
2969 self.client_slot_epochs.insert(client_key, client_epoch);
2970 self.module_slot_epochs.insert(module_key, module_epoch);
2971 self.next_client_channel
2972 .insert(connection_id, next_channel(client_channel));
2973 self.next_module_channel
2974 .insert(endpoint, next_channel(module_channel));
2975 Ok((client_channel, client_epoch, module_channel, module_epoch))
2976 }
2977
2978 fn allocate_control_corr(
2979 &mut self,
2980 endpoint: ModuleEndpointId,
2981 ) -> Result<u64, ForwardingError> {
2982 let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
2983 if candidate == 0 {
2984 self.closing_connections.insert(endpoint.connection_id);
2985 return Err(ForwardingError::RelayCorrelationExhausted);
2986 }
2987 self.next_control_corr.insert(
2988 endpoint,
2989 if candidate == u64::MAX {
2990 0
2991 } else {
2992 candidate + 1
2993 },
2994 );
2995 Ok(candidate)
2996 }
2997}
2998
2999fn connection_has_client_routes_locked(
3000 inner: &ForwardingInner,
3001 connection_id: ConnectionId,
3002) -> bool {
3003 inner
3004 .client_to_module
3005 .keys()
3006 .any(|key| key.connection_id == connection_id)
3007 || inner
3008 .reserved_client
3009 .keys()
3010 .any(|key| key.connection_id == connection_id)
3011}
3012
3013fn check_module_connection_role_locked(
3014 inner: &ForwardingInner,
3015 connection_id: ConnectionId,
3016) -> Result<(), ForwardingError> {
3017 if inner.endpoint_by_connection.contains_key(&connection_id)
3018 || connection_has_client_routes_locked(inner, connection_id)
3019 {
3020 return Err(ForwardingError::ConnectionRoleConflict { connection_id });
3021 }
3022 Ok(())
3023}
3024
3025fn next_channel(channel: u16) -> u16 {
3026 let next = channel.wrapping_add(1);
3027 if next == 0 {
3028 1
3029 } else {
3030 next
3031 }
3032}
3033
3034fn endpoint_routes_locked(
3035 inner: &ForwardingInner,
3036 endpoint: ModuleEndpointId,
3037) -> Vec<EndpointRoute> {
3038 let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
3039 let draining = drain_reason.is_some();
3040 let mut routes = inner
3041 .module_to_client
3042 .iter()
3043 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3044 .map(|(_, route)| EndpointRoute {
3045 goodbye_target: GoodbyeTarget {
3046 connection_id: route.client_connection_id,
3047 sink: route.client_sink.clone(),
3048 negotiated_ver: route.client_negotiated_ver,
3049 channel: route.client_channel,
3050 epoch: route.client_epoch,
3051 kind: GoodbyeTargetKind::Client,
3052 module_id: Some(route.module_id.clone()),
3053 },
3054 principal: route.principal.clone(),
3055 bound_at: route.bound_at,
3056 draining,
3057 drain_reason,
3058 })
3059 .collect::<Vec<_>>();
3060 routes.sort_by_key(|route| {
3061 (
3062 route.goodbye_target.connection_id.get(),
3063 route.goodbye_target.channel,
3064 route.goodbye_target.epoch,
3065 )
3066 });
3067 routes
3068}
3069
3070fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
3074 let key = (scope.owner.clone(), scope.scope_ref.clone());
3075 match inner.scope_tags.get(&key) {
3076 Some(current) if *current == scope.tag => None,
3077 Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
3078 subc_protocol::error_codes::SCOPE_CHANGED,
3079 format!(
3080 "scope '{}' of {} changed while the route was being bound; re-open it",
3081 scope.scope_ref, scope.owner
3082 ),
3083 )),
3084 _ => Some(ErrorBody::new(
3085 subc_protocol::error_codes::SCOPE_ENDED,
3086 format!(
3087 "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
3088 scope.scope_ref, scope.owner, scope.tag.scope_epoch
3089 ),
3090 )),
3091 }
3092}
3093
3094fn release_reserved_route_locked(
3095 inner: &mut ForwardingInner,
3096 client_key: ClientRouteKey,
3097 module_key: ModuleRouteKey,
3098) {
3099 if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
3100 inner.reserved_client.remove(&client_key);
3101 }
3102 if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
3103 inner.reserved_module.remove(&module_key);
3104 }
3105 inner.status.retain(|(key, _), _| *key != client_key);
3106}
3107
3108fn release_client_route_locked(
3109 inner: &mut ForwardingInner,
3110 client_key: ClientRouteKey,
3111 expected_epoch: u32,
3112) -> RouteRelease {
3113 let Some(route) = inner.client_to_module.get(&client_key) else {
3114 return RouteRelease::Absent;
3115 };
3116 if route.client_epoch != expected_epoch {
3117 return RouteRelease::Stale;
3118 }
3119 let route = inner
3120 .client_to_module
3121 .remove(&client_key)
3122 .expect("route checked under the same forwarding lock");
3123 route.flow.close();
3124 if let Some(scope) = &route.scope {
3125 scope.release_bind();
3126 }
3127 inner.module_to_client.remove(&ModuleRouteKey {
3128 endpoint: route.module_endpoint,
3129 channel: route.module_channel,
3130 });
3131 inner.operator_confirms.route_closed(ModuleRouteKey {
3132 endpoint: route.module_endpoint,
3133 channel: route.module_channel,
3134 });
3135 inner.status.remove(&(client_key, expected_epoch));
3136 RouteRelease::Removed(GoodbyeTarget {
3137 connection_id: route.module_endpoint.connection_id,
3138 sink: route.module_sink.clone(),
3139 negotiated_ver: route.module_negotiated_ver,
3140 channel: route.module_channel,
3141 epoch: route.module_epoch,
3142 kind: GoodbyeTargetKind::Module,
3143 module_id: Some(route.module_id.clone()),
3144 })
3145}
3146
3147fn release_module_route_locked(
3148 inner: &mut ForwardingInner,
3149 module_key: ModuleRouteKey,
3150 expected_epoch: u32,
3151) -> RouteRelease {
3152 let Some(route) = inner.module_to_client.get(&module_key) else {
3153 return RouteRelease::Absent;
3154 };
3155 if route.module_epoch != expected_epoch {
3156 return RouteRelease::Stale;
3157 }
3158 let route = inner
3159 .module_to_client
3160 .remove(&module_key)
3161 .expect("route checked under the same forwarding lock");
3162 inner.operator_confirms.route_closed(module_key);
3163 route.flow.close();
3164 if let Some(scope) = &route.scope {
3165 scope.release_bind();
3166 }
3167 let client_key = ClientRouteKey {
3168 connection_id: route.client_connection_id,
3169 channel: route.client_channel,
3170 };
3171 inner.client_to_module.remove(&client_key);
3172 inner.status.remove(&(client_key, route.client_epoch));
3173 RouteRelease::Removed(GoodbyeTarget {
3174 connection_id: route.client_connection_id,
3175 sink: route.client_sink.clone(),
3176 negotiated_ver: route.client_negotiated_ver,
3177 channel: route.client_channel,
3178 epoch: route.client_epoch,
3179 kind: GoodbyeTargetKind::Client,
3180 module_id: Some(route.module_id.clone()),
3181 })
3182}
3183
3184fn commit_route_locked(
3185 inner: &mut ForwardingInner,
3186 pending: PendingRouteBindRelayEntry,
3187) -> Result<Option<GoodbyeTarget>, ForwardingError> {
3188 let reservation = pending.reservation;
3189 if inner
3190 .closing_connections
3191 .contains(&reservation.client_key.connection_id)
3192 {
3193 return Err(ForwardingError::ConnectionClosing {
3194 connection_id: reservation.client_key.connection_id,
3195 });
3196 }
3197 let module_id = inner
3198 .module_id_by_endpoint
3199 .get(&reservation.module_key.endpoint)
3200 .cloned()
3201 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3202 if inner
3203 .draining_endpoints
3204 .contains_key(&reservation.module_key.endpoint)
3205 {
3206 return Err(ForwardingError::ModuleReloading { module_id });
3207 }
3208 if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
3209 || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
3210 {
3211 return Err(ForwardingError::UnknownReservation {
3212 client_channel: reservation.client_key.channel,
3213 module_channel: reservation.module_key.channel,
3214 });
3215 }
3216 let module = inner
3217 .modules_by_id
3218 .get(&module_id)
3219 .filter(|module| module.endpoint == reservation.module_key.endpoint)
3220 .cloned()
3221 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3222 let binding = Arc::new(RouteBinding {
3223 client_connection_id: reservation.client_key.connection_id,
3224 client_sink: pending.client_sink,
3225 client_negotiated_ver: pending.client_negotiated_ver,
3226 client_channel: reservation.client_key.channel,
3227 client_epoch: reservation.client_epoch,
3228 module_id,
3229 module_endpoint: reservation.module_key.endpoint,
3230 module_sink: module.sink,
3231 module_negotiated_ver: module.negotiated_ver,
3232 module_channel: reservation.module_key.channel,
3233 module_epoch: reservation.module_epoch,
3234 principal: pending.principal,
3235 project_root: reservation.project_root.clone(),
3236 bound_at: Instant::now(),
3237 flow: Arc::new(ChannelFlow::new(window_for(&module.concurrency))),
3238 scope: pending.scope,
3239 });
3240 inner
3241 .client_to_module
3242 .insert(reservation.client_key, Arc::clone(&binding));
3243 inner
3244 .module_to_client
3245 .insert(reservation.module_key, Arc::clone(&binding));
3246 let previous_published = inner
3247 .last_published_epoch
3248 .insert(reservation.client_key, reservation.client_epoch);
3249
3250 if let Some(scope) = &binding.scope {
3254 scope.record_served(&binding.module_id);
3255 }
3256
3257 let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
3262 if client_writer_closed {
3263 let abandoned = abandoned_route_target(inner, &reservation);
3267 if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
3268 route.flow.close();
3269 }
3270 inner.module_to_client.remove(&reservation.module_key);
3271 inner
3272 .status
3273 .remove(&(reservation.client_key, reservation.client_epoch));
3274 match previous_published {
3275 Some(epoch) => {
3276 inner
3277 .last_published_epoch
3278 .insert(reservation.client_key, epoch);
3279 }
3280 None => {
3281 inner.last_published_epoch.remove(&reservation.client_key);
3282 }
3283 }
3284 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3285 "client egress closed during route publication".to_string(),
3286 ));
3287 return Ok(abandoned);
3288 }
3289
3290 if let Some(scope) = &binding.scope {
3291 scope.commit_presence();
3292 }
3293 let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
3294 Ok(None)
3295}
3296
3297fn module_connection_for_endpoint_locked(
3305 inner: &ForwardingInner,
3306 endpoint: ModuleEndpointId,
3307) -> Option<&ModuleConnection> {
3308 let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
3309 inner
3310 .modules_by_id
3311 .get(module_id)
3312 .filter(|module| module.endpoint == endpoint)
3313 .or_else(|| {
3314 inner
3315 .candidates_by_id
3316 .get(module_id)
3317 .filter(|module| module.endpoint == endpoint)
3318 })
3319 .or_else(|| inner.superseded_endpoints.get(&endpoint))
3320}
3321
3322fn abandoned_route_target(
3323 inner: &ForwardingInner,
3324 reservation: &RouteReservation,
3325) -> Option<GoodbyeTarget> {
3326 let module_id = inner
3327 .module_id_by_endpoint
3328 .get(&reservation.module_key.endpoint)?;
3329 let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
3330 (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
3331 connection_id: module.endpoint.connection_id,
3332 sink: module.sink.clone(),
3333 negotiated_ver: module.negotiated_ver,
3334 channel: reservation.module_key.channel,
3335 epoch: reservation.module_epoch,
3336 kind: GoodbyeTargetKind::Module,
3337 module_id: Some(module_id.clone()),
3338 })
3339}
3340
3341fn enqueue_hello_ack_locked(
3346 sink: &FrameSink,
3347 connection_id: ConnectionId,
3348 hello_ack: Option<Frame>,
3349) -> Result<(), ForwardingError> {
3350 let Some(hello_ack) = hello_ack else {
3351 return Ok(());
3352 };
3353 sink.try_send(hello_ack)
3354 .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
3355}
3356
3357fn remove_module_connection_locked(
3358 inner: &mut ForwardingInner,
3359 endpoint: ModuleEndpointId,
3360) -> ConnectionCleanup {
3361 inner.operator_confirms.module_closed(endpoint);
3364 inner.draining_endpoints.remove(&endpoint);
3365 inner.daemon_drain_results.remove(&endpoint);
3366 let module_id = inner.module_id_by_endpoint.remove(&endpoint);
3367 if let Some(module_id) = module_id.as_ref() {
3368 if inner
3369 .modules_by_id
3370 .get(module_id)
3371 .is_some_and(|module| module.endpoint == endpoint)
3372 {
3373 inner.modules_by_id.remove(module_id);
3374 }
3375 if inner
3376 .candidates_by_id
3377 .get(module_id)
3378 .is_some_and(|module| module.endpoint == endpoint)
3379 {
3380 inner.candidates_by_id.remove(module_id);
3381 }
3382 }
3383 inner.superseded_endpoints.remove(&endpoint);
3384 inner.endpoint_by_connection.remove(&endpoint.connection_id);
3385 inner.next_module_channel.remove(&endpoint);
3386 inner.next_control_corr.remove(&endpoint);
3387 inner
3388 .health_probe_tombstones
3389 .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
3390 inner
3391 .module_slot_epochs
3392 .retain(|key, _| key.endpoint != endpoint);
3393 let reserved_module_keys: Vec<ModuleRouteKey> = inner
3394 .reserved_module
3395 .keys()
3396 .filter(|module_key| module_key.endpoint == endpoint)
3397 .copied()
3398 .collect();
3399 for module_key in reserved_module_keys {
3400 if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
3401 release_reserved_route_locked(inner, client_key, module_key);
3402 }
3403 }
3404
3405 let pending_keys: Vec<_> = inner
3406 .pending_relays
3407 .keys()
3408 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3409 .copied()
3410 .collect();
3411 let pending: Vec<_> = pending_keys
3412 .into_iter()
3413 .filter_map(|key| {
3414 inner
3415 .pending_relays
3416 .remove(&key)
3417 .map(|pending| (key, pending))
3418 })
3419 .collect();
3420 let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
3421 for (key, pending) in pending {
3422 if !pending.relay_enqueued {
3426 inner.relays_released_before_enqueue.insert(key);
3427 }
3428 let module_label = module_id.as_deref().unwrap_or("unknown");
3429 let _ = pending
3430 .sender
3431 .send(RouteBindRelayOutcome::ModuleGone(format!(
3432 "module '{module_label}' connection closed during route.bind relay"
3433 )));
3434 }
3435
3436 let pending_control_keys: Vec<_> = inner
3437 .pending_control_rpcs
3438 .keys()
3439 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3440 .copied()
3441 .collect();
3442 let pending_control: Vec<_> = pending_control_keys
3443 .into_iter()
3444 .filter_map(|key| inner.pending_control_rpcs.remove(&key))
3445 .collect();
3446 for pending in pending_control {
3447 let module_label = module_id.as_deref().unwrap_or("unknown");
3448 let _ = pending
3449 .sender
3450 .send(ModuleControlRpcOutcome::ModuleGone(format!(
3451 "module '{module_label}' connection closed during module-control RPC"
3452 )));
3453 }
3454
3455 let module_routes = inner
3456 .module_to_client
3457 .iter()
3458 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3459 .map(|(module_key, route)| (*module_key, route.module_epoch))
3460 .collect::<Vec<_>>();
3461 let mut released = Vec::with_capacity(module_routes.len());
3462 for (module_key, epoch) in module_routes {
3463 if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
3464 {
3465 released.push(target);
3466 }
3467 }
3468 ConnectionCleanup {
3469 released,
3470 abandoned_relays,
3471 }
3472}
3473
3474#[derive(Debug, Clone, Copy)]
3475struct RequestCredit {
3476 subscription: bool,
3477 excluded_from_drain: bool,
3478}
3479
3480#[derive(Debug, Default)]
3481struct CreditLedger {
3482 by_corr: HashMap<u64, Vec<RequestCredit>>,
3483}
3484
3485impl CreditLedger {
3486 fn acquire(&mut self, corr: u64, subscription: bool) {
3487 self.by_corr.entry(corr).or_default().push(RequestCredit {
3488 subscription,
3489 excluded_from_drain: false,
3490 });
3491 }
3492
3493 fn release(&mut self, corr: u64) -> bool {
3494 let Some(credits) = self.by_corr.get_mut(&corr) else {
3495 return false;
3496 };
3497 let released = credits.pop().is_some();
3498 if credits.is_empty() {
3499 self.by_corr.remove(&corr);
3500 }
3501 released
3502 }
3503
3504 fn capture_subscription_exclusions(&mut self) -> u32 {
3505 let mut excluded = 0u32;
3506 for credit in self.by_corr.values_mut().flatten() {
3507 if credit.subscription && !credit.excluded_from_drain {
3508 credit.excluded_from_drain = true;
3509 excluded = excluded.saturating_add(1);
3510 }
3511 }
3512 excluded
3513 }
3514
3515 #[cfg(test)]
3516 fn in_flight(&self) -> usize {
3517 self.by_corr.values().map(Vec::len).sum()
3518 }
3519
3520 fn drain_in_flight(&self) -> usize {
3521 self.by_corr
3522 .values()
3523 .flatten()
3524 .filter(|credit| !credit.excluded_from_drain)
3525 .count()
3526 }
3527
3528 fn drain_held_corrs(&self) -> Vec<u64> {
3531 let mut corrs = self
3532 .by_corr
3533 .iter()
3534 .flat_map(|(corr, credits)| {
3535 credits
3536 .iter()
3537 .filter(|credit| !credit.excluded_from_drain)
3538 .map(move |_| *corr)
3539 })
3540 .collect::<Vec<_>>();
3541 corrs.sort_unstable();
3542 corrs
3543 }
3544}
3545
3546#[derive(Debug, Default)]
3547struct ChannelFlowState {
3548 closed: bool,
3549 credits: CreditLedger,
3550}
3551
3552#[derive(Debug)]
3554pub(crate) struct ChannelFlow {
3555 sem: Semaphore,
3556 window: usize,
3557 state: Mutex<ChannelFlowState>,
3558}
3559
3560impl ChannelFlow {
3561 pub(crate) fn new(window: usize) -> Self {
3562 debug_assert!(window > 0, "flow-control window must be non-zero");
3563 Self {
3564 sem: Semaphore::new(window),
3565 window,
3566 state: Mutex::new(ChannelFlowState::default()),
3567 }
3568 }
3569
3570 #[cfg(test)]
3571 pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
3572 self.acquire_tagged(0, false).await
3573 }
3574
3575 pub(crate) async fn acquire_tagged(
3576 &self,
3577 corr: u64,
3578 subscription: bool,
3579 ) -> Result<(), ChannelFlowClosed> {
3580 let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
3581 let mut state = self
3582 .state
3583 .lock()
3584 .unwrap_or_else(|poisoned| poisoned.into_inner());
3585 if state.closed {
3586 return Err(ChannelFlowClosed);
3587 }
3588 state.credits.acquire(corr, subscription);
3589 permit.forget();
3590 Ok(())
3591 }
3592
3593 #[cfg(test)]
3594 pub(crate) fn release(&self) {
3595 self.release_corr(0);
3596 }
3597
3598 pub(crate) fn release_corr(&self, corr: u64) {
3599 let released = self
3600 .state
3601 .lock()
3602 .unwrap_or_else(|poisoned| poisoned.into_inner())
3603 .credits
3604 .release(corr);
3605 if !released {
3606 warn!(
3610 window = self.window,
3611 available = self.sem.available_permits(),
3612 "flow-control over-release ignored"
3613 );
3614 return;
3615 }
3616 if !self.sem.is_closed() {
3617 self.sem.add_permits(1);
3618 }
3619 }
3620
3621 #[cfg(test)]
3622 pub(crate) fn in_flight(&self) -> usize {
3623 self.state
3624 .lock()
3625 .unwrap_or_else(|poisoned| poisoned.into_inner())
3626 .credits
3627 .in_flight()
3628 }
3629
3630 pub(crate) fn drain_in_flight(&self) -> usize {
3631 self.state
3632 .lock()
3633 .unwrap_or_else(|poisoned| poisoned.into_inner())
3634 .credits
3635 .drain_in_flight()
3636 }
3637
3638 pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
3639 self.state
3640 .lock()
3641 .unwrap_or_else(|poisoned| poisoned.into_inner())
3642 .credits
3643 .drain_held_corrs()
3644 }
3645
3646 #[cfg(test)]
3647 pub(crate) fn available_permits(&self) -> usize {
3648 self.sem.available_permits()
3649 }
3650
3651 pub(crate) fn begin_drain(&self) -> u32 {
3652 let mut state = self
3653 .state
3654 .lock()
3655 .unwrap_or_else(|poisoned| poisoned.into_inner());
3656 state.closed = true;
3657 self.sem.close();
3658 state.credits.capture_subscription_exclusions()
3659 }
3660
3661 pub(crate) fn close(&self) {
3662 self.state
3663 .lock()
3664 .unwrap_or_else(|poisoned| poisoned.into_inner())
3665 .closed = true;
3666 self.sem.close();
3667 }
3668}
3669
3670#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3671pub(crate) struct ChannelFlowClosed;
3672
3673impl fmt::Display for ChannelFlowClosed {
3674 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3675 write!(f, "flow-control window closed")
3676 }
3677}
3678
3679impl Error for ChannelFlowClosed {}
3680
3681fn window_for(concurrency: &Concurrency) -> usize {
3682 match concurrency {
3683 Concurrency::Serial => 1,
3684 Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
3685 Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
3686 }
3687}
3688
3689#[derive(Debug, Clone, PartialEq, Eq)]
3690pub enum ForwardingError {
3691 ConnectionRoleConflict {
3694 connection_id: ConnectionId,
3695 },
3696 NoModuleConnection,
3697 ModuleReloading {
3698 module_id: String,
3699 },
3700 StaleModuleEndpoint,
3701 UnknownReservation {
3702 client_channel: u16,
3703 module_channel: u16,
3704 },
3705 ClientRouteChannelExhausted {
3706 connection_id: ConnectionId,
3707 },
3708 ModuleRouteChannelExhausted {
3709 endpoint: ModuleEndpointId,
3710 },
3711 RelayCorrelationExhausted,
3712 ConnectionClosing {
3713 connection_id: ConnectionId,
3714 },
3715 ClientEgressClosed {
3716 connection_id: ConnectionId,
3717 },
3718 RouteOpenBuild(String),
3719 CandidateSlotOccupied {
3721 module_id: String,
3722 },
3723 ModuleEgressUnavailable {
3726 connection_id: ConnectionId,
3727 },
3728 Poisoned,
3729}
3730
3731impl fmt::Display for ForwardingError {
3732 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3733 match self {
3734 Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
3735 Self::NoModuleConnection => write!(f, "no module connection is registered"),
3736 Self::ModuleReloading { module_id } => {
3737 write!(f, "module_id '{module_id}' is reloading")
3738 }
3739 Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
3740 Self::UnknownReservation {
3741 client_channel,
3742 module_channel,
3743 } => write!(
3744 f,
3745 "route reservation client channel {client_channel} / module channel {module_channel} was not found"
3746 ),
3747 Self::ClientRouteChannelExhausted { connection_id } => write!(
3748 f,
3749 "no client route channels are available for connection {}",
3750 connection_id.get()
3751 ),
3752 Self::ModuleRouteChannelExhausted { endpoint } => write!(
3753 f,
3754 "no module route channels are available for endpoint generation {} on connection {}",
3755 endpoint.generation,
3756 endpoint.connection_id.get()
3757 ),
3758 Self::RelayCorrelationExhausted => {
3759 write!(f, "module control correlation ids are exhausted")
3760 }
3761 Self::ConnectionClosing { connection_id } => write!(
3762 f,
3763 "connection {} is closing and cannot accept route allocation",
3764 connection_id.get()
3765 ),
3766 Self::ClientEgressClosed { connection_id } => write!(
3767 f,
3768 "client connection {} egress is closed",
3769 connection_id.get()
3770 ),
3771 Self::RouteOpenBuild(message) => {
3772 write!(f, "failed to prebuild route.open response: {message}")
3773 }
3774 Self::CandidateSlotOccupied { module_id } => write!(
3775 f,
3776 "module_id '{module_id}' already has a swap candidate registered"
3777 ),
3778 Self::ModuleEgressUnavailable { connection_id } => write!(
3779 f,
3780 "module connection {} egress is unavailable; HELLO_ACK could not be queued",
3781 connection_id.get()
3782 ),
3783 Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
3784 }
3785 }
3786}
3787
3788impl Error for ForwardingError {}
3789
3790#[cfg(test)]
3791mod tests {
3792 use std::time::Duration;
3793
3794 use super::*;
3795 use tokio::sync::mpsc;
3796
3797 #[test]
3798 fn ordinary_long_running_request_is_not_excluded_from_drain() {
3799 let mut ledger = CreditLedger::default();
3800 ledger.acquire(1, false);
3801
3802 assert_eq!(ledger.capture_subscription_exclusions(), 0);
3803 assert_eq!(ledger.drain_in_flight(), 1);
3804 }
3805
3806 #[test]
3807 fn bit_set_subscription_is_excluded_and_counted() {
3808 let mut ledger = CreditLedger::default();
3809 ledger.acquire(1, true);
3810
3811 assert_eq!(ledger.capture_subscription_exclusions(), 1);
3812 assert_eq!(ledger.drain_in_flight(), 0);
3813 }
3814
3815 #[test]
3816 fn subscription_opened_after_drain_snapshot_is_not_excluded() {
3817 let mut ledger = CreditLedger::default();
3818 ledger.acquire(1, true);
3819 assert_eq!(ledger.capture_subscription_exclusions(), 1);
3820
3821 ledger.acquire(2, true);
3822
3823 assert_eq!(ledger.drain_in_flight(), 1);
3824 }
3825
3826 #[test]
3827 fn drain_with_no_subscriptions_reports_zero_excluded() {
3828 let mut ledger = CreditLedger::default();
3829 assert_eq!(ledger.capture_subscription_exclusions(), 0);
3830 }
3831
3832 fn test_hello_ack(corr: u64) -> Frame {
3833 Frame::build(
3834 FrameType::HelloAck,
3835 Flags::new(false, Priority::Passive, false),
3836 0,
3837 0,
3838 corr,
3839 Vec::new(),
3840 )
3841 .unwrap()
3842 }
3843
3844 #[test]
3848 fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
3849 let forwarding = ForwardingTable::default();
3850
3851 let (closed_tx, closed_rx) = mpsc::channel(8);
3852 drop(closed_rx);
3853 let closed = ConnectionId::new(1);
3854 assert_eq!(
3855 forwarding.register_module_connection_acked(
3856 closed,
3857 "closed".to_string(),
3858 2,
3859 Concurrency::ModuleManaged,
3860 FrameSink::new(closed_tx),
3861 test_hello_ack(1),
3862 ),
3863 Err(ForwardingError::ModuleEgressUnavailable {
3864 connection_id: closed
3865 })
3866 );
3867
3868 let (full_tx, _full_rx) = mpsc::channel(1);
3869 let full_sink = FrameSink::new(full_tx);
3870 full_sink.try_send(test_hello_ack(99)).unwrap();
3871 let full = ConnectionId::new(2);
3872 assert_eq!(
3873 forwarding.register_module_connection_acked(
3874 full,
3875 "full".to_string(),
3876 2,
3877 Concurrency::ModuleManaged,
3878 full_sink.clone(),
3879 test_hello_ack(2),
3880 ),
3881 Err(ForwardingError::ModuleEgressUnavailable {
3882 connection_id: full
3883 })
3884 );
3885 assert_eq!(
3886 forwarding.register_candidate_module_connection_acked(
3887 full,
3888 "full".to_string(),
3889 2,
3890 Concurrency::ModuleManaged,
3891 full_sink,
3892 test_hello_ack(3),
3893 ),
3894 Err(ForwardingError::ModuleEgressUnavailable {
3895 connection_id: full
3896 })
3897 );
3898
3899 for (connection, module_id) in [(closed, "closed"), (full, "full")] {
3900 assert_eq!(
3901 forwarding
3902 .module_endpoint_for_connection(connection)
3903 .unwrap(),
3904 None
3905 );
3906 let (client_tx, _client_rx) = mpsc::channel(8);
3907 assert_eq!(
3908 forwarding
3909 .begin_route_bind_relay_for_test(
3910 ConnectionId::new(50),
3911 FrameSink::new(client_tx),
3912 1,
3913 module_id,
3914 )
3915 .err(),
3916 Some(ForwardingError::NoModuleConnection)
3917 );
3918 }
3919 assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
3920 }
3921
3922 #[test]
3925 fn acked_registration_queues_the_hello_ack_first() {
3926 let forwarding = ForwardingTable::default();
3927 let (active_tx, mut active_rx) = mpsc::channel(8);
3928 forwarding
3929 .register_module_connection_acked(
3930 ConnectionId::new(1),
3931 "acked".to_string(),
3932 2,
3933 Concurrency::ModuleManaged,
3934 FrameSink::new(active_tx),
3935 test_hello_ack(11),
3936 )
3937 .unwrap();
3938 let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
3939 forwarding
3940 .register_candidate_module_connection_acked(
3941 ConnectionId::new(2),
3942 "acked".to_string(),
3943 2,
3944 Concurrency::ModuleManaged,
3945 FrameSink::new(candidate_tx),
3946 test_hello_ack(12),
3947 )
3948 .unwrap();
3949
3950 let active_first = active_rx.try_recv().unwrap().frame;
3951 assert_eq!(active_first.header.ty, FrameType::HelloAck);
3952 assert_eq!(active_first.header.corr, 11);
3953 let candidate_first = candidate_rx.try_recv().unwrap().frame;
3954 assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
3955 assert_eq!(candidate_first.header.corr, 12);
3956 }
3957
3958 #[test]
3959 fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
3960 let forwarding = ForwardingTable::default();
3961 let module_connection = ConnectionId::new(10);
3962 let exhausted_client = ConnectionId::new(20);
3963 let second_client = ConnectionId::new(30);
3964 let (module_tx, _module_rx) = mpsc::channel(1);
3965 let endpoint = forwarding
3966 .register_module_connection(
3967 module_connection,
3968 "route-limit-provider".to_string(),
3969 1,
3970 Concurrency::ModuleManaged,
3971 FrameSink::new(module_tx),
3972 )
3973 .unwrap();
3974
3975 {
3976 let mut inner = forwarding.inner.write().unwrap();
3977 for channel in 1..=u16::MAX {
3978 inner.reserved_client.insert(
3979 ClientRouteKey {
3980 connection_id: exhausted_client,
3981 channel,
3982 },
3983 ModuleRouteKey {
3984 endpoint,
3985 channel: 1,
3986 },
3987 );
3988 }
3989 }
3990
3991 let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
3992 let err = forwarding
3993 .begin_route_bind_relay_for_test(
3994 exhausted_client,
3995 FrameSink::new(exhausted_tx),
3996 1,
3997 "route-limit-provider",
3998 )
3999 .unwrap_err();
4000 assert!(matches!(
4001 err,
4002 ForwardingError::ClientRouteChannelExhausted { connection_id }
4003 if connection_id == exhausted_client
4004 ));
4005
4006 let (second_tx, _second_rx) = mpsc::channel(1);
4007 let pending = forwarding
4008 .begin_route_bind_relay_for_test(
4009 second_client,
4010 FrameSink::new(second_tx),
4011 2,
4012 "route-limit-provider",
4013 )
4014 .unwrap();
4015 assert_eq!(pending.client_channel, 1);
4016 }
4017
4018 #[test]
4019 fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
4020 let forwarding = ForwardingTable::default();
4021 let module_connection = ConnectionId::new(40);
4022 let client = ConnectionId::new(50);
4023 let (module_tx, _module_rx) = mpsc::channel(1);
4024 forwarding
4025 .register_module_connection(
4026 module_connection,
4027 "slot-reuse-provider".to_string(),
4028 1,
4029 Concurrency::ModuleManaged,
4030 FrameSink::new(module_tx),
4031 )
4032 .unwrap();
4033
4034 let (client_tx, _client_rx) = mpsc::channel(1);
4035 let client_sink = FrameSink::new(client_tx);
4036 let mut wrapped_channel = None;
4037 for index in 0..=usize::from(u16::MAX) {
4038 let pending = forwarding
4039 .begin_route_bind_relay_for_test(
4040 client,
4041 client_sink.clone(),
4042 index as u64 + 1,
4043 "slot-reuse-provider",
4044 )
4045 .unwrap();
4046 if index == usize::from(u16::MAX) {
4047 wrapped_channel = Some(pending.module_channel);
4048 }
4049 forwarding
4050 .abort_pending_relay(
4051 pending.endpoint,
4052 pending.corr,
4053 RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
4054 )
4055 .unwrap();
4056 }
4057
4058 assert_eq!(wrapped_channel, Some(1));
4059 }
4060
4061 #[test]
4062 fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
4063 let forwarding = ForwardingTable::default();
4064 let client = ConnectionId::new(60);
4065 forwarding
4066 .inner
4067 .write()
4068 .unwrap()
4069 .next_client_channel
4070 .insert(client, 41);
4071
4072 let released = forwarding.cleanup_connection(client).unwrap();
4073
4074 assert!(released.is_empty());
4075 assert!(!forwarding
4076 .inner
4077 .read()
4078 .unwrap()
4079 .next_client_channel
4080 .contains_key(&client));
4081 }
4082
4083 #[test]
4084 fn stale_module_cleanup_preserves_fast_reconnect_successor() {
4085 let forwarding = ForwardingTable::default();
4086 let module_id = "fast-reconnect-provider";
4087 let first_connection = ConnectionId::new(70);
4088 let second_connection = ConnectionId::new(80);
4089 let (first_tx, _first_rx) = mpsc::channel(1);
4090 let first_endpoint = forwarding
4091 .register_module_connection(
4092 first_connection,
4093 module_id.to_string(),
4094 1,
4095 Concurrency::ModuleManaged,
4096 FrameSink::new(first_tx),
4097 )
4098 .unwrap();
4099 let (second_tx, _second_rx) = mpsc::channel(1);
4100 let second_endpoint = forwarding
4101 .register_module_connection(
4102 second_connection,
4103 module_id.to_string(),
4104 1,
4105 Concurrency::ModuleManaged,
4106 FrameSink::new(second_tx),
4107 )
4108 .unwrap();
4109 assert_ne!(first_endpoint, second_endpoint);
4110
4111 let released = forwarding.cleanup_connection(first_connection).unwrap();
4112
4113 assert!(released.is_empty());
4114 assert_eq!(
4115 forwarding
4116 .inner
4117 .read()
4118 .unwrap()
4119 .modules_by_id
4120 .get(module_id)
4121 .map(|module| module.endpoint),
4122 Some(second_endpoint)
4123 );
4124 assert!(forwarding.has_live_module_connection(module_id).unwrap());
4125 let control_rpc = forwarding
4126 .begin_module_control_rpc_for(
4127 module_id,
4128 "health.check",
4129 Instant::now() + Duration::from_secs(1),
4130 )
4131 .unwrap();
4132 assert_eq!(control_rpc.endpoint, second_endpoint);
4133 }
4134
4135 fn route_fixture(
4136 module_id: &str,
4137 ) -> (
4138 ForwardingTable,
4139 ConnectionId,
4140 ModuleEndpointId,
4141 ConnectionId,
4142 FrameSink,
4143 mpsc::Receiver<crate::router::OutboundFrame>,
4144 ) {
4145 let forwarding = ForwardingTable::default();
4146 let module_connection = ConnectionId::new(100);
4147 let client_connection = ConnectionId::new(200);
4148 let (module_tx, _module_rx) = mpsc::channel(8);
4149 let endpoint = forwarding
4150 .register_module_connection(
4151 module_connection,
4152 module_id.to_string(),
4153 2,
4154 Concurrency::ModuleManaged,
4155 FrameSink::new(module_tx),
4156 )
4157 .unwrap();
4158 let (client_tx, client_rx) = mpsc::channel(8);
4159 (
4160 forwarding,
4161 module_connection,
4162 endpoint,
4163 client_connection,
4164 FrameSink::new(client_tx),
4165 client_rx,
4166 )
4167 }
4168
4169 #[test]
4170 #[cfg(unix)]
4171 fn daemon_drain_gates_current_and_racing_provider_registrations() {
4172 let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
4173 assert_eq!(
4174 forwarding.begin_daemon_drain().unwrap(),
4175 [("provider".to_string(), endpoint)]
4176 );
4177 assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
4178 assert!(matches!(
4179 forwarding.register_module_connection(
4180 ConnectionId::new(300),
4181 "late-provider".into(),
4182 2,
4183 Concurrency::ModuleManaged,
4184 sink,
4185 ),
4186 Err(ForwardingError::ConnectionClosing { .. })
4187 ));
4188 }
4189
4190 #[test]
4191 #[cfg(unix)]
4192 fn daemon_drain_results_are_endpoint_fenced_and_removed_on_cleanup() {
4193 let (forwarding, connection, endpoint, _, _, _) = route_fixture("provider");
4194 assert_eq!(
4195 forwarding
4196 .daemon_drain_result_for_connection(connection)
4197 .unwrap(),
4198 None
4199 );
4200 forwarding
4201 .record_daemon_drain_result(endpoint, true)
4202 .unwrap();
4203 forwarding.begin_daemon_drain().unwrap();
4204 assert_eq!(
4205 forwarding
4206 .daemon_drain_result_for_connection(connection)
4207 .unwrap(),
4208 Some(false)
4209 );
4210 let replacement = ModuleEndpointId {
4211 generation: endpoint.generation + 1,
4212 ..endpoint
4213 };
4214 forwarding
4215 .record_daemon_drain_result(replacement, true)
4216 .unwrap();
4217 assert_eq!(
4218 forwarding
4219 .daemon_drain_result_for_connection(connection)
4220 .unwrap(),
4221 Some(false)
4222 );
4223 forwarding
4224 .record_daemon_drain_result(endpoint, true)
4225 .unwrap();
4226 assert_eq!(
4227 forwarding
4228 .daemon_drain_result_for_connection(connection)
4229 .unwrap(),
4230 Some(true)
4231 );
4232 assert_eq!(
4233 forwarding
4234 .daemon_drain_result_for_connection(ConnectionId::new(300))
4235 .unwrap(),
4236 Some(false)
4237 );
4238 forwarding.cleanup_connection(connection).unwrap();
4239 forwarding
4240 .record_daemon_drain_result(endpoint, true)
4241 .unwrap();
4242 assert_eq!(
4243 forwarding
4244 .daemon_drain_result_for_connection(connection)
4245 .unwrap(),
4246 Some(false)
4247 );
4248 assert!(forwarding
4249 .read_inner()
4250 .unwrap()
4251 .daemon_drain_results
4252 .is_empty());
4253 }
4254
4255 #[test]
4256 #[cfg(unix)]
4257 fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
4258 let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
4259 route_fixture("provider");
4260 let mut pending =
4261 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
4262 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
4263 forwarding.begin_daemon_drain().unwrap();
4264 let completion = forwarding
4265 .complete_pending_relay(
4266 module_connection,
4267 pending.corr,
4268 RouteBindRelayOutcome::Accepted,
4269 )
4270 .expect("draining admission is not a fatal module error");
4271 assert!(completion.settled);
4272 let abandoned = completion
4273 .abandoned
4274 .expect("late accepted bind needs a route GOODBYE");
4275 assert_eq!(abandoned.channel, pending.module_channel);
4276 assert!(matches!(pending.receiver.try_recv().unwrap(),
4277 RouteBindRelayOutcome::Rejected(error) if error.code == "module_reloading"));
4278 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4279 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4280 assert!(
4281 rx.try_recv().is_err(),
4282 "no route.open may be published during drain"
4283 );
4284 assert_eq!(
4285 forwarding
4286 .module_endpoint_for_connection(module_connection)
4287 .unwrap(),
4288 Some(endpoint)
4289 );
4290 assert!(
4291 !forwarding
4292 .complete_pending_relay(
4293 module_connection,
4294 pending.corr,
4295 RouteBindRelayOutcome::Accepted,
4296 )
4297 .unwrap()
4298 .settled
4299 );
4300 }
4301
4302 fn test_ping(corr: u64) -> Frame {
4303 Frame::build(
4304 FrameType::Ping,
4305 Flags::new(false, Priority::Passive, false),
4306 0,
4307 0,
4308 corr,
4309 Vec::new(),
4310 )
4311 .unwrap()
4312 }
4313
4314 fn begin_test_route(
4315 forwarding: &ForwardingTable,
4316 client_connection: ConnectionId,
4317 client_sink: FrameSink,
4318 corr: u64,
4319 module_id: &str,
4320 ) -> PendingRouteBindRelay {
4321 forwarding
4322 .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
4323 .unwrap()
4324 }
4325
4326 #[test]
4327 fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
4328 for candidate in [false, true] {
4329 for register_first in [false, true] {
4330 let (forwarding, module_connection, _, client_connection, sink, _rx) =
4331 route_fixture("target");
4332 let register = || {
4333 if candidate {
4334 forwarding.register_candidate_module_connection(
4335 client_connection,
4336 "source".into(),
4337 2,
4338 Concurrency::ModuleManaged,
4339 sink.clone(),
4340 )
4341 } else {
4342 forwarding.register_module_connection(
4343 client_connection,
4344 "source".into(),
4345 2,
4346 Concurrency::ModuleManaged,
4347 sink.clone(),
4348 )
4349 }
4350 };
4351 if register_first {
4352 register().unwrap();
4353 assert!(
4354 forwarding
4355 .begin_route_bind_relay_for_test(
4356 client_connection,
4357 sink.clone(),
4358 1,
4359 "target",
4360 )
4361 .is_err(),
4362 "a deferred route.open cannot reserve after HELLO"
4363 );
4364 assert!(forwarding
4365 .cleanup_connection(client_connection)
4366 .unwrap()
4367 .is_empty());
4368 } else {
4369 let pending =
4370 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
4371 assert!(
4372 register().is_err(),
4373 "HELLO cannot register after a route reservation"
4374 );
4375 forwarding
4376 .complete_pending_relay(
4377 module_connection,
4378 pending.corr,
4379 RouteBindRelayOutcome::Accepted,
4380 )
4381 .unwrap();
4382 assert_eq!(
4383 forwarding
4384 .cleanup_connection(client_connection)
4385 .unwrap()
4386 .len(),
4387 1
4388 );
4389 }
4390 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4391 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4392 }
4393 }
4394 }
4395
4396 #[tokio::test]
4402 async fn pending_route_open_completes_behind_queued_data_frames() {
4403 assert_eq!(
4404 crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
4405 8,
4406 "the per-connection pending route.open limit is its own constant"
4407 );
4408 let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
4409 route_fixture("open-behind-data");
4410 let (sink, mut client_rx) = crate::server::connection_egress();
4411 const DATA_FRAMES: usize = 1_000;
4412 let data = |corr: u64| {
4413 Frame::build(
4414 FrameType::StreamData,
4415 Flags::new(false, Priority::Interactive, false),
4416 9,
4417 1,
4418 corr,
4419 vec![b'x'; 200],
4420 )
4421 .unwrap()
4422 };
4423 for corr in 0..DATA_FRAMES as u64 {
4424 sink.try_send(data(corr)).unwrap();
4425 }
4426 let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
4427 assert_eq!(sink.backlog().queued_bytes, data_bytes);
4428
4429 let pending = tokio::time::timeout(
4430 Duration::from_secs(5),
4431 forwarding.begin_route_bind_relay_for(
4432 client,
4433 sink.clone(),
4434 subc_protocol::PROTOCOL_VERSION,
4435 4_242,
4436 "open-behind-data",
4437 Principal::Direct,
4438 None,
4439 None,
4440 Instant::now() + Duration::from_secs(60),
4441 ),
4442 )
4443 .await
4444 .expect("reserving the route.open slot must not wait behind data frames")
4445 .unwrap();
4446 forwarding
4447 .complete_pending_relay(
4448 module_connection,
4449 pending.corr,
4450 RouteBindRelayOutcome::Accepted,
4451 )
4452 .unwrap();
4453
4454 let backlog = sink.backlog();
4455 assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
4456 assert!(
4457 backlog.queued_bytes > data_bytes,
4458 "the route.open response must be counted in queued bytes: {backlog:?}"
4459 );
4460 for corr in 0..DATA_FRAMES as u64 {
4461 assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
4462 }
4463 let open = client_rx.recv().await.unwrap();
4464 assert_eq!(open.header.corr, 4_242);
4465 assert_eq!(open.header.ty, FrameType::Response);
4466 drop(open);
4467 assert_eq!(sink.backlog().queued_bytes, 0);
4468 assert_eq!(sink.backlog().queued_frames, 0);
4469 }
4470
4471 #[tokio::test]
4476 async fn drain_holdouts_count_held_requests_and_name_the_connection() {
4477 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4478 route_fixture("holdouts");
4479 let mut bound = |corr| {
4480 let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
4481 forwarding
4482 .complete_pending_relay(
4483 module_connection,
4484 route.corr,
4485 RouteBindRelayOutcome::Accepted,
4486 )
4487 .unwrap();
4488 client_rx.try_recv().unwrap();
4489 match forwarding
4490 .lookup_data_route(client, route.client_channel, route.client_epoch)
4491 .unwrap()
4492 {
4493 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4494 other => panic!("expected live route, got {other:?}"),
4495 }
4496 };
4497 let holding = bound(61);
4498 let _idle = bound(62);
4499 holding.flow.acquire_tagged(7, false).await.unwrap();
4500 holding.flow.acquire_tagged(2, false).await.unwrap();
4501 holding.flow.acquire_tagged(3, true).await.unwrap();
4502 forwarding
4503 .begin_module_drain("holdouts", RouteCloseReason::Restart)
4504 .unwrap();
4505
4506 let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
4507 assert_eq!(
4508 holdouts,
4509 DrainHoldouts {
4510 requests: 2,
4511 routes: 1,
4512 total_routes: 2,
4513 top_connections: vec![(client.get(), 2)],
4514 held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
4517 }
4518 );
4519 }
4520
4521 #[test]
4522 fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
4523 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4524 route_fixture("census");
4525 let pending = begin_test_route(&forwarding, client, sink, 1, "census");
4526 forwarding
4527 .complete_pending_relay(
4528 module_connection,
4529 pending.corr,
4530 RouteBindRelayOutcome::Accepted,
4531 )
4532 .unwrap();
4533
4534 let routes = forwarding.endpoint_routes(endpoint).unwrap();
4535 assert_eq!(routes.len(), 1);
4536 assert!(matches!(routes[0].principal, Principal::Direct));
4537 assert_eq!(routes[0].goodbye_target.connection_id, client);
4538 assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
4539 assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
4540 assert!(!routes[0].draining);
4541
4542 forwarding
4543 .begin_module_drain("census", RouteCloseReason::Restart)
4544 .unwrap();
4545 let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
4546 assert_eq!(draining_routes.len(), 1);
4547 assert!(draining_routes[0].draining);
4548 }
4549
4550 #[test]
4551 fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
4552 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
4553 let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
4554 assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
4555 forwarding
4556 .abort_pending_relay(
4557 first.endpoint,
4558 first.corr,
4559 RouteBindRelayOutcome::ModuleGone("abort".into()),
4560 )
4561 .unwrap();
4562 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4563 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4564
4565 let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
4566 assert_eq!(second.client_channel, first.client_channel);
4567 assert_eq!(second.module_channel, first.module_channel);
4568 assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
4569 }
4570
4571 #[test]
4572 fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
4573 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4574 route_fixture("epoch-release");
4575 let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
4576 forwarding
4577 .complete_pending_relay(
4578 module_connection,
4579 first.corr,
4580 RouteBindRelayOutcome::Accepted,
4581 )
4582 .unwrap();
4583 assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
4584 assert!(matches!(
4585 forwarding
4586 .release_client_route(client, first.client_channel, first.client_epoch)
4587 .unwrap(),
4588 RouteRelease::Removed(_)
4589 ));
4590 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4591 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4592
4593 let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
4594 forwarding
4595 .complete_pending_relay(
4596 module_connection,
4597 second.corr,
4598 RouteBindRelayOutcome::Accepted,
4599 )
4600 .unwrap();
4601 assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
4602 assert!(matches!(
4603 forwarding
4604 .release_client_route(client, second.client_channel, first.client_epoch)
4605 .unwrap(),
4606 RouteRelease::Stale
4607 ));
4608 assert!(!forwarding
4609 .cache_status(
4610 endpoint,
4611 second.module_channel,
4612 first.module_epoch,
4613 "stale".into(),
4614 )
4615 .unwrap());
4616 assert!(forwarding
4617 .cache_status(
4618 endpoint,
4619 second.module_channel,
4620 second.module_epoch,
4621 "current".into(),
4622 )
4623 .unwrap());
4624 match forwarding
4625 .route_poll_snapshot(client, second.client_channel, second.client_epoch)
4626 .unwrap()
4627 {
4628 RoutePollSnapshot::Bound { status, .. } => {
4629 assert_eq!(status.as_deref(), Some("current"));
4630 }
4631 RoutePollSnapshot::Absent => panic!("successor binding was removed"),
4632 }
4633 let counters = forwarding.counters().snapshot();
4634 assert_eq!(counters["route_released_epoch_fenced"], 1);
4635 assert_eq!(counters["route_release_stale_skipped"], 1);
4636 }
4637
4638 #[test]
4639 fn max_epoch_reservation_retires_only_that_slot() {
4640 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
4641 forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
4642 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
4643 let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
4644 assert_eq!(
4645 (final_use.client_channel, final_use.client_epoch),
4646 (7, u32::MAX)
4647 );
4648 assert_eq!(
4649 (final_use.module_channel, final_use.module_epoch),
4650 (9, u32::MAX)
4651 );
4652 forwarding
4653 .abort_pending_relay(
4654 endpoint,
4655 final_use.corr,
4656 RouteBindRelayOutcome::ModuleGone("abort".into()),
4657 )
4658 .unwrap();
4659 forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
4660 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
4661 let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
4662 assert_ne!(next.client_channel, 7);
4663 assert_ne!(next.module_channel, 9);
4664 assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
4665 }
4666
4667 #[test]
4668 fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
4669 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4670 route_fixture("corr-shared");
4671 let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
4672 assert_eq!(bind.corr, 1);
4673 forwarding
4674 .abort_pending_relay(
4675 endpoint,
4676 bind.corr,
4677 RouteBindRelayOutcome::ModuleGone("abort".into()),
4678 )
4679 .unwrap();
4680 let rpc = forwarding
4681 .begin_module_control_rpc_for(
4682 "corr-shared",
4683 "health.check",
4684 Instant::now() - Duration::from_millis(1),
4685 )
4686 .unwrap();
4687 assert_eq!(rpc.corr, 2);
4688 assert_eq!(
4689 forwarding
4690 .complete_module_control_rpc(
4691 module_connection,
4692 rpc.corr,
4693 Some("health.check"),
4694 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
4695 status: subc_protocol::session::HealthStatus::Ok,
4696 detail: None,
4697 metrics: None,
4698 }),
4699 )
4700 .unwrap(),
4701 ModuleControlRpcCompletion::Settled
4702 );
4703 assert!(matches!(
4704 rpc.receiver.blocking_recv().unwrap(),
4705 ModuleControlRpcOutcome::DeadlineElapsed
4706 ));
4707 }
4708
4709 #[tokio::test(start_paused = true)]
4710 async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
4711 let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
4712 let probe_started_at = Instant::now();
4713 let rpc = forwarding
4714 .begin_health_probe_rpc_for(
4715 "tombstone-ttl",
4716 "health.check",
4717 probe_started_at,
4718 probe_started_at + Duration::from_secs(5),
4719 )
4720 .unwrap();
4721 assert!(forwarding
4722 .tombstone_health_probe_rpc(endpoint, rpc.corr)
4723 .unwrap());
4724 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
4725
4726 tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
4727 tokio::task::yield_now().await;
4728
4729 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
4730 }
4731
4732 #[test]
4733 fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
4734 let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
4735 let mut close = forwarding.register_connection_close(endpoint.connection_id);
4736 forwarding.inject_control_corr(endpoint, u64::MAX);
4737 let final_rpc = forwarding
4738 .begin_module_control_rpc_for(
4739 "corr-max",
4740 "health.check",
4741 Instant::now() + Duration::from_secs(1),
4742 )
4743 .unwrap();
4744 assert_eq!(final_rpc.corr, u64::MAX);
4745 forwarding
4746 .cancel_module_control_rpc(endpoint, final_rpc.corr)
4747 .unwrap();
4748 assert!(matches!(
4749 forwarding.begin_module_control_rpc_for(
4750 "corr-max",
4751 "health.check",
4752 Instant::now() + Duration::from_secs(1),
4753 ),
4754 Err(ForwardingError::RelayCorrelationExhausted)
4755 ));
4756 assert!(close.try_recv().is_ok());
4757 }
4758
4759 #[test]
4760 fn publication_epoch_controls_delivery_failure_escalation() {
4761 fn setup_successor(
4762 commit_successor: Option<bool>,
4763 ) -> (ForwardingTable, ConnectionId, u16, u32) {
4764 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4765 route_fixture("escalation");
4766 let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
4767 forwarding
4768 .complete_pending_relay(
4769 module_connection,
4770 first.corr,
4771 RouteBindRelayOutcome::Accepted,
4772 )
4773 .unwrap();
4774 client_rx.try_recv().unwrap();
4775 assert!(matches!(
4776 forwarding
4777 .release_client_route(client, first.client_channel, first.client_epoch)
4778 .unwrap(),
4779 RouteRelease::Removed(_)
4780 ));
4781 if let Some(commit_successor) = commit_successor {
4782 forwarding.inject_client_slot_epoch(
4783 client,
4784 first.client_channel,
4785 first.client_epoch,
4786 );
4787 forwarding.inject_module_slot_epoch(
4788 endpoint,
4789 first.module_channel,
4790 first.module_epoch,
4791 );
4792 let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
4793 if commit_successor {
4794 forwarding
4795 .complete_pending_relay(
4796 module_connection,
4797 successor.corr,
4798 RouteBindRelayOutcome::Accepted,
4799 )
4800 .unwrap();
4801 client_rx.try_recv().unwrap();
4802 } else {
4803 forwarding
4804 .abort_pending_relay(
4805 endpoint,
4806 successor.corr,
4807 RouteBindRelayOutcome::ModuleGone("abort".into()),
4808 )
4809 .unwrap();
4810 }
4811 }
4812 (forwarding, client, first.client_channel, first.client_epoch)
4813 }
4814
4815 let probe_sink = FrameSink::new(mpsc::channel(1).0);
4816 let (no_successor, client, channel, epoch) = setup_successor(None);
4817 let mut close = no_successor.register_connection_close(client);
4818 assert!(no_successor
4819 .escalate_client_delivery_failure(
4820 client,
4821 channel,
4822 epoch,
4823 CloseReason::new("delivery", "failed"),
4824 UndeliveredFrame {
4825 module_id: None,
4826 sink: &probe_sink,
4827 },
4828 )
4829 .unwrap());
4830 assert!(close.try_recv().is_ok());
4831
4832 let (aborted, client, channel, epoch) = setup_successor(Some(false));
4833 let mut close = aborted.register_connection_close(client);
4834 assert!(aborted
4835 .escalate_client_delivery_failure(
4836 client,
4837 channel,
4838 epoch,
4839 CloseReason::new("delivery", "failed"),
4840 UndeliveredFrame {
4841 module_id: None,
4842 sink: &probe_sink,
4843 },
4844 )
4845 .unwrap());
4846 assert!(close.try_recv().is_ok());
4847
4848 let (published, client, channel, epoch) = setup_successor(Some(true));
4849 let mut close = published.register_connection_close(client);
4850 assert!(!published
4851 .escalate_client_delivery_failure(
4852 client,
4853 channel,
4854 epoch,
4855 CloseReason::new("delivery", "stale failure"),
4856 UndeliveredFrame {
4857 module_id: None,
4858 sink: &probe_sink,
4859 },
4860 )
4861 .unwrap());
4862 assert!(close.try_recv().is_err());
4863 }
4864
4865 #[test]
4866 fn route_concentration_separates_client_count_from_routes_per_client() {
4867 let (forwarding, module_connection, _, client, sink, _client_rx) =
4871 route_fixture("concentration");
4872 assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
4873
4874 for corr in [70_u64, 71] {
4875 let pending =
4876 begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
4877 forwarding
4878 .complete_pending_relay(
4879 module_connection,
4880 pending.corr,
4881 RouteBindRelayOutcome::Accepted,
4882 )
4883 .unwrap();
4884 }
4885
4886 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
4888 assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
4889 }
4890
4891 #[test]
4892 fn cleanup_and_accepted_resolution_have_one_lock_winner() {
4893 let (forwarding, module_connection, _, client, sink, mut client_rx) =
4894 route_fixture("cleanup-race");
4895 let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
4896 forwarding
4897 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4898 .unwrap();
4899 let released = forwarding.cleanup_connection(client).unwrap();
4900 assert_eq!(released.len(), 1);
4901 let completion = forwarding
4902 .complete_pending_relay(
4903 module_connection,
4904 pending.corr,
4905 RouteBindRelayOutcome::Accepted,
4906 )
4907 .unwrap();
4908 assert!(!completion.settled);
4909 assert!(client_rx.try_recv().is_err());
4910 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4911
4912 let (forwarding, module_connection, _, client, sink, mut client_rx) =
4913 route_fixture("accepted-race");
4914 let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
4915 forwarding
4916 .complete_pending_relay(
4917 module_connection,
4918 pending.corr,
4919 RouteBindRelayOutcome::Accepted,
4920 )
4921 .unwrap();
4922 assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
4923 let released = forwarding.cleanup_connection(client).unwrap();
4924 assert_eq!(released.len(), 1);
4925 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4926 }
4927
4928 #[test]
4929 fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
4930 let (forwarding, module_connection, _, client, sink, mut client_rx) =
4931 route_fixture("drain-gap");
4932 let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
4933 forwarding
4934 .complete_pending_relay(
4935 module_connection,
4936 live.corr,
4937 RouteBindRelayOutcome::Accepted,
4938 )
4939 .unwrap();
4940 client_rx.try_recv().unwrap();
4941 let binding = match forwarding
4942 .lookup_data_route(client, live.client_channel, live.client_epoch)
4943 .unwrap()
4944 {
4945 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4946 other => panic!("expected live route, got {other:?}"),
4947 };
4948
4949 let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
4950 forwarding
4951 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4952 .unwrap();
4953 let control_rpc = forwarding
4954 .begin_module_control_rpc_for(
4955 "drain-gap",
4956 "health.check",
4957 Instant::now() + Duration::from_secs(1),
4958 )
4959 .unwrap();
4960 let target = forwarding
4961 .begin_module_drain("drain-gap", RouteCloseReason::Reload)
4962 .unwrap()
4963 .unwrap();
4964 assert!(matches!(
4965 control_rpc.receiver.blocking_recv().unwrap(),
4966 ModuleControlRpcOutcome::ModuleGone(_)
4967 ));
4968 assert_eq!(target.abandoned_bindings.len(), 1);
4969 assert!(binding.flow.sem.is_closed());
4970 assert!(
4971 !forwarding
4972 .complete_pending_relay(
4973 module_connection,
4974 pending.corr,
4975 RouteBindRelayOutcome::Accepted,
4976 )
4977 .unwrap()
4978 .settled
4979 );
4980 assert!(matches!(
4981 forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
4982 Err(ForwardingError::ModuleReloading { .. })
4983 ));
4984 let released = forwarding
4985 .release_module_endpoint_routes(target.endpoint)
4986 .unwrap();
4987 assert_eq!(released.len(), 1);
4988 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4989 }
4990
4991 #[test]
4999 fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
5000 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5001 route_fixture("closing-client");
5002
5003 let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
5006 forwarding
5007 .complete_pending_relay(
5008 module_connection,
5009 live.corr,
5010 RouteBindRelayOutcome::Accepted,
5011 )
5012 .unwrap();
5013 client_rx.try_recv().unwrap();
5014
5015 let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
5017 forwarding
5018 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5019 .unwrap();
5020
5021 assert!(forwarding
5023 .escalate_client_delivery_failure(
5024 client,
5025 live.client_channel,
5026 live.client_epoch,
5027 CloseReason::new(
5028 "module_to_client_delivery_failed",
5029 "client egress refused a module frame",
5030 ),
5031 UndeliveredFrame {
5032 module_id: None,
5033 sink: &sink,
5034 },
5035 )
5036 .unwrap());
5037 assert!(!sink.is_closed());
5038
5039 let completion = forwarding
5040 .complete_pending_relay(
5041 module_connection,
5042 pending.corr,
5043 RouteBindRelayOutcome::Accepted,
5044 )
5045 .expect("a closing client must not turn a module's ack into an error");
5046
5047 assert!(completion.settled);
5048 let abandoned = completion
5049 .abandoned
5050 .expect("the module must be told to drop the binding it just created");
5051 assert_eq!(abandoned.connection_id, module_connection);
5052 assert_eq!(abandoned.channel, pending.module_channel);
5053 assert_eq!(abandoned.epoch, pending.module_epoch);
5054 assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
5055 assert!(matches!(
5056 pending.receiver.blocking_recv().unwrap(),
5057 RouteBindRelayOutcome::ModuleGone(_)
5058 ));
5059 assert!(client_rx.try_recv().is_err());
5062 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
5063
5064 assert!(forwarding
5067 .has_live_module_connection("closing-client")
5068 .unwrap());
5069 let cotenant = ConnectionId::new(201);
5070 let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
5071 let cotenant_route = begin_test_route(
5072 &forwarding,
5073 cotenant,
5074 FrameSink::new(cotenant_tx),
5075 62,
5076 "closing-client",
5077 );
5078 assert_eq!(cotenant_route.endpoint, endpoint);
5079 forwarding
5080 .complete_pending_relay(
5081 module_connection,
5082 cotenant_route.corr,
5083 RouteBindRelayOutcome::Accepted,
5084 )
5085 .unwrap();
5086 assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
5087 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5088 }
5089
5090 #[test]
5091 fn pending_route_permit_is_released_on_rejection_and_abort() {
5092 let forwarding = ForwardingTable::default();
5093 let module_connection = ConnectionId::new(300);
5094 let client = ConnectionId::new(301);
5095 let (module_tx, _module_rx) = mpsc::channel(1);
5096 let endpoint = forwarding
5097 .register_module_connection(
5098 module_connection,
5099 "permit".into(),
5100 2,
5101 Concurrency::ModuleManaged,
5102 FrameSink::new(module_tx),
5103 )
5104 .unwrap();
5105 let (client_tx, mut client_rx) = mpsc::channel(1);
5106 let sink = FrameSink::new(client_tx);
5107 let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
5108 assert!(sink.try_send(test_ping(999)).is_err());
5109 forwarding
5110 .complete_pending_relay(
5111 module_connection,
5112 rejected.corr,
5113 RouteBindRelayOutcome::Rejected(ErrorBody {
5114 code: "no".into(),
5115 message: "rejected".into(),
5116 detail: None,
5117 }),
5118 )
5119 .unwrap();
5120 sink.try_send(test_ping(1000)).unwrap();
5121 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
5122
5123 let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
5124 assert!(sink.try_send(test_ping(1001)).is_err());
5125 forwarding
5126 .abort_pending_relay(
5127 endpoint,
5128 aborted.corr,
5129 RouteBindRelayOutcome::ModuleGone("abort".into()),
5130 )
5131 .unwrap();
5132 sink.try_send(test_ping(1002)).unwrap();
5133 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
5134
5135 let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
5136 forwarding
5137 .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
5138 .unwrap();
5139 drop(client_rx);
5140 let completion = forwarding
5141 .complete_pending_relay(
5142 module_connection,
5143 receiver_closed.corr,
5144 RouteBindRelayOutcome::Accepted,
5145 )
5146 .unwrap();
5147 assert!(completion.abandoned.is_some());
5148 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5149 }
5150
5151 #[test]
5157 fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
5158 let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
5159 route_fixture("closing-set-leak");
5160
5161 const CONNECTIONS: u64 = 32;
5162 for index in 0..CONNECTIONS {
5163 let client = ConnectionId::new(1000 + index);
5164 let (client_tx, _client_rx) = mpsc::channel(8);
5165 let route = begin_test_route(
5166 &forwarding,
5167 client,
5168 FrameSink::new(client_tx),
5169 index + 1,
5170 "closing-set-leak",
5171 );
5172 forwarding
5173 .complete_pending_relay(
5174 module_connection,
5175 route.corr,
5176 RouteBindRelayOutcome::Accepted,
5177 )
5178 .unwrap();
5179 forwarding.cleanup_connection(client).unwrap();
5180 }
5181 forwarding.cleanup_connection(module_connection).unwrap();
5182
5183 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5184 }
5185
5186 #[test]
5193 fn closing_connection_is_refused_new_work_until_cleanup_completes() {
5194 let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
5195 route_fixture("closing-gate");
5196
5197 let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
5200 forwarding
5201 .complete_pending_relay(
5202 module_connection,
5203 live.corr,
5204 RouteBindRelayOutcome::Accepted,
5205 )
5206 .unwrap();
5207 client_rx.try_recv().unwrap();
5208
5209 assert!(forwarding
5212 .escalate_client_delivery_failure(
5213 client,
5214 live.client_channel,
5215 live.client_epoch,
5216 CloseReason::new(
5217 "module_to_client_delivery_failed",
5218 "client egress refused a module frame",
5219 ),
5220 UndeliveredFrame {
5221 module_id: None,
5222 sink: &sink,
5223 },
5224 )
5225 .unwrap());
5226 assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
5227
5228 assert!(matches!(
5230 forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
5231 Err(ForwardingError::ConnectionClosing { connection_id })
5232 if connection_id == client
5233 ));
5234 let (late_tx, _late_rx) = mpsc::channel(1);
5236 assert!(matches!(
5237 forwarding.register_module_connection(
5238 client,
5239 "late-module".into(),
5240 2,
5241 Concurrency::ModuleManaged,
5242 FrameSink::new(late_tx),
5243 ),
5244 Err(ForwardingError::ConnectionClosing { connection_id })
5245 if connection_id == client
5246 ));
5247
5248 forwarding.cleanup_connection(client).unwrap();
5252 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5253 }
5254}
5255
5256#[cfg(test)]
5259mod swap_slot_tests {
5260 use std::time::Duration;
5261
5262 use super::*;
5263 use tokio::sync::mpsc;
5264
5265 const MODULE_ID: &str = "swapped";
5266
5267 struct SwapFixture {
5268 forwarding: ForwardingTable,
5269 incumbent_connection: ConnectionId,
5270 incumbent: ModuleEndpointId,
5271 candidate_connection: ConnectionId,
5272 candidate: ModuleEndpointId,
5273 _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
5274 }
5275
5276 fn swap_fixture() -> SwapFixture {
5277 let forwarding = ForwardingTable::default();
5278 let incumbent_connection = ConnectionId::new(100);
5279 let candidate_connection = ConnectionId::new(110);
5280 let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
5281 let incumbent = forwarding
5282 .register_module_connection(
5283 incumbent_connection,
5284 MODULE_ID.to_string(),
5285 2,
5286 Concurrency::ModuleManaged,
5287 FrameSink::new(incumbent_tx),
5288 )
5289 .unwrap();
5290 let (candidate_tx, candidate_rx) = mpsc::channel(8);
5291 let candidate = forwarding
5292 .register_candidate_module_connection(
5293 candidate_connection,
5294 MODULE_ID.to_string(),
5295 2,
5296 Concurrency::ModuleManaged,
5297 FrameSink::new(candidate_tx),
5298 )
5299 .unwrap();
5300 SwapFixture {
5301 forwarding,
5302 incumbent_connection,
5303 incumbent,
5304 candidate_connection,
5305 candidate,
5306 _module_rxs: vec![incumbent_rx, candidate_rx],
5307 }
5308 }
5309
5310 fn client(
5311 raw: u64,
5312 ) -> (
5313 ConnectionId,
5314 FrameSink,
5315 mpsc::Receiver<crate::router::OutboundFrame>,
5316 ) {
5317 let (tx, rx) = mpsc::channel(8);
5318 (ConnectionId::new(raw), FrameSink::new(tx), rx)
5319 }
5320
5321 fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
5322 forwarding
5323 .read_inner()
5324 .unwrap()
5325 .client_to_module
5326 .values()
5327 .map(|route| route.module_endpoint)
5328 .collect()
5329 }
5330
5331 #[test]
5332 fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
5333 let fixture = swap_fixture();
5334 let forwarding = &fixture.forwarding;
5335 assert_ne!(fixture.incumbent, fixture.candidate);
5336
5337 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5339 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5340 let (client_connection, client_sink, _client_rx) = client(200);
5341 let pending = forwarding
5342 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5343 .unwrap();
5344 assert_eq!(pending.endpoint, fixture.incumbent);
5345 let rpc = forwarding
5346 .begin_module_control_rpc_for(
5347 MODULE_ID,
5348 "health.check",
5349 Instant::now() + Duration::from_secs(1),
5350 )
5351 .unwrap();
5352 assert_eq!(rpc.endpoint, fixture.incumbent);
5353 let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
5354 assert_eq!(census.len(), 1, "the census lists one endpoint per id");
5355
5356 assert_eq!(
5358 forwarding
5359 .module_endpoint_for_connection(fixture.candidate_connection)
5360 .unwrap(),
5361 Some(fixture.candidate)
5362 );
5363 assert_eq!(
5364 forwarding
5365 .module_id_for_connection(fixture.candidate_connection)
5366 .unwrap()
5367 .as_deref(),
5368 Some(MODULE_ID)
5369 );
5370
5371 let (other_tx, _other_rx) = mpsc::channel(1);
5373 assert_eq!(
5374 forwarding.register_candidate_module_connection(
5375 ConnectionId::new(120),
5376 MODULE_ID.to_string(),
5377 2,
5378 Concurrency::ModuleManaged,
5379 FrameSink::new(other_tx),
5380 ),
5381 Err(ForwardingError::CandidateSlotOccupied {
5382 module_id: MODULE_ID.to_string()
5383 })
5384 );
5385 }
5386
5387 #[test]
5391 fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
5392 let fixture = swap_fixture();
5393 let forwarding = &fixture.forwarding;
5394 let (early_client, early_sink, _early_rx) = client(200);
5395 let mut early = forwarding
5396 .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
5397 .unwrap();
5398 assert_eq!(early.endpoint, fixture.incumbent);
5399 assert_eq!(
5400 forwarding
5401 .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
5402 .unwrap(),
5403 RouteBindRelayMark::Marked
5404 );
5405
5406 let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5407 assert_eq!(
5408 cutover,
5409 ForwardingCutover {
5410 promoted: fixture.candidate,
5411 incumbent: Some(fixture.incumbent),
5412 }
5413 );
5414
5415 let (late_client, late_sink, _late_rx) = client(201);
5417 let late = forwarding
5418 .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
5419 .unwrap();
5420 assert_eq!(
5421 late.endpoint, fixture.candidate,
5422 "a route.open after cutover was reserved on the incumbent"
5423 );
5424
5425 let completion = forwarding
5427 .complete_pending_relay(
5428 fixture.incumbent_connection,
5429 early.corr,
5430 RouteBindRelayOutcome::Accepted,
5431 )
5432 .expect("a superseded endpoint's ack is not an error on its connection");
5433 assert!(completion.settled);
5434 assert!(
5435 !committed_endpoints(forwarding).contains(&fixture.incumbent),
5436 "a relay reserved before cutover committed a route on the incumbent"
5437 );
5438 let goodbye = completion
5439 .abandoned
5440 .expect("the incumbent is told to drop the binding it just created");
5441 assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
5442 assert_eq!(goodbye.channel, early.module_channel);
5443 assert_eq!(goodbye.epoch, early.module_epoch);
5444 assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
5445 match early.receiver.try_recv() {
5446 Ok(RouteBindRelayOutcome::Rejected(body)) => assert_eq!(body.code, "module_reloading"),
5447 other => panic!("expected a retryable module_reloading answer, got {other:?}"),
5448 }
5449 assert!(matches!(
5450 forwarding
5451 .lookup_data_route(early_client, early.client_channel, early.client_epoch)
5452 .unwrap(),
5453 DataRoute::Client(DataRouteState::Absent)
5454 ));
5455
5456 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
5459 forwarding
5460 .complete_pending_relay(
5461 fixture.candidate_connection,
5462 late.corr,
5463 RouteBindRelayOutcome::Accepted,
5464 )
5465 .unwrap();
5466 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5467 assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
5468 }
5469
5470 #[test]
5471 fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
5472 let fixture = swap_fixture();
5473 let forwarding = &fixture.forwarding;
5474 let (bound_client, bound_sink, _bound_rx) = client(200);
5476 let bound = forwarding
5477 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5478 .unwrap();
5479 forwarding
5480 .complete_pending_relay(
5481 fixture.incumbent_connection,
5482 bound.corr,
5483 RouteBindRelayOutcome::Accepted,
5484 )
5485 .unwrap();
5486 let (pending_client, pending_sink, _pending_rx) = client(201);
5487 let mut in_flight = forwarding
5488 .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
5489 .unwrap();
5490 forwarding
5491 .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
5492 .unwrap();
5493
5494 let incumbent = forwarding
5495 .cutover_candidate(MODULE_ID)
5496 .unwrap()
5497 .unwrap()
5498 .incumbent
5499 .unwrap();
5500 let target = forwarding
5501 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
5502 .unwrap()
5503 .expect("the superseded incumbent is still registered");
5504
5505 assert_eq!(target.endpoint, fixture.incumbent);
5506 assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
5507 assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
5508 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5509 assert_eq!(target.abandoned_bindings.len(), 1);
5510 assert_eq!(
5511 target.abandoned_bindings[0].channel,
5512 in_flight.module_channel
5513 );
5514 assert!(matches!(
5515 in_flight.receiver.try_recv(),
5516 Ok(RouteBindRelayOutcome::Rejected(body)) if body.code == "module_reloading"
5517 ));
5518 assert_eq!(
5519 forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
5520 1,
5521 "the incumbent's bound route stays until its drain finishes"
5522 );
5523
5524 let (next_client, next_sink, _next_rx) = client(202);
5525 let next = forwarding
5526 .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
5527 .expect("the promoted candidate keeps accepting routes");
5528 assert_eq!(next.endpoint, fixture.candidate);
5529 }
5530
5531 #[test]
5537 fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
5538 let forwarding = ForwardingTable::default();
5539 let first_connection = ConnectionId::new(70);
5540 let (first_tx, _first_rx) = mpsc::channel(8);
5541 forwarding
5542 .register_module_connection(
5543 first_connection,
5544 MODULE_ID.to_string(),
5545 2,
5546 Concurrency::ModuleManaged,
5547 FrameSink::new(first_tx),
5548 )
5549 .unwrap();
5550 let (client_connection, client_sink, _client_rx) = client(200);
5551 let mut pending = forwarding
5552 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5553 .unwrap();
5554 let (second_tx, _second_rx) = mpsc::channel(8);
5555 forwarding
5556 .register_module_connection(
5557 ConnectionId::new(80),
5558 MODULE_ID.to_string(),
5559 2,
5560 Concurrency::ModuleManaged,
5561 FrameSink::new(second_tx),
5562 )
5563 .unwrap();
5564
5565 assert_eq!(
5566 forwarding
5567 .complete_pending_relay(
5568 first_connection,
5569 pending.corr,
5570 RouteBindRelayOutcome::Accepted
5571 )
5572 .unwrap_err(),
5573 ForwardingError::StaleModuleEndpoint
5574 );
5575 assert!(committed_endpoints(&forwarding).is_empty());
5576 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5577 assert!(matches!(
5578 pending.receiver.try_recv(),
5579 Err(oneshot::error::TryRecvError::Closed)
5580 ));
5581 }
5582
5583 #[test]
5584 fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
5585 let fixture = swap_fixture();
5587 let forwarding = &fixture.forwarding;
5588 assert!(forwarding
5589 .cleanup_connection(fixture.candidate_connection)
5590 .unwrap()
5591 .is_empty());
5592 assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
5593 let (client_connection, client_sink, _client_rx) = client(200);
5594 assert_eq!(
5595 forwarding
5596 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5597 .unwrap()
5598 .endpoint,
5599 fixture.incumbent
5600 );
5601
5602 let fixture = swap_fixture();
5605 let forwarding = &fixture.forwarding;
5606 let (bound_client, bound_sink, _bound_rx) = client(200);
5607 let bound = forwarding
5608 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5609 .unwrap();
5610 forwarding
5611 .complete_pending_relay(
5612 fixture.incumbent_connection,
5613 bound.corr,
5614 RouteBindRelayOutcome::Accepted,
5615 )
5616 .unwrap();
5617 forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5618 let released = forwarding
5619 .cleanup_connection(fixture.incumbent_connection)
5620 .unwrap();
5621 assert_eq!(released.len(), 1);
5622 assert_eq!(released[0].connection_id, bound_client);
5623 assert!(forwarding
5624 .read_inner()
5625 .unwrap()
5626 .superseded_endpoints
5627 .is_empty());
5628 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5629 let (next_client, next_sink, _next_rx) = client(201);
5630 assert_eq!(
5631 forwarding
5632 .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
5633 .unwrap()
5634 .endpoint,
5635 fixture.candidate
5636 );
5637 }
5638}