1use std::{
2 collections::{BTreeMap, HashMap, HashSet},
3 error::Error,
4 fmt,
5 sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
6 time::Duration,
7};
8
9use subc_control::{ClientControlResponse, RouteCloseReason};
10use subc_protocol::{
11 manifest::Concurrency,
12 session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
13 ErrorBody, Flags, FrameType, Principal, Priority,
14};
15use tokio::sync::{oneshot, Semaphore};
16use tokio::time::Instant;
17use tracing::{debug, info, warn};
18
19use crate::{
20 control::{RouteBindBreakers, RouteBindConcurrency},
21 observability::DaemonCounters,
22 registry::ConnectionId,
23 router::FrameSink,
24 scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
25 Frame, ProjectRootId,
26};
27
28const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
30
31const STATELESS_PARALLEL_WINDOW: usize = 1024;
33
34const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
38
39#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
44pub struct ModuleEndpointId {
45 pub connection_id: ConnectionId,
46 pub generation: u64,
47}
48
49#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
51pub(crate) struct ClientRouteKey {
52 pub connection_id: ConnectionId,
53 pub channel: u16,
54}
55
56#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
58pub(crate) struct ModuleRouteKey {
59 pub endpoint: ModuleEndpointId,
60 pub channel: u16,
61}
62
63#[derive(Debug)]
64pub(crate) struct RouteBinding {
65 pub client_connection_id: ConnectionId,
66 pub client_sink: FrameSink,
67 pub client_negotiated_ver: u8,
68 pub client_channel: u16,
69 pub client_epoch: u32,
70 pub module_id: String,
71 pub module_endpoint: ModuleEndpointId,
72 pub module_sink: FrameSink,
73 pub module_negotiated_ver: u8,
74 pub module_channel: u16,
75 pub module_epoch: u32,
76 pub principal: Principal,
77 pub project_root: Option<ProjectRootId>,
78 pub bound_at: Instant,
79 pub flow: Arc<ChannelFlow>,
80 pub scope: Option<BoundScope>,
83}
84
85#[derive(Debug, Clone)]
86pub(crate) enum DataRoute {
87 Client(DataRouteState),
88 Module(DataRouteState),
89}
90
91#[derive(Debug, Clone)]
92pub(crate) enum DataRouteState {
93 Bound(Arc<RouteBinding>),
94 Reserved,
95 EpochMismatch,
96 Absent,
97}
98
99#[derive(Debug, Clone, Copy, PartialEq, Eq)]
128pub(crate) enum GoodbyeTargetKind {
129 Client,
130 Module,
131}
132
133#[derive(Debug, Clone)]
134pub(crate) struct GoodbyeTarget {
135 pub connection_id: ConnectionId,
136 pub sink: FrameSink,
137 pub negotiated_ver: u8,
138 pub channel: u16,
139 pub epoch: u32,
140 pub kind: GoodbyeTargetKind,
141 pub module_id: Option<String>,
146}
147
148#[derive(Debug, Clone, Copy)]
151pub(crate) struct UndeliveredFrame<'a> {
152 pub module_id: Option<&'a str>,
154 pub sink: &'a FrameSink,
156}
157
158pub(crate) fn principal_label(principal: &Principal) -> String {
160 match principal {
161 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
162 Principal::Direct => "direct".to_string(),
163 other => format!("{other:?}"),
164 }
165}
166
167fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
170 let labels = inner
171 .client_to_module
172 .iter()
173 .filter(|(key, _)| key.connection_id == connection_id)
174 .map(|(_, route)| principal_label(&route.principal))
175 .collect::<std::collections::BTreeSet<_>>();
176 if labels.is_empty() {
177 "none".to_string()
178 } else {
179 labels.into_iter().collect::<Vec<_>>().join(",")
180 }
181}
182
183impl GoodbyeTarget {
184 pub(crate) fn close_on_delivery_failure(&self) -> bool {
187 matches!(self.kind, GoodbyeTargetKind::Client)
188 }
189}
190
191pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
204
205pub(crate) fn send_module_route_goodbye(
217 counters: &DaemonCounters,
218 sink: &FrameSink,
219 frame: Frame,
220 module_id: Option<&str>,
221 context: &'static str,
222) {
223 let channel = frame.header.channel;
224 let epoch = frame.header.epoch;
225 let Err(err) = sink.try_send(frame.clone()) else {
226 return;
227 };
228 let runtime = match tokio::runtime::Handle::try_current() {
231 Ok(runtime) if !sink.is_closed() => runtime,
232 _ => {
233 counters.increment_goodbye_relay_module_dropped(module_id);
234 warn!(
235 module_id = module_id.unwrap_or("unknown"),
236 route_channel = channel,
237 route_epoch = epoch,
238 error = %err,
239 context,
240 "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
241 );
242 return;
243 }
244 };
245 debug!(
246 module_id = module_id.unwrap_or("unknown"),
247 route_channel = channel,
248 route_epoch = epoch,
249 error = %err,
250 context,
251 "module egress queue refused route GOODBYE; delivering it once the module frees room"
252 );
253 let counters = counters.clone();
254 let sink = sink.clone();
255 let module_id = module_id.map(str::to_string);
256 runtime.spawn(async move {
257 let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
258 let why = match outcome {
259 Ok(Ok(())) => {
260 debug!(
261 module_id = module_id.as_deref().unwrap_or("unknown"),
262 route_channel = channel,
263 route_epoch = epoch,
264 context,
265 "late route GOODBYE delivered to module"
266 );
267 return;
268 }
269 Ok(Err(err)) => err.to_string(),
270 Err(_) => format!(
271 "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
272 ),
273 };
274 counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
275 warn!(
276 module_id = module_id.as_deref().unwrap_or("unknown"),
277 route_channel = channel,
278 route_epoch = epoch,
279 error = %why,
280 context,
281 "route GOODBYE to module dropped under backpressure; not closing shared module connection"
282 );
283 });
284}
285
286#[derive(Debug, Clone)]
292pub(crate) struct EndpointRoute {
293 pub goodbye_target: GoodbyeTarget,
294 pub principal: Principal,
295 pub bound_at: Instant,
296 pub draining: bool,
297 pub drain_reason: Option<RouteCloseReason>,
301}
302
303#[derive(Debug, Clone)]
306pub(crate) struct ScopeDrainedRoute {
307 pub reason: RouteCloseReason,
308 pub scope: BoundScope,
309 pub module_id: String,
310 pub client: GoodbyeTarget,
311 pub module: GoodbyeTarget,
312}
313
314#[derive(Debug)]
315pub(crate) struct PendingRouteBindRelay {
316 pub endpoint: ModuleEndpointId,
317 pub module_sink: FrameSink,
318 pub negotiated_ver: u8,
319 pub client_channel: u16,
320 pub client_epoch: u32,
321 pub module_channel: u16,
322 pub module_epoch: u32,
323 pub corr: u64,
324 pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
325}
326
327#[derive(Debug, Clone)]
328pub(crate) struct ModuleDrainTarget {
329 pub endpoint: ModuleEndpointId,
330 pub sink: FrameSink,
331 pub negotiated_ver: u8,
332 pub abandoned_bindings: Vec<GoodbyeTarget>,
333 pub excluded_subscriptions: u32,
334}
335
336#[cfg(unix)]
339#[derive(Debug, Clone)]
340pub(crate) struct ModuleConnectionTarget {
341 pub module_id: String,
342 pub endpoint: ModuleEndpointId,
343 pub sink: FrameSink,
344 pub negotiated_ver: u8,
345}
346
347#[derive(Debug, Clone)]
348pub(crate) enum RouteBindRelayOutcome {
349 Accepted,
350 Rejected(ErrorBody),
351 ModuleGone(String),
352}
353
354#[derive(Debug, Clone, Copy, PartialEq, Eq)]
356pub(crate) struct ForwardingCutover {
357 pub promoted: ModuleEndpointId,
359 pub incumbent: Option<ModuleEndpointId>,
362}
363
364#[derive(Debug)]
366pub(crate) struct ConnectionCleanup {
367 pub released: Vec<GoodbyeTarget>,
369 pub abandoned_relays: u32,
372}
373
374#[derive(Debug, Clone)]
375pub(crate) struct PendingRelayCompletion {
376 pub settled: bool,
377 pub abandoned: Option<GoodbyeTarget>,
378}
379
380#[derive(Debug)]
381pub(crate) struct PendingModuleControlRpc {
382 pub endpoint: ModuleEndpointId,
383 pub module_sink: FrameSink,
384 pub negotiated_ver: u8,
385 pub corr: u64,
386 pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
387}
388
389#[derive(Debug, Clone)]
390pub(crate) enum ModuleControlRpcOutcome {
391 Response(ModuleControlResponse),
392 Rejected(ErrorBody),
393 ModuleGone(String),
394 MalformedResponse(String),
395 UnexpectedOp { expected: String, actual: String },
396 DeadlineElapsed,
397}
398
399#[derive(Debug, Clone, PartialEq, Eq)]
400pub(crate) enum ModuleControlRpcCompletion {
401 Unknown,
402 Settled,
403 LateHealthAnswer {
404 module_id: String,
405 latency: Duration,
406 },
407}
408
409#[derive(Debug)]
410struct PendingModuleControlRpcEntry {
411 expected_op: String,
412 deadline: Instant,
413 health_probe_started_at: Option<Instant>,
414 sender: oneshot::Sender<ModuleControlRpcOutcome>,
415}
416
417#[derive(Debug)]
418struct HealthProbeTombstone {
419 expected_op: String,
420 module_id: String,
421 probe_started_at: Instant,
422 expires_at: Instant,
423}
424
425#[derive(Debug, Clone)]
426struct RouteReservation {
427 client_key: ClientRouteKey,
428 module_key: ModuleRouteKey,
429 client_epoch: u32,
430 module_epoch: u32,
431 project_root: Option<ProjectRootId>,
432}
433
434#[derive(Debug)]
435struct PendingRouteBindRelayEntry {
436 reservation: RouteReservation,
437 client_sink: FrameSink,
438 client_negotiated_ver: u8,
439 client_permit: crate::router::EgressPermit,
440 route_open_frame: Frame,
441 principal: Principal,
442 scope: Option<BoundScope>,
445 deadline: Instant,
446 relay_enqueued: bool,
457 sender: oneshot::Sender<RouteBindRelayOutcome>,
458}
459
460#[derive(Debug, Clone, Copy, PartialEq, Eq)]
463pub(crate) enum RouteBindRelayMark {
464 Marked,
466 SettledByReply,
470 ReleasedWithoutReply,
474}
475
476#[derive(Debug, Clone)]
477pub(crate) enum RouteRelease {
478 Removed(GoodbyeTarget),
479 Stale,
480 Absent,
481}
482
483#[derive(Debug, Clone)]
484pub(crate) enum RoutePollSnapshot {
485 Bound {
486 module_id: String,
487 status: Option<String>,
488 },
489 Absent,
490}
491
492#[derive(Debug, Clone)]
493struct ModuleConnection {
494 endpoint: ModuleEndpointId,
495 sink: FrameSink,
496 negotiated_ver: u8,
497 concurrency: Concurrency,
498}
499
500#[derive(Debug, Default)]
501struct ForwardingInner {
502 operator_confirms: Arc<crate::operator_confirm::OperatorConfirms>,
503 daemon_draining: bool,
504 daemon_drain_results: HashMap<ModuleEndpointId, bool>,
509 modules_by_id: HashMap<String, ModuleConnection>,
513 candidates_by_id: HashMap<String, ModuleConnection>,
519 superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
525 endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
526 module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
527 draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
531 closing_connections: HashSet<ConnectionId>,
532 next_generation: u64,
533 reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
534 reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
535 next_client_channel: HashMap<ConnectionId, u16>,
536 next_module_channel: HashMap<ModuleEndpointId, u16>,
537 client_slot_epochs: HashMap<ClientRouteKey, u32>,
538 module_slot_epochs: HashMap<ModuleRouteKey, u32>,
539 last_published_epoch: HashMap<ClientRouteKey, u32>,
540 client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
541 module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
542 status: HashMap<(ClientRouteKey, u32), String>,
543 pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
544 relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
549 scope_tags: HashMap<(String, String), ScopeTag>,
554 next_control_corr: HashMap<ModuleEndpointId, u64>,
555 pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
556 health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
557}
558
559#[derive(Debug, Clone)]
560pub(crate) struct CloseReason {
561 code: &'static str,
562 message: String,
563}
564
565impl CloseReason {
566 pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
567 Self {
568 code,
569 message: message.into(),
570 }
571 }
572}
573
574impl fmt::Display for CloseReason {
575 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
576 write!(f, "{}: {}", self.code, self.message)
577 }
578}
579
580pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
581
582#[derive(Debug, Default)]
584pub struct ForwardingTable {
585 inner: Arc<RwLock<ForwardingInner>>,
586 #[cfg(feature = "test-support")]
587 route_closing_hold: Mutex<Option<oneshot::Receiver<()>>>,
588 close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
589 counters: DaemonCounters,
590 route_bind_breakers: RouteBindBreakers,
595 route_bind_concurrency: RouteBindConcurrency,
598 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
602}
603
604impl ForwardingTable {
605 #[cfg(feature = "test-support")]
647 pub fn configure_operator_script(
648 &self,
649 script: &std::path::Path,
650 events: Option<&std::path::Path>,
651 limits: Option<crate::operator_confirm::OperatorLimits>,
652 ) {
653 self.operator_confirms()
654 .configure_script(script, events, limits);
655 }
656
657 #[cfg(feature = "test-support")]
661 pub fn hold_route_closing_for_test(&self) -> oneshot::Sender<()> {
662 let (release, hold) = oneshot::channel();
663 *self.route_closing_hold.lock().unwrap() = Some(hold);
664 release
665 }
666
667 #[cfg(feature = "test-support")]
668 pub(crate) async fn wait_before_route_closing_for_test(&self) {
669 let hold = self.route_closing_hold.lock().unwrap().take();
670 if let Some(hold) = hold {
671 let _ = hold.await;
672 }
673 }
674
675 pub(crate) fn operator_confirms(&self) -> Arc<crate::operator_confirm::OperatorConfirms> {
676 Arc::clone(
677 &self
678 .inner
679 .read()
680 .unwrap_or_else(|p| p.into_inner())
681 .operator_confirms,
682 )
683 }
684
685 #[cfg(test)]
686 pub(crate) fn inject_operator_principal(&self, key: ModuleRouteKey, principal: Principal) {
687 let mut inner = self.write_inner().unwrap();
688 let old = inner.module_to_client.remove(&key).unwrap();
689 let client = ClientRouteKey {
690 connection_id: old.client_connection_id,
691 channel: old.client_channel,
692 };
693 inner.client_to_module.remove(&client);
694 let mut binding = Arc::try_unwrap(old).expect("test binding has no outstanding readers");
695 binding.principal = principal;
696 let binding = Arc::new(binding);
697 inner.client_to_module.insert(client, Arc::clone(&binding));
698 inner.module_to_client.insert(key, binding);
699 }
700
701 pub(crate) fn with_operator_route<T>(
705 &self,
706 connection_id: ConnectionId,
707 channel: u16,
708 epoch: u32,
709 admit: impl FnOnce(Option<&RouteBinding>) -> T,
710 ) -> Result<T, ForwardingError> {
711 let inner = self.read_inner()?;
712 let binding = inner
713 .endpoint_by_connection
714 .get(&connection_id)
715 .and_then(|endpoint| {
716 inner.module_to_client.get(&ModuleRouteKey {
717 endpoint: *endpoint,
718 channel,
719 })
720 })
721 .filter(|binding| binding.module_epoch == epoch);
722 Ok(admit(binding.map(Arc::as_ref)))
723 }
724
725 pub(crate) fn counters(&self) -> DaemonCounters {
726 self.counters.clone()
727 }
728
729 pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
730 self.route_bind_breakers.clone()
731 }
732
733 pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
734 self.route_bind_concurrency.clone()
735 }
736
737 pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
738 Arc::clone(&self.route_outages)
739 }
740
741 pub(crate) fn register_connection_close(
742 &self,
743 connection_id: ConnectionId,
744 ) -> ConnectionCloseReceiver {
745 let (sender, receiver) = oneshot::channel();
746 let replaced = self
747 .lock_close_registry()
748 .insert(connection_id, sender)
749 .is_some();
750 if replaced {
751 warn!(
752 connection_id = connection_id.get(),
753 "replaced existing connection close registration"
754 );
755 }
756 receiver
757 }
758
759 pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
760 self.lock_close_registry().remove(&connection_id);
761 }
762
763 #[cfg(unix)]
772 pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
773 let senders: Vec<_> = self.lock_close_registry().drain().collect();
774 let count = senders.len();
775 for (_, sender) in senders {
776 let _ = sender.send(reason.clone());
777 }
778 count
779 }
780
781 #[cfg(unix)]
785 pub(crate) fn module_connections(
786 &self,
787 ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
788 let inner = self.read_inner()?;
789 let mut seen = HashSet::new();
790 Ok(inner
791 .modules_by_id
792 .values()
793 .chain(inner.candidates_by_id.values())
794 .chain(inner.superseded_endpoints.values())
795 .filter(|module| seen.insert(module.endpoint))
796 .map(|module| ModuleConnectionTarget {
797 module_id: inner
798 .module_id_by_endpoint
799 .get(&module.endpoint)
800 .cloned()
801 .unwrap_or_default(),
802 endpoint: module.endpoint,
803 sink: module.sink.clone(),
804 negotiated_ver: module.negotiated_ver,
805 })
806 .collect())
807 }
808
809 pub(crate) fn request_connection_close(
813 &self,
814 connection_id: ConnectionId,
815 reason: CloseReason,
816 ) -> bool {
817 let sender = self.lock_close_registry().remove(&connection_id);
818 if let Some(sender) = sender {
819 debug!(
820 connection_id = connection_id.get(),
821 close_reason = %reason,
822 "requesting connection close"
823 );
824 let _ = sender.send(reason);
825 true
826 } else {
827 debug!(
828 connection_id = connection_id.get(),
829 close_reason = %reason,
830 "connection close request ignored for inactive connection"
831 );
832 false
833 }
834 }
835
836 pub fn register_module_connection(
837 &self,
838 connection_id: ConnectionId,
839 module_id: String,
840 negotiated_ver: u8,
841 concurrency: Concurrency,
842 sink: FrameSink,
843 ) -> Result<ModuleEndpointId, ForwardingError> {
844 self.register_module_connection_inner(
845 connection_id,
846 module_id,
847 negotiated_ver,
848 concurrency,
849 sink,
850 None,
851 )
852 }
853
854 pub(crate) fn register_module_connection_acked(
867 &self,
868 connection_id: ConnectionId,
869 module_id: String,
870 negotiated_ver: u8,
871 concurrency: Concurrency,
872 sink: FrameSink,
873 hello_ack: Frame,
874 ) -> Result<ModuleEndpointId, ForwardingError> {
875 self.register_module_connection_inner(
876 connection_id,
877 module_id,
878 negotiated_ver,
879 concurrency,
880 sink,
881 Some(hello_ack),
882 )
883 }
884
885 fn register_module_connection_inner(
886 &self,
887 connection_id: ConnectionId,
888 module_id: String,
889 negotiated_ver: u8,
890 concurrency: Concurrency,
891 sink: FrameSink,
892 hello_ack: Option<Frame>,
893 ) -> Result<ModuleEndpointId, ForwardingError> {
894 let mut inner = self.write_inner()?;
895 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
896 return Err(ForwardingError::ConnectionClosing { connection_id });
897 }
898 check_module_connection_role_locked(&inner, connection_id)?;
899 enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
902
903 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
904 let endpoint = ModuleEndpointId {
905 connection_id,
906 generation: inner.next_generation,
907 };
908 inner.endpoint_by_connection.insert(connection_id, endpoint);
909 inner
910 .module_id_by_endpoint
911 .insert(endpoint, module_id.clone());
912 inner.next_module_channel.insert(endpoint, 1);
913 inner.next_control_corr.insert(endpoint, 1);
914 inner.modules_by_id.insert(
915 module_id.clone(),
916 ModuleConnection {
917 endpoint,
918 sink,
919 negotiated_ver,
920 concurrency,
921 },
922 );
923 drop(inner);
924
925 if let Some(discarded) = self
938 .route_bind_breakers
939 .reset_for_new_module_connection(&module_id)
940 {
941 info!(
942 module_id = %module_id,
943 discarded_consecutive_timeouts = discarded,
944 "route.bind breaker state discarded: a new module connection replaced the process it described"
945 );
946 }
947 Ok(endpoint)
948 }
949
950 #[cfg(test)]
964 pub(crate) fn register_candidate_module_connection(
965 &self,
966 connection_id: ConnectionId,
967 module_id: String,
968 negotiated_ver: u8,
969 concurrency: Concurrency,
970 sink: FrameSink,
971 ) -> Result<ModuleEndpointId, ForwardingError> {
972 self.register_candidate_module_connection_inner(
973 connection_id,
974 module_id,
975 negotiated_ver,
976 concurrency,
977 sink,
978 None,
979 )
980 }
981
982 pub(crate) fn register_candidate_module_connection_acked(
989 &self,
990 connection_id: ConnectionId,
991 module_id: String,
992 negotiated_ver: u8,
993 concurrency: Concurrency,
994 sink: FrameSink,
995 hello_ack: Frame,
996 ) -> Result<ModuleEndpointId, ForwardingError> {
997 self.register_candidate_module_connection_inner(
998 connection_id,
999 module_id,
1000 negotiated_ver,
1001 concurrency,
1002 sink,
1003 Some(hello_ack),
1004 )
1005 }
1006
1007 fn register_candidate_module_connection_inner(
1008 &self,
1009 connection_id: ConnectionId,
1010 module_id: String,
1011 negotiated_ver: u8,
1012 concurrency: Concurrency,
1013 sink: FrameSink,
1014 hello_ack: Option<Frame>,
1015 ) -> Result<ModuleEndpointId, ForwardingError> {
1016 let mut inner = self.write_inner()?;
1017 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1018 return Err(ForwardingError::ConnectionClosing { connection_id });
1019 }
1020 check_module_connection_role_locked(&inner, connection_id)?;
1021 if inner.candidates_by_id.contains_key(&module_id) {
1022 return Err(ForwardingError::CandidateSlotOccupied { module_id });
1023 }
1024 enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1027
1028 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1029 let endpoint = ModuleEndpointId {
1030 connection_id,
1031 generation: inner.next_generation,
1032 };
1033 inner.endpoint_by_connection.insert(connection_id, endpoint);
1034 inner
1035 .module_id_by_endpoint
1036 .insert(endpoint, module_id.clone());
1037 inner.next_module_channel.insert(endpoint, 1);
1038 inner.next_control_corr.insert(endpoint, 1);
1039 inner.candidates_by_id.insert(
1040 module_id,
1041 ModuleConnection {
1042 endpoint,
1043 sink,
1044 negotiated_ver,
1045 concurrency,
1046 },
1047 );
1048 Ok(endpoint)
1049 }
1050
1051 pub(crate) fn cutover_candidate(
1069 &self,
1070 module_id: &str,
1071 ) -> Result<Option<ForwardingCutover>, ForwardingError> {
1072 let mut inner = self.write_inner()?;
1073 if inner.daemon_draining {
1074 return Err(ForwardingError::ModuleReloading {
1075 module_id: module_id.to_string(),
1076 });
1077 }
1078 let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
1079 return Ok(None);
1080 };
1081 let promoted = candidate.endpoint;
1082 let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
1083 let incumbent = incumbent.map(|incumbent| {
1084 let endpoint = incumbent.endpoint;
1085 inner.superseded_endpoints.insert(endpoint, incumbent);
1086 endpoint
1087 });
1088 drop(inner);
1089
1090 if let Some(discarded) = self
1094 .route_bind_breakers
1095 .reset_for_new_module_connection(module_id)
1096 {
1097 info!(
1098 module_id = %module_id,
1099 discarded_consecutive_timeouts = discarded,
1100 "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
1101 );
1102 }
1103 Ok(Some(ForwardingCutover {
1104 promoted,
1105 incumbent,
1106 }))
1107 }
1108
1109 #[allow(clippy::too_many_arguments)]
1110 pub(crate) async fn begin_route_bind_relay_for(
1111 &self,
1112 client_connection_id: ConnectionId,
1113 client_sink: FrameSink,
1114 client_negotiated_ver: u8,
1115 client_corr: u64,
1116 module_id: &str,
1117 principal: Principal,
1118 scope: Option<BoundScope>,
1119 project_root: Option<ProjectRootId>,
1120 deadline: Instant,
1121 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1122 let client_permit =
1126 client_sink
1127 .reserve_owned()
1128 .await
1129 .map_err(|_| ForwardingError::ClientEgressClosed {
1130 connection_id: client_connection_id,
1131 })?;
1132 self.begin_route_bind_relay_inner(
1133 client_connection_id,
1134 client_sink,
1135 client_negotiated_ver,
1136 client_corr,
1137 module_id,
1138 principal,
1139 scope,
1140 project_root,
1141 deadline,
1142 client_permit,
1143 )
1144 }
1145
1146 #[cfg(test)]
1147 pub(crate) fn begin_route_bind_relay_for_test(
1148 &self,
1149 client_connection_id: ConnectionId,
1150 client_sink: FrameSink,
1151 client_corr: u64,
1152 module_id: &str,
1153 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1154 let permit =
1155 client_sink
1156 .try_reserve_owned()
1157 .map_err(|_| ForwardingError::ClientEgressClosed {
1158 connection_id: client_connection_id,
1159 })?;
1160 self.begin_route_bind_relay_inner(
1161 client_connection_id,
1162 client_sink,
1163 subc_protocol::PROTOCOL_VERSION,
1164 client_corr,
1165 module_id,
1166 Principal::Direct,
1167 None,
1168 None,
1169 Instant::now() + std::time::Duration::from_secs(60),
1170 permit,
1171 )
1172 }
1173
1174 pub(crate) fn begin_module_control_rpc_for(
1175 &self,
1176 module_id: &str,
1177 expected_op: &str,
1178 deadline: Instant,
1179 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1180 self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1181 }
1182
1183 pub(crate) fn begin_health_probe_rpc_for(
1184 &self,
1185 module_id: &str,
1186 expected_op: &str,
1187 probe_started_at: Instant,
1188 deadline: Instant,
1189 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1190 self.begin_module_control_rpc_inner(
1191 module_id,
1192 expected_op,
1193 deadline,
1194 Some(probe_started_at),
1195 false,
1196 )
1197 }
1198
1199 pub(crate) fn begin_drain_health_probe_rpc_for(
1200 &self,
1201 module_id: &str,
1202 expected_op: &str,
1203 probe_started_at: Instant,
1204 deadline: Instant,
1205 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1206 self.begin_module_control_rpc_inner(
1207 module_id,
1208 expected_op,
1209 deadline,
1210 Some(probe_started_at),
1211 true,
1212 )
1213 }
1214
1215 pub(crate) fn begin_endpoint_health_probe_rpc_for(
1223 &self,
1224 endpoint: ModuleEndpointId,
1225 expected_op: &str,
1226 probe_started_at: Instant,
1227 deadline: Instant,
1228 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1229 let inner = self.write_inner()?;
1230 let module = module_connection_for_endpoint_locked(&inner, endpoint)
1231 .cloned()
1232 .ok_or(ForwardingError::NoModuleConnection)?;
1233 let module_id = inner
1234 .module_id_by_endpoint
1235 .get(&endpoint)
1236 .cloned()
1237 .unwrap_or_default();
1238 self.begin_control_rpc_locked(
1241 inner,
1242 &module_id,
1243 module,
1244 expected_op,
1245 deadline,
1246 Some(probe_started_at),
1247 true,
1248 )
1249 }
1250
1251 fn begin_module_control_rpc_inner(
1252 &self,
1253 module_id: &str,
1254 expected_op: &str,
1255 deadline: Instant,
1256 health_probe_started_at: Option<Instant>,
1257 allow_draining: bool,
1258 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1259 let inner = self.write_inner()?;
1260 let module = inner
1261 .modules_by_id
1262 .get(module_id)
1263 .cloned()
1264 .ok_or(ForwardingError::NoModuleConnection)?;
1265 self.begin_control_rpc_locked(
1266 inner,
1267 module_id,
1268 module,
1269 expected_op,
1270 deadline,
1271 health_probe_started_at,
1272 allow_draining,
1273 )
1274 }
1275
1276 #[allow(clippy::too_many_arguments)]
1277 fn begin_control_rpc_locked(
1278 &self,
1279 mut inner: RwLockWriteGuard<'_, ForwardingInner>,
1280 module_id: &str,
1281 module: ModuleConnection,
1282 expected_op: &str,
1283 deadline: Instant,
1284 health_probe_started_at: Option<Instant>,
1285 allow_draining: bool,
1286 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1287 if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1288 return Err(ForwardingError::ModuleReloading {
1289 module_id: module_id.to_string(),
1290 });
1291 }
1292 if inner
1293 .closing_connections
1294 .contains(&module.endpoint.connection_id)
1295 {
1296 return Err(ForwardingError::ConnectionClosing {
1297 connection_id: module.endpoint.connection_id,
1298 });
1299 }
1300 if health_probe_started_at.is_some() {
1301 inner
1305 .health_probe_tombstones
1306 .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1307 }
1308 let corr = match inner.allocate_control_corr(module.endpoint) {
1309 Ok(corr) => corr,
1310 Err(err) => {
1311 drop(inner);
1312 self.request_connection_close(
1313 module.endpoint.connection_id,
1314 CloseReason::new(
1315 "control_correlation_exhausted",
1316 "daemon-originated channel-0 correlation space exhausted",
1317 ),
1318 );
1319 return Err(err);
1320 }
1321 };
1322 let (sender, receiver) = oneshot::channel();
1323 inner.pending_control_rpcs.insert(
1324 (module.endpoint, corr),
1325 PendingModuleControlRpcEntry {
1326 expected_op: expected_op.to_string(),
1327 deadline,
1328 health_probe_started_at,
1329 sender,
1330 },
1331 );
1332
1333 Ok(PendingModuleControlRpc {
1334 endpoint: module.endpoint,
1335 module_sink: module.sink,
1336 negotiated_ver: module.negotiated_ver,
1337 corr,
1338 receiver,
1339 })
1340 }
1341
1342 #[allow(clippy::too_many_arguments)]
1343 fn begin_route_bind_relay_inner(
1344 &self,
1345 client_connection_id: ConnectionId,
1346 client_sink: FrameSink,
1347 client_negotiated_ver: u8,
1348 client_corr: u64,
1349 expected_module_id: &str,
1350 principal: Principal,
1351 scope: Option<BoundScope>,
1352 project_root: Option<ProjectRootId>,
1353 deadline: Instant,
1354 client_permit: crate::router::EgressPermit,
1355 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1356 let mut inner = self.write_inner()?;
1357 if inner
1361 .endpoint_by_connection
1362 .contains_key(&client_connection_id)
1363 {
1364 return Err(ForwardingError::ConnectionRoleConflict {
1365 connection_id: client_connection_id,
1366 });
1367 }
1368 if inner.closing_connections.contains(&client_connection_id) {
1369 return Err(ForwardingError::ConnectionClosing {
1370 connection_id: client_connection_id,
1371 });
1372 }
1373 let module = inner
1374 .modules_by_id
1375 .get(expected_module_id)
1376 .cloned()
1377 .ok_or(ForwardingError::NoModuleConnection)?;
1378 if inner.draining_endpoints.contains_key(&module.endpoint) {
1379 return Err(ForwardingError::ModuleReloading {
1380 module_id: expected_module_id.to_string(),
1381 });
1382 }
1383 if inner
1384 .closing_connections
1385 .contains(&module.endpoint.connection_id)
1386 {
1387 return Err(ForwardingError::ConnectionClosing {
1388 connection_id: module.endpoint.connection_id,
1389 });
1390 }
1391
1392 let corr = match inner.allocate_control_corr(module.endpoint) {
1393 Ok(corr) => corr,
1394 Err(err) => {
1395 drop(inner);
1396 self.request_connection_close(
1397 module.endpoint.connection_id,
1398 CloseReason::new(
1399 "control_correlation_exhausted",
1400 "daemon-originated channel-0 correlation space exhausted",
1401 ),
1402 );
1403 return Err(err);
1404 }
1405 };
1406 let (client_channel, client_epoch, module_channel, module_epoch) =
1407 inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1408 let client_key = ClientRouteKey {
1409 connection_id: client_connection_id,
1410 channel: client_channel,
1411 };
1412 let module_key = ModuleRouteKey {
1413 endpoint: module.endpoint,
1414 channel: module_channel,
1415 };
1416 let reservation = RouteReservation {
1417 client_key,
1418 module_key,
1419 client_epoch,
1420 module_epoch,
1421 project_root,
1422 };
1423 let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
1424 route_channel: client_channel,
1425 route_epoch: client_epoch,
1426 })
1427 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1428 let route_open_frame = Frame::build_with_version(
1429 client_negotiated_ver,
1430 FrameType::Response,
1431 Flags::new(false, Priority::Passive, false),
1432 0,
1433 0,
1434 client_corr,
1435 response_body,
1436 )
1437 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1438 let (sender, receiver) = oneshot::channel();
1439 inner.reserved_client.insert(client_key, module_key);
1440 inner.reserved_module.insert(module_key, client_key);
1441 inner.pending_relays.insert(
1442 (module.endpoint, corr),
1443 PendingRouteBindRelayEntry {
1444 reservation,
1445 client_sink,
1446 client_negotiated_ver,
1447 client_permit,
1448 route_open_frame,
1449 principal,
1450 scope,
1451 deadline,
1452 relay_enqueued: false,
1453 sender,
1454 },
1455 );
1456
1457 Ok(PendingRouteBindRelay {
1458 endpoint: module.endpoint,
1459 module_sink: module.sink,
1460 negotiated_ver: module.negotiated_ver,
1461 client_channel,
1462 client_epoch,
1463 module_channel,
1464 module_epoch,
1465 corr,
1466 receiver,
1467 })
1468 }
1469
1470 pub(crate) fn mark_route_bind_relay_enqueued(
1480 &self,
1481 endpoint: ModuleEndpointId,
1482 corr: u64,
1483 ) -> Result<RouteBindRelayMark, ForwardingError> {
1484 let mut inner = self.write_inner()?;
1485 if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
1486 pending.relay_enqueued = true;
1487 return Ok(RouteBindRelayMark::Marked);
1488 }
1489 if inner
1490 .relays_released_before_enqueue
1491 .remove(&(endpoint, corr))
1492 {
1493 return Ok(RouteBindRelayMark::ReleasedWithoutReply);
1494 }
1495 Ok(RouteBindRelayMark::SettledByReply)
1496 }
1497
1498 pub(crate) fn release_client_route(
1499 &self,
1500 client_connection_id: ConnectionId,
1501 client_channel: u16,
1502 expected_epoch: u32,
1503 ) -> Result<RouteRelease, ForwardingError> {
1504 let mut inner = self.write_inner()?;
1505 let release = release_client_route_locked(
1506 &mut inner,
1507 ClientRouteKey {
1508 connection_id: client_connection_id,
1509 channel: client_channel,
1510 },
1511 expected_epoch,
1512 );
1513 self.record_route_release(&release);
1514 Ok(release)
1515 }
1516
1517 pub(crate) fn release_module_route(
1518 &self,
1519 module_connection_id: ConnectionId,
1520 module_channel: u16,
1521 expected_epoch: u32,
1522 ) -> Result<RouteRelease, ForwardingError> {
1523 let mut inner = self.write_inner()?;
1524 let Some(endpoint) = inner
1525 .endpoint_by_connection
1526 .get(&module_connection_id)
1527 .copied()
1528 else {
1529 return Ok(RouteRelease::Absent);
1530 };
1531 let release = release_module_route_locked(
1532 &mut inner,
1533 ModuleRouteKey {
1534 endpoint,
1535 channel: module_channel,
1536 },
1537 expected_epoch,
1538 );
1539 self.record_route_release(&release);
1540 Ok(release)
1541 }
1542
1543 pub(crate) fn abort_pending_relay(
1544 &self,
1545 endpoint: ModuleEndpointId,
1546 corr: u64,
1547 outcome: RouteBindRelayOutcome,
1548 ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
1549 let mut inner = self.write_inner()?;
1550 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1551 inner
1556 .relays_released_before_enqueue
1557 .remove(&(endpoint, corr));
1558 return Ok(None);
1559 };
1560 release_reserved_route_locked(
1561 &mut inner,
1562 pending.reservation.client_key,
1563 pending.reservation.module_key,
1564 );
1565 let target = pending
1566 .relay_enqueued
1567 .then(|| abandoned_route_target(&inner, &pending.reservation));
1568 let _ = pending.sender.send(outcome);
1569 Ok(target.flatten())
1570 }
1571
1572 pub(crate) fn cancel_module_control_rpc(
1573 &self,
1574 endpoint: ModuleEndpointId,
1575 corr: u64,
1576 ) -> Result<(), ForwardingError> {
1577 self.write_inner()?
1578 .pending_control_rpcs
1579 .remove(&(endpoint, corr));
1580 Ok(())
1581 }
1582
1583 pub(crate) fn tombstone_health_probe_rpc(
1584 &self,
1585 endpoint: ModuleEndpointId,
1586 corr: u64,
1587 ) -> Result<bool, ForwardingError> {
1588 let key = (endpoint, corr);
1589 let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
1590 {
1591 let mut inner = self.write_inner()?;
1592 let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
1593 return Ok(false);
1594 };
1595 let Some(probe_started_at) = pending.health_probe_started_at else {
1596 inner.pending_control_rpcs.insert(key, pending);
1597 return Ok(false);
1598 };
1599 let module_id = inner
1600 .module_id_by_endpoint
1601 .get(&endpoint)
1602 .cloned()
1603 .unwrap_or_else(|| "unknown".to_string());
1604 inner.health_probe_tombstones.insert(
1605 key,
1606 HealthProbeTombstone {
1607 expected_op: pending.expected_op,
1608 module_id,
1609 probe_started_at,
1610 expires_at,
1611 },
1612 );
1613 }
1614 self.schedule_health_probe_tombstone_expiration(key, expires_at);
1615 Ok(true)
1616 }
1617
1618 fn schedule_health_probe_tombstone_expiration(
1619 &self,
1620 key: (ModuleEndpointId, u64),
1621 expires_at: Instant,
1622 ) {
1623 let inner = Arc::downgrade(&self.inner);
1624 tokio::spawn(async move {
1625 tokio::time::sleep_until(expires_at).await;
1626 let Some(inner) = inner.upgrade() else {
1627 return;
1628 };
1629 let Ok(mut inner) = inner.write() else {
1630 return;
1631 };
1632 let expired = inner
1633 .health_probe_tombstones
1634 .get(&key)
1635 .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
1636 if expired {
1637 inner.health_probe_tombstones.remove(&key);
1638 }
1639 });
1640 }
1641
1642 pub(crate) fn complete_pending_relay(
1643 &self,
1644 connection_id: ConnectionId,
1645 corr: u64,
1646 outcome: RouteBindRelayOutcome,
1647 ) -> Result<PendingRelayCompletion, ForwardingError> {
1648 let mut inner = self.write_inner()?;
1649 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1650 return Ok(PendingRelayCompletion {
1651 settled: false,
1652 abandoned: None,
1653 });
1654 };
1655 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1656 return Ok(PendingRelayCompletion {
1657 settled: false,
1658 abandoned: None,
1659 });
1660 };
1661
1662 if Instant::now() >= pending.deadline {
1663 release_reserved_route_locked(
1664 &mut inner,
1665 pending.reservation.client_key,
1666 pending.reservation.module_key,
1667 );
1668 let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
1669 .then(|| abandoned_route_target(&inner, &pending.reservation))
1670 .flatten();
1671 let _ = pending
1672 .sender
1673 .send(RouteBindRelayOutcome::Rejected(ErrorBody {
1674 code: "module_timeout".to_string(),
1675 message: "route.bind response arrived after its daemon deadline".to_string(),
1676 detail: None,
1677 }));
1678 return Ok(PendingRelayCompletion {
1679 settled: true,
1680 abandoned,
1681 });
1682 }
1683
1684 match outcome {
1685 RouteBindRelayOutcome::Accepted
1704 if pending.client_sink.is_closed()
1705 || inner
1706 .closing_connections
1707 .contains(&pending.reservation.client_key.connection_id) =>
1708 {
1709 let reason = if pending.client_sink.is_closed() {
1710 "client egress closed before route publication"
1711 } else {
1712 "client connection is closing before route publication"
1713 };
1714 release_reserved_route_locked(
1715 &mut inner,
1716 pending.reservation.client_key,
1717 pending.reservation.module_key,
1718 );
1719 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1723 let _ = pending
1724 .sender
1725 .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
1726 return Ok(PendingRelayCompletion {
1727 settled: true,
1728 abandoned,
1729 });
1730 }
1731 RouteBindRelayOutcome::Accepted
1754 if inner.superseded_endpoints.contains_key(&endpoint)
1755 || inner.draining_endpoints.contains_key(&endpoint) =>
1756 {
1757 release_reserved_route_locked(
1758 &mut inner,
1759 pending.reservation.client_key,
1760 pending.reservation.module_key,
1761 );
1762 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1765 let module_id = inner
1766 .module_id_by_endpoint
1767 .get(&endpoint)
1768 .cloned()
1769 .unwrap_or_else(|| "unknown".to_string());
1770 let _ = pending
1771 .sender
1772 .send(RouteBindRelayOutcome::Rejected(ErrorBody::new(
1773 "module_reloading",
1774 format!("module_id '{module_id}' is reloading"),
1775 )));
1776 return Ok(PendingRelayCompletion {
1777 settled: true,
1778 abandoned,
1779 });
1780 }
1781 RouteBindRelayOutcome::Accepted
1790 if pending
1791 .scope
1792 .as_ref()
1793 .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
1794 {
1795 let refusal = pending
1796 .scope
1797 .as_ref()
1798 .and_then(|scope| scope_refusal_locked(&inner, scope))
1799 .expect("guard matched a refusal under the same lock");
1800 release_reserved_route_locked(
1801 &mut inner,
1802 pending.reservation.client_key,
1803 pending.reservation.module_key,
1804 );
1805 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1806 let _ = pending
1807 .sender
1808 .send(RouteBindRelayOutcome::Rejected(refusal));
1809 return Ok(PendingRelayCompletion {
1810 settled: true,
1811 abandoned,
1812 });
1813 }
1814 RouteBindRelayOutcome::Accepted => {
1815 let abandoned = commit_route_locked(&mut inner, pending)?;
1816 return Ok(PendingRelayCompletion {
1817 settled: true,
1818 abandoned,
1819 });
1820 }
1821 terminal => {
1822 release_reserved_route_locked(
1823 &mut inner,
1824 pending.reservation.client_key,
1825 pending.reservation.module_key,
1826 );
1827 let _ = pending.sender.send(terminal);
1828 }
1829 }
1830 Ok(PendingRelayCompletion {
1831 settled: true,
1832 abandoned: None,
1833 })
1834 }
1835
1836 pub(crate) fn pending_module_control_op(
1837 &self,
1838 connection_id: ConnectionId,
1839 corr: u64,
1840 ) -> Result<Option<String>, ForwardingError> {
1841 let inner = self.read_inner()?;
1842 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1843 return Ok(None);
1844 };
1845 let key = (endpoint, corr);
1846 Ok(inner
1847 .pending_control_rpcs
1848 .get(&key)
1849 .map(|pending| pending.expected_op.clone())
1850 .or_else(|| {
1851 inner
1852 .health_probe_tombstones
1853 .get(&key)
1854 .filter(|tombstone| tombstone.expires_at > Instant::now())
1855 .map(|tombstone| tombstone.expected_op.clone())
1856 }))
1857 }
1858
1859 pub(crate) fn complete_module_control_rpc(
1860 &self,
1861 connection_id: ConnectionId,
1862 corr: u64,
1863 actual_op: Option<&str>,
1864 outcome: ModuleControlRpcOutcome,
1865 ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
1866 let now = Instant::now();
1867 let mut inner = self.write_inner()?;
1868 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1869 return Ok(ModuleControlRpcCompletion::Unknown);
1870 };
1871 let key = (endpoint, corr);
1872 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
1873 if now >= pending.deadline {
1874 let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
1875 ModuleControlRpcCompletion::LateHealthAnswer {
1876 module_id: inner
1877 .module_id_by_endpoint
1878 .get(&endpoint)
1879 .cloned()
1880 .unwrap_or_else(|| "unknown".to_string()),
1881 latency: now.saturating_duration_since(probe_started_at),
1882 }
1883 });
1884 let _ = pending
1885 .sender
1886 .send(ModuleControlRpcOutcome::DeadlineElapsed);
1887 return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
1888 }
1889 let outcome = match actual_op {
1890 Some(actual) if actual != pending.expected_op => {
1891 ModuleControlRpcOutcome::UnexpectedOp {
1892 expected: pending.expected_op,
1893 actual: actual.to_string(),
1894 }
1895 }
1896 _ => outcome,
1897 };
1898 let _ = pending.sender.send(outcome);
1899 return Ok(ModuleControlRpcCompletion::Settled);
1900 }
1901
1902 let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
1903 return Ok(ModuleControlRpcCompletion::Unknown);
1904 };
1905 if tombstone.expires_at <= now {
1906 return Ok(ModuleControlRpcCompletion::Unknown);
1907 }
1908 Ok(ModuleControlRpcCompletion::LateHealthAnswer {
1909 module_id: tombstone.module_id,
1910 latency: now.saturating_duration_since(tombstone.probe_started_at),
1911 })
1912 }
1913
1914 #[cfg(test)]
1915 pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
1916 Ok(self.read_inner()?.health_probe_tombstones.len())
1917 }
1918
1919 #[cfg(test)]
1920 pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
1921 Ok(self.read_inner()?.closing_connections.len())
1922 }
1923
1924 #[cfg(test)]
1927 pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
1928 let inner = self.read_inner()?;
1929 Ok((inner.reserved_client.len(), inner.reserved_module.len()))
1930 }
1931
1932 pub(crate) fn module_endpoint_for_connection(
1933 &self,
1934 connection_id: ConnectionId,
1935 ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
1936 Ok(self
1937 .read_inner()?
1938 .endpoint_by_connection
1939 .get(&connection_id)
1940 .copied())
1941 }
1942
1943 pub(crate) fn module_id_for_connection(
1946 &self,
1947 connection_id: ConnectionId,
1948 ) -> Result<Option<String>, ForwardingError> {
1949 let inner = self.read_inner()?;
1950 Ok(inner
1951 .endpoint_by_connection
1952 .get(&connection_id)
1953 .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
1954 .cloned())
1955 }
1956
1957 pub(crate) fn module_route_epoch_was_allocated(
1964 &self,
1965 connection_id: ConnectionId,
1966 channel: u16,
1967 epoch: u32,
1968 ) -> Result<bool, ForwardingError> {
1969 let inner = self.read_inner()?;
1970 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1971 return Ok(false);
1972 };
1973 Ok(inner
1974 .module_slot_epochs
1975 .get(&ModuleRouteKey { endpoint, channel })
1976 .is_some_and(|last| epoch != 0 && epoch <= *last))
1977 }
1978
1979 pub(crate) fn has_live_module_connection(
1980 &self,
1981 module_id: &str,
1982 ) -> Result<bool, ForwardingError> {
1983 Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
1984 }
1985
1986 pub(crate) fn lookup_data_route(
1987 &self,
1988 connection_id: ConnectionId,
1989 channel: u16,
1990 epoch: u32,
1991 ) -> Result<DataRoute, ForwardingError> {
1992 let inner = self.read_inner()?;
1993 let state = if let Some(endpoint) =
1994 inner.endpoint_by_connection.get(&connection_id).copied()
1995 {
1996 let key = ModuleRouteKey { endpoint, channel };
1997 match inner.module_to_client.get(&key) {
1998 Some(route) if route.module_epoch == epoch => {
1999 DataRouteState::Bound(Arc::clone(route))
2000 }
2001 Some(_) => DataRouteState::EpochMismatch,
2002 None if inner.reserved_module.contains_key(&key)
2003 && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
2004 {
2005 DataRouteState::Reserved
2006 }
2007 None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
2008 None => DataRouteState::Absent,
2009 }
2010 } else {
2011 let key = ClientRouteKey {
2012 connection_id,
2013 channel,
2014 };
2015 match inner.client_to_module.get(&key) {
2016 Some(route) if route.client_epoch == epoch => {
2017 DataRouteState::Bound(Arc::clone(route))
2018 }
2019 Some(_) => DataRouteState::EpochMismatch,
2020 None if inner.reserved_client.contains_key(&key)
2021 && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
2022 {
2023 DataRouteState::Reserved
2024 }
2025 None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
2026 None => DataRouteState::Absent,
2027 }
2028 };
2029 Ok(
2030 if inner.endpoint_by_connection.contains_key(&connection_id) {
2031 DataRoute::Module(state)
2032 } else {
2033 DataRoute::Client(state)
2034 },
2035 )
2036 }
2037
2038 #[cfg(test)]
2039 pub(crate) fn inject_client_slot_epoch(
2040 &self,
2041 connection_id: ConnectionId,
2042 channel: u16,
2043 last_epoch: u32,
2044 ) {
2045 let mut inner = self.write_inner().expect("forwarding lock");
2046 inner.client_slot_epochs.insert(
2047 ClientRouteKey {
2048 connection_id,
2049 channel,
2050 },
2051 last_epoch,
2052 );
2053 inner.next_client_channel.insert(connection_id, channel);
2054 }
2055
2056 #[cfg(test)]
2057 pub(crate) fn inject_module_slot_epoch(
2058 &self,
2059 endpoint: ModuleEndpointId,
2060 channel: u16,
2061 last_epoch: u32,
2062 ) {
2063 let mut inner = self.write_inner().expect("forwarding lock");
2064 inner
2065 .module_slot_epochs
2066 .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
2067 inner.next_module_channel.insert(endpoint, channel);
2068 }
2069
2070 #[cfg(test)]
2071 pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
2072 self.write_inner()
2073 .expect("forwarding lock")
2074 .next_control_corr
2075 .insert(endpoint, next_corr);
2076 }
2077
2078 pub(crate) fn cache_status(
2079 &self,
2080 endpoint: ModuleEndpointId,
2081 module_channel: u16,
2082 module_epoch: u32,
2083 status: String,
2084 ) -> Result<bool, ForwardingError> {
2085 let mut inner = self.write_inner()?;
2086 if !inner.module_id_by_endpoint.contains_key(&endpoint) {
2087 return Err(ForwardingError::StaleModuleEndpoint);
2088 }
2089
2090 let module_key = ModuleRouteKey {
2091 endpoint,
2092 channel: module_channel,
2093 };
2094 let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
2095 (route.module_epoch == module_epoch).then_some((
2096 ClientRouteKey {
2097 connection_id: route.client_connection_id,
2098 channel: route.client_channel,
2099 },
2100 route.client_epoch,
2101 ))
2102 } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
2103 (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
2104 client_key,
2105 inner
2106 .client_slot_epochs
2107 .get(&client_key)
2108 .copied()
2109 .unwrap_or(0),
2110 ))
2111 } else {
2112 None
2113 };
2114
2115 if let Some(handle) = handle {
2116 inner.status.insert(handle, status);
2117 Ok(true)
2118 } else {
2119 debug!(
2120 module_channel,
2121 module_epoch,
2122 generation = endpoint.generation,
2123 connection_id = endpoint.connection_id.get(),
2124 "dropping stale status update for module route handle"
2125 );
2126 Ok(false)
2127 }
2128 }
2129
2130 pub(crate) fn route_poll_snapshot(
2131 &self,
2132 client_connection_id: ConnectionId,
2133 client_channel: u16,
2134 client_epoch: u32,
2135 ) -> Result<RoutePollSnapshot, ForwardingError> {
2136 let inner = self.read_inner()?;
2137 let client_key = ClientRouteKey {
2138 connection_id: client_connection_id,
2139 channel: client_channel,
2140 };
2141 let Some(route) = inner.client_to_module.get(&client_key) else {
2142 return Ok(RoutePollSnapshot::Absent);
2143 };
2144 if route.client_epoch != client_epoch
2145 || !inner
2146 .module_id_by_endpoint
2147 .contains_key(&route.module_endpoint)
2148 {
2149 return Ok(RoutePollSnapshot::Absent);
2150 }
2151 Ok(RoutePollSnapshot::Bound {
2152 module_id: route.module_id.clone(),
2153 status: inner.status.get(&(client_key, client_epoch)).cloned(),
2154 })
2155 }
2156
2157 pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2158 Ok(self.read_inner()?.client_to_module.len())
2159 }
2160
2161 pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2171 let inner = self.read_inner()?;
2172 let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2173 for key in inner.client_to_module.keys() {
2174 *per_connection.entry(key.connection_id).or_insert(0) += 1;
2175 }
2176 let max = per_connection.values().copied().max().unwrap_or(0);
2177 Ok((per_connection.len(), max))
2178 }
2179
2180 pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2181 let inner = self.read_inner()?;
2182 Ok(inner
2183 .client_to_module
2184 .keys()
2185 .any(|key| key.channel == route_channel))
2186 }
2187
2188 pub(crate) fn daemon_drain_result_for_connection(
2192 &self,
2193 connection_id: ConnectionId,
2194 ) -> Result<Option<bool>, ForwardingError> {
2195 let inner = self.read_inner()?;
2196 Ok(inner.daemon_draining.then(|| {
2197 inner
2198 .endpoint_by_connection
2199 .get(&connection_id)
2200 .and_then(|endpoint| inner.daemon_drain_results.get(endpoint))
2201 .copied()
2202 .unwrap_or(false)
2203 }))
2204 }
2205
2206 #[cfg(unix)]
2207 pub(crate) fn record_daemon_drain_result(
2208 &self,
2209 endpoint: ModuleEndpointId,
2210 drained: bool,
2211 ) -> Result<(), ForwardingError> {
2212 let mut inner = self.write_inner()?;
2213 if let Some(result) = inner.daemon_drain_results.get_mut(&endpoint) {
2217 *result = drained;
2218 }
2219 Ok(())
2220 }
2221
2222 #[cfg(unix)]
2225 pub(crate) fn begin_daemon_drain(
2226 &self,
2227 ) -> Result<Vec<(String, ModuleEndpointId)>, ForwardingError> {
2228 let mut inner = self.write_inner()?;
2229 if inner.daemon_draining {
2230 return Ok(Vec::new());
2231 }
2232 inner.daemon_draining = true;
2233 let modules = inner
2234 .modules_by_id
2235 .iter()
2236 .map(|(id, module)| (id.clone(), module.endpoint))
2237 .collect::<Vec<_>>();
2238 for (_, endpoint) in &modules {
2239 inner.daemon_drain_results.insert(*endpoint, false);
2240 inner
2241 .draining_endpoints
2242 .insert(*endpoint, RouteCloseReason::Restart);
2243 }
2244 let off_slot_endpoints = inner
2249 .candidates_by_id
2250 .values()
2251 .map(|module| module.endpoint)
2252 .chain(inner.superseded_endpoints.keys().copied())
2253 .collect::<Vec<_>>();
2254 for endpoint in off_slot_endpoints {
2255 inner
2256 .draining_endpoints
2257 .insert(endpoint, RouteCloseReason::Restart);
2258 }
2259 Ok(modules)
2260 }
2261
2262 pub(crate) fn begin_module_drain(
2269 &self,
2270 module_id: &str,
2271 reason: RouteCloseReason,
2272 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2273 let mut inner = self.write_inner()?;
2274 let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2275 return Ok(None);
2276 };
2277 Ok(Some(begin_drain_locked(
2278 &mut inner, module_id, module, reason,
2279 )))
2280 }
2281
2282 pub(crate) fn begin_endpoint_drain(
2289 &self,
2290 endpoint: ModuleEndpointId,
2291 reason: RouteCloseReason,
2292 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2293 let mut inner = self.write_inner()?;
2294 let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2295 return Ok(None);
2296 };
2297 let module_id = inner
2298 .module_id_by_endpoint
2299 .get(&endpoint)
2300 .cloned()
2301 .expect("an endpoint resolved to a module connection has a module id");
2302 Ok(Some(begin_drain_locked(
2303 &mut inner, &module_id, module, reason,
2304 )))
2305 }
2306}
2307
2308fn begin_drain_locked(
2312 inner: &mut ForwardingInner,
2313 module_id: &str,
2314 module: ModuleConnection,
2315 reason: RouteCloseReason,
2316) -> ModuleDrainTarget {
2317 {
2318 let endpoint = module.endpoint;
2319 inner.draining_endpoints.insert(endpoint, reason);
2320
2321 let flows = inner
2322 .client_to_module
2323 .values()
2324 .filter(|route| route.module_endpoint == endpoint)
2325 .map(|route| Arc::clone(&route.flow))
2326 .collect::<Vec<_>>();
2327 let excluded_subscriptions = flows
2328 .into_iter()
2329 .map(|flow| flow.begin_drain())
2330 .fold(0u32, u32::saturating_add);
2331
2332 let pending_keys = inner
2333 .pending_relays
2334 .keys()
2335 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2336 .copied()
2337 .collect::<Vec<_>>();
2338 let mut abandoned_bindings = Vec::new();
2339 for key in pending_keys {
2340 let Some(pending) = inner.pending_relays.remove(&key) else {
2341 continue;
2342 };
2343 release_reserved_route_locked(
2344 inner,
2345 pending.reservation.client_key,
2346 pending.reservation.module_key,
2347 );
2348 if pending.relay_enqueued {
2349 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2350 abandoned_bindings.push(target);
2351 }
2352 } else {
2353 inner.relays_released_before_enqueue.insert(key);
2357 }
2358 let _ = pending
2359 .sender
2360 .send(RouteBindRelayOutcome::Rejected(ErrorBody::new(
2361 "module_reloading",
2362 format!("module_id '{module_id}' is reloading"),
2363 )));
2364 }
2365
2366 let pending_control_keys = inner
2367 .pending_control_rpcs
2368 .keys()
2369 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2370 .copied()
2371 .collect::<Vec<_>>();
2372 for key in pending_control_keys {
2373 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2374 let _ = pending
2375 .sender
2376 .send(ModuleControlRpcOutcome::ModuleGone(format!(
2377 "module '{module_id}' began draining during module-control RPC"
2378 )));
2379 }
2380 }
2381
2382 ModuleDrainTarget {
2383 endpoint,
2384 sink: module.sink,
2385 negotiated_ver: module.negotiated_ver,
2386 abandoned_bindings,
2387 excluded_subscriptions,
2388 }
2389 }
2390}
2391
2392#[derive(Debug, Default, PartialEq, Eq)]
2395pub(crate) struct DrainHoldouts {
2396 pub(crate) requests: usize,
2399 pub(crate) routes: usize,
2401 pub(crate) total_routes: usize,
2403 pub(crate) top_connections: Vec<(u64, usize)>,
2406 pub(crate) held: Vec<(u16, u64)>,
2415}
2416
2417pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
2419
2420impl ForwardingTable {
2421 pub(crate) fn endpoint_drain_holdouts(
2423 &self,
2424 endpoint: ModuleEndpointId,
2425 ) -> Result<DrainHoldouts, ForwardingError> {
2426 let inner = self.read_inner()?;
2427 let mut holdouts = DrainHoldouts::default();
2428 let mut by_connection: HashMap<u64, usize> = HashMap::new();
2429 for (key, route) in &inner.client_to_module {
2430 if route.module_endpoint != endpoint {
2431 continue;
2432 }
2433 holdouts.total_routes += 1;
2434 let held = route.flow.drain_in_flight();
2435 if held == 0 {
2436 continue;
2437 }
2438 holdouts.requests += held;
2439 holdouts.routes += 1;
2440 *by_connection.entry(key.connection_id.get()).or_default() += held;
2441 holdouts.held.extend(
2442 route
2443 .flow
2444 .drain_held_corrs()
2445 .into_iter()
2446 .map(|corr| (route.module_channel, corr)),
2447 );
2448 }
2449 holdouts.held.sort_unstable();
2450 holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
2451 let mut connections = by_connection.into_iter().collect::<Vec<_>>();
2452 connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
2453 connections.truncate(3);
2454 holdouts.top_connections = connections;
2455 Ok(holdouts)
2456 }
2457
2458 pub(crate) fn endpoint_in_flight_count(
2459 &self,
2460 endpoint: ModuleEndpointId,
2461 ) -> Result<usize, ForwardingError> {
2462 let inner = self.read_inner()?;
2463 Ok(inner
2464 .client_to_module
2465 .values()
2466 .filter(|route| route.module_endpoint == endpoint)
2467 .map(|route| route.flow.drain_in_flight())
2468 .sum())
2469 }
2470
2471 pub(crate) fn endpoint_is_draining(
2472 &self,
2473 endpoint: ModuleEndpointId,
2474 ) -> Result<bool, ForwardingError> {
2475 Ok(self
2476 .read_inner()?
2477 .draining_endpoints
2478 .contains_key(&endpoint))
2479 }
2480
2481 pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
2482 let inner = self.read_inner()?;
2483 Ok(inner
2484 .modules_by_id
2485 .get(module_id)
2486 .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
2487 }
2488
2489 pub(crate) fn release_module_endpoint_routes(
2490 &self,
2491 endpoint: ModuleEndpointId,
2492 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2493 let mut inner = self.write_inner()?;
2494 let routes = inner
2495 .module_to_client
2496 .iter()
2497 .filter(|(module_key, _)| module_key.endpoint == endpoint)
2498 .map(|(module_key, route)| (*module_key, route.module_epoch))
2499 .collect::<Vec<_>>();
2500 let mut released = Vec::with_capacity(routes.len());
2501 for (module_key, epoch) in routes {
2502 if let RouteRelease::Removed(target) =
2503 release_module_route_locked(&mut inner, module_key, epoch)
2504 {
2505 released.push(target);
2506 }
2507 }
2508 Ok(released)
2509 }
2510
2511 pub(crate) fn endpoint_routes(
2517 &self,
2518 endpoint: ModuleEndpointId,
2519 ) -> Result<Vec<EndpointRoute>, ForwardingError> {
2520 let inner = self.read_inner()?;
2521 Ok(endpoint_routes_locked(&inner, endpoint))
2522 }
2523
2524 pub(crate) fn route_census(
2526 &self,
2527 module_id: Option<&str>,
2528 ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
2529 let inner = self.read_inner()?;
2530 let mut endpoints = inner
2531 .modules_by_id
2532 .iter()
2533 .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
2534 .map(|(id, module)| (id.clone(), module.endpoint))
2535 .collect::<Vec<_>>();
2536 endpoints.sort_by(|left, right| left.0.cmp(&right.0));
2537 Ok(endpoints
2538 .into_iter()
2539 .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
2540 .collect())
2541 }
2542
2543 pub(crate) fn live_roots(
2545 &self,
2546 module_id: &str,
2547 ) -> Result<ModuleControlResponseToModule, ForwardingError> {
2548 let inner = self.read_inner()?;
2549 let endpoint = inner
2550 .modules_by_id
2551 .get(module_id)
2552 .map(|module| module.endpoint);
2553 let mut roots = BTreeMap::new();
2554 let mut unknown_root_bindings = 0;
2555 let mut total_bindings = 0;
2556 if let Some(endpoint) = endpoint {
2557 for binding in inner
2558 .module_to_client
2559 .values()
2560 .filter(|binding| binding.module_endpoint == endpoint)
2561 {
2562 total_bindings += 1;
2563 if let Some(root) = &binding.project_root {
2564 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2565 entry.0 += 1;
2566 } else {
2567 unknown_root_bindings += 1;
2568 }
2569 }
2570 for pending in inner
2571 .pending_relays
2572 .values()
2573 .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
2574 {
2575 total_bindings += 1;
2576 if let Some(root) = &pending.reservation.project_root {
2577 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2578 entry.1 += 1;
2579 } else {
2580 unknown_root_bindings += 1;
2581 }
2582 }
2583 }
2584 Ok(ModuleControlResponseToModule::LiveRoots {
2585 roots: roots
2586 .into_iter()
2587 .map(|(project_root, (bound, pending))| LiveRoot {
2588 project_root,
2589 bound,
2590 pending,
2591 })
2592 .collect(),
2593 unknown_root_bindings,
2594 total_bindings,
2595 })
2596 }
2597
2598 pub(crate) fn connection_has_client_routes(
2604 &self,
2605 connection_id: ConnectionId,
2606 ) -> Result<bool, ForwardingError> {
2607 let inner = self.read_inner()?;
2608 Ok(connection_has_client_routes_locked(&inner, connection_id))
2609 }
2610
2611 pub(crate) fn cleanup_connection(
2612 &self,
2613 connection_id: ConnectionId,
2614 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2615 self.cleanup_connection_counted(connection_id)
2616 .map(|cleanup| cleanup.released)
2617 }
2618
2619 pub(crate) fn cleanup_connection_counted(
2624 &self,
2625 connection_id: ConnectionId,
2626 ) -> Result<ConnectionCleanup, ForwardingError> {
2627 let mut inner = self.write_inner()?;
2628 inner.closing_connections.insert(connection_id);
2629 let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
2630 remove_module_connection_locked(&mut inner, endpoint)
2631 } else {
2632 ConnectionCleanup {
2633 released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
2634 abandoned_relays: 0,
2635 }
2636 };
2637 inner.closing_connections.remove(&connection_id);
2646 Ok(cleanup)
2647 }
2648
2649 fn cleanup_client_connection_locked(
2650 inner: &mut ForwardingInner,
2651 connection_id: ConnectionId,
2652 ) -> Vec<GoodbyeTarget> {
2653 let routes = inner
2654 .client_to_module
2655 .iter()
2656 .filter(|(key, _)| key.connection_id == connection_id)
2657 .map(|(key, route)| (*key, route.client_epoch))
2658 .collect::<Vec<_>>();
2659 let mut released = Vec::with_capacity(routes.len());
2660 for (client_key, epoch) in routes {
2661 if let RouteRelease::Removed(target) =
2662 release_client_route_locked(inner, client_key, epoch)
2663 {
2664 released.push(target);
2665 }
2666 }
2667
2668 let pending_keys = inner
2669 .pending_relays
2670 .iter()
2671 .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
2672 .map(|(key, _)| *key)
2673 .collect::<Vec<_>>();
2674 for key in pending_keys {
2675 let Some(pending) = inner.pending_relays.remove(&key) else {
2676 continue;
2677 };
2678 release_reserved_route_locked(
2679 inner,
2680 pending.reservation.client_key,
2681 pending.reservation.module_key,
2682 );
2683 if pending.relay_enqueued {
2684 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2685 released.push(target);
2686 }
2687 } else {
2688 inner.relays_released_before_enqueue.insert(key);
2692 }
2693 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
2694 "client connection closed during route.bind relay".to_string(),
2695 ));
2696 }
2697
2698 let orphaned = inner
2699 .reserved_client
2700 .iter()
2701 .filter(|(key, _)| key.connection_id == connection_id)
2702 .map(|(client, module)| (*client, *module))
2703 .collect::<Vec<_>>();
2704 for (client_key, module_key) in orphaned {
2705 release_reserved_route_locked(inner, client_key, module_key);
2706 }
2707 inner.next_client_channel.remove(&connection_id);
2708 inner
2709 .client_slot_epochs
2710 .retain(|key, _| key.connection_id != connection_id);
2711 inner
2712 .last_published_epoch
2713 .retain(|key, _| key.connection_id != connection_id);
2714 inner
2715 .status
2716 .retain(|(key, _), _| key.connection_id != connection_id);
2717
2718 released
2719 }
2720
2721 pub(crate) fn escalate_client_delivery_failure(
2730 &self,
2731 connection_id: ConnectionId,
2732 channel: u16,
2733 expected_epoch: u32,
2734 reason: CloseReason,
2735 undelivered: UndeliveredFrame<'_>,
2736 ) -> Result<bool, ForwardingError> {
2737 self.escalate_client_delivery_failure_diagnosed(
2738 connection_id,
2739 channel,
2740 expected_epoch,
2741 reason,
2742 undelivered,
2743 None,
2744 )
2745 }
2746
2747 pub(crate) fn escalate_client_delivery_failure_diagnosed(
2748 &self,
2749 connection_id: ConnectionId,
2750 channel: u16,
2751 expected_epoch: u32,
2752 reason: CloseReason,
2753 undelivered: UndeliveredFrame<'_>,
2754 diagnosis: Option<&crate::router::ClientDeliveryDiagnosis>,
2755 ) -> Result<bool, ForwardingError> {
2756 let principals = {
2757 let mut inner = self.write_inner()?;
2758 let key = ClientRouteKey {
2759 connection_id,
2760 channel,
2761 };
2762 if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
2763 None
2764 } else {
2765 inner.closing_connections.insert(connection_id);
2766 Some(connection_principals_locked(&inner, connection_id))
2767 }
2768 };
2769 let Some(principals) = principals else {
2770 return Ok(false);
2771 };
2772 let backlog = undelivered.sink.backlog();
2773 let close_reason = reason.to_string();
2774 if self.request_connection_close(connection_id, reason) {
2775 warn!(
2776 connection_id = connection_id.get(),
2777 principals = %principals,
2778 module_id = undelivered.module_id.unwrap_or("unknown"),
2779 client_channel = channel,
2780 queued_bytes = backlog.queued_bytes,
2781 queued_frames = backlog.queued_frames,
2782 frame_type = ?diagnosis.map(|failure| failure.frame_type),
2783 frame_flags = ?diagnosis.map(|failure| failure.flags.0),
2784 close_cause = diagnosis.map(|failure| failure.cause).unwrap_or("delivery_failed"),
2785 oldest_queued_ms = backlog
2786 .oldest_age
2787 .map(|age| age.as_millis() as u64)
2788 .unwrap_or(0),
2789 close_reason = %close_reason,
2790 "closing client connection: its egress queue could not take a frame"
2791 );
2792 }
2793 Ok(true)
2794 }
2795
2796 pub(crate) fn publish_scope_changes(
2810 &self,
2811 changes: &[ScopeTagChange],
2812 ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
2813 let mut inner = self.write_inner()?;
2814 let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
2815 for change in changes {
2816 let key = (change.owner.clone(), change.scope_ref.clone());
2817 match change.after {
2818 Some(tag) => {
2819 inner.scope_tags.insert(key, tag);
2820 }
2821 None => {
2822 inner.scope_tags.remove(&key);
2823 }
2824 }
2825 by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
2826 }
2827 let mut selected = Vec::new();
2828 for (client_key, route) in &inner.client_to_module {
2829 let Some(scope) = &route.scope else {
2830 continue;
2831 };
2832 let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
2833 else {
2834 continue;
2835 };
2836 if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
2837 continue;
2838 }
2839 let reason = match &change.drain {
2840 ScopeDrain::Nothing => continue,
2841 ScopeDrain::All(reason) => *reason,
2842 ScopeDrain::Carriers(narrowed) => {
2843 let owner = Principal::Reserved {
2844 module_id: scope.owner.clone(),
2845 };
2846 let hit = route.principal != owner
2847 && narrowed.iter().any(|(principal, allowed)| {
2848 *principal == route.principal
2849 && allowed
2850 .as_ref()
2851 .is_none_or(|targets| !targets.contains(&route.module_id))
2852 });
2853 if !hit {
2854 continue;
2855 }
2856 RouteCloseReason::ScopeCarrierRemoved
2857 }
2858 };
2859 selected.push((*client_key, route.client_epoch, reason));
2860 }
2861 let mut drained = Vec::new();
2862 for (client_key, client_epoch, reason) in selected {
2863 let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
2864 continue;
2865 };
2866 let release = release_client_route_locked(&mut inner, client_key, client_epoch);
2867 self.record_route_release(&release);
2868 if let RouteRelease::Removed(module) = release {
2869 drained.push(ScopeDrainedRoute {
2870 reason,
2871 scope: route
2872 .scope
2873 .clone()
2874 .expect("selected scope drain has a bound scope"),
2875 module_id: route.module_id.clone(),
2876 client: GoodbyeTarget {
2877 connection_id: route.client_connection_id,
2878 sink: route.client_sink.clone(),
2879 negotiated_ver: route.client_negotiated_ver,
2880 channel: route.client_channel,
2881 epoch: route.client_epoch,
2882 kind: GoodbyeTargetKind::Client,
2883 module_id: Some(route.module_id.clone()),
2884 },
2885 module,
2886 });
2887 }
2888 }
2889 Ok(drained)
2890 }
2891
2892 #[cfg(test)]
2894 pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
2895 self.read_inner()
2896 .ok()?
2897 .scope_tags
2898 .get(&(owner.to_string(), scope_ref.to_string()))
2899 .copied()
2900 }
2901
2902 fn record_route_release(&self, release: &RouteRelease) {
2903 match release {
2904 RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
2905 RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
2906 RouteRelease::Absent => {}
2907 }
2908 }
2909
2910 fn read_inner(&self) -> Result<RwLockReadGuard<'_, ForwardingInner>, ForwardingError> {
2911 self.inner.read().map_err(|_| ForwardingError::Poisoned)
2912 }
2913
2914 fn write_inner(&self) -> Result<RwLockWriteGuard<'_, ForwardingInner>, ForwardingError> {
2915 self.inner.write().map_err(|_| ForwardingError::Poisoned)
2916 }
2917
2918 fn lock_close_registry(
2919 &self,
2920 ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
2921 self.close_registry
2922 .lock()
2923 .unwrap_or_else(|poisoned| poisoned.into_inner())
2924 }
2925}
2926
2927impl ForwardingInner {
2928 fn allocate_route_slots(
2929 &mut self,
2930 connection_id: ConnectionId,
2931 endpoint: ModuleEndpointId,
2932 ) -> Result<(u16, u32, u16, u32), ForwardingError> {
2933 let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
2934 let mut client_channel = client_start;
2935 let client_channel = loop {
2936 let key = ClientRouteKey {
2937 connection_id,
2938 channel: client_channel,
2939 };
2940 let eligible = !self.client_to_module.contains_key(&key)
2941 && !self.reserved_client.contains_key(&key)
2942 && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2943 if eligible {
2944 break client_channel;
2945 }
2946 client_channel = next_channel(client_channel);
2947 if client_channel == client_start {
2948 return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
2949 }
2950 };
2951
2952 let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
2953 let mut module_channel = module_start;
2954 let module_channel = loop {
2955 let key = ModuleRouteKey {
2956 endpoint,
2957 channel: module_channel,
2958 };
2959 let eligible = !self.module_to_client.contains_key(&key)
2960 && !self.reserved_module.contains_key(&key)
2961 && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2962 if eligible {
2963 break module_channel;
2964 }
2965 module_channel = next_channel(module_channel);
2966 if module_channel == module_start {
2967 return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
2968 }
2969 };
2970
2971 let client_key = ClientRouteKey {
2972 connection_id,
2973 channel: client_channel,
2974 };
2975 let module_key = ModuleRouteKey {
2976 endpoint,
2977 channel: module_channel,
2978 };
2979 let client_epoch = self
2980 .client_slot_epochs
2981 .get(&client_key)
2982 .copied()
2983 .unwrap_or(0)
2984 + 1;
2985 let module_epoch = self
2986 .module_slot_epochs
2987 .get(&module_key)
2988 .copied()
2989 .unwrap_or(0)
2990 + 1;
2991 self.client_slot_epochs.insert(client_key, client_epoch);
2992 self.module_slot_epochs.insert(module_key, module_epoch);
2993 self.next_client_channel
2994 .insert(connection_id, next_channel(client_channel));
2995 self.next_module_channel
2996 .insert(endpoint, next_channel(module_channel));
2997 Ok((client_channel, client_epoch, module_channel, module_epoch))
2998 }
2999
3000 fn allocate_control_corr(
3001 &mut self,
3002 endpoint: ModuleEndpointId,
3003 ) -> Result<u64, ForwardingError> {
3004 let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
3005 if candidate == 0 {
3006 self.closing_connections.insert(endpoint.connection_id);
3007 return Err(ForwardingError::RelayCorrelationExhausted);
3008 }
3009 self.next_control_corr.insert(
3010 endpoint,
3011 if candidate == u64::MAX {
3012 0
3013 } else {
3014 candidate + 1
3015 },
3016 );
3017 Ok(candidate)
3018 }
3019}
3020
3021fn connection_has_client_routes_locked(
3022 inner: &ForwardingInner,
3023 connection_id: ConnectionId,
3024) -> bool {
3025 inner
3026 .client_to_module
3027 .keys()
3028 .any(|key| key.connection_id == connection_id)
3029 || inner
3030 .reserved_client
3031 .keys()
3032 .any(|key| key.connection_id == connection_id)
3033}
3034
3035fn check_module_connection_role_locked(
3036 inner: &ForwardingInner,
3037 connection_id: ConnectionId,
3038) -> Result<(), ForwardingError> {
3039 if inner.endpoint_by_connection.contains_key(&connection_id)
3040 || connection_has_client_routes_locked(inner, connection_id)
3041 {
3042 return Err(ForwardingError::ConnectionRoleConflict { connection_id });
3043 }
3044 Ok(())
3045}
3046
3047fn next_channel(channel: u16) -> u16 {
3048 let next = channel.wrapping_add(1);
3049 if next == 0 {
3050 1
3051 } else {
3052 next
3053 }
3054}
3055
3056fn endpoint_routes_locked(
3057 inner: &ForwardingInner,
3058 endpoint: ModuleEndpointId,
3059) -> Vec<EndpointRoute> {
3060 let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
3061 let draining = drain_reason.is_some();
3062 let mut routes = inner
3063 .module_to_client
3064 .iter()
3065 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3066 .map(|(_, route)| EndpointRoute {
3067 goodbye_target: GoodbyeTarget {
3068 connection_id: route.client_connection_id,
3069 sink: route.client_sink.clone(),
3070 negotiated_ver: route.client_negotiated_ver,
3071 channel: route.client_channel,
3072 epoch: route.client_epoch,
3073 kind: GoodbyeTargetKind::Client,
3074 module_id: Some(route.module_id.clone()),
3075 },
3076 principal: route.principal.clone(),
3077 bound_at: route.bound_at,
3078 draining,
3079 drain_reason,
3080 })
3081 .collect::<Vec<_>>();
3082 routes.sort_by_key(|route| {
3083 (
3084 route.goodbye_target.connection_id.get(),
3085 route.goodbye_target.channel,
3086 route.goodbye_target.epoch,
3087 )
3088 });
3089 routes
3090}
3091
3092fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
3096 let key = (scope.owner.clone(), scope.scope_ref.clone());
3097 match inner.scope_tags.get(&key) {
3098 Some(current) if *current == scope.tag => None,
3099 Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
3100 subc_protocol::error_codes::SCOPE_CHANGED,
3101 format!(
3102 "scope '{}' of {} changed while the route was being bound; re-open it",
3103 scope.scope_ref, scope.owner
3104 ),
3105 )),
3106 _ => Some(ErrorBody::new(
3107 subc_protocol::error_codes::SCOPE_ENDED,
3108 format!(
3109 "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
3110 scope.scope_ref, scope.owner, scope.tag.scope_epoch
3111 ),
3112 )),
3113 }
3114}
3115
3116fn release_reserved_route_locked(
3117 inner: &mut ForwardingInner,
3118 client_key: ClientRouteKey,
3119 module_key: ModuleRouteKey,
3120) {
3121 if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
3122 inner.reserved_client.remove(&client_key);
3123 }
3124 if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
3125 inner.reserved_module.remove(&module_key);
3126 }
3127 inner.status.retain(|(key, _), _| *key != client_key);
3128}
3129
3130fn release_client_route_locked(
3131 inner: &mut ForwardingInner,
3132 client_key: ClientRouteKey,
3133 expected_epoch: u32,
3134) -> RouteRelease {
3135 let Some(route) = inner.client_to_module.get(&client_key) else {
3136 return RouteRelease::Absent;
3137 };
3138 if route.client_epoch != expected_epoch {
3139 return RouteRelease::Stale;
3140 }
3141 let route = inner
3142 .client_to_module
3143 .remove(&client_key)
3144 .expect("route checked under the same forwarding lock");
3145 route.flow.close();
3146 if let Some(scope) = &route.scope {
3147 scope.release_bind();
3148 }
3149 inner.module_to_client.remove(&ModuleRouteKey {
3150 endpoint: route.module_endpoint,
3151 channel: route.module_channel,
3152 });
3153 inner.operator_confirms.route_closed(ModuleRouteKey {
3154 endpoint: route.module_endpoint,
3155 channel: route.module_channel,
3156 });
3157 inner.status.remove(&(client_key, expected_epoch));
3158 RouteRelease::Removed(GoodbyeTarget {
3159 connection_id: route.module_endpoint.connection_id,
3160 sink: route.module_sink.clone(),
3161 negotiated_ver: route.module_negotiated_ver,
3162 channel: route.module_channel,
3163 epoch: route.module_epoch,
3164 kind: GoodbyeTargetKind::Module,
3165 module_id: Some(route.module_id.clone()),
3166 })
3167}
3168
3169fn release_module_route_locked(
3170 inner: &mut ForwardingInner,
3171 module_key: ModuleRouteKey,
3172 expected_epoch: u32,
3173) -> RouteRelease {
3174 let Some(route) = inner.module_to_client.get(&module_key) else {
3175 return RouteRelease::Absent;
3176 };
3177 if route.module_epoch != expected_epoch {
3178 return RouteRelease::Stale;
3179 }
3180 let route = inner
3181 .module_to_client
3182 .remove(&module_key)
3183 .expect("route checked under the same forwarding lock");
3184 inner.operator_confirms.route_closed(module_key);
3185 route.flow.close();
3186 if let Some(scope) = &route.scope {
3187 scope.release_bind();
3188 }
3189 let client_key = ClientRouteKey {
3190 connection_id: route.client_connection_id,
3191 channel: route.client_channel,
3192 };
3193 inner.client_to_module.remove(&client_key);
3194 inner.status.remove(&(client_key, route.client_epoch));
3195 RouteRelease::Removed(GoodbyeTarget {
3196 connection_id: route.client_connection_id,
3197 sink: route.client_sink.clone(),
3198 negotiated_ver: route.client_negotiated_ver,
3199 channel: route.client_channel,
3200 epoch: route.client_epoch,
3201 kind: GoodbyeTargetKind::Client,
3202 module_id: Some(route.module_id.clone()),
3203 })
3204}
3205
3206fn commit_route_locked(
3207 inner: &mut ForwardingInner,
3208 pending: PendingRouteBindRelayEntry,
3209) -> Result<Option<GoodbyeTarget>, ForwardingError> {
3210 let reservation = pending.reservation;
3211 if inner
3212 .closing_connections
3213 .contains(&reservation.client_key.connection_id)
3214 {
3215 return Err(ForwardingError::ConnectionClosing {
3216 connection_id: reservation.client_key.connection_id,
3217 });
3218 }
3219 let module_id = inner
3220 .module_id_by_endpoint
3221 .get(&reservation.module_key.endpoint)
3222 .cloned()
3223 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3224 if inner
3225 .draining_endpoints
3226 .contains_key(&reservation.module_key.endpoint)
3227 {
3228 return Err(ForwardingError::ModuleReloading { module_id });
3229 }
3230 if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
3231 || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
3232 {
3233 return Err(ForwardingError::UnknownReservation {
3234 client_channel: reservation.client_key.channel,
3235 module_channel: reservation.module_key.channel,
3236 });
3237 }
3238 let module = inner
3239 .modules_by_id
3240 .get(&module_id)
3241 .filter(|module| module.endpoint == reservation.module_key.endpoint)
3242 .cloned()
3243 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3244 let binding = Arc::new(RouteBinding {
3245 client_connection_id: reservation.client_key.connection_id,
3246 client_sink: pending.client_sink,
3247 client_negotiated_ver: pending.client_negotiated_ver,
3248 client_channel: reservation.client_key.channel,
3249 client_epoch: reservation.client_epoch,
3250 module_id,
3251 module_endpoint: reservation.module_key.endpoint,
3252 module_sink: module.sink,
3253 module_negotiated_ver: module.negotiated_ver,
3254 module_channel: reservation.module_key.channel,
3255 module_epoch: reservation.module_epoch,
3256 principal: pending.principal,
3257 project_root: reservation.project_root.clone(),
3258 bound_at: Instant::now(),
3259 flow: Arc::new(ChannelFlow::new(window_for(&module.concurrency))),
3260 scope: pending.scope,
3261 });
3262 inner
3263 .client_to_module
3264 .insert(reservation.client_key, Arc::clone(&binding));
3265 inner
3266 .module_to_client
3267 .insert(reservation.module_key, Arc::clone(&binding));
3268 let previous_published = inner
3269 .last_published_epoch
3270 .insert(reservation.client_key, reservation.client_epoch);
3271
3272 if let Some(scope) = &binding.scope {
3276 scope.record_served(&binding.module_id);
3277 }
3278
3279 let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
3284 if client_writer_closed {
3285 let abandoned = abandoned_route_target(inner, &reservation);
3289 if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
3290 route.flow.close();
3291 }
3292 inner.module_to_client.remove(&reservation.module_key);
3293 inner
3294 .status
3295 .remove(&(reservation.client_key, reservation.client_epoch));
3296 match previous_published {
3297 Some(epoch) => {
3298 inner
3299 .last_published_epoch
3300 .insert(reservation.client_key, epoch);
3301 }
3302 None => {
3303 inner.last_published_epoch.remove(&reservation.client_key);
3304 }
3305 }
3306 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3307 "client egress closed during route publication".to_string(),
3308 ));
3309 return Ok(abandoned);
3310 }
3311
3312 if let Some(scope) = &binding.scope {
3313 scope.commit_presence();
3314 }
3315 let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
3316 Ok(None)
3317}
3318
3319fn module_connection_for_endpoint_locked(
3327 inner: &ForwardingInner,
3328 endpoint: ModuleEndpointId,
3329) -> Option<&ModuleConnection> {
3330 let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
3331 inner
3332 .modules_by_id
3333 .get(module_id)
3334 .filter(|module| module.endpoint == endpoint)
3335 .or_else(|| {
3336 inner
3337 .candidates_by_id
3338 .get(module_id)
3339 .filter(|module| module.endpoint == endpoint)
3340 })
3341 .or_else(|| inner.superseded_endpoints.get(&endpoint))
3342}
3343
3344fn abandoned_route_target(
3345 inner: &ForwardingInner,
3346 reservation: &RouteReservation,
3347) -> Option<GoodbyeTarget> {
3348 let module_id = inner
3349 .module_id_by_endpoint
3350 .get(&reservation.module_key.endpoint)?;
3351 let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
3352 (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
3353 connection_id: module.endpoint.connection_id,
3354 sink: module.sink.clone(),
3355 negotiated_ver: module.negotiated_ver,
3356 channel: reservation.module_key.channel,
3357 epoch: reservation.module_epoch,
3358 kind: GoodbyeTargetKind::Module,
3359 module_id: Some(module_id.clone()),
3360 })
3361}
3362
3363fn enqueue_hello_ack_locked(
3368 sink: &FrameSink,
3369 connection_id: ConnectionId,
3370 hello_ack: Option<Frame>,
3371) -> Result<(), ForwardingError> {
3372 let Some(hello_ack) = hello_ack else {
3373 return Ok(());
3374 };
3375 sink.try_send(hello_ack)
3376 .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
3377}
3378
3379fn remove_module_connection_locked(
3380 inner: &mut ForwardingInner,
3381 endpoint: ModuleEndpointId,
3382) -> ConnectionCleanup {
3383 inner.operator_confirms.module_closed(endpoint);
3386 inner.draining_endpoints.remove(&endpoint);
3387 inner.daemon_drain_results.remove(&endpoint);
3388 let module_id = inner.module_id_by_endpoint.remove(&endpoint);
3389 if let Some(module_id) = module_id.as_ref() {
3390 if inner
3391 .modules_by_id
3392 .get(module_id)
3393 .is_some_and(|module| module.endpoint == endpoint)
3394 {
3395 inner.modules_by_id.remove(module_id);
3396 }
3397 if inner
3398 .candidates_by_id
3399 .get(module_id)
3400 .is_some_and(|module| module.endpoint == endpoint)
3401 {
3402 inner.candidates_by_id.remove(module_id);
3403 }
3404 }
3405 inner.superseded_endpoints.remove(&endpoint);
3406 inner.endpoint_by_connection.remove(&endpoint.connection_id);
3407 inner.next_module_channel.remove(&endpoint);
3408 inner.next_control_corr.remove(&endpoint);
3409 inner
3410 .health_probe_tombstones
3411 .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
3412 inner
3413 .module_slot_epochs
3414 .retain(|key, _| key.endpoint != endpoint);
3415 let reserved_module_keys: Vec<ModuleRouteKey> = inner
3416 .reserved_module
3417 .keys()
3418 .filter(|module_key| module_key.endpoint == endpoint)
3419 .copied()
3420 .collect();
3421 for module_key in reserved_module_keys {
3422 if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
3423 release_reserved_route_locked(inner, client_key, module_key);
3424 }
3425 }
3426
3427 let pending_keys: Vec<_> = inner
3428 .pending_relays
3429 .keys()
3430 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3431 .copied()
3432 .collect();
3433 let pending: Vec<_> = pending_keys
3434 .into_iter()
3435 .filter_map(|key| {
3436 inner
3437 .pending_relays
3438 .remove(&key)
3439 .map(|pending| (key, pending))
3440 })
3441 .collect();
3442 let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
3443 for (key, pending) in pending {
3444 if !pending.relay_enqueued {
3448 inner.relays_released_before_enqueue.insert(key);
3449 }
3450 let module_label = module_id.as_deref().unwrap_or("unknown");
3451 let _ = pending
3452 .sender
3453 .send(RouteBindRelayOutcome::ModuleGone(format!(
3454 "module '{module_label}' connection closed during route.bind relay"
3455 )));
3456 }
3457
3458 let pending_control_keys: Vec<_> = inner
3459 .pending_control_rpcs
3460 .keys()
3461 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3462 .copied()
3463 .collect();
3464 let pending_control: Vec<_> = pending_control_keys
3465 .into_iter()
3466 .filter_map(|key| inner.pending_control_rpcs.remove(&key))
3467 .collect();
3468 for pending in pending_control {
3469 let module_label = module_id.as_deref().unwrap_or("unknown");
3470 let _ = pending
3471 .sender
3472 .send(ModuleControlRpcOutcome::ModuleGone(format!(
3473 "module '{module_label}' connection closed during module-control RPC"
3474 )));
3475 }
3476
3477 let module_routes = inner
3478 .module_to_client
3479 .iter()
3480 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3481 .map(|(module_key, route)| (*module_key, route.module_epoch))
3482 .collect::<Vec<_>>();
3483 let mut released = Vec::with_capacity(module_routes.len());
3484 for (module_key, epoch) in module_routes {
3485 if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
3486 {
3487 released.push(target);
3488 }
3489 }
3490 ConnectionCleanup {
3491 released,
3492 abandoned_relays,
3493 }
3494}
3495
3496#[derive(Debug, Clone, Copy)]
3497struct RequestCredit {
3498 subscription: bool,
3499 excluded_from_drain: bool,
3500}
3501
3502#[derive(Debug, Default)]
3503struct CreditLedger {
3504 by_corr: HashMap<u64, Vec<RequestCredit>>,
3505}
3506
3507impl CreditLedger {
3508 fn acquire(&mut self, corr: u64, subscription: bool) {
3509 self.by_corr.entry(corr).or_default().push(RequestCredit {
3510 subscription,
3511 excluded_from_drain: false,
3512 });
3513 }
3514
3515 fn release(&mut self, corr: u64) -> bool {
3516 let Some(credits) = self.by_corr.get_mut(&corr) else {
3517 return false;
3518 };
3519 let released = credits.pop().is_some();
3520 if credits.is_empty() {
3521 self.by_corr.remove(&corr);
3522 }
3523 released
3524 }
3525
3526 fn capture_subscription_exclusions(&mut self) -> u32 {
3527 let mut excluded = 0u32;
3528 for credit in self.by_corr.values_mut().flatten() {
3529 if credit.subscription && !credit.excluded_from_drain {
3530 credit.excluded_from_drain = true;
3531 excluded = excluded.saturating_add(1);
3532 }
3533 }
3534 excluded
3535 }
3536
3537 #[cfg(test)]
3538 fn in_flight(&self) -> usize {
3539 self.by_corr.values().map(Vec::len).sum()
3540 }
3541
3542 fn drain_in_flight(&self) -> usize {
3543 self.by_corr
3544 .values()
3545 .flatten()
3546 .filter(|credit| !credit.excluded_from_drain)
3547 .count()
3548 }
3549
3550 fn drain_held_corrs(&self) -> Vec<u64> {
3553 let mut corrs = self
3554 .by_corr
3555 .iter()
3556 .flat_map(|(corr, credits)| {
3557 credits
3558 .iter()
3559 .filter(|credit| !credit.excluded_from_drain)
3560 .map(move |_| *corr)
3561 })
3562 .collect::<Vec<_>>();
3563 corrs.sort_unstable();
3564 corrs
3565 }
3566}
3567
3568#[derive(Debug, Default)]
3569struct ChannelFlowState {
3570 closed: bool,
3571 credits: CreditLedger,
3572}
3573
3574#[derive(Debug)]
3576pub(crate) struct ChannelFlow {
3577 sem: Semaphore,
3578 window: usize,
3579 state: Mutex<ChannelFlowState>,
3580}
3581
3582impl ChannelFlow {
3583 pub(crate) fn new(window: usize) -> Self {
3584 debug_assert!(window > 0, "flow-control window must be non-zero");
3585 Self {
3586 sem: Semaphore::new(window),
3587 window,
3588 state: Mutex::new(ChannelFlowState::default()),
3589 }
3590 }
3591
3592 #[cfg(test)]
3593 pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
3594 self.acquire_tagged(0, false).await
3595 }
3596
3597 pub(crate) async fn acquire_tagged(
3598 &self,
3599 corr: u64,
3600 subscription: bool,
3601 ) -> Result<(), ChannelFlowClosed> {
3602 let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
3603 let mut state = self
3604 .state
3605 .lock()
3606 .unwrap_or_else(|poisoned| poisoned.into_inner());
3607 if state.closed {
3608 return Err(ChannelFlowClosed);
3609 }
3610 state.credits.acquire(corr, subscription);
3611 permit.forget();
3612 Ok(())
3613 }
3614
3615 #[cfg(test)]
3616 pub(crate) fn release(&self) {
3617 self.release_corr(0);
3618 }
3619
3620 pub(crate) fn release_corr(&self, corr: u64) {
3621 let released = self
3622 .state
3623 .lock()
3624 .unwrap_or_else(|poisoned| poisoned.into_inner())
3625 .credits
3626 .release(corr);
3627 if !released {
3628 warn!(
3632 window = self.window,
3633 available = self.sem.available_permits(),
3634 "flow-control over-release ignored"
3635 );
3636 return;
3637 }
3638 if !self.sem.is_closed() {
3639 self.sem.add_permits(1);
3640 }
3641 }
3642
3643 #[cfg(test)]
3644 pub(crate) fn in_flight(&self) -> usize {
3645 self.state
3646 .lock()
3647 .unwrap_or_else(|poisoned| poisoned.into_inner())
3648 .credits
3649 .in_flight()
3650 }
3651
3652 pub(crate) fn drain_in_flight(&self) -> usize {
3653 self.state
3654 .lock()
3655 .unwrap_or_else(|poisoned| poisoned.into_inner())
3656 .credits
3657 .drain_in_flight()
3658 }
3659
3660 pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
3661 self.state
3662 .lock()
3663 .unwrap_or_else(|poisoned| poisoned.into_inner())
3664 .credits
3665 .drain_held_corrs()
3666 }
3667
3668 #[cfg(test)]
3669 pub(crate) fn available_permits(&self) -> usize {
3670 self.sem.available_permits()
3671 }
3672
3673 pub(crate) fn begin_drain(&self) -> u32 {
3674 let mut state = self
3675 .state
3676 .lock()
3677 .unwrap_or_else(|poisoned| poisoned.into_inner());
3678 state.closed = true;
3679 self.sem.close();
3680 state.credits.capture_subscription_exclusions()
3681 }
3682
3683 pub(crate) fn close(&self) {
3684 self.state
3685 .lock()
3686 .unwrap_or_else(|poisoned| poisoned.into_inner())
3687 .closed = true;
3688 self.sem.close();
3689 }
3690}
3691
3692#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3693pub(crate) struct ChannelFlowClosed;
3694
3695impl fmt::Display for ChannelFlowClosed {
3696 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3697 write!(f, "flow-control window closed")
3698 }
3699}
3700
3701impl Error for ChannelFlowClosed {}
3702
3703fn window_for(concurrency: &Concurrency) -> usize {
3704 match concurrency {
3705 Concurrency::Serial => 1,
3706 Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
3707 Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
3708 }
3709}
3710
3711#[derive(Debug, Clone, PartialEq, Eq)]
3712pub enum ForwardingError {
3713 ConnectionRoleConflict {
3716 connection_id: ConnectionId,
3717 },
3718 NoModuleConnection,
3719 ModuleReloading {
3720 module_id: String,
3721 },
3722 StaleModuleEndpoint,
3723 UnknownReservation {
3724 client_channel: u16,
3725 module_channel: u16,
3726 },
3727 ClientRouteChannelExhausted {
3728 connection_id: ConnectionId,
3729 },
3730 ModuleRouteChannelExhausted {
3731 endpoint: ModuleEndpointId,
3732 },
3733 RelayCorrelationExhausted,
3734 ConnectionClosing {
3735 connection_id: ConnectionId,
3736 },
3737 ClientEgressClosed {
3738 connection_id: ConnectionId,
3739 },
3740 RouteOpenBuild(String),
3741 CandidateSlotOccupied {
3743 module_id: String,
3744 },
3745 ModuleEgressUnavailable {
3748 connection_id: ConnectionId,
3749 },
3750 Poisoned,
3751}
3752
3753impl fmt::Display for ForwardingError {
3754 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3755 match self {
3756 Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
3757 Self::NoModuleConnection => write!(f, "no module connection is registered"),
3758 Self::ModuleReloading { module_id } => {
3759 write!(f, "module_id '{module_id}' is reloading")
3760 }
3761 Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
3762 Self::UnknownReservation {
3763 client_channel,
3764 module_channel,
3765 } => write!(
3766 f,
3767 "route reservation client channel {client_channel} / module channel {module_channel} was not found"
3768 ),
3769 Self::ClientRouteChannelExhausted { connection_id } => write!(
3770 f,
3771 "no client route channels are available for connection {}",
3772 connection_id.get()
3773 ),
3774 Self::ModuleRouteChannelExhausted { endpoint } => write!(
3775 f,
3776 "no module route channels are available for endpoint generation {} on connection {}",
3777 endpoint.generation,
3778 endpoint.connection_id.get()
3779 ),
3780 Self::RelayCorrelationExhausted => {
3781 write!(f, "module control correlation ids are exhausted")
3782 }
3783 Self::ConnectionClosing { connection_id } => write!(
3784 f,
3785 "connection {} is closing and cannot accept route allocation",
3786 connection_id.get()
3787 ),
3788 Self::ClientEgressClosed { connection_id } => write!(
3789 f,
3790 "client connection {} egress is closed",
3791 connection_id.get()
3792 ),
3793 Self::RouteOpenBuild(message) => {
3794 write!(f, "failed to prebuild route.open response: {message}")
3795 }
3796 Self::CandidateSlotOccupied { module_id } => write!(
3797 f,
3798 "module_id '{module_id}' already has a swap candidate registered"
3799 ),
3800 Self::ModuleEgressUnavailable { connection_id } => write!(
3801 f,
3802 "module connection {} egress is unavailable; HELLO_ACK could not be queued",
3803 connection_id.get()
3804 ),
3805 Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
3806 }
3807 }
3808}
3809
3810impl Error for ForwardingError {}
3811
3812#[cfg(test)]
3813mod tests {
3814 use std::time::Duration;
3815
3816 use super::*;
3817 use tokio::sync::mpsc;
3818
3819 #[test]
3820 fn ordinary_long_running_request_is_not_excluded_from_drain() {
3821 let mut ledger = CreditLedger::default();
3822 ledger.acquire(1, false);
3823
3824 assert_eq!(ledger.capture_subscription_exclusions(), 0);
3825 assert_eq!(ledger.drain_in_flight(), 1);
3826 }
3827
3828 #[test]
3829 fn bit_set_subscription_is_excluded_and_counted() {
3830 let mut ledger = CreditLedger::default();
3831 ledger.acquire(1, true);
3832
3833 assert_eq!(ledger.capture_subscription_exclusions(), 1);
3834 assert_eq!(ledger.drain_in_flight(), 0);
3835 }
3836
3837 #[test]
3838 fn subscription_opened_after_drain_snapshot_is_not_excluded() {
3839 let mut ledger = CreditLedger::default();
3840 ledger.acquire(1, true);
3841 assert_eq!(ledger.capture_subscription_exclusions(), 1);
3842
3843 ledger.acquire(2, true);
3844
3845 assert_eq!(ledger.drain_in_flight(), 1);
3846 }
3847
3848 #[test]
3849 fn drain_with_no_subscriptions_reports_zero_excluded() {
3850 let mut ledger = CreditLedger::default();
3851 assert_eq!(ledger.capture_subscription_exclusions(), 0);
3852 }
3853
3854 fn test_hello_ack(corr: u64) -> Frame {
3855 Frame::build(
3856 FrameType::HelloAck,
3857 Flags::new(false, Priority::Passive, false),
3858 0,
3859 0,
3860 corr,
3861 Vec::new(),
3862 )
3863 .unwrap()
3864 }
3865
3866 #[test]
3870 fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
3871 let forwarding = ForwardingTable::default();
3872
3873 let (closed_tx, closed_rx) = mpsc::channel(8);
3874 drop(closed_rx);
3875 let closed = ConnectionId::new(1);
3876 assert_eq!(
3877 forwarding.register_module_connection_acked(
3878 closed,
3879 "closed".to_string(),
3880 2,
3881 Concurrency::ModuleManaged,
3882 FrameSink::new(closed_tx),
3883 test_hello_ack(1),
3884 ),
3885 Err(ForwardingError::ModuleEgressUnavailable {
3886 connection_id: closed
3887 })
3888 );
3889
3890 let (full_tx, _full_rx) = mpsc::channel(1);
3891 let full_sink = FrameSink::new(full_tx);
3892 full_sink.try_send(test_hello_ack(99)).unwrap();
3893 let full = ConnectionId::new(2);
3894 assert_eq!(
3895 forwarding.register_module_connection_acked(
3896 full,
3897 "full".to_string(),
3898 2,
3899 Concurrency::ModuleManaged,
3900 full_sink.clone(),
3901 test_hello_ack(2),
3902 ),
3903 Err(ForwardingError::ModuleEgressUnavailable {
3904 connection_id: full
3905 })
3906 );
3907 assert_eq!(
3908 forwarding.register_candidate_module_connection_acked(
3909 full,
3910 "full".to_string(),
3911 2,
3912 Concurrency::ModuleManaged,
3913 full_sink,
3914 test_hello_ack(3),
3915 ),
3916 Err(ForwardingError::ModuleEgressUnavailable {
3917 connection_id: full
3918 })
3919 );
3920
3921 for (connection, module_id) in [(closed, "closed"), (full, "full")] {
3922 assert_eq!(
3923 forwarding
3924 .module_endpoint_for_connection(connection)
3925 .unwrap(),
3926 None
3927 );
3928 let (client_tx, _client_rx) = mpsc::channel(8);
3929 assert_eq!(
3930 forwarding
3931 .begin_route_bind_relay_for_test(
3932 ConnectionId::new(50),
3933 FrameSink::new(client_tx),
3934 1,
3935 module_id,
3936 )
3937 .err(),
3938 Some(ForwardingError::NoModuleConnection)
3939 );
3940 }
3941 assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
3942 }
3943
3944 #[test]
3947 fn acked_registration_queues_the_hello_ack_first() {
3948 let forwarding = ForwardingTable::default();
3949 let (active_tx, mut active_rx) = mpsc::channel(8);
3950 forwarding
3951 .register_module_connection_acked(
3952 ConnectionId::new(1),
3953 "acked".to_string(),
3954 2,
3955 Concurrency::ModuleManaged,
3956 FrameSink::new(active_tx),
3957 test_hello_ack(11),
3958 )
3959 .unwrap();
3960 let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
3961 forwarding
3962 .register_candidate_module_connection_acked(
3963 ConnectionId::new(2),
3964 "acked".to_string(),
3965 2,
3966 Concurrency::ModuleManaged,
3967 FrameSink::new(candidate_tx),
3968 test_hello_ack(12),
3969 )
3970 .unwrap();
3971
3972 let active_first = active_rx.try_recv().unwrap().frame;
3973 assert_eq!(active_first.header.ty, FrameType::HelloAck);
3974 assert_eq!(active_first.header.corr, 11);
3975 let candidate_first = candidate_rx.try_recv().unwrap().frame;
3976 assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
3977 assert_eq!(candidate_first.header.corr, 12);
3978 }
3979
3980 #[test]
3981 fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
3982 let forwarding = ForwardingTable::default();
3983 let module_connection = ConnectionId::new(10);
3984 let exhausted_client = ConnectionId::new(20);
3985 let second_client = ConnectionId::new(30);
3986 let (module_tx, _module_rx) = mpsc::channel(1);
3987 let endpoint = forwarding
3988 .register_module_connection(
3989 module_connection,
3990 "route-limit-provider".to_string(),
3991 1,
3992 Concurrency::ModuleManaged,
3993 FrameSink::new(module_tx),
3994 )
3995 .unwrap();
3996
3997 {
3998 let mut inner = forwarding.inner.write().unwrap();
3999 for channel in 1..=u16::MAX {
4000 inner.reserved_client.insert(
4001 ClientRouteKey {
4002 connection_id: exhausted_client,
4003 channel,
4004 },
4005 ModuleRouteKey {
4006 endpoint,
4007 channel: 1,
4008 },
4009 );
4010 }
4011 }
4012
4013 let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
4014 let err = forwarding
4015 .begin_route_bind_relay_for_test(
4016 exhausted_client,
4017 FrameSink::new(exhausted_tx),
4018 1,
4019 "route-limit-provider",
4020 )
4021 .unwrap_err();
4022 assert!(matches!(
4023 err,
4024 ForwardingError::ClientRouteChannelExhausted { connection_id }
4025 if connection_id == exhausted_client
4026 ));
4027
4028 let (second_tx, _second_rx) = mpsc::channel(1);
4029 let pending = forwarding
4030 .begin_route_bind_relay_for_test(
4031 second_client,
4032 FrameSink::new(second_tx),
4033 2,
4034 "route-limit-provider",
4035 )
4036 .unwrap();
4037 assert_eq!(pending.client_channel, 1);
4038 }
4039
4040 #[test]
4041 fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
4042 let forwarding = ForwardingTable::default();
4043 let module_connection = ConnectionId::new(40);
4044 let client = ConnectionId::new(50);
4045 let (module_tx, _module_rx) = mpsc::channel(1);
4046 forwarding
4047 .register_module_connection(
4048 module_connection,
4049 "slot-reuse-provider".to_string(),
4050 1,
4051 Concurrency::ModuleManaged,
4052 FrameSink::new(module_tx),
4053 )
4054 .unwrap();
4055
4056 let (client_tx, _client_rx) = mpsc::channel(1);
4057 let client_sink = FrameSink::new(client_tx);
4058 let mut wrapped_channel = None;
4059 for index in 0..=usize::from(u16::MAX) {
4060 let pending = forwarding
4061 .begin_route_bind_relay_for_test(
4062 client,
4063 client_sink.clone(),
4064 index as u64 + 1,
4065 "slot-reuse-provider",
4066 )
4067 .unwrap();
4068 if index == usize::from(u16::MAX) {
4069 wrapped_channel = Some(pending.module_channel);
4070 }
4071 forwarding
4072 .abort_pending_relay(
4073 pending.endpoint,
4074 pending.corr,
4075 RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
4076 )
4077 .unwrap();
4078 }
4079
4080 assert_eq!(wrapped_channel, Some(1));
4081 }
4082
4083 #[test]
4084 fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
4085 let forwarding = ForwardingTable::default();
4086 let client = ConnectionId::new(60);
4087 forwarding
4088 .inner
4089 .write()
4090 .unwrap()
4091 .next_client_channel
4092 .insert(client, 41);
4093
4094 let released = forwarding.cleanup_connection(client).unwrap();
4095
4096 assert!(released.is_empty());
4097 assert!(!forwarding
4098 .inner
4099 .read()
4100 .unwrap()
4101 .next_client_channel
4102 .contains_key(&client));
4103 }
4104
4105 #[test]
4106 fn stale_module_cleanup_preserves_fast_reconnect_successor() {
4107 let forwarding = ForwardingTable::default();
4108 let module_id = "fast-reconnect-provider";
4109 let first_connection = ConnectionId::new(70);
4110 let second_connection = ConnectionId::new(80);
4111 let (first_tx, _first_rx) = mpsc::channel(1);
4112 let first_endpoint = forwarding
4113 .register_module_connection(
4114 first_connection,
4115 module_id.to_string(),
4116 1,
4117 Concurrency::ModuleManaged,
4118 FrameSink::new(first_tx),
4119 )
4120 .unwrap();
4121 let (second_tx, _second_rx) = mpsc::channel(1);
4122 let second_endpoint = forwarding
4123 .register_module_connection(
4124 second_connection,
4125 module_id.to_string(),
4126 1,
4127 Concurrency::ModuleManaged,
4128 FrameSink::new(second_tx),
4129 )
4130 .unwrap();
4131 assert_ne!(first_endpoint, second_endpoint);
4132
4133 let released = forwarding.cleanup_connection(first_connection).unwrap();
4134
4135 assert!(released.is_empty());
4136 assert_eq!(
4137 forwarding
4138 .inner
4139 .read()
4140 .unwrap()
4141 .modules_by_id
4142 .get(module_id)
4143 .map(|module| module.endpoint),
4144 Some(second_endpoint)
4145 );
4146 assert!(forwarding.has_live_module_connection(module_id).unwrap());
4147 let control_rpc = forwarding
4148 .begin_module_control_rpc_for(
4149 module_id,
4150 "health.check",
4151 Instant::now() + Duration::from_secs(1),
4152 )
4153 .unwrap();
4154 assert_eq!(control_rpc.endpoint, second_endpoint);
4155 }
4156
4157 fn route_fixture(
4158 module_id: &str,
4159 ) -> (
4160 ForwardingTable,
4161 ConnectionId,
4162 ModuleEndpointId,
4163 ConnectionId,
4164 FrameSink,
4165 mpsc::Receiver<crate::router::OutboundFrame>,
4166 ) {
4167 let forwarding = ForwardingTable::default();
4168 let module_connection = ConnectionId::new(100);
4169 let client_connection = ConnectionId::new(200);
4170 let (module_tx, _module_rx) = mpsc::channel(8);
4171 let endpoint = forwarding
4172 .register_module_connection(
4173 module_connection,
4174 module_id.to_string(),
4175 2,
4176 Concurrency::ModuleManaged,
4177 FrameSink::new(module_tx),
4178 )
4179 .unwrap();
4180 let (client_tx, client_rx) = mpsc::channel(8);
4181 (
4182 forwarding,
4183 module_connection,
4184 endpoint,
4185 client_connection,
4186 FrameSink::new(client_tx),
4187 client_rx,
4188 )
4189 }
4190
4191 #[test]
4192 #[cfg(unix)]
4193 fn daemon_drain_gates_current_and_racing_provider_registrations() {
4194 let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
4195 assert_eq!(
4196 forwarding.begin_daemon_drain().unwrap(),
4197 [("provider".to_string(), endpoint)]
4198 );
4199 assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
4200 assert!(matches!(
4201 forwarding.register_module_connection(
4202 ConnectionId::new(300),
4203 "late-provider".into(),
4204 2,
4205 Concurrency::ModuleManaged,
4206 sink,
4207 ),
4208 Err(ForwardingError::ConnectionClosing { .. })
4209 ));
4210 }
4211
4212 #[test]
4213 #[cfg(unix)]
4214 fn daemon_drain_results_are_endpoint_fenced_and_removed_on_cleanup() {
4215 let (forwarding, connection, endpoint, _, _, _) = route_fixture("provider");
4216 assert_eq!(
4217 forwarding
4218 .daemon_drain_result_for_connection(connection)
4219 .unwrap(),
4220 None
4221 );
4222 forwarding
4223 .record_daemon_drain_result(endpoint, true)
4224 .unwrap();
4225 forwarding.begin_daemon_drain().unwrap();
4226 assert_eq!(
4227 forwarding
4228 .daemon_drain_result_for_connection(connection)
4229 .unwrap(),
4230 Some(false)
4231 );
4232 let replacement = ModuleEndpointId {
4233 generation: endpoint.generation + 1,
4234 ..endpoint
4235 };
4236 forwarding
4237 .record_daemon_drain_result(replacement, true)
4238 .unwrap();
4239 assert_eq!(
4240 forwarding
4241 .daemon_drain_result_for_connection(connection)
4242 .unwrap(),
4243 Some(false)
4244 );
4245 forwarding
4246 .record_daemon_drain_result(endpoint, true)
4247 .unwrap();
4248 assert_eq!(
4249 forwarding
4250 .daemon_drain_result_for_connection(connection)
4251 .unwrap(),
4252 Some(true)
4253 );
4254 assert_eq!(
4255 forwarding
4256 .daemon_drain_result_for_connection(ConnectionId::new(300))
4257 .unwrap(),
4258 Some(false)
4259 );
4260 forwarding.cleanup_connection(connection).unwrap();
4261 forwarding
4262 .record_daemon_drain_result(endpoint, true)
4263 .unwrap();
4264 assert_eq!(
4265 forwarding
4266 .daemon_drain_result_for_connection(connection)
4267 .unwrap(),
4268 Some(false)
4269 );
4270 assert!(forwarding
4271 .read_inner()
4272 .unwrap()
4273 .daemon_drain_results
4274 .is_empty());
4275 }
4276
4277 #[test]
4278 #[cfg(unix)]
4279 fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
4280 let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
4281 route_fixture("provider");
4282 let mut pending =
4283 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
4284 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
4285 forwarding.begin_daemon_drain().unwrap();
4286 let completion = forwarding
4287 .complete_pending_relay(
4288 module_connection,
4289 pending.corr,
4290 RouteBindRelayOutcome::Accepted,
4291 )
4292 .expect("draining admission is not a fatal module error");
4293 assert!(completion.settled);
4294 let abandoned = completion
4295 .abandoned
4296 .expect("late accepted bind needs a route GOODBYE");
4297 assert_eq!(abandoned.channel, pending.module_channel);
4298 assert!(matches!(pending.receiver.try_recv().unwrap(),
4299 RouteBindRelayOutcome::Rejected(error) if error.code == "module_reloading"));
4300 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4301 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4302 assert!(
4303 rx.try_recv().is_err(),
4304 "no route.open may be published during drain"
4305 );
4306 assert_eq!(
4307 forwarding
4308 .module_endpoint_for_connection(module_connection)
4309 .unwrap(),
4310 Some(endpoint)
4311 );
4312 assert!(
4313 !forwarding
4314 .complete_pending_relay(
4315 module_connection,
4316 pending.corr,
4317 RouteBindRelayOutcome::Accepted,
4318 )
4319 .unwrap()
4320 .settled
4321 );
4322 }
4323
4324 fn test_ping(corr: u64) -> Frame {
4325 Frame::build(
4326 FrameType::Ping,
4327 Flags::new(false, Priority::Passive, false),
4328 0,
4329 0,
4330 corr,
4331 Vec::new(),
4332 )
4333 .unwrap()
4334 }
4335
4336 fn begin_test_route(
4337 forwarding: &ForwardingTable,
4338 client_connection: ConnectionId,
4339 client_sink: FrameSink,
4340 corr: u64,
4341 module_id: &str,
4342 ) -> PendingRouteBindRelay {
4343 forwarding
4344 .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
4345 .unwrap()
4346 }
4347
4348 #[test]
4349 fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
4350 for candidate in [false, true] {
4351 for register_first in [false, true] {
4352 let (forwarding, module_connection, _, client_connection, sink, _rx) =
4353 route_fixture("target");
4354 let register = || {
4355 if candidate {
4356 forwarding.register_candidate_module_connection(
4357 client_connection,
4358 "source".into(),
4359 2,
4360 Concurrency::ModuleManaged,
4361 sink.clone(),
4362 )
4363 } else {
4364 forwarding.register_module_connection(
4365 client_connection,
4366 "source".into(),
4367 2,
4368 Concurrency::ModuleManaged,
4369 sink.clone(),
4370 )
4371 }
4372 };
4373 if register_first {
4374 register().unwrap();
4375 assert!(
4376 forwarding
4377 .begin_route_bind_relay_for_test(
4378 client_connection,
4379 sink.clone(),
4380 1,
4381 "target",
4382 )
4383 .is_err(),
4384 "a deferred route.open cannot reserve after HELLO"
4385 );
4386 assert!(forwarding
4387 .cleanup_connection(client_connection)
4388 .unwrap()
4389 .is_empty());
4390 } else {
4391 let pending =
4392 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
4393 assert!(
4394 register().is_err(),
4395 "HELLO cannot register after a route reservation"
4396 );
4397 forwarding
4398 .complete_pending_relay(
4399 module_connection,
4400 pending.corr,
4401 RouteBindRelayOutcome::Accepted,
4402 )
4403 .unwrap();
4404 assert_eq!(
4405 forwarding
4406 .cleanup_connection(client_connection)
4407 .unwrap()
4408 .len(),
4409 1
4410 );
4411 }
4412 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4413 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4414 }
4415 }
4416 }
4417
4418 #[tokio::test]
4424 async fn pending_route_open_completes_behind_queued_data_frames() {
4425 assert_eq!(
4426 crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
4427 8,
4428 "the per-connection pending route.open limit is its own constant"
4429 );
4430 let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
4431 route_fixture("open-behind-data");
4432 let (sink, mut client_rx) = crate::server::connection_egress();
4433 const DATA_FRAMES: usize = 1_000;
4434 let data = |corr: u64| {
4435 Frame::build(
4436 FrameType::StreamData,
4437 Flags::new(false, Priority::Interactive, false),
4438 9,
4439 1,
4440 corr,
4441 vec![b'x'; 200],
4442 )
4443 .unwrap()
4444 };
4445 for corr in 0..DATA_FRAMES as u64 {
4446 sink.try_send(data(corr)).unwrap();
4447 }
4448 let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
4449 assert_eq!(sink.backlog().queued_bytes, data_bytes);
4450
4451 let pending = tokio::time::timeout(
4452 Duration::from_secs(5),
4453 forwarding.begin_route_bind_relay_for(
4454 client,
4455 sink.clone(),
4456 subc_protocol::PROTOCOL_VERSION,
4457 4_242,
4458 "open-behind-data",
4459 Principal::Direct,
4460 None,
4461 None,
4462 Instant::now() + Duration::from_secs(60),
4463 ),
4464 )
4465 .await
4466 .expect("reserving the route.open slot must not wait behind data frames")
4467 .unwrap();
4468 forwarding
4469 .complete_pending_relay(
4470 module_connection,
4471 pending.corr,
4472 RouteBindRelayOutcome::Accepted,
4473 )
4474 .unwrap();
4475
4476 let backlog = sink.backlog();
4477 assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
4478 assert!(
4479 backlog.queued_bytes > data_bytes,
4480 "the route.open response must be counted in queued bytes: {backlog:?}"
4481 );
4482 for corr in 0..DATA_FRAMES as u64 {
4483 assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
4484 }
4485 let open = client_rx.recv().await.unwrap();
4486 assert_eq!(open.header.corr, 4_242);
4487 assert_eq!(open.header.ty, FrameType::Response);
4488 drop(open);
4489 assert_eq!(sink.backlog().queued_bytes, 0);
4490 assert_eq!(sink.backlog().queued_frames, 0);
4491 }
4492
4493 #[tokio::test]
4498 async fn drain_holdouts_count_held_requests_and_name_the_connection() {
4499 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4500 route_fixture("holdouts");
4501 let mut bound = |corr| {
4502 let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
4503 forwarding
4504 .complete_pending_relay(
4505 module_connection,
4506 route.corr,
4507 RouteBindRelayOutcome::Accepted,
4508 )
4509 .unwrap();
4510 client_rx.try_recv().unwrap();
4511 match forwarding
4512 .lookup_data_route(client, route.client_channel, route.client_epoch)
4513 .unwrap()
4514 {
4515 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4516 other => panic!("expected live route, got {other:?}"),
4517 }
4518 };
4519 let holding = bound(61);
4520 let _idle = bound(62);
4521 holding.flow.acquire_tagged(7, false).await.unwrap();
4522 holding.flow.acquire_tagged(2, false).await.unwrap();
4523 holding.flow.acquire_tagged(3, true).await.unwrap();
4524 forwarding
4525 .begin_module_drain("holdouts", RouteCloseReason::Restart)
4526 .unwrap();
4527
4528 let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
4529 assert_eq!(
4530 holdouts,
4531 DrainHoldouts {
4532 requests: 2,
4533 routes: 1,
4534 total_routes: 2,
4535 top_connections: vec![(client.get(), 2)],
4536 held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
4539 }
4540 );
4541 }
4542
4543 #[test]
4544 fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
4545 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4546 route_fixture("census");
4547 let pending = begin_test_route(&forwarding, client, sink, 1, "census");
4548 forwarding
4549 .complete_pending_relay(
4550 module_connection,
4551 pending.corr,
4552 RouteBindRelayOutcome::Accepted,
4553 )
4554 .unwrap();
4555
4556 let routes = forwarding.endpoint_routes(endpoint).unwrap();
4557 assert_eq!(routes.len(), 1);
4558 assert!(matches!(routes[0].principal, Principal::Direct));
4559 assert_eq!(routes[0].goodbye_target.connection_id, client);
4560 assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
4561 assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
4562 assert!(!routes[0].draining);
4563
4564 forwarding
4565 .begin_module_drain("census", RouteCloseReason::Restart)
4566 .unwrap();
4567 let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
4568 assert_eq!(draining_routes.len(), 1);
4569 assert!(draining_routes[0].draining);
4570 }
4571
4572 #[test]
4573 fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
4574 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
4575 let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
4576 assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
4577 forwarding
4578 .abort_pending_relay(
4579 first.endpoint,
4580 first.corr,
4581 RouteBindRelayOutcome::ModuleGone("abort".into()),
4582 )
4583 .unwrap();
4584 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4585 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4586
4587 let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
4588 assert_eq!(second.client_channel, first.client_channel);
4589 assert_eq!(second.module_channel, first.module_channel);
4590 assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
4591 }
4592
4593 #[test]
4594 fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
4595 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4596 route_fixture("epoch-release");
4597 let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
4598 forwarding
4599 .complete_pending_relay(
4600 module_connection,
4601 first.corr,
4602 RouteBindRelayOutcome::Accepted,
4603 )
4604 .unwrap();
4605 assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
4606 assert!(matches!(
4607 forwarding
4608 .release_client_route(client, first.client_channel, first.client_epoch)
4609 .unwrap(),
4610 RouteRelease::Removed(_)
4611 ));
4612 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4613 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4614
4615 let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
4616 forwarding
4617 .complete_pending_relay(
4618 module_connection,
4619 second.corr,
4620 RouteBindRelayOutcome::Accepted,
4621 )
4622 .unwrap();
4623 assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
4624 assert!(matches!(
4625 forwarding
4626 .release_client_route(client, second.client_channel, first.client_epoch)
4627 .unwrap(),
4628 RouteRelease::Stale
4629 ));
4630 assert!(!forwarding
4631 .cache_status(
4632 endpoint,
4633 second.module_channel,
4634 first.module_epoch,
4635 "stale".into(),
4636 )
4637 .unwrap());
4638 assert!(forwarding
4639 .cache_status(
4640 endpoint,
4641 second.module_channel,
4642 second.module_epoch,
4643 "current".into(),
4644 )
4645 .unwrap());
4646 match forwarding
4647 .route_poll_snapshot(client, second.client_channel, second.client_epoch)
4648 .unwrap()
4649 {
4650 RoutePollSnapshot::Bound { status, .. } => {
4651 assert_eq!(status.as_deref(), Some("current"));
4652 }
4653 RoutePollSnapshot::Absent => panic!("successor binding was removed"),
4654 }
4655 let counters = forwarding.counters().snapshot();
4656 assert_eq!(counters["route_released_epoch_fenced"], 1);
4657 assert_eq!(counters["route_release_stale_skipped"], 1);
4658 }
4659
4660 #[test]
4661 fn max_epoch_reservation_retires_only_that_slot() {
4662 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
4663 forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
4664 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
4665 let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
4666 assert_eq!(
4667 (final_use.client_channel, final_use.client_epoch),
4668 (7, u32::MAX)
4669 );
4670 assert_eq!(
4671 (final_use.module_channel, final_use.module_epoch),
4672 (9, u32::MAX)
4673 );
4674 forwarding
4675 .abort_pending_relay(
4676 endpoint,
4677 final_use.corr,
4678 RouteBindRelayOutcome::ModuleGone("abort".into()),
4679 )
4680 .unwrap();
4681 forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
4682 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
4683 let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
4684 assert_ne!(next.client_channel, 7);
4685 assert_ne!(next.module_channel, 9);
4686 assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
4687 }
4688
4689 #[test]
4690 fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
4691 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4692 route_fixture("corr-shared");
4693 let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
4694 assert_eq!(bind.corr, 1);
4695 forwarding
4696 .abort_pending_relay(
4697 endpoint,
4698 bind.corr,
4699 RouteBindRelayOutcome::ModuleGone("abort".into()),
4700 )
4701 .unwrap();
4702 let rpc = forwarding
4703 .begin_module_control_rpc_for(
4704 "corr-shared",
4705 "health.check",
4706 Instant::now() - Duration::from_millis(1),
4707 )
4708 .unwrap();
4709 assert_eq!(rpc.corr, 2);
4710 assert_eq!(
4711 forwarding
4712 .complete_module_control_rpc(
4713 module_connection,
4714 rpc.corr,
4715 Some("health.check"),
4716 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
4717 status: subc_protocol::session::HealthStatus::Ok,
4718 detail: None,
4719 metrics: None,
4720 }),
4721 )
4722 .unwrap(),
4723 ModuleControlRpcCompletion::Settled
4724 );
4725 assert!(matches!(
4726 rpc.receiver.blocking_recv().unwrap(),
4727 ModuleControlRpcOutcome::DeadlineElapsed
4728 ));
4729 }
4730
4731 #[tokio::test(start_paused = true)]
4732 async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
4733 let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
4734 let probe_started_at = Instant::now();
4735 let rpc = forwarding
4736 .begin_health_probe_rpc_for(
4737 "tombstone-ttl",
4738 "health.check",
4739 probe_started_at,
4740 probe_started_at + Duration::from_secs(5),
4741 )
4742 .unwrap();
4743 assert!(forwarding
4744 .tombstone_health_probe_rpc(endpoint, rpc.corr)
4745 .unwrap());
4746 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
4747
4748 tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
4749 tokio::task::yield_now().await;
4750
4751 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
4752 }
4753
4754 #[test]
4755 fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
4756 let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
4757 let mut close = forwarding.register_connection_close(endpoint.connection_id);
4758 forwarding.inject_control_corr(endpoint, u64::MAX);
4759 let final_rpc = forwarding
4760 .begin_module_control_rpc_for(
4761 "corr-max",
4762 "health.check",
4763 Instant::now() + Duration::from_secs(1),
4764 )
4765 .unwrap();
4766 assert_eq!(final_rpc.corr, u64::MAX);
4767 forwarding
4768 .cancel_module_control_rpc(endpoint, final_rpc.corr)
4769 .unwrap();
4770 assert!(matches!(
4771 forwarding.begin_module_control_rpc_for(
4772 "corr-max",
4773 "health.check",
4774 Instant::now() + Duration::from_secs(1),
4775 ),
4776 Err(ForwardingError::RelayCorrelationExhausted)
4777 ));
4778 assert!(close.try_recv().is_ok());
4779 }
4780
4781 #[test]
4782 fn publication_epoch_controls_delivery_failure_escalation() {
4783 fn setup_successor(
4784 commit_successor: Option<bool>,
4785 ) -> (ForwardingTable, ConnectionId, u16, u32) {
4786 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4787 route_fixture("escalation");
4788 let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
4789 forwarding
4790 .complete_pending_relay(
4791 module_connection,
4792 first.corr,
4793 RouteBindRelayOutcome::Accepted,
4794 )
4795 .unwrap();
4796 client_rx.try_recv().unwrap();
4797 assert!(matches!(
4798 forwarding
4799 .release_client_route(client, first.client_channel, first.client_epoch)
4800 .unwrap(),
4801 RouteRelease::Removed(_)
4802 ));
4803 if let Some(commit_successor) = commit_successor {
4804 forwarding.inject_client_slot_epoch(
4805 client,
4806 first.client_channel,
4807 first.client_epoch,
4808 );
4809 forwarding.inject_module_slot_epoch(
4810 endpoint,
4811 first.module_channel,
4812 first.module_epoch,
4813 );
4814 let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
4815 if commit_successor {
4816 forwarding
4817 .complete_pending_relay(
4818 module_connection,
4819 successor.corr,
4820 RouteBindRelayOutcome::Accepted,
4821 )
4822 .unwrap();
4823 client_rx.try_recv().unwrap();
4824 } else {
4825 forwarding
4826 .abort_pending_relay(
4827 endpoint,
4828 successor.corr,
4829 RouteBindRelayOutcome::ModuleGone("abort".into()),
4830 )
4831 .unwrap();
4832 }
4833 }
4834 (forwarding, client, first.client_channel, first.client_epoch)
4835 }
4836
4837 let probe_sink = FrameSink::new(mpsc::channel(1).0);
4838 let (no_successor, client, channel, epoch) = setup_successor(None);
4839 let mut close = no_successor.register_connection_close(client);
4840 assert!(no_successor
4841 .escalate_client_delivery_failure(
4842 client,
4843 channel,
4844 epoch,
4845 CloseReason::new("delivery", "failed"),
4846 UndeliveredFrame {
4847 module_id: None,
4848 sink: &probe_sink,
4849 },
4850 )
4851 .unwrap());
4852 assert!(close.try_recv().is_ok());
4853
4854 let (aborted, client, channel, epoch) = setup_successor(Some(false));
4855 let mut close = aborted.register_connection_close(client);
4856 assert!(aborted
4857 .escalate_client_delivery_failure(
4858 client,
4859 channel,
4860 epoch,
4861 CloseReason::new("delivery", "failed"),
4862 UndeliveredFrame {
4863 module_id: None,
4864 sink: &probe_sink,
4865 },
4866 )
4867 .unwrap());
4868 assert!(close.try_recv().is_ok());
4869
4870 let (published, client, channel, epoch) = setup_successor(Some(true));
4871 let mut close = published.register_connection_close(client);
4872 assert!(!published
4873 .escalate_client_delivery_failure(
4874 client,
4875 channel,
4876 epoch,
4877 CloseReason::new("delivery", "stale failure"),
4878 UndeliveredFrame {
4879 module_id: None,
4880 sink: &probe_sink,
4881 },
4882 )
4883 .unwrap());
4884 assert!(close.try_recv().is_err());
4885 }
4886
4887 #[test]
4888 fn route_concentration_separates_client_count_from_routes_per_client() {
4889 let (forwarding, module_connection, _, client, sink, _client_rx) =
4893 route_fixture("concentration");
4894 assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
4895
4896 for corr in [70_u64, 71] {
4897 let pending =
4898 begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
4899 forwarding
4900 .complete_pending_relay(
4901 module_connection,
4902 pending.corr,
4903 RouteBindRelayOutcome::Accepted,
4904 )
4905 .unwrap();
4906 }
4907
4908 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
4910 assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
4911 }
4912
4913 #[test]
4914 fn cleanup_and_accepted_resolution_have_one_lock_winner() {
4915 let (forwarding, module_connection, _, client, sink, mut client_rx) =
4916 route_fixture("cleanup-race");
4917 let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
4918 forwarding
4919 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4920 .unwrap();
4921 let released = forwarding.cleanup_connection(client).unwrap();
4922 assert_eq!(released.len(), 1);
4923 let completion = forwarding
4924 .complete_pending_relay(
4925 module_connection,
4926 pending.corr,
4927 RouteBindRelayOutcome::Accepted,
4928 )
4929 .unwrap();
4930 assert!(!completion.settled);
4931 assert!(client_rx.try_recv().is_err());
4932 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4933
4934 let (forwarding, module_connection, _, client, sink, mut client_rx) =
4935 route_fixture("accepted-race");
4936 let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
4937 forwarding
4938 .complete_pending_relay(
4939 module_connection,
4940 pending.corr,
4941 RouteBindRelayOutcome::Accepted,
4942 )
4943 .unwrap();
4944 assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
4945 let released = forwarding.cleanup_connection(client).unwrap();
4946 assert_eq!(released.len(), 1);
4947 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4948 }
4949
4950 #[test]
4951 fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
4952 let (forwarding, module_connection, _, client, sink, mut client_rx) =
4953 route_fixture("drain-gap");
4954 let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
4955 forwarding
4956 .complete_pending_relay(
4957 module_connection,
4958 live.corr,
4959 RouteBindRelayOutcome::Accepted,
4960 )
4961 .unwrap();
4962 client_rx.try_recv().unwrap();
4963 let binding = match forwarding
4964 .lookup_data_route(client, live.client_channel, live.client_epoch)
4965 .unwrap()
4966 {
4967 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4968 other => panic!("expected live route, got {other:?}"),
4969 };
4970
4971 let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
4972 forwarding
4973 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4974 .unwrap();
4975 let control_rpc = forwarding
4976 .begin_module_control_rpc_for(
4977 "drain-gap",
4978 "health.check",
4979 Instant::now() + Duration::from_secs(1),
4980 )
4981 .unwrap();
4982 let target = forwarding
4983 .begin_module_drain("drain-gap", RouteCloseReason::Reload)
4984 .unwrap()
4985 .unwrap();
4986 assert!(matches!(
4987 control_rpc.receiver.blocking_recv().unwrap(),
4988 ModuleControlRpcOutcome::ModuleGone(_)
4989 ));
4990 assert_eq!(target.abandoned_bindings.len(), 1);
4991 assert!(binding.flow.sem.is_closed());
4992 assert!(
4993 !forwarding
4994 .complete_pending_relay(
4995 module_connection,
4996 pending.corr,
4997 RouteBindRelayOutcome::Accepted,
4998 )
4999 .unwrap()
5000 .settled
5001 );
5002 assert!(matches!(
5003 forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
5004 Err(ForwardingError::ModuleReloading { .. })
5005 ));
5006 let released = forwarding
5007 .release_module_endpoint_routes(target.endpoint)
5008 .unwrap();
5009 assert_eq!(released.len(), 1);
5010 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5011 }
5012
5013 #[test]
5021 fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
5022 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5023 route_fixture("closing-client");
5024
5025 let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
5028 forwarding
5029 .complete_pending_relay(
5030 module_connection,
5031 live.corr,
5032 RouteBindRelayOutcome::Accepted,
5033 )
5034 .unwrap();
5035 client_rx.try_recv().unwrap();
5036
5037 let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
5039 forwarding
5040 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5041 .unwrap();
5042
5043 assert!(forwarding
5045 .escalate_client_delivery_failure(
5046 client,
5047 live.client_channel,
5048 live.client_epoch,
5049 CloseReason::new(
5050 "module_to_client_delivery_failed",
5051 "client egress refused a module frame",
5052 ),
5053 UndeliveredFrame {
5054 module_id: None,
5055 sink: &sink,
5056 },
5057 )
5058 .unwrap());
5059 assert!(!sink.is_closed());
5060
5061 let completion = forwarding
5062 .complete_pending_relay(
5063 module_connection,
5064 pending.corr,
5065 RouteBindRelayOutcome::Accepted,
5066 )
5067 .expect("a closing client must not turn a module's ack into an error");
5068
5069 assert!(completion.settled);
5070 let abandoned = completion
5071 .abandoned
5072 .expect("the module must be told to drop the binding it just created");
5073 assert_eq!(abandoned.connection_id, module_connection);
5074 assert_eq!(abandoned.channel, pending.module_channel);
5075 assert_eq!(abandoned.epoch, pending.module_epoch);
5076 assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
5077 assert!(matches!(
5078 pending.receiver.blocking_recv().unwrap(),
5079 RouteBindRelayOutcome::ModuleGone(_)
5080 ));
5081 assert!(client_rx.try_recv().is_err());
5084 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
5085
5086 assert!(forwarding
5089 .has_live_module_connection("closing-client")
5090 .unwrap());
5091 let cotenant = ConnectionId::new(201);
5092 let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
5093 let cotenant_route = begin_test_route(
5094 &forwarding,
5095 cotenant,
5096 FrameSink::new(cotenant_tx),
5097 62,
5098 "closing-client",
5099 );
5100 assert_eq!(cotenant_route.endpoint, endpoint);
5101 forwarding
5102 .complete_pending_relay(
5103 module_connection,
5104 cotenant_route.corr,
5105 RouteBindRelayOutcome::Accepted,
5106 )
5107 .unwrap();
5108 assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
5109 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5110 }
5111
5112 #[test]
5113 fn pending_route_permit_is_released_on_rejection_and_abort() {
5114 let forwarding = ForwardingTable::default();
5115 let module_connection = ConnectionId::new(300);
5116 let client = ConnectionId::new(301);
5117 let (module_tx, _module_rx) = mpsc::channel(1);
5118 let endpoint = forwarding
5119 .register_module_connection(
5120 module_connection,
5121 "permit".into(),
5122 2,
5123 Concurrency::ModuleManaged,
5124 FrameSink::new(module_tx),
5125 )
5126 .unwrap();
5127 let (client_tx, mut client_rx) = mpsc::channel(1);
5128 let sink = FrameSink::new(client_tx);
5129 let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
5130 assert!(sink.try_send(test_ping(999)).is_err());
5131 forwarding
5132 .complete_pending_relay(
5133 module_connection,
5134 rejected.corr,
5135 RouteBindRelayOutcome::Rejected(ErrorBody {
5136 code: "no".into(),
5137 message: "rejected".into(),
5138 detail: None,
5139 }),
5140 )
5141 .unwrap();
5142 sink.try_send(test_ping(1000)).unwrap();
5143 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
5144
5145 let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
5146 assert!(sink.try_send(test_ping(1001)).is_err());
5147 forwarding
5148 .abort_pending_relay(
5149 endpoint,
5150 aborted.corr,
5151 RouteBindRelayOutcome::ModuleGone("abort".into()),
5152 )
5153 .unwrap();
5154 sink.try_send(test_ping(1002)).unwrap();
5155 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
5156
5157 let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
5158 forwarding
5159 .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
5160 .unwrap();
5161 drop(client_rx);
5162 let completion = forwarding
5163 .complete_pending_relay(
5164 module_connection,
5165 receiver_closed.corr,
5166 RouteBindRelayOutcome::Accepted,
5167 )
5168 .unwrap();
5169 assert!(completion.abandoned.is_some());
5170 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5171 }
5172
5173 #[test]
5179 fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
5180 let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
5181 route_fixture("closing-set-leak");
5182
5183 const CONNECTIONS: u64 = 32;
5184 for index in 0..CONNECTIONS {
5185 let client = ConnectionId::new(1000 + index);
5186 let (client_tx, _client_rx) = mpsc::channel(8);
5187 let route = begin_test_route(
5188 &forwarding,
5189 client,
5190 FrameSink::new(client_tx),
5191 index + 1,
5192 "closing-set-leak",
5193 );
5194 forwarding
5195 .complete_pending_relay(
5196 module_connection,
5197 route.corr,
5198 RouteBindRelayOutcome::Accepted,
5199 )
5200 .unwrap();
5201 forwarding.cleanup_connection(client).unwrap();
5202 }
5203 forwarding.cleanup_connection(module_connection).unwrap();
5204
5205 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5206 }
5207
5208 #[test]
5215 fn closing_connection_is_refused_new_work_until_cleanup_completes() {
5216 let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
5217 route_fixture("closing-gate");
5218
5219 let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
5222 forwarding
5223 .complete_pending_relay(
5224 module_connection,
5225 live.corr,
5226 RouteBindRelayOutcome::Accepted,
5227 )
5228 .unwrap();
5229 client_rx.try_recv().unwrap();
5230
5231 assert!(forwarding
5234 .escalate_client_delivery_failure(
5235 client,
5236 live.client_channel,
5237 live.client_epoch,
5238 CloseReason::new(
5239 "module_to_client_delivery_failed",
5240 "client egress refused a module frame",
5241 ),
5242 UndeliveredFrame {
5243 module_id: None,
5244 sink: &sink,
5245 },
5246 )
5247 .unwrap());
5248 assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
5249
5250 assert!(matches!(
5252 forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
5253 Err(ForwardingError::ConnectionClosing { connection_id })
5254 if connection_id == client
5255 ));
5256 let (late_tx, _late_rx) = mpsc::channel(1);
5258 assert!(matches!(
5259 forwarding.register_module_connection(
5260 client,
5261 "late-module".into(),
5262 2,
5263 Concurrency::ModuleManaged,
5264 FrameSink::new(late_tx),
5265 ),
5266 Err(ForwardingError::ConnectionClosing { connection_id })
5267 if connection_id == client
5268 ));
5269
5270 forwarding.cleanup_connection(client).unwrap();
5274 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5275 }
5276}
5277
5278#[cfg(test)]
5281mod swap_slot_tests {
5282 use std::time::Duration;
5283
5284 use super::*;
5285 use tokio::sync::mpsc;
5286
5287 const MODULE_ID: &str = "swapped";
5288
5289 struct SwapFixture {
5290 forwarding: ForwardingTable,
5291 incumbent_connection: ConnectionId,
5292 incumbent: ModuleEndpointId,
5293 candidate_connection: ConnectionId,
5294 candidate: ModuleEndpointId,
5295 _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
5296 }
5297
5298 fn swap_fixture() -> SwapFixture {
5299 let forwarding = ForwardingTable::default();
5300 let incumbent_connection = ConnectionId::new(100);
5301 let candidate_connection = ConnectionId::new(110);
5302 let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
5303 let incumbent = forwarding
5304 .register_module_connection(
5305 incumbent_connection,
5306 MODULE_ID.to_string(),
5307 2,
5308 Concurrency::ModuleManaged,
5309 FrameSink::new(incumbent_tx),
5310 )
5311 .unwrap();
5312 let (candidate_tx, candidate_rx) = mpsc::channel(8);
5313 let candidate = forwarding
5314 .register_candidate_module_connection(
5315 candidate_connection,
5316 MODULE_ID.to_string(),
5317 2,
5318 Concurrency::ModuleManaged,
5319 FrameSink::new(candidate_tx),
5320 )
5321 .unwrap();
5322 SwapFixture {
5323 forwarding,
5324 incumbent_connection,
5325 incumbent,
5326 candidate_connection,
5327 candidate,
5328 _module_rxs: vec![incumbent_rx, candidate_rx],
5329 }
5330 }
5331
5332 fn client(
5333 raw: u64,
5334 ) -> (
5335 ConnectionId,
5336 FrameSink,
5337 mpsc::Receiver<crate::router::OutboundFrame>,
5338 ) {
5339 let (tx, rx) = mpsc::channel(8);
5340 (ConnectionId::new(raw), FrameSink::new(tx), rx)
5341 }
5342
5343 fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
5344 forwarding
5345 .read_inner()
5346 .unwrap()
5347 .client_to_module
5348 .values()
5349 .map(|route| route.module_endpoint)
5350 .collect()
5351 }
5352
5353 #[test]
5354 fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
5355 let fixture = swap_fixture();
5356 let forwarding = &fixture.forwarding;
5357 assert_ne!(fixture.incumbent, fixture.candidate);
5358
5359 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5361 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5362 let (client_connection, client_sink, _client_rx) = client(200);
5363 let pending = forwarding
5364 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5365 .unwrap();
5366 assert_eq!(pending.endpoint, fixture.incumbent);
5367 let rpc = forwarding
5368 .begin_module_control_rpc_for(
5369 MODULE_ID,
5370 "health.check",
5371 Instant::now() + Duration::from_secs(1),
5372 )
5373 .unwrap();
5374 assert_eq!(rpc.endpoint, fixture.incumbent);
5375 let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
5376 assert_eq!(census.len(), 1, "the census lists one endpoint per id");
5377
5378 assert_eq!(
5380 forwarding
5381 .module_endpoint_for_connection(fixture.candidate_connection)
5382 .unwrap(),
5383 Some(fixture.candidate)
5384 );
5385 assert_eq!(
5386 forwarding
5387 .module_id_for_connection(fixture.candidate_connection)
5388 .unwrap()
5389 .as_deref(),
5390 Some(MODULE_ID)
5391 );
5392
5393 let (other_tx, _other_rx) = mpsc::channel(1);
5395 assert_eq!(
5396 forwarding.register_candidate_module_connection(
5397 ConnectionId::new(120),
5398 MODULE_ID.to_string(),
5399 2,
5400 Concurrency::ModuleManaged,
5401 FrameSink::new(other_tx),
5402 ),
5403 Err(ForwardingError::CandidateSlotOccupied {
5404 module_id: MODULE_ID.to_string()
5405 })
5406 );
5407 }
5408
5409 #[test]
5413 fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
5414 let fixture = swap_fixture();
5415 let forwarding = &fixture.forwarding;
5416 let (early_client, early_sink, _early_rx) = client(200);
5417 let mut early = forwarding
5418 .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
5419 .unwrap();
5420 assert_eq!(early.endpoint, fixture.incumbent);
5421 assert_eq!(
5422 forwarding
5423 .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
5424 .unwrap(),
5425 RouteBindRelayMark::Marked
5426 );
5427
5428 let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5429 assert_eq!(
5430 cutover,
5431 ForwardingCutover {
5432 promoted: fixture.candidate,
5433 incumbent: Some(fixture.incumbent),
5434 }
5435 );
5436
5437 let (late_client, late_sink, _late_rx) = client(201);
5439 let late = forwarding
5440 .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
5441 .unwrap();
5442 assert_eq!(
5443 late.endpoint, fixture.candidate,
5444 "a route.open after cutover was reserved on the incumbent"
5445 );
5446
5447 let completion = forwarding
5449 .complete_pending_relay(
5450 fixture.incumbent_connection,
5451 early.corr,
5452 RouteBindRelayOutcome::Accepted,
5453 )
5454 .expect("a superseded endpoint's ack is not an error on its connection");
5455 assert!(completion.settled);
5456 assert!(
5457 !committed_endpoints(forwarding).contains(&fixture.incumbent),
5458 "a relay reserved before cutover committed a route on the incumbent"
5459 );
5460 let goodbye = completion
5461 .abandoned
5462 .expect("the incumbent is told to drop the binding it just created");
5463 assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
5464 assert_eq!(goodbye.channel, early.module_channel);
5465 assert_eq!(goodbye.epoch, early.module_epoch);
5466 assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
5467 match early.receiver.try_recv() {
5468 Ok(RouteBindRelayOutcome::Rejected(body)) => assert_eq!(body.code, "module_reloading"),
5469 other => panic!("expected a retryable module_reloading answer, got {other:?}"),
5470 }
5471 assert!(matches!(
5472 forwarding
5473 .lookup_data_route(early_client, early.client_channel, early.client_epoch)
5474 .unwrap(),
5475 DataRoute::Client(DataRouteState::Absent)
5476 ));
5477
5478 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
5481 forwarding
5482 .complete_pending_relay(
5483 fixture.candidate_connection,
5484 late.corr,
5485 RouteBindRelayOutcome::Accepted,
5486 )
5487 .unwrap();
5488 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5489 assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
5490 }
5491
5492 #[test]
5493 fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
5494 let fixture = swap_fixture();
5495 let forwarding = &fixture.forwarding;
5496 let (bound_client, bound_sink, _bound_rx) = client(200);
5498 let bound = forwarding
5499 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5500 .unwrap();
5501 forwarding
5502 .complete_pending_relay(
5503 fixture.incumbent_connection,
5504 bound.corr,
5505 RouteBindRelayOutcome::Accepted,
5506 )
5507 .unwrap();
5508 let (pending_client, pending_sink, _pending_rx) = client(201);
5509 let mut in_flight = forwarding
5510 .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
5511 .unwrap();
5512 forwarding
5513 .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
5514 .unwrap();
5515
5516 let incumbent = forwarding
5517 .cutover_candidate(MODULE_ID)
5518 .unwrap()
5519 .unwrap()
5520 .incumbent
5521 .unwrap();
5522 let target = forwarding
5523 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
5524 .unwrap()
5525 .expect("the superseded incumbent is still registered");
5526
5527 assert_eq!(target.endpoint, fixture.incumbent);
5528 assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
5529 assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
5530 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5531 assert_eq!(target.abandoned_bindings.len(), 1);
5532 assert_eq!(
5533 target.abandoned_bindings[0].channel,
5534 in_flight.module_channel
5535 );
5536 assert!(matches!(
5537 in_flight.receiver.try_recv(),
5538 Ok(RouteBindRelayOutcome::Rejected(body)) if body.code == "module_reloading"
5539 ));
5540 assert_eq!(
5541 forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
5542 1,
5543 "the incumbent's bound route stays until its drain finishes"
5544 );
5545
5546 let (next_client, next_sink, _next_rx) = client(202);
5547 let next = forwarding
5548 .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
5549 .expect("the promoted candidate keeps accepting routes");
5550 assert_eq!(next.endpoint, fixture.candidate);
5551 }
5552
5553 #[test]
5559 fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
5560 let forwarding = ForwardingTable::default();
5561 let first_connection = ConnectionId::new(70);
5562 let (first_tx, _first_rx) = mpsc::channel(8);
5563 forwarding
5564 .register_module_connection(
5565 first_connection,
5566 MODULE_ID.to_string(),
5567 2,
5568 Concurrency::ModuleManaged,
5569 FrameSink::new(first_tx),
5570 )
5571 .unwrap();
5572 let (client_connection, client_sink, _client_rx) = client(200);
5573 let mut pending = forwarding
5574 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5575 .unwrap();
5576 let (second_tx, _second_rx) = mpsc::channel(8);
5577 forwarding
5578 .register_module_connection(
5579 ConnectionId::new(80),
5580 MODULE_ID.to_string(),
5581 2,
5582 Concurrency::ModuleManaged,
5583 FrameSink::new(second_tx),
5584 )
5585 .unwrap();
5586
5587 assert_eq!(
5588 forwarding
5589 .complete_pending_relay(
5590 first_connection,
5591 pending.corr,
5592 RouteBindRelayOutcome::Accepted
5593 )
5594 .unwrap_err(),
5595 ForwardingError::StaleModuleEndpoint
5596 );
5597 assert!(committed_endpoints(&forwarding).is_empty());
5598 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5599 assert!(matches!(
5600 pending.receiver.try_recv(),
5601 Err(oneshot::error::TryRecvError::Closed)
5602 ));
5603 }
5604
5605 #[test]
5606 fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
5607 let fixture = swap_fixture();
5609 let forwarding = &fixture.forwarding;
5610 assert!(forwarding
5611 .cleanup_connection(fixture.candidate_connection)
5612 .unwrap()
5613 .is_empty());
5614 assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
5615 let (client_connection, client_sink, _client_rx) = client(200);
5616 assert_eq!(
5617 forwarding
5618 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5619 .unwrap()
5620 .endpoint,
5621 fixture.incumbent
5622 );
5623
5624 let fixture = swap_fixture();
5627 let forwarding = &fixture.forwarding;
5628 let (bound_client, bound_sink, _bound_rx) = client(200);
5629 let bound = forwarding
5630 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5631 .unwrap();
5632 forwarding
5633 .complete_pending_relay(
5634 fixture.incumbent_connection,
5635 bound.corr,
5636 RouteBindRelayOutcome::Accepted,
5637 )
5638 .unwrap();
5639 forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5640 let released = forwarding
5641 .cleanup_connection(fixture.incumbent_connection)
5642 .unwrap();
5643 assert_eq!(released.len(), 1);
5644 assert_eq!(released[0].connection_id, bound_client);
5645 assert!(forwarding
5646 .read_inner()
5647 .unwrap()
5648 .superseded_endpoints
5649 .is_empty());
5650 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5651 let (next_client, next_sink, _next_rx) = client(201);
5652 assert_eq!(
5653 forwarding
5654 .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
5655 .unwrap()
5656 .endpoint,
5657 fixture.candidate
5658 );
5659 }
5660}