1use std::{
2 collections::{BTreeMap, HashMap, HashSet},
3 error::Error,
4 fmt,
5 sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
6 time::Duration,
7};
8
9use subc_control::{ClientControlResponse, RouteCloseReason};
10use subc_protocol::{
11 manifest::Concurrency,
12 session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
13 ErrorBody, Flags, FrameType, Principal, Priority,
14};
15use tokio::sync::{oneshot, Semaphore};
16use tokio::time::Instant;
17use tracing::{debug, info, warn};
18
19use crate::{
20 control::{RouteBindBreakers, RouteBindConcurrency},
21 observability::DaemonCounters,
22 registry::ConnectionId,
23 router::FrameSink,
24 scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
25 Frame, ProjectRootId,
26};
27
28const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
30
31const STATELESS_PARALLEL_WINDOW: usize = 1024;
33
34const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
38
39#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
44pub struct ModuleEndpointId {
45 pub connection_id: ConnectionId,
46 pub generation: u64,
47}
48
49#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
51pub(crate) struct ClientRouteKey {
52 pub connection_id: ConnectionId,
53 pub channel: u16,
54}
55
56#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
58pub(crate) struct ModuleRouteKey {
59 pub endpoint: ModuleEndpointId,
60 pub channel: u16,
61}
62
63#[derive(Debug)]
64pub(crate) struct RouteBinding {
65 pub client_connection_id: ConnectionId,
66 pub client_sink: FrameSink,
67 pub client_negotiated_ver: u8,
68 pub client_channel: u16,
69 pub client_epoch: u32,
70 pub module_id: String,
71 pub module_endpoint: ModuleEndpointId,
72 pub module_sink: FrameSink,
73 pub module_negotiated_ver: u8,
74 pub module_channel: u16,
75 pub module_epoch: u32,
76 pub principal: Principal,
77 pub project_root: Option<ProjectRootId>,
78 pub bound_at: Instant,
79 pub flow: Arc<ChannelFlow>,
80 pub scope: Option<BoundScope>,
83}
84
85#[derive(Debug, Clone)]
86pub(crate) enum DataRoute {
87 Client(DataRouteState),
88 Module(DataRouteState),
89}
90
91#[derive(Debug, Clone)]
92pub(crate) enum DataRouteState {
93 Bound(Arc<RouteBinding>),
94 Reserved,
95 EpochMismatch,
96 Absent,
97}
98
99#[derive(Debug, Clone, Copy, PartialEq, Eq)]
128pub(crate) enum GoodbyeTargetKind {
129 Client,
130 Module,
131}
132
133#[derive(Debug, Clone)]
134pub(crate) struct GoodbyeTarget {
135 pub connection_id: ConnectionId,
136 pub sink: FrameSink,
137 pub negotiated_ver: u8,
138 pub channel: u16,
139 pub epoch: u32,
140 pub kind: GoodbyeTargetKind,
141 pub module_id: Option<String>,
146}
147
148#[derive(Debug, Clone, Copy)]
151pub(crate) struct UndeliveredFrame<'a> {
152 pub module_id: Option<&'a str>,
154 pub sink: &'a FrameSink,
156}
157
158pub(crate) fn principal_label(principal: &Principal) -> String {
160 match principal {
161 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
162 Principal::Direct => "direct".to_string(),
163 other => format!("{other:?}"),
164 }
165}
166
167fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
170 let labels = inner
171 .client_to_module
172 .iter()
173 .filter(|(key, _)| key.connection_id == connection_id)
174 .map(|(_, route)| principal_label(&route.principal))
175 .collect::<std::collections::BTreeSet<_>>();
176 if labels.is_empty() {
177 "none".to_string()
178 } else {
179 labels.into_iter().collect::<Vec<_>>().join(",")
180 }
181}
182
183impl GoodbyeTarget {
184 pub(crate) fn close_on_delivery_failure(&self) -> bool {
187 matches!(self.kind, GoodbyeTargetKind::Client)
188 }
189}
190
191pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
204
205pub(crate) fn send_module_route_goodbye(
217 counters: &DaemonCounters,
218 sink: &FrameSink,
219 frame: Frame,
220 module_id: Option<&str>,
221 context: &'static str,
222) {
223 let channel = frame.header.channel;
224 let epoch = frame.header.epoch;
225 let Err(err) = sink.try_send(frame.clone()) else {
226 return;
227 };
228 let runtime = match tokio::runtime::Handle::try_current() {
231 Ok(runtime) if !sink.is_closed() => runtime,
232 _ => {
233 counters.increment_goodbye_relay_module_dropped(module_id);
234 warn!(
235 module_id = module_id.unwrap_or("unknown"),
236 route_channel = channel,
237 route_epoch = epoch,
238 error = %err,
239 context,
240 "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
241 );
242 return;
243 }
244 };
245 debug!(
246 module_id = module_id.unwrap_or("unknown"),
247 route_channel = channel,
248 route_epoch = epoch,
249 error = %err,
250 context,
251 "module egress queue refused route GOODBYE; delivering it once the module frees room"
252 );
253 let counters = counters.clone();
254 let sink = sink.clone();
255 let module_id = module_id.map(str::to_string);
256 runtime.spawn(async move {
257 let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
258 let why = match outcome {
259 Ok(Ok(())) => {
260 debug!(
261 module_id = module_id.as_deref().unwrap_or("unknown"),
262 route_channel = channel,
263 route_epoch = epoch,
264 context,
265 "late route GOODBYE delivered to module"
266 );
267 return;
268 }
269 Ok(Err(err)) => err.to_string(),
270 Err(_) => format!(
271 "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
272 ),
273 };
274 counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
275 warn!(
276 module_id = module_id.as_deref().unwrap_or("unknown"),
277 route_channel = channel,
278 route_epoch = epoch,
279 error = %why,
280 context,
281 "route GOODBYE to module dropped under backpressure; not closing shared module connection"
282 );
283 });
284}
285
286#[derive(Debug, Clone)]
292pub(crate) struct EndpointRoute {
293 pub goodbye_target: GoodbyeTarget,
294 pub principal: Principal,
295 pub bound_at: Instant,
296 pub draining: bool,
297 pub drain_reason: Option<RouteCloseReason>,
301}
302
303#[derive(Debug, Clone)]
306pub(crate) struct ScopeDrainedRoute {
307 pub reason: RouteCloseReason,
308 pub scope: BoundScope,
309 pub module_id: String,
310 pub client: GoodbyeTarget,
311 pub module: GoodbyeTarget,
312}
313
314#[derive(Debug)]
315pub(crate) struct PendingRouteBindRelay {
316 pub endpoint: ModuleEndpointId,
317 pub module_sink: FrameSink,
318 pub negotiated_ver: u8,
319 pub client_channel: u16,
320 pub client_epoch: u32,
321 pub module_channel: u16,
322 pub module_epoch: u32,
323 pub corr: u64,
324 pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
325}
326
327#[derive(Debug, Clone)]
328pub(crate) struct ModuleDrainTarget {
329 pub endpoint: ModuleEndpointId,
330 pub sink: FrameSink,
331 pub negotiated_ver: u8,
332 pub abandoned_bindings: Vec<GoodbyeTarget>,
333 pub excluded_subscriptions: u32,
334}
335
336#[cfg(unix)]
339#[derive(Debug, Clone)]
340pub(crate) struct ModuleConnectionTarget {
341 pub module_id: String,
342 pub endpoint: ModuleEndpointId,
343 pub sink: FrameSink,
344 pub negotiated_ver: u8,
345}
346
347#[derive(Debug, Clone)]
348pub(crate) enum RouteBindRelayOutcome {
349 Accepted,
350 Rejected(ErrorBody),
351 ModuleGone(String),
352}
353
354#[derive(Debug, Clone, Copy, PartialEq, Eq)]
356pub(crate) struct ForwardingCutover {
357 pub promoted: ModuleEndpointId,
359 pub incumbent: Option<ModuleEndpointId>,
362}
363
364#[derive(Debug)]
366pub(crate) struct ConnectionCleanup {
367 pub released: Vec<GoodbyeTarget>,
369 pub abandoned_relays: u32,
372}
373
374#[derive(Debug, Clone)]
375pub(crate) struct PendingRelayCompletion {
376 pub settled: bool,
377 pub abandoned: Option<GoodbyeTarget>,
378}
379
380#[derive(Debug)]
381pub(crate) struct PendingModuleControlRpc {
382 pub endpoint: ModuleEndpointId,
383 pub module_sink: FrameSink,
384 pub negotiated_ver: u8,
385 pub corr: u64,
386 pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
387}
388
389#[derive(Debug, Clone)]
390pub(crate) enum ModuleControlRpcOutcome {
391 Response(ModuleControlResponse),
392 Rejected(ErrorBody),
393 ModuleGone(String),
394 MalformedResponse(String),
395 UnexpectedOp { expected: String, actual: String },
396 DeadlineElapsed,
397}
398
399#[derive(Debug, Clone, PartialEq, Eq)]
400pub(crate) enum ModuleControlRpcCompletion {
401 Unknown,
402 Settled,
403 LateHealthAnswer {
404 module_id: String,
405 latency: Duration,
406 },
407}
408
409#[derive(Debug)]
410struct PendingModuleControlRpcEntry {
411 expected_op: String,
412 deadline: Instant,
413 health_probe_started_at: Option<Instant>,
414 sender: oneshot::Sender<ModuleControlRpcOutcome>,
415}
416
417#[derive(Debug)]
418struct HealthProbeTombstone {
419 expected_op: String,
420 module_id: String,
421 probe_started_at: Instant,
422 expires_at: Instant,
423}
424
425#[derive(Debug, Clone)]
426struct RouteReservation {
427 client_key: ClientRouteKey,
428 module_key: ModuleRouteKey,
429 client_epoch: u32,
430 module_epoch: u32,
431 project_root: Option<ProjectRootId>,
432}
433
434#[derive(Debug)]
435struct PendingRouteBindRelayEntry {
436 reservation: RouteReservation,
437 client_sink: FrameSink,
438 client_negotiated_ver: u8,
439 client_permit: crate::router::EgressPermit,
440 route_open_frame: Frame,
441 principal: Principal,
442 scope: Option<BoundScope>,
445 deadline: Instant,
446 relay_enqueued: bool,
457 sender: oneshot::Sender<RouteBindRelayOutcome>,
458}
459
460#[derive(Debug, Clone, Copy, PartialEq, Eq)]
463pub(crate) enum RouteBindRelayMark {
464 Marked,
466 SettledByReply,
470 ReleasedWithoutReply,
474}
475
476#[derive(Debug, Clone)]
477pub(crate) enum RouteRelease {
478 Removed(GoodbyeTarget),
479 Stale,
480 Absent,
481}
482
483#[derive(Debug, Clone)]
484pub(crate) enum RoutePollSnapshot {
485 Bound {
486 module_id: String,
487 status: Option<String>,
488 },
489 Absent,
490}
491
492#[derive(Debug, Clone)]
493struct ModuleConnection {
494 endpoint: ModuleEndpointId,
495 sink: FrameSink,
496 negotiated_ver: u8,
497 concurrency: Concurrency,
498}
499
500#[derive(Debug, Default)]
501struct ForwardingInner {
502 operator_confirms: Arc<crate::operator_confirm::OperatorConfirms>,
503 daemon_draining: bool,
504 daemon_drain_results: HashMap<ModuleEndpointId, bool>,
509 modules_by_id: HashMap<String, ModuleConnection>,
513 candidates_by_id: HashMap<String, ModuleConnection>,
519 superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
525 endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
526 module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
527 draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
531 closing_connections: HashSet<ConnectionId>,
532 next_generation: u64,
533 reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
534 reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
535 next_client_channel: HashMap<ConnectionId, u16>,
536 next_module_channel: HashMap<ModuleEndpointId, u16>,
537 client_slot_epochs: HashMap<ClientRouteKey, u32>,
538 module_slot_epochs: HashMap<ModuleRouteKey, u32>,
539 last_published_epoch: HashMap<ClientRouteKey, u32>,
540 client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
541 module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
542 status: HashMap<(ClientRouteKey, u32), String>,
543 pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
544 relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
549 scope_tags: HashMap<(String, String), ScopeTag>,
554 next_control_corr: HashMap<ModuleEndpointId, u64>,
555 pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
556 health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
557}
558
559#[derive(Debug, Clone)]
560pub(crate) struct CloseReason {
561 code: &'static str,
562 message: String,
563}
564
565impl CloseReason {
566 pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
567 Self {
568 code,
569 message: message.into(),
570 }
571 }
572}
573
574impl fmt::Display for CloseReason {
575 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
576 write!(f, "{}: {}", self.code, self.message)
577 }
578}
579
580pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
581
582#[derive(Debug, Default)]
584pub struct ForwardingTable {
585 inner: Arc<RwLock<ForwardingInner>>,
586 #[cfg(feature = "test-support")]
587 route_closing_hold: Mutex<Option<oneshot::Receiver<()>>>,
588 close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
589 counters: DaemonCounters,
590 route_bind_breakers: RouteBindBreakers,
595 route_bind_concurrency: RouteBindConcurrency,
598 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
602}
603
604impl ForwardingTable {
605 #[cfg(feature = "test-support")]
647 pub fn configure_operator_script(
648 &self,
649 script: &std::path::Path,
650 events: Option<&std::path::Path>,
651 limits: Option<crate::operator_confirm::OperatorLimits>,
652 ) {
653 self.operator_confirms()
654 .configure_script(script, events, limits);
655 }
656
657 #[cfg(feature = "test-support")]
661 pub fn hold_route_closing_for_test(&self) -> oneshot::Sender<()> {
662 let (release, hold) = oneshot::channel();
663 *self.route_closing_hold.lock().unwrap() = Some(hold);
664 release
665 }
666
667 #[cfg(feature = "test-support")]
668 pub(crate) async fn wait_before_route_closing_for_test(&self) {
669 let hold = self.route_closing_hold.lock().unwrap().take();
670 if let Some(hold) = hold {
671 let _ = hold.await;
672 }
673 }
674
675 pub(crate) fn operator_confirms(&self) -> Arc<crate::operator_confirm::OperatorConfirms> {
676 Arc::clone(
677 &self
678 .inner
679 .read()
680 .unwrap_or_else(|p| p.into_inner())
681 .operator_confirms,
682 )
683 }
684
685 #[cfg(test)]
686 pub(crate) fn inject_operator_principal(&self, key: ModuleRouteKey, principal: Principal) {
687 let mut inner = self.write_inner().unwrap();
688 let old = inner.module_to_client.remove(&key).unwrap();
689 let client = ClientRouteKey {
690 connection_id: old.client_connection_id,
691 channel: old.client_channel,
692 };
693 inner.client_to_module.remove(&client);
694 let mut binding = Arc::try_unwrap(old).expect("test binding has no outstanding readers");
695 binding.principal = principal;
696 let binding = Arc::new(binding);
697 inner.client_to_module.insert(client, Arc::clone(&binding));
698 inner.module_to_client.insert(key, binding);
699 }
700
701 pub(crate) fn with_operator_route<T>(
705 &self,
706 connection_id: ConnectionId,
707 channel: u16,
708 epoch: u32,
709 admit: impl FnOnce(Option<&RouteBinding>) -> T,
710 ) -> Result<T, ForwardingError> {
711 let inner = self.read_inner()?;
712 let binding = inner
713 .endpoint_by_connection
714 .get(&connection_id)
715 .and_then(|endpoint| {
716 inner.module_to_client.get(&ModuleRouteKey {
717 endpoint: *endpoint,
718 channel,
719 })
720 })
721 .filter(|binding| binding.module_epoch == epoch);
722 Ok(admit(binding.map(Arc::as_ref)))
723 }
724
725 pub(crate) fn counters(&self) -> DaemonCounters {
726 self.counters.clone()
727 }
728
729 pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
730 self.route_bind_breakers.clone()
731 }
732
733 pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
734 self.route_bind_concurrency.clone()
735 }
736
737 pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
738 Arc::clone(&self.route_outages)
739 }
740
741 pub(crate) fn register_connection_close(
742 &self,
743 connection_id: ConnectionId,
744 ) -> ConnectionCloseReceiver {
745 let (sender, receiver) = oneshot::channel();
746 let replaced = self
747 .lock_close_registry()
748 .insert(connection_id, sender)
749 .is_some();
750 if replaced {
751 warn!(
752 connection_id = connection_id.get(),
753 "replaced existing connection close registration"
754 );
755 }
756 receiver
757 }
758
759 pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
760 self.lock_close_registry().remove(&connection_id);
761 }
762
763 #[cfg(unix)]
772 pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
773 let senders: Vec<_> = self.lock_close_registry().drain().collect();
774 let count = senders.len();
775 for (_, sender) in senders {
776 let _ = sender.send(reason.clone());
777 }
778 count
779 }
780
781 #[cfg(unix)]
785 pub(crate) fn module_connections(
786 &self,
787 ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
788 let inner = self.read_inner()?;
789 let mut seen = HashSet::new();
790 Ok(inner
791 .modules_by_id
792 .values()
793 .chain(inner.candidates_by_id.values())
794 .chain(inner.superseded_endpoints.values())
795 .filter(|module| seen.insert(module.endpoint))
796 .map(|module| ModuleConnectionTarget {
797 module_id: inner
798 .module_id_by_endpoint
799 .get(&module.endpoint)
800 .cloned()
801 .unwrap_or_default(),
802 endpoint: module.endpoint,
803 sink: module.sink.clone(),
804 negotiated_ver: module.negotiated_ver,
805 })
806 .collect())
807 }
808
809 pub(crate) fn request_connection_close(
813 &self,
814 connection_id: ConnectionId,
815 reason: CloseReason,
816 ) -> bool {
817 let sender = self.lock_close_registry().remove(&connection_id);
818 if let Some(sender) = sender {
819 debug!(
820 connection_id = connection_id.get(),
821 close_reason = %reason,
822 "requesting connection close"
823 );
824 let _ = sender.send(reason);
825 true
826 } else {
827 debug!(
828 connection_id = connection_id.get(),
829 close_reason = %reason,
830 "connection close request ignored for inactive connection"
831 );
832 false
833 }
834 }
835
836 pub fn register_module_connection(
837 &self,
838 connection_id: ConnectionId,
839 module_id: String,
840 negotiated_ver: u8,
841 concurrency: Concurrency,
842 sink: FrameSink,
843 ) -> Result<ModuleEndpointId, ForwardingError> {
844 self.register_module_connection_inner(
845 connection_id,
846 module_id,
847 negotiated_ver,
848 concurrency,
849 sink,
850 None,
851 )
852 }
853
854 pub(crate) fn register_module_connection_acked(
867 &self,
868 connection_id: ConnectionId,
869 module_id: String,
870 negotiated_ver: u8,
871 concurrency: Concurrency,
872 sink: FrameSink,
873 hello_ack: Frame,
874 ) -> Result<ModuleEndpointId, ForwardingError> {
875 self.register_module_connection_inner(
876 connection_id,
877 module_id,
878 negotiated_ver,
879 concurrency,
880 sink,
881 Some(hello_ack),
882 )
883 }
884
885 fn register_module_connection_inner(
886 &self,
887 connection_id: ConnectionId,
888 module_id: String,
889 negotiated_ver: u8,
890 concurrency: Concurrency,
891 sink: FrameSink,
892 hello_ack: Option<Frame>,
893 ) -> Result<ModuleEndpointId, ForwardingError> {
894 let mut inner = self.write_inner()?;
895 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
896 return Err(ForwardingError::ConnectionClosing { connection_id });
897 }
898 check_module_connection_role_locked(&inner, connection_id)?;
899 enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
902
903 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
904 let endpoint = ModuleEndpointId {
905 connection_id,
906 generation: inner.next_generation,
907 };
908 inner.endpoint_by_connection.insert(connection_id, endpoint);
909 inner
910 .module_id_by_endpoint
911 .insert(endpoint, module_id.clone());
912 inner.next_module_channel.insert(endpoint, 1);
913 inner.next_control_corr.insert(endpoint, 1);
914 inner.modules_by_id.insert(
915 module_id.clone(),
916 ModuleConnection {
917 endpoint,
918 sink,
919 negotiated_ver,
920 concurrency,
921 },
922 );
923 drop(inner);
924
925 if let Some(discarded) = self
938 .route_bind_breakers
939 .reset_for_new_module_connection(&module_id)
940 {
941 info!(
942 module_id = %module_id,
943 discarded_consecutive_timeouts = discarded,
944 "route.bind breaker state discarded: a new module connection replaced the process it described"
945 );
946 }
947 Ok(endpoint)
948 }
949
950 #[cfg(test)]
964 pub(crate) fn register_candidate_module_connection(
965 &self,
966 connection_id: ConnectionId,
967 module_id: String,
968 negotiated_ver: u8,
969 concurrency: Concurrency,
970 sink: FrameSink,
971 ) -> Result<ModuleEndpointId, ForwardingError> {
972 self.register_candidate_module_connection_inner(
973 connection_id,
974 module_id,
975 negotiated_ver,
976 concurrency,
977 sink,
978 None,
979 )
980 }
981
982 pub(crate) fn register_candidate_module_connection_acked(
989 &self,
990 connection_id: ConnectionId,
991 module_id: String,
992 negotiated_ver: u8,
993 concurrency: Concurrency,
994 sink: FrameSink,
995 hello_ack: Frame,
996 ) -> Result<ModuleEndpointId, ForwardingError> {
997 self.register_candidate_module_connection_inner(
998 connection_id,
999 module_id,
1000 negotiated_ver,
1001 concurrency,
1002 sink,
1003 Some(hello_ack),
1004 )
1005 }
1006
1007 fn register_candidate_module_connection_inner(
1008 &self,
1009 connection_id: ConnectionId,
1010 module_id: String,
1011 negotiated_ver: u8,
1012 concurrency: Concurrency,
1013 sink: FrameSink,
1014 hello_ack: Option<Frame>,
1015 ) -> Result<ModuleEndpointId, ForwardingError> {
1016 let mut inner = self.write_inner()?;
1017 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1018 return Err(ForwardingError::ConnectionClosing { connection_id });
1019 }
1020 check_module_connection_role_locked(&inner, connection_id)?;
1021 if inner.candidates_by_id.contains_key(&module_id) {
1022 return Err(ForwardingError::CandidateSlotOccupied { module_id });
1023 }
1024 enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1027
1028 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1029 let endpoint = ModuleEndpointId {
1030 connection_id,
1031 generation: inner.next_generation,
1032 };
1033 inner.endpoint_by_connection.insert(connection_id, endpoint);
1034 inner
1035 .module_id_by_endpoint
1036 .insert(endpoint, module_id.clone());
1037 inner.next_module_channel.insert(endpoint, 1);
1038 inner.next_control_corr.insert(endpoint, 1);
1039 inner.candidates_by_id.insert(
1040 module_id,
1041 ModuleConnection {
1042 endpoint,
1043 sink,
1044 negotiated_ver,
1045 concurrency,
1046 },
1047 );
1048 Ok(endpoint)
1049 }
1050
1051 pub(crate) fn cutover_candidate(
1069 &self,
1070 module_id: &str,
1071 ) -> Result<Option<ForwardingCutover>, ForwardingError> {
1072 let mut inner = self.write_inner()?;
1073 if inner.daemon_draining {
1074 return Err(ForwardingError::ModuleReloading {
1075 module_id: module_id.to_string(),
1076 });
1077 }
1078 let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
1079 return Ok(None);
1080 };
1081 let promoted = candidate.endpoint;
1082 let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
1083 let incumbent = incumbent.map(|incumbent| {
1084 let endpoint = incumbent.endpoint;
1085 inner.superseded_endpoints.insert(endpoint, incumbent);
1086 endpoint
1087 });
1088 drop(inner);
1089
1090 if let Some(discarded) = self
1094 .route_bind_breakers
1095 .reset_for_new_module_connection(module_id)
1096 {
1097 info!(
1098 module_id = %module_id,
1099 discarded_consecutive_timeouts = discarded,
1100 "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
1101 );
1102 }
1103 Ok(Some(ForwardingCutover {
1104 promoted,
1105 incumbent,
1106 }))
1107 }
1108
1109 #[allow(clippy::too_many_arguments)]
1110 pub(crate) async fn begin_route_bind_relay_for(
1111 &self,
1112 client_connection_id: ConnectionId,
1113 client_sink: FrameSink,
1114 client_negotiated_ver: u8,
1115 client_corr: u64,
1116 module_id: &str,
1117 principal: Principal,
1118 scope: Option<BoundScope>,
1119 project_root: Option<ProjectRootId>,
1120 deadline: Instant,
1121 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1122 let client_permit =
1126 client_sink
1127 .reserve_owned()
1128 .await
1129 .map_err(|_| ForwardingError::ClientEgressClosed {
1130 connection_id: client_connection_id,
1131 })?;
1132 self.begin_route_bind_relay_inner(
1133 client_connection_id,
1134 client_sink,
1135 client_negotiated_ver,
1136 client_corr,
1137 module_id,
1138 principal,
1139 scope,
1140 project_root,
1141 deadline,
1142 client_permit,
1143 )
1144 }
1145
1146 #[cfg(test)]
1147 pub(crate) fn begin_route_bind_relay_for_test(
1148 &self,
1149 client_connection_id: ConnectionId,
1150 client_sink: FrameSink,
1151 client_corr: u64,
1152 module_id: &str,
1153 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1154 let permit =
1155 client_sink
1156 .try_reserve_owned()
1157 .map_err(|_| ForwardingError::ClientEgressClosed {
1158 connection_id: client_connection_id,
1159 })?;
1160 self.begin_route_bind_relay_inner(
1161 client_connection_id,
1162 client_sink,
1163 subc_protocol::PROTOCOL_VERSION,
1164 client_corr,
1165 module_id,
1166 Principal::Direct,
1167 None,
1168 None,
1169 Instant::now() + std::time::Duration::from_secs(60),
1170 permit,
1171 )
1172 }
1173
1174 pub(crate) fn begin_module_control_rpc_for(
1175 &self,
1176 module_id: &str,
1177 expected_op: &str,
1178 deadline: Instant,
1179 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1180 self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1181 }
1182
1183 pub(crate) fn begin_health_probe_rpc_for(
1184 &self,
1185 module_id: &str,
1186 expected_op: &str,
1187 probe_started_at: Instant,
1188 deadline: Instant,
1189 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1190 self.begin_module_control_rpc_inner(
1191 module_id,
1192 expected_op,
1193 deadline,
1194 Some(probe_started_at),
1195 false,
1196 )
1197 }
1198
1199 pub(crate) fn begin_drain_health_probe_rpc_for(
1200 &self,
1201 module_id: &str,
1202 expected_op: &str,
1203 probe_started_at: Instant,
1204 deadline: Instant,
1205 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1206 self.begin_module_control_rpc_inner(
1207 module_id,
1208 expected_op,
1209 deadline,
1210 Some(probe_started_at),
1211 true,
1212 )
1213 }
1214
1215 pub(crate) fn begin_endpoint_health_probe_rpc_for(
1223 &self,
1224 endpoint: ModuleEndpointId,
1225 expected_op: &str,
1226 probe_started_at: Instant,
1227 deadline: Instant,
1228 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1229 let inner = self.write_inner()?;
1230 let module = module_connection_for_endpoint_locked(&inner, endpoint)
1231 .cloned()
1232 .ok_or(ForwardingError::NoModuleConnection)?;
1233 let module_id = inner
1234 .module_id_by_endpoint
1235 .get(&endpoint)
1236 .cloned()
1237 .unwrap_or_default();
1238 self.begin_control_rpc_locked(
1241 inner,
1242 &module_id,
1243 module,
1244 expected_op,
1245 deadline,
1246 Some(probe_started_at),
1247 true,
1248 )
1249 }
1250
1251 fn begin_module_control_rpc_inner(
1252 &self,
1253 module_id: &str,
1254 expected_op: &str,
1255 deadline: Instant,
1256 health_probe_started_at: Option<Instant>,
1257 allow_draining: bool,
1258 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1259 let inner = self.write_inner()?;
1260 let module = inner
1261 .modules_by_id
1262 .get(module_id)
1263 .cloned()
1264 .ok_or(ForwardingError::NoModuleConnection)?;
1265 self.begin_control_rpc_locked(
1266 inner,
1267 module_id,
1268 module,
1269 expected_op,
1270 deadline,
1271 health_probe_started_at,
1272 allow_draining,
1273 )
1274 }
1275
1276 #[allow(clippy::too_many_arguments)]
1277 fn begin_control_rpc_locked(
1278 &self,
1279 mut inner: RwLockWriteGuard<'_, ForwardingInner>,
1280 module_id: &str,
1281 module: ModuleConnection,
1282 expected_op: &str,
1283 deadline: Instant,
1284 health_probe_started_at: Option<Instant>,
1285 allow_draining: bool,
1286 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1287 if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1288 return Err(ForwardingError::ModuleReloading {
1289 module_id: module_id.to_string(),
1290 });
1291 }
1292 if inner
1293 .closing_connections
1294 .contains(&module.endpoint.connection_id)
1295 {
1296 return Err(ForwardingError::ConnectionClosing {
1297 connection_id: module.endpoint.connection_id,
1298 });
1299 }
1300 if health_probe_started_at.is_some() {
1301 inner
1305 .health_probe_tombstones
1306 .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1307 }
1308 let corr = match inner.allocate_control_corr(module.endpoint) {
1309 Ok(corr) => corr,
1310 Err(err) => {
1311 drop(inner);
1312 self.request_connection_close(
1313 module.endpoint.connection_id,
1314 CloseReason::new(
1315 "control_correlation_exhausted",
1316 "daemon-originated channel-0 correlation space exhausted",
1317 ),
1318 );
1319 return Err(err);
1320 }
1321 };
1322 let (sender, receiver) = oneshot::channel();
1323 inner.pending_control_rpcs.insert(
1324 (module.endpoint, corr),
1325 PendingModuleControlRpcEntry {
1326 expected_op: expected_op.to_string(),
1327 deadline,
1328 health_probe_started_at,
1329 sender,
1330 },
1331 );
1332
1333 Ok(PendingModuleControlRpc {
1334 endpoint: module.endpoint,
1335 module_sink: module.sink,
1336 negotiated_ver: module.negotiated_ver,
1337 corr,
1338 receiver,
1339 })
1340 }
1341
1342 #[allow(clippy::too_many_arguments)]
1343 fn begin_route_bind_relay_inner(
1344 &self,
1345 client_connection_id: ConnectionId,
1346 client_sink: FrameSink,
1347 client_negotiated_ver: u8,
1348 client_corr: u64,
1349 expected_module_id: &str,
1350 principal: Principal,
1351 scope: Option<BoundScope>,
1352 project_root: Option<ProjectRootId>,
1353 deadline: Instant,
1354 client_permit: crate::router::EgressPermit,
1355 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1356 let mut inner = self.write_inner()?;
1357 if inner
1361 .endpoint_by_connection
1362 .contains_key(&client_connection_id)
1363 {
1364 return Err(ForwardingError::ConnectionRoleConflict {
1365 connection_id: client_connection_id,
1366 });
1367 }
1368 if inner.closing_connections.contains(&client_connection_id) {
1369 return Err(ForwardingError::ConnectionClosing {
1370 connection_id: client_connection_id,
1371 });
1372 }
1373 let module = inner
1374 .modules_by_id
1375 .get(expected_module_id)
1376 .cloned()
1377 .ok_or(ForwardingError::NoModuleConnection)?;
1378 if inner.draining_endpoints.contains_key(&module.endpoint) {
1379 return Err(ForwardingError::ModuleReloading {
1380 module_id: expected_module_id.to_string(),
1381 });
1382 }
1383 if inner
1384 .closing_connections
1385 .contains(&module.endpoint.connection_id)
1386 {
1387 return Err(ForwardingError::ConnectionClosing {
1388 connection_id: module.endpoint.connection_id,
1389 });
1390 }
1391
1392 let corr = match inner.allocate_control_corr(module.endpoint) {
1393 Ok(corr) => corr,
1394 Err(err) => {
1395 drop(inner);
1396 self.request_connection_close(
1397 module.endpoint.connection_id,
1398 CloseReason::new(
1399 "control_correlation_exhausted",
1400 "daemon-originated channel-0 correlation space exhausted",
1401 ),
1402 );
1403 return Err(err);
1404 }
1405 };
1406 let (client_channel, client_epoch, module_channel, module_epoch) =
1407 inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1408 let client_key = ClientRouteKey {
1409 connection_id: client_connection_id,
1410 channel: client_channel,
1411 };
1412 let module_key = ModuleRouteKey {
1413 endpoint: module.endpoint,
1414 channel: module_channel,
1415 };
1416 let reservation = RouteReservation {
1417 client_key,
1418 module_key,
1419 client_epoch,
1420 module_epoch,
1421 project_root,
1422 };
1423 let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
1424 route_channel: client_channel,
1425 route_epoch: client_epoch,
1426 })
1427 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1428 let route_open_frame = Frame::build_with_version(
1429 client_negotiated_ver,
1430 FrameType::Response,
1431 Flags::new(false, Priority::Passive, false),
1432 0,
1433 0,
1434 client_corr,
1435 response_body,
1436 )
1437 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1438 let (sender, receiver) = oneshot::channel();
1439 inner.reserved_client.insert(client_key, module_key);
1440 inner.reserved_module.insert(module_key, client_key);
1441 inner.pending_relays.insert(
1442 (module.endpoint, corr),
1443 PendingRouteBindRelayEntry {
1444 reservation,
1445 client_sink,
1446 client_negotiated_ver,
1447 client_permit,
1448 route_open_frame,
1449 principal,
1450 scope,
1451 deadline,
1452 relay_enqueued: false,
1453 sender,
1454 },
1455 );
1456
1457 Ok(PendingRouteBindRelay {
1458 endpoint: module.endpoint,
1459 module_sink: module.sink,
1460 negotiated_ver: module.negotiated_ver,
1461 client_channel,
1462 client_epoch,
1463 module_channel,
1464 module_epoch,
1465 corr,
1466 receiver,
1467 })
1468 }
1469
1470 pub(crate) fn mark_route_bind_relay_enqueued(
1480 &self,
1481 endpoint: ModuleEndpointId,
1482 corr: u64,
1483 ) -> Result<RouteBindRelayMark, ForwardingError> {
1484 let mut inner = self.write_inner()?;
1485 if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
1486 pending.relay_enqueued = true;
1487 return Ok(RouteBindRelayMark::Marked);
1488 }
1489 if inner
1490 .relays_released_before_enqueue
1491 .remove(&(endpoint, corr))
1492 {
1493 return Ok(RouteBindRelayMark::ReleasedWithoutReply);
1494 }
1495 Ok(RouteBindRelayMark::SettledByReply)
1496 }
1497
1498 pub(crate) fn release_client_route(
1499 &self,
1500 client_connection_id: ConnectionId,
1501 client_channel: u16,
1502 expected_epoch: u32,
1503 ) -> Result<RouteRelease, ForwardingError> {
1504 let mut inner = self.write_inner()?;
1505 let release = release_client_route_locked(
1506 &mut inner,
1507 ClientRouteKey {
1508 connection_id: client_connection_id,
1509 channel: client_channel,
1510 },
1511 expected_epoch,
1512 );
1513 self.record_route_release(&release);
1514 Ok(release)
1515 }
1516
1517 pub(crate) fn release_module_route(
1518 &self,
1519 module_connection_id: ConnectionId,
1520 module_channel: u16,
1521 expected_epoch: u32,
1522 ) -> Result<RouteRelease, ForwardingError> {
1523 let mut inner = self.write_inner()?;
1524 let Some(endpoint) = inner
1525 .endpoint_by_connection
1526 .get(&module_connection_id)
1527 .copied()
1528 else {
1529 return Ok(RouteRelease::Absent);
1530 };
1531 let release = release_module_route_locked(
1532 &mut inner,
1533 ModuleRouteKey {
1534 endpoint,
1535 channel: module_channel,
1536 },
1537 expected_epoch,
1538 );
1539 self.record_route_release(&release);
1540 Ok(release)
1541 }
1542
1543 pub(crate) fn abort_pending_relay(
1544 &self,
1545 endpoint: ModuleEndpointId,
1546 corr: u64,
1547 outcome: RouteBindRelayOutcome,
1548 ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
1549 let mut inner = self.write_inner()?;
1550 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1551 inner
1556 .relays_released_before_enqueue
1557 .remove(&(endpoint, corr));
1558 return Ok(None);
1559 };
1560 release_reserved_route_locked(
1561 &mut inner,
1562 pending.reservation.client_key,
1563 pending.reservation.module_key,
1564 );
1565 let target = pending
1566 .relay_enqueued
1567 .then(|| abandoned_route_target(&inner, &pending.reservation));
1568 let _ = pending.sender.send(outcome);
1569 Ok(target.flatten())
1570 }
1571
1572 pub(crate) fn cancel_module_control_rpc(
1573 &self,
1574 endpoint: ModuleEndpointId,
1575 corr: u64,
1576 ) -> Result<(), ForwardingError> {
1577 self.write_inner()?
1578 .pending_control_rpcs
1579 .remove(&(endpoint, corr));
1580 Ok(())
1581 }
1582
1583 pub(crate) fn tombstone_health_probe_rpc(
1584 &self,
1585 endpoint: ModuleEndpointId,
1586 corr: u64,
1587 ) -> Result<bool, ForwardingError> {
1588 let key = (endpoint, corr);
1589 let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
1590 {
1591 let mut inner = self.write_inner()?;
1592 let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
1593 return Ok(false);
1594 };
1595 let Some(probe_started_at) = pending.health_probe_started_at else {
1596 inner.pending_control_rpcs.insert(key, pending);
1597 return Ok(false);
1598 };
1599 let module_id = inner
1600 .module_id_by_endpoint
1601 .get(&endpoint)
1602 .cloned()
1603 .unwrap_or_else(|| "unknown".to_string());
1604 inner.health_probe_tombstones.insert(
1605 key,
1606 HealthProbeTombstone {
1607 expected_op: pending.expected_op,
1608 module_id,
1609 probe_started_at,
1610 expires_at,
1611 },
1612 );
1613 }
1614 self.schedule_health_probe_tombstone_expiration(key, expires_at);
1615 Ok(true)
1616 }
1617
1618 fn schedule_health_probe_tombstone_expiration(
1619 &self,
1620 key: (ModuleEndpointId, u64),
1621 expires_at: Instant,
1622 ) {
1623 let inner = Arc::downgrade(&self.inner);
1624 tokio::spawn(async move {
1625 tokio::time::sleep_until(expires_at).await;
1626 let Some(inner) = inner.upgrade() else {
1627 return;
1628 };
1629 let Ok(mut inner) = inner.write() else {
1630 return;
1631 };
1632 let expired = inner
1633 .health_probe_tombstones
1634 .get(&key)
1635 .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
1636 if expired {
1637 inner.health_probe_tombstones.remove(&key);
1638 }
1639 });
1640 }
1641
1642 pub(crate) fn complete_pending_relay(
1643 &self,
1644 connection_id: ConnectionId,
1645 corr: u64,
1646 outcome: RouteBindRelayOutcome,
1647 ) -> Result<PendingRelayCompletion, ForwardingError> {
1648 let mut inner = self.write_inner()?;
1649 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1650 return Ok(PendingRelayCompletion {
1651 settled: false,
1652 abandoned: None,
1653 });
1654 };
1655 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1656 return Ok(PendingRelayCompletion {
1657 settled: false,
1658 abandoned: None,
1659 });
1660 };
1661
1662 if Instant::now() >= pending.deadline {
1663 release_reserved_route_locked(
1664 &mut inner,
1665 pending.reservation.client_key,
1666 pending.reservation.module_key,
1667 );
1668 let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
1669 .then(|| abandoned_route_target(&inner, &pending.reservation))
1670 .flatten();
1671 let _ = pending
1672 .sender
1673 .send(RouteBindRelayOutcome::Rejected(ErrorBody {
1674 code: "module_timeout".to_string(),
1675 message: "route.bind response arrived after its daemon deadline".to_string(),
1676 detail: None,
1677 }));
1678 return Ok(PendingRelayCompletion {
1679 settled: true,
1680 abandoned,
1681 });
1682 }
1683
1684 match outcome {
1685 RouteBindRelayOutcome::Accepted
1704 if pending.client_sink.is_closed()
1705 || inner
1706 .closing_connections
1707 .contains(&pending.reservation.client_key.connection_id) =>
1708 {
1709 let reason = if pending.client_sink.is_closed() {
1710 "client egress closed before route publication"
1711 } else {
1712 "client connection is closing before route publication"
1713 };
1714 release_reserved_route_locked(
1715 &mut inner,
1716 pending.reservation.client_key,
1717 pending.reservation.module_key,
1718 );
1719 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1723 let _ = pending
1724 .sender
1725 .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
1726 return Ok(PendingRelayCompletion {
1727 settled: true,
1728 abandoned,
1729 });
1730 }
1731 RouteBindRelayOutcome::Accepted
1754 if inner.superseded_endpoints.contains_key(&endpoint)
1755 || inner.draining_endpoints.contains_key(&endpoint) =>
1756 {
1757 release_reserved_route_locked(
1758 &mut inner,
1759 pending.reservation.client_key,
1760 pending.reservation.module_key,
1761 );
1762 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1765 let module_id = inner
1766 .module_id_by_endpoint
1767 .get(&endpoint)
1768 .cloned()
1769 .unwrap_or_else(|| "unknown".to_string());
1770 let _ = pending
1771 .sender
1772 .send(RouteBindRelayOutcome::Rejected(ErrorBody::new(
1773 "module_reloading",
1774 format!("module_id '{module_id}' is reloading"),
1775 )));
1776 return Ok(PendingRelayCompletion {
1777 settled: true,
1778 abandoned,
1779 });
1780 }
1781 RouteBindRelayOutcome::Accepted
1790 if pending
1791 .scope
1792 .as_ref()
1793 .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
1794 {
1795 let refusal = pending
1796 .scope
1797 .as_ref()
1798 .and_then(|scope| scope_refusal_locked(&inner, scope))
1799 .expect("guard matched a refusal under the same lock");
1800 release_reserved_route_locked(
1801 &mut inner,
1802 pending.reservation.client_key,
1803 pending.reservation.module_key,
1804 );
1805 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1806 let _ = pending
1807 .sender
1808 .send(RouteBindRelayOutcome::Rejected(refusal));
1809 return Ok(PendingRelayCompletion {
1810 settled: true,
1811 abandoned,
1812 });
1813 }
1814 RouteBindRelayOutcome::Accepted => {
1815 let abandoned = commit_route_locked(&mut inner, pending)?;
1816 return Ok(PendingRelayCompletion {
1817 settled: true,
1818 abandoned,
1819 });
1820 }
1821 terminal => {
1822 release_reserved_route_locked(
1823 &mut inner,
1824 pending.reservation.client_key,
1825 pending.reservation.module_key,
1826 );
1827 let _ = pending.sender.send(terminal);
1828 }
1829 }
1830 Ok(PendingRelayCompletion {
1831 settled: true,
1832 abandoned: None,
1833 })
1834 }
1835
1836 pub(crate) fn pending_module_control_op(
1837 &self,
1838 connection_id: ConnectionId,
1839 corr: u64,
1840 ) -> Result<Option<String>, ForwardingError> {
1841 let inner = self.read_inner()?;
1842 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1843 return Ok(None);
1844 };
1845 let key = (endpoint, corr);
1846 Ok(inner
1847 .pending_control_rpcs
1848 .get(&key)
1849 .map(|pending| pending.expected_op.clone())
1850 .or_else(|| {
1851 inner
1852 .health_probe_tombstones
1853 .get(&key)
1854 .filter(|tombstone| tombstone.expires_at > Instant::now())
1855 .map(|tombstone| tombstone.expected_op.clone())
1856 }))
1857 }
1858
1859 pub(crate) fn complete_module_control_rpc(
1860 &self,
1861 connection_id: ConnectionId,
1862 corr: u64,
1863 actual_op: Option<&str>,
1864 outcome: ModuleControlRpcOutcome,
1865 ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
1866 let now = Instant::now();
1867 let mut inner = self.write_inner()?;
1868 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1869 return Ok(ModuleControlRpcCompletion::Unknown);
1870 };
1871 let key = (endpoint, corr);
1872 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
1873 if now >= pending.deadline {
1874 let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
1875 ModuleControlRpcCompletion::LateHealthAnswer {
1876 module_id: inner
1877 .module_id_by_endpoint
1878 .get(&endpoint)
1879 .cloned()
1880 .unwrap_or_else(|| "unknown".to_string()),
1881 latency: now.saturating_duration_since(probe_started_at),
1882 }
1883 });
1884 let _ = pending
1885 .sender
1886 .send(ModuleControlRpcOutcome::DeadlineElapsed);
1887 return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
1888 }
1889 let outcome = match actual_op {
1890 Some(actual) if actual != pending.expected_op => {
1891 ModuleControlRpcOutcome::UnexpectedOp {
1892 expected: pending.expected_op,
1893 actual: actual.to_string(),
1894 }
1895 }
1896 _ => outcome,
1897 };
1898 let _ = pending.sender.send(outcome);
1899 return Ok(ModuleControlRpcCompletion::Settled);
1900 }
1901
1902 let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
1903 return Ok(ModuleControlRpcCompletion::Unknown);
1904 };
1905 if tombstone.expires_at <= now {
1906 return Ok(ModuleControlRpcCompletion::Unknown);
1907 }
1908 Ok(ModuleControlRpcCompletion::LateHealthAnswer {
1909 module_id: tombstone.module_id,
1910 latency: now.saturating_duration_since(tombstone.probe_started_at),
1911 })
1912 }
1913
1914 #[cfg(test)]
1915 pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
1916 Ok(self.read_inner()?.health_probe_tombstones.len())
1917 }
1918
1919 #[cfg(test)]
1920 pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
1921 Ok(self.read_inner()?.closing_connections.len())
1922 }
1923
1924 #[cfg(test)]
1927 pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
1928 let inner = self.read_inner()?;
1929 Ok((inner.reserved_client.len(), inner.reserved_module.len()))
1930 }
1931
1932 pub(crate) fn module_endpoint_for_connection(
1933 &self,
1934 connection_id: ConnectionId,
1935 ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
1936 Ok(self
1937 .read_inner()?
1938 .endpoint_by_connection
1939 .get(&connection_id)
1940 .copied())
1941 }
1942
1943 pub(crate) fn module_id_for_connection(
1946 &self,
1947 connection_id: ConnectionId,
1948 ) -> Result<Option<String>, ForwardingError> {
1949 let inner = self.read_inner()?;
1950 Ok(inner
1951 .endpoint_by_connection
1952 .get(&connection_id)
1953 .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
1954 .cloned())
1955 }
1956
1957 pub(crate) fn module_route_epoch_was_allocated(
1964 &self,
1965 connection_id: ConnectionId,
1966 channel: u16,
1967 epoch: u32,
1968 ) -> Result<bool, ForwardingError> {
1969 let inner = self.read_inner()?;
1970 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1971 return Ok(false);
1972 };
1973 Ok(inner
1974 .module_slot_epochs
1975 .get(&ModuleRouteKey { endpoint, channel })
1976 .is_some_and(|last| epoch != 0 && epoch <= *last))
1977 }
1978
1979 pub(crate) fn has_live_module_connection(
1980 &self,
1981 module_id: &str,
1982 ) -> Result<bool, ForwardingError> {
1983 Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
1984 }
1985
1986 pub(crate) fn lookup_data_route(
1987 &self,
1988 connection_id: ConnectionId,
1989 channel: u16,
1990 epoch: u32,
1991 ) -> Result<DataRoute, ForwardingError> {
1992 let inner = self.read_inner()?;
1993 let state = if let Some(endpoint) =
1994 inner.endpoint_by_connection.get(&connection_id).copied()
1995 {
1996 let key = ModuleRouteKey { endpoint, channel };
1997 match inner.module_to_client.get(&key) {
1998 Some(route) if route.module_epoch == epoch => {
1999 DataRouteState::Bound(Arc::clone(route))
2000 }
2001 Some(_) => DataRouteState::EpochMismatch,
2002 None if inner.reserved_module.contains_key(&key)
2003 && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
2004 {
2005 DataRouteState::Reserved
2006 }
2007 None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
2008 None => DataRouteState::Absent,
2009 }
2010 } else {
2011 let key = ClientRouteKey {
2012 connection_id,
2013 channel,
2014 };
2015 match inner.client_to_module.get(&key) {
2016 Some(route) if route.client_epoch == epoch => {
2017 DataRouteState::Bound(Arc::clone(route))
2018 }
2019 Some(_) => DataRouteState::EpochMismatch,
2020 None if inner.reserved_client.contains_key(&key)
2021 && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
2022 {
2023 DataRouteState::Reserved
2024 }
2025 None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
2026 None => DataRouteState::Absent,
2027 }
2028 };
2029 Ok(
2030 if inner.endpoint_by_connection.contains_key(&connection_id) {
2031 DataRoute::Module(state)
2032 } else {
2033 DataRoute::Client(state)
2034 },
2035 )
2036 }
2037
2038 #[cfg(test)]
2039 pub(crate) fn inject_client_slot_epoch(
2040 &self,
2041 connection_id: ConnectionId,
2042 channel: u16,
2043 last_epoch: u32,
2044 ) {
2045 let mut inner = self.write_inner().expect("forwarding lock");
2046 inner.client_slot_epochs.insert(
2047 ClientRouteKey {
2048 connection_id,
2049 channel,
2050 },
2051 last_epoch,
2052 );
2053 inner.next_client_channel.insert(connection_id, channel);
2054 }
2055
2056 #[cfg(test)]
2057 pub(crate) fn inject_module_slot_epoch(
2058 &self,
2059 endpoint: ModuleEndpointId,
2060 channel: u16,
2061 last_epoch: u32,
2062 ) {
2063 let mut inner = self.write_inner().expect("forwarding lock");
2064 inner
2065 .module_slot_epochs
2066 .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
2067 inner.next_module_channel.insert(endpoint, channel);
2068 }
2069
2070 #[cfg(test)]
2071 pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
2072 self.write_inner()
2073 .expect("forwarding lock")
2074 .next_control_corr
2075 .insert(endpoint, next_corr);
2076 }
2077
2078 pub(crate) fn cache_status(
2079 &self,
2080 endpoint: ModuleEndpointId,
2081 module_channel: u16,
2082 module_epoch: u32,
2083 status: String,
2084 ) -> Result<bool, ForwardingError> {
2085 let mut inner = self.write_inner()?;
2086 if !inner.module_id_by_endpoint.contains_key(&endpoint) {
2087 return Err(ForwardingError::StaleModuleEndpoint);
2088 }
2089
2090 let module_key = ModuleRouteKey {
2091 endpoint,
2092 channel: module_channel,
2093 };
2094 let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
2095 (route.module_epoch == module_epoch).then_some((
2096 ClientRouteKey {
2097 connection_id: route.client_connection_id,
2098 channel: route.client_channel,
2099 },
2100 route.client_epoch,
2101 ))
2102 } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
2103 (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
2104 client_key,
2105 inner
2106 .client_slot_epochs
2107 .get(&client_key)
2108 .copied()
2109 .unwrap_or(0),
2110 ))
2111 } else {
2112 None
2113 };
2114
2115 if let Some(handle) = handle {
2116 inner.status.insert(handle, status);
2117 Ok(true)
2118 } else {
2119 debug!(
2120 module_channel,
2121 module_epoch,
2122 generation = endpoint.generation,
2123 connection_id = endpoint.connection_id.get(),
2124 "dropping stale status update for module route handle"
2125 );
2126 Ok(false)
2127 }
2128 }
2129
2130 pub(crate) fn route_poll_snapshot(
2131 &self,
2132 client_connection_id: ConnectionId,
2133 client_channel: u16,
2134 client_epoch: u32,
2135 ) -> Result<RoutePollSnapshot, ForwardingError> {
2136 let inner = self.read_inner()?;
2137 let client_key = ClientRouteKey {
2138 connection_id: client_connection_id,
2139 channel: client_channel,
2140 };
2141 let Some(route) = inner.client_to_module.get(&client_key) else {
2142 return Ok(RoutePollSnapshot::Absent);
2143 };
2144 if route.client_epoch != client_epoch
2145 || !inner
2146 .module_id_by_endpoint
2147 .contains_key(&route.module_endpoint)
2148 {
2149 return Ok(RoutePollSnapshot::Absent);
2150 }
2151 Ok(RoutePollSnapshot::Bound {
2152 module_id: route.module_id.clone(),
2153 status: inner.status.get(&(client_key, client_epoch)).cloned(),
2154 })
2155 }
2156
2157 pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2158 Ok(self.read_inner()?.client_to_module.len())
2159 }
2160
2161 pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2171 let inner = self.read_inner()?;
2172 let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2173 for key in inner.client_to_module.keys() {
2174 *per_connection.entry(key.connection_id).or_insert(0) += 1;
2175 }
2176 let max = per_connection.values().copied().max().unwrap_or(0);
2177 Ok((per_connection.len(), max))
2178 }
2179
2180 pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2181 let inner = self.read_inner()?;
2182 Ok(inner
2183 .client_to_module
2184 .keys()
2185 .any(|key| key.channel == route_channel))
2186 }
2187
2188 pub(crate) fn daemon_drain_result_for_connection(
2192 &self,
2193 connection_id: ConnectionId,
2194 ) -> Result<Option<bool>, ForwardingError> {
2195 let inner = self.read_inner()?;
2196 Ok(inner.daemon_draining.then(|| {
2197 inner
2198 .endpoint_by_connection
2199 .get(&connection_id)
2200 .and_then(|endpoint| inner.daemon_drain_results.get(endpoint))
2201 .copied()
2202 .unwrap_or(false)
2203 }))
2204 }
2205
2206 #[cfg(unix)]
2207 pub(crate) fn record_daemon_drain_result(
2208 &self,
2209 endpoint: ModuleEndpointId,
2210 drained: bool,
2211 ) -> Result<(), ForwardingError> {
2212 let mut inner = self.write_inner()?;
2213 if let Some(result) = inner.daemon_drain_results.get_mut(&endpoint) {
2217 *result = drained;
2218 }
2219 Ok(())
2220 }
2221
2222 #[cfg(unix)]
2225 pub(crate) fn begin_daemon_drain(
2226 &self,
2227 ) -> Result<Vec<(String, ModuleEndpointId)>, ForwardingError> {
2228 let mut inner = self.write_inner()?;
2229 if inner.daemon_draining {
2230 return Ok(Vec::new());
2231 }
2232 inner.daemon_draining = true;
2233 let modules = inner
2234 .modules_by_id
2235 .iter()
2236 .map(|(id, module)| (id.clone(), module.endpoint))
2237 .collect::<Vec<_>>();
2238 for (_, endpoint) in &modules {
2239 inner.daemon_drain_results.insert(*endpoint, false);
2240 inner
2241 .draining_endpoints
2242 .insert(*endpoint, RouteCloseReason::Restart);
2243 }
2244 let off_slot_endpoints = inner
2249 .candidates_by_id
2250 .values()
2251 .map(|module| module.endpoint)
2252 .chain(inner.superseded_endpoints.keys().copied())
2253 .collect::<Vec<_>>();
2254 for endpoint in off_slot_endpoints {
2255 inner
2256 .draining_endpoints
2257 .insert(endpoint, RouteCloseReason::Restart);
2258 }
2259 Ok(modules)
2260 }
2261
2262 pub(crate) fn begin_module_drain(
2269 &self,
2270 module_id: &str,
2271 reason: RouteCloseReason,
2272 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2273 let mut inner = self.write_inner()?;
2274 let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2275 return Ok(None);
2276 };
2277 Ok(Some(begin_drain_locked(
2278 &mut inner, module_id, module, reason,
2279 )))
2280 }
2281
2282 pub(crate) fn begin_endpoint_drain(
2289 &self,
2290 endpoint: ModuleEndpointId,
2291 reason: RouteCloseReason,
2292 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2293 let mut inner = self.write_inner()?;
2294 let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2295 return Ok(None);
2296 };
2297 let module_id = inner
2298 .module_id_by_endpoint
2299 .get(&endpoint)
2300 .cloned()
2301 .expect("an endpoint resolved to a module connection has a module id");
2302 Ok(Some(begin_drain_locked(
2303 &mut inner, &module_id, module, reason,
2304 )))
2305 }
2306}
2307
2308fn begin_drain_locked(
2312 inner: &mut ForwardingInner,
2313 module_id: &str,
2314 module: ModuleConnection,
2315 reason: RouteCloseReason,
2316) -> ModuleDrainTarget {
2317 {
2318 let endpoint = module.endpoint;
2319 inner.draining_endpoints.insert(endpoint, reason);
2320
2321 let flows = inner
2322 .client_to_module
2323 .values()
2324 .filter(|route| route.module_endpoint == endpoint)
2325 .map(|route| Arc::clone(&route.flow))
2326 .collect::<Vec<_>>();
2327 let excluded_subscriptions = flows
2328 .into_iter()
2329 .map(|flow| flow.begin_drain())
2330 .fold(0u32, u32::saturating_add);
2331
2332 let pending_keys = inner
2333 .pending_relays
2334 .keys()
2335 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2336 .copied()
2337 .collect::<Vec<_>>();
2338 let mut abandoned_bindings = Vec::new();
2339 for key in pending_keys {
2340 let Some(pending) = inner.pending_relays.remove(&key) else {
2341 continue;
2342 };
2343 release_reserved_route_locked(
2344 inner,
2345 pending.reservation.client_key,
2346 pending.reservation.module_key,
2347 );
2348 if pending.relay_enqueued {
2349 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2350 abandoned_bindings.push(target);
2351 }
2352 } else {
2353 inner.relays_released_before_enqueue.insert(key);
2357 }
2358 let _ = pending
2359 .sender
2360 .send(RouteBindRelayOutcome::Rejected(ErrorBody::new(
2361 "module_reloading",
2362 format!("module_id '{module_id}' is reloading"),
2363 )));
2364 }
2365
2366 let pending_control_keys = inner
2367 .pending_control_rpcs
2368 .keys()
2369 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2370 .copied()
2371 .collect::<Vec<_>>();
2372 for key in pending_control_keys {
2373 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2374 let _ = pending
2375 .sender
2376 .send(ModuleControlRpcOutcome::ModuleGone(format!(
2377 "module '{module_id}' began draining during module-control RPC"
2378 )));
2379 }
2380 }
2381
2382 ModuleDrainTarget {
2383 endpoint,
2384 sink: module.sink,
2385 negotiated_ver: module.negotiated_ver,
2386 abandoned_bindings,
2387 excluded_subscriptions,
2388 }
2389 }
2390}
2391
2392#[derive(Debug, Default, PartialEq, Eq)]
2395pub(crate) struct DrainHoldouts {
2396 pub(crate) requests: usize,
2399 pub(crate) routes: usize,
2401 pub(crate) total_routes: usize,
2403 pub(crate) top_connections: Vec<(u64, usize)>,
2406 pub(crate) held: Vec<(u16, u64)>,
2415}
2416
2417pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
2419
2420impl ForwardingTable {
2421 pub(crate) fn endpoint_drain_holdouts(
2423 &self,
2424 endpoint: ModuleEndpointId,
2425 ) -> Result<DrainHoldouts, ForwardingError> {
2426 let inner = self.read_inner()?;
2427 let mut holdouts = DrainHoldouts::default();
2428 let mut by_connection: HashMap<u64, usize> = HashMap::new();
2429 for (key, route) in &inner.client_to_module {
2430 if route.module_endpoint != endpoint {
2431 continue;
2432 }
2433 holdouts.total_routes += 1;
2434 let held = route.flow.drain_in_flight();
2435 if held == 0 {
2436 continue;
2437 }
2438 holdouts.requests += held;
2439 holdouts.routes += 1;
2440 *by_connection.entry(key.connection_id.get()).or_default() += held;
2441 holdouts.held.extend(
2442 route
2443 .flow
2444 .drain_held_corrs()
2445 .into_iter()
2446 .map(|corr| (route.module_channel, corr)),
2447 );
2448 }
2449 holdouts.held.sort_unstable();
2450 holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
2451 let mut connections = by_connection.into_iter().collect::<Vec<_>>();
2452 connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
2453 connections.truncate(3);
2454 holdouts.top_connections = connections;
2455 Ok(holdouts)
2456 }
2457
2458 pub(crate) fn endpoint_in_flight_count(
2459 &self,
2460 endpoint: ModuleEndpointId,
2461 ) -> Result<usize, ForwardingError> {
2462 let inner = self.read_inner()?;
2463 Ok(inner
2464 .client_to_module
2465 .values()
2466 .filter(|route| route.module_endpoint == endpoint)
2467 .map(|route| route.flow.drain_in_flight())
2468 .sum())
2469 }
2470
2471 pub(crate) fn endpoint_is_draining(
2472 &self,
2473 endpoint: ModuleEndpointId,
2474 ) -> Result<bool, ForwardingError> {
2475 Ok(self
2476 .read_inner()?
2477 .draining_endpoints
2478 .contains_key(&endpoint))
2479 }
2480
2481 pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
2482 let inner = self.read_inner()?;
2483 Ok(inner
2484 .modules_by_id
2485 .get(module_id)
2486 .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
2487 }
2488
2489 pub(crate) fn release_module_endpoint_routes(
2490 &self,
2491 endpoint: ModuleEndpointId,
2492 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2493 let mut inner = self.write_inner()?;
2494 let routes = inner
2495 .module_to_client
2496 .iter()
2497 .filter(|(module_key, _)| module_key.endpoint == endpoint)
2498 .map(|(module_key, route)| (*module_key, route.module_epoch))
2499 .collect::<Vec<_>>();
2500 let mut released = Vec::with_capacity(routes.len());
2501 for (module_key, epoch) in routes {
2502 if let RouteRelease::Removed(target) =
2503 release_module_route_locked(&mut inner, module_key, epoch)
2504 {
2505 released.push(target);
2506 }
2507 }
2508 Ok(released)
2509 }
2510
2511 pub(crate) fn endpoint_routes(
2517 &self,
2518 endpoint: ModuleEndpointId,
2519 ) -> Result<Vec<EndpointRoute>, ForwardingError> {
2520 let inner = self.read_inner()?;
2521 Ok(endpoint_routes_locked(&inner, endpoint))
2522 }
2523
2524 pub(crate) fn route_census(
2526 &self,
2527 module_id: Option<&str>,
2528 ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
2529 let inner = self.read_inner()?;
2530 let mut endpoints = inner
2531 .modules_by_id
2532 .iter()
2533 .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
2534 .map(|(id, module)| (id.clone(), module.endpoint))
2535 .collect::<Vec<_>>();
2536 endpoints.sort_by(|left, right| left.0.cmp(&right.0));
2537 Ok(endpoints
2538 .into_iter()
2539 .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
2540 .collect())
2541 }
2542
2543 pub(crate) fn live_roots(
2545 &self,
2546 module_id: &str,
2547 ) -> Result<ModuleControlResponseToModule, ForwardingError> {
2548 let inner = self.read_inner()?;
2549 let endpoint = inner
2550 .modules_by_id
2551 .get(module_id)
2552 .map(|module| module.endpoint);
2553 let mut roots = BTreeMap::new();
2554 let mut unknown_root_bindings = 0;
2555 let mut total_bindings = 0;
2556 if let Some(endpoint) = endpoint {
2557 for binding in inner
2558 .module_to_client
2559 .values()
2560 .filter(|binding| binding.module_endpoint == endpoint)
2561 {
2562 total_bindings += 1;
2563 let workdir = binding
2564 .scope
2565 .as_ref()
2566 .and_then(|scope| scope.workdir.as_deref())
2567 .map(std::path::Path::new);
2568 if let Some(root) = &binding.project_root {
2569 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2570 entry.0 += 1;
2571 } else if workdir.is_none() {
2572 unknown_root_bindings += 1;
2573 }
2574 if let Some(workdir) = workdir.filter(|workdir| {
2575 binding.project_root.as_ref().map(ProjectRootId::as_path) != Some(*workdir)
2576 }) {
2577 roots.entry(workdir.to_path_buf()).or_insert((0, 0)).0 += 1;
2578 }
2579 }
2580 for pending in inner
2581 .pending_relays
2582 .values()
2583 .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
2584 {
2585 total_bindings += 1;
2586 if let Some(root) = &pending.reservation.project_root {
2587 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2588 entry.1 += 1;
2589 } else {
2590 unknown_root_bindings += 1;
2591 }
2592 }
2593 }
2594 Ok(ModuleControlResponseToModule::LiveRoots {
2595 roots: roots
2596 .into_iter()
2597 .map(|(project_root, (bound, pending))| LiveRoot {
2598 project_root,
2599 bound,
2600 pending,
2601 })
2602 .collect(),
2603 unknown_root_bindings,
2604 total_bindings,
2605 })
2606 }
2607
2608 pub(crate) fn connection_has_client_routes(
2614 &self,
2615 connection_id: ConnectionId,
2616 ) -> Result<bool, ForwardingError> {
2617 let inner = self.read_inner()?;
2618 Ok(connection_has_client_routes_locked(&inner, connection_id))
2619 }
2620
2621 pub(crate) fn cleanup_connection(
2622 &self,
2623 connection_id: ConnectionId,
2624 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2625 self.cleanup_connection_counted(connection_id)
2626 .map(|cleanup| cleanup.released)
2627 }
2628
2629 pub(crate) fn cleanup_connection_counted(
2634 &self,
2635 connection_id: ConnectionId,
2636 ) -> Result<ConnectionCleanup, ForwardingError> {
2637 let mut inner = self.write_inner()?;
2638 inner.closing_connections.insert(connection_id);
2639 let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
2640 remove_module_connection_locked(&mut inner, endpoint)
2641 } else {
2642 ConnectionCleanup {
2643 released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
2644 abandoned_relays: 0,
2645 }
2646 };
2647 inner.closing_connections.remove(&connection_id);
2656 Ok(cleanup)
2657 }
2658
2659 fn cleanup_client_connection_locked(
2660 inner: &mut ForwardingInner,
2661 connection_id: ConnectionId,
2662 ) -> Vec<GoodbyeTarget> {
2663 let routes = inner
2664 .client_to_module
2665 .iter()
2666 .filter(|(key, _)| key.connection_id == connection_id)
2667 .map(|(key, route)| (*key, route.client_epoch))
2668 .collect::<Vec<_>>();
2669 let mut released = Vec::with_capacity(routes.len());
2670 for (client_key, epoch) in routes {
2671 if let RouteRelease::Removed(target) =
2672 release_client_route_locked(inner, client_key, epoch)
2673 {
2674 released.push(target);
2675 }
2676 }
2677
2678 let pending_keys = inner
2679 .pending_relays
2680 .iter()
2681 .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
2682 .map(|(key, _)| *key)
2683 .collect::<Vec<_>>();
2684 for key in pending_keys {
2685 let Some(pending) = inner.pending_relays.remove(&key) else {
2686 continue;
2687 };
2688 release_reserved_route_locked(
2689 inner,
2690 pending.reservation.client_key,
2691 pending.reservation.module_key,
2692 );
2693 if pending.relay_enqueued {
2694 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2695 released.push(target);
2696 }
2697 } else {
2698 inner.relays_released_before_enqueue.insert(key);
2702 }
2703 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
2704 "client connection closed during route.bind relay".to_string(),
2705 ));
2706 }
2707
2708 let orphaned = inner
2709 .reserved_client
2710 .iter()
2711 .filter(|(key, _)| key.connection_id == connection_id)
2712 .map(|(client, module)| (*client, *module))
2713 .collect::<Vec<_>>();
2714 for (client_key, module_key) in orphaned {
2715 release_reserved_route_locked(inner, client_key, module_key);
2716 }
2717 inner.next_client_channel.remove(&connection_id);
2718 inner
2719 .client_slot_epochs
2720 .retain(|key, _| key.connection_id != connection_id);
2721 inner
2722 .last_published_epoch
2723 .retain(|key, _| key.connection_id != connection_id);
2724 inner
2725 .status
2726 .retain(|(key, _), _| key.connection_id != connection_id);
2727
2728 released
2729 }
2730
2731 pub(crate) fn escalate_client_delivery_failure(
2740 &self,
2741 connection_id: ConnectionId,
2742 channel: u16,
2743 expected_epoch: u32,
2744 reason: CloseReason,
2745 undelivered: UndeliveredFrame<'_>,
2746 ) -> Result<bool, ForwardingError> {
2747 self.escalate_client_delivery_failure_diagnosed(
2748 connection_id,
2749 channel,
2750 expected_epoch,
2751 reason,
2752 undelivered,
2753 None,
2754 )
2755 }
2756
2757 pub(crate) fn escalate_client_delivery_failure_diagnosed(
2758 &self,
2759 connection_id: ConnectionId,
2760 channel: u16,
2761 expected_epoch: u32,
2762 reason: CloseReason,
2763 undelivered: UndeliveredFrame<'_>,
2764 diagnosis: Option<&crate::router::ClientDeliveryDiagnosis>,
2765 ) -> Result<bool, ForwardingError> {
2766 let principals = {
2767 let mut inner = self.write_inner()?;
2768 let key = ClientRouteKey {
2769 connection_id,
2770 channel,
2771 };
2772 if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
2773 None
2774 } else {
2775 inner.closing_connections.insert(connection_id);
2776 Some(connection_principals_locked(&inner, connection_id))
2777 }
2778 };
2779 let Some(principals) = principals else {
2780 return Ok(false);
2781 };
2782 let backlog = undelivered.sink.backlog();
2783 let close_reason = reason.to_string();
2784 if self.request_connection_close(connection_id, reason) {
2785 warn!(
2786 connection_id = connection_id.get(),
2787 principals = %principals,
2788 module_id = undelivered.module_id.unwrap_or("unknown"),
2789 client_channel = channel,
2790 queued_bytes = backlog.queued_bytes,
2791 queued_frames = backlog.queued_frames,
2792 frame_type = ?diagnosis.map(|failure| failure.frame_type),
2793 frame_flags = ?diagnosis.map(|failure| failure.flags.0),
2794 close_cause = diagnosis.map(|failure| failure.cause).unwrap_or("delivery_failed"),
2795 oldest_queued_ms = backlog
2796 .oldest_age
2797 .map(|age| age.as_millis() as u64)
2798 .unwrap_or(0),
2799 close_reason = %close_reason,
2800 "closing client connection: its egress queue could not take a frame"
2801 );
2802 }
2803 Ok(true)
2804 }
2805
2806 pub(crate) fn publish_scope_changes(
2820 &self,
2821 changes: &[ScopeTagChange],
2822 ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
2823 let mut inner = self.write_inner()?;
2824 let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
2825 for change in changes {
2826 let key = (change.owner.clone(), change.scope_ref.clone());
2827 match change.after {
2828 Some(tag) => {
2829 inner.scope_tags.insert(key, tag);
2830 }
2831 None => {
2832 inner.scope_tags.remove(&key);
2833 }
2834 }
2835 by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
2836 }
2837 let mut selected = Vec::new();
2838 for (client_key, route) in &inner.client_to_module {
2839 let Some(scope) = &route.scope else {
2840 continue;
2841 };
2842 let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
2843 else {
2844 continue;
2845 };
2846 if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
2847 continue;
2848 }
2849 let reason = match &change.drain {
2850 ScopeDrain::Nothing => continue,
2851 ScopeDrain::All(reason) => *reason,
2852 ScopeDrain::Carriers(narrowed) => {
2853 let owner = Principal::Reserved {
2854 module_id: scope.owner.clone(),
2855 };
2856 let hit = route.principal != owner
2857 && narrowed.iter().any(|(principal, allowed)| {
2858 *principal == route.principal
2859 && allowed
2860 .as_ref()
2861 .is_none_or(|targets| !targets.contains(&route.module_id))
2862 });
2863 if !hit {
2864 continue;
2865 }
2866 RouteCloseReason::ScopeCarrierRemoved
2867 }
2868 };
2869 selected.push((*client_key, route.client_epoch, reason));
2870 }
2871 let mut drained = Vec::new();
2872 for (client_key, client_epoch, reason) in selected {
2873 let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
2874 continue;
2875 };
2876 let release = release_client_route_locked(&mut inner, client_key, client_epoch);
2877 self.record_route_release(&release);
2878 if let RouteRelease::Removed(module) = release {
2879 drained.push(ScopeDrainedRoute {
2880 reason,
2881 scope: route
2882 .scope
2883 .clone()
2884 .expect("selected scope drain has a bound scope"),
2885 module_id: route.module_id.clone(),
2886 client: GoodbyeTarget {
2887 connection_id: route.client_connection_id,
2888 sink: route.client_sink.clone(),
2889 negotiated_ver: route.client_negotiated_ver,
2890 channel: route.client_channel,
2891 epoch: route.client_epoch,
2892 kind: GoodbyeTargetKind::Client,
2893 module_id: Some(route.module_id.clone()),
2894 },
2895 module,
2896 });
2897 }
2898 }
2899 Ok(drained)
2900 }
2901
2902 #[cfg(test)]
2904 pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
2905 self.read_inner()
2906 .ok()?
2907 .scope_tags
2908 .get(&(owner.to_string(), scope_ref.to_string()))
2909 .copied()
2910 }
2911
2912 fn record_route_release(&self, release: &RouteRelease) {
2913 match release {
2914 RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
2915 RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
2916 RouteRelease::Absent => {}
2917 }
2918 }
2919
2920 fn read_inner(&self) -> Result<RwLockReadGuard<'_, ForwardingInner>, ForwardingError> {
2921 self.inner.read().map_err(|_| ForwardingError::Poisoned)
2922 }
2923
2924 fn write_inner(&self) -> Result<RwLockWriteGuard<'_, ForwardingInner>, ForwardingError> {
2925 self.inner.write().map_err(|_| ForwardingError::Poisoned)
2926 }
2927
2928 fn lock_close_registry(
2929 &self,
2930 ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
2931 self.close_registry
2932 .lock()
2933 .unwrap_or_else(|poisoned| poisoned.into_inner())
2934 }
2935}
2936
2937impl ForwardingInner {
2938 fn allocate_route_slots(
2939 &mut self,
2940 connection_id: ConnectionId,
2941 endpoint: ModuleEndpointId,
2942 ) -> Result<(u16, u32, u16, u32), ForwardingError> {
2943 let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
2944 let mut client_channel = client_start;
2945 let client_channel = loop {
2946 let key = ClientRouteKey {
2947 connection_id,
2948 channel: client_channel,
2949 };
2950 let eligible = !self.client_to_module.contains_key(&key)
2951 && !self.reserved_client.contains_key(&key)
2952 && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2953 if eligible {
2954 break client_channel;
2955 }
2956 client_channel = next_channel(client_channel);
2957 if client_channel == client_start {
2958 return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
2959 }
2960 };
2961
2962 let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
2963 let mut module_channel = module_start;
2964 let module_channel = loop {
2965 let key = ModuleRouteKey {
2966 endpoint,
2967 channel: module_channel,
2968 };
2969 let eligible = !self.module_to_client.contains_key(&key)
2970 && !self.reserved_module.contains_key(&key)
2971 && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2972 if eligible {
2973 break module_channel;
2974 }
2975 module_channel = next_channel(module_channel);
2976 if module_channel == module_start {
2977 return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
2978 }
2979 };
2980
2981 let client_key = ClientRouteKey {
2982 connection_id,
2983 channel: client_channel,
2984 };
2985 let module_key = ModuleRouteKey {
2986 endpoint,
2987 channel: module_channel,
2988 };
2989 let client_epoch = self
2990 .client_slot_epochs
2991 .get(&client_key)
2992 .copied()
2993 .unwrap_or(0)
2994 + 1;
2995 let module_epoch = self
2996 .module_slot_epochs
2997 .get(&module_key)
2998 .copied()
2999 .unwrap_or(0)
3000 + 1;
3001 self.client_slot_epochs.insert(client_key, client_epoch);
3002 self.module_slot_epochs.insert(module_key, module_epoch);
3003 self.next_client_channel
3004 .insert(connection_id, next_channel(client_channel));
3005 self.next_module_channel
3006 .insert(endpoint, next_channel(module_channel));
3007 Ok((client_channel, client_epoch, module_channel, module_epoch))
3008 }
3009
3010 fn allocate_control_corr(
3011 &mut self,
3012 endpoint: ModuleEndpointId,
3013 ) -> Result<u64, ForwardingError> {
3014 let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
3015 if candidate == 0 {
3016 self.closing_connections.insert(endpoint.connection_id);
3017 return Err(ForwardingError::RelayCorrelationExhausted);
3018 }
3019 self.next_control_corr.insert(
3020 endpoint,
3021 if candidate == u64::MAX {
3022 0
3023 } else {
3024 candidate + 1
3025 },
3026 );
3027 Ok(candidate)
3028 }
3029}
3030
3031fn connection_has_client_routes_locked(
3032 inner: &ForwardingInner,
3033 connection_id: ConnectionId,
3034) -> bool {
3035 inner
3036 .client_to_module
3037 .keys()
3038 .any(|key| key.connection_id == connection_id)
3039 || inner
3040 .reserved_client
3041 .keys()
3042 .any(|key| key.connection_id == connection_id)
3043}
3044
3045fn check_module_connection_role_locked(
3046 inner: &ForwardingInner,
3047 connection_id: ConnectionId,
3048) -> Result<(), ForwardingError> {
3049 if inner.endpoint_by_connection.contains_key(&connection_id)
3050 || connection_has_client_routes_locked(inner, connection_id)
3051 {
3052 return Err(ForwardingError::ConnectionRoleConflict { connection_id });
3053 }
3054 Ok(())
3055}
3056
3057fn next_channel(channel: u16) -> u16 {
3058 let next = channel.wrapping_add(1);
3059 if next == 0 {
3060 1
3061 } else {
3062 next
3063 }
3064}
3065
3066fn endpoint_routes_locked(
3067 inner: &ForwardingInner,
3068 endpoint: ModuleEndpointId,
3069) -> Vec<EndpointRoute> {
3070 let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
3071 let draining = drain_reason.is_some();
3072 let mut routes = inner
3073 .module_to_client
3074 .iter()
3075 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3076 .map(|(_, route)| EndpointRoute {
3077 goodbye_target: GoodbyeTarget {
3078 connection_id: route.client_connection_id,
3079 sink: route.client_sink.clone(),
3080 negotiated_ver: route.client_negotiated_ver,
3081 channel: route.client_channel,
3082 epoch: route.client_epoch,
3083 kind: GoodbyeTargetKind::Client,
3084 module_id: Some(route.module_id.clone()),
3085 },
3086 principal: route.principal.clone(),
3087 bound_at: route.bound_at,
3088 draining,
3089 drain_reason,
3090 })
3091 .collect::<Vec<_>>();
3092 routes.sort_by_key(|route| {
3093 (
3094 route.goodbye_target.connection_id.get(),
3095 route.goodbye_target.channel,
3096 route.goodbye_target.epoch,
3097 )
3098 });
3099 routes
3100}
3101
3102fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
3106 let key = (scope.owner.clone(), scope.scope_ref.clone());
3107 match inner.scope_tags.get(&key) {
3108 Some(current) if *current == scope.tag => None,
3109 Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
3110 subc_protocol::error_codes::SCOPE_CHANGED,
3111 format!(
3112 "scope '{}' of {} changed while the route was being bound; re-open it",
3113 scope.scope_ref, scope.owner
3114 ),
3115 )),
3116 _ => Some(ErrorBody::new(
3117 subc_protocol::error_codes::SCOPE_ENDED,
3118 format!(
3119 "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
3120 scope.scope_ref, scope.owner, scope.tag.scope_epoch
3121 ),
3122 )),
3123 }
3124}
3125
3126fn release_reserved_route_locked(
3127 inner: &mut ForwardingInner,
3128 client_key: ClientRouteKey,
3129 module_key: ModuleRouteKey,
3130) {
3131 if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
3132 inner.reserved_client.remove(&client_key);
3133 }
3134 if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
3135 inner.reserved_module.remove(&module_key);
3136 }
3137 inner.status.retain(|(key, _), _| *key != client_key);
3138}
3139
3140fn release_client_route_locked(
3141 inner: &mut ForwardingInner,
3142 client_key: ClientRouteKey,
3143 expected_epoch: u32,
3144) -> RouteRelease {
3145 let Some(route) = inner.client_to_module.get(&client_key) else {
3146 return RouteRelease::Absent;
3147 };
3148 if route.client_epoch != expected_epoch {
3149 return RouteRelease::Stale;
3150 }
3151 let route = inner
3152 .client_to_module
3153 .remove(&client_key)
3154 .expect("route checked under the same forwarding lock");
3155 route.flow.close();
3156 if let Some(scope) = &route.scope {
3157 scope.release_bind();
3158 }
3159 inner.module_to_client.remove(&ModuleRouteKey {
3160 endpoint: route.module_endpoint,
3161 channel: route.module_channel,
3162 });
3163 inner.operator_confirms.route_closed(ModuleRouteKey {
3164 endpoint: route.module_endpoint,
3165 channel: route.module_channel,
3166 });
3167 inner.status.remove(&(client_key, expected_epoch));
3168 RouteRelease::Removed(GoodbyeTarget {
3169 connection_id: route.module_endpoint.connection_id,
3170 sink: route.module_sink.clone(),
3171 negotiated_ver: route.module_negotiated_ver,
3172 channel: route.module_channel,
3173 epoch: route.module_epoch,
3174 kind: GoodbyeTargetKind::Module,
3175 module_id: Some(route.module_id.clone()),
3176 })
3177}
3178
3179fn release_module_route_locked(
3180 inner: &mut ForwardingInner,
3181 module_key: ModuleRouteKey,
3182 expected_epoch: u32,
3183) -> RouteRelease {
3184 let Some(route) = inner.module_to_client.get(&module_key) else {
3185 return RouteRelease::Absent;
3186 };
3187 if route.module_epoch != expected_epoch {
3188 return RouteRelease::Stale;
3189 }
3190 let route = inner
3191 .module_to_client
3192 .remove(&module_key)
3193 .expect("route checked under the same forwarding lock");
3194 inner.operator_confirms.route_closed(module_key);
3195 route.flow.close();
3196 if let Some(scope) = &route.scope {
3197 scope.release_bind();
3198 }
3199 let client_key = ClientRouteKey {
3200 connection_id: route.client_connection_id,
3201 channel: route.client_channel,
3202 };
3203 inner.client_to_module.remove(&client_key);
3204 inner.status.remove(&(client_key, route.client_epoch));
3205 RouteRelease::Removed(GoodbyeTarget {
3206 connection_id: route.client_connection_id,
3207 sink: route.client_sink.clone(),
3208 negotiated_ver: route.client_negotiated_ver,
3209 channel: route.client_channel,
3210 epoch: route.client_epoch,
3211 kind: GoodbyeTargetKind::Client,
3212 module_id: Some(route.module_id.clone()),
3213 })
3214}
3215
3216fn commit_route_locked(
3217 inner: &mut ForwardingInner,
3218 pending: PendingRouteBindRelayEntry,
3219) -> Result<Option<GoodbyeTarget>, ForwardingError> {
3220 let reservation = pending.reservation;
3221 if inner
3222 .closing_connections
3223 .contains(&reservation.client_key.connection_id)
3224 {
3225 return Err(ForwardingError::ConnectionClosing {
3226 connection_id: reservation.client_key.connection_id,
3227 });
3228 }
3229 let module_id = inner
3230 .module_id_by_endpoint
3231 .get(&reservation.module_key.endpoint)
3232 .cloned()
3233 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3234 if inner
3235 .draining_endpoints
3236 .contains_key(&reservation.module_key.endpoint)
3237 {
3238 return Err(ForwardingError::ModuleReloading { module_id });
3239 }
3240 if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
3241 || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
3242 {
3243 return Err(ForwardingError::UnknownReservation {
3244 client_channel: reservation.client_key.channel,
3245 module_channel: reservation.module_key.channel,
3246 });
3247 }
3248 let module = inner
3249 .modules_by_id
3250 .get(&module_id)
3251 .filter(|module| module.endpoint == reservation.module_key.endpoint)
3252 .cloned()
3253 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3254 let binding = Arc::new(RouteBinding {
3255 client_connection_id: reservation.client_key.connection_id,
3256 client_sink: pending.client_sink,
3257 client_negotiated_ver: pending.client_negotiated_ver,
3258 client_channel: reservation.client_key.channel,
3259 client_epoch: reservation.client_epoch,
3260 module_id,
3261 module_endpoint: reservation.module_key.endpoint,
3262 module_sink: module.sink,
3263 module_negotiated_ver: module.negotiated_ver,
3264 module_channel: reservation.module_key.channel,
3265 module_epoch: reservation.module_epoch,
3266 principal: pending.principal,
3267 project_root: reservation.project_root.clone(),
3268 bound_at: Instant::now(),
3269 flow: Arc::new(ChannelFlow::new(window_for(&module.concurrency))),
3270 scope: pending.scope,
3271 });
3272 inner
3273 .client_to_module
3274 .insert(reservation.client_key, Arc::clone(&binding));
3275 inner
3276 .module_to_client
3277 .insert(reservation.module_key, Arc::clone(&binding));
3278 let previous_published = inner
3279 .last_published_epoch
3280 .insert(reservation.client_key, reservation.client_epoch);
3281
3282 if let Some(scope) = &binding.scope {
3286 scope.record_served(&binding.module_id);
3287 }
3288
3289 let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
3294 if client_writer_closed {
3295 let abandoned = abandoned_route_target(inner, &reservation);
3299 if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
3300 route.flow.close();
3301 }
3302 inner.module_to_client.remove(&reservation.module_key);
3303 inner
3304 .status
3305 .remove(&(reservation.client_key, reservation.client_epoch));
3306 match previous_published {
3307 Some(epoch) => {
3308 inner
3309 .last_published_epoch
3310 .insert(reservation.client_key, epoch);
3311 }
3312 None => {
3313 inner.last_published_epoch.remove(&reservation.client_key);
3314 }
3315 }
3316 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3317 "client egress closed during route publication".to_string(),
3318 ));
3319 return Ok(abandoned);
3320 }
3321
3322 if let Some(scope) = &binding.scope {
3323 scope.commit_presence();
3324 }
3325 let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
3326 Ok(None)
3327}
3328
3329fn module_connection_for_endpoint_locked(
3337 inner: &ForwardingInner,
3338 endpoint: ModuleEndpointId,
3339) -> Option<&ModuleConnection> {
3340 let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
3341 inner
3342 .modules_by_id
3343 .get(module_id)
3344 .filter(|module| module.endpoint == endpoint)
3345 .or_else(|| {
3346 inner
3347 .candidates_by_id
3348 .get(module_id)
3349 .filter(|module| module.endpoint == endpoint)
3350 })
3351 .or_else(|| inner.superseded_endpoints.get(&endpoint))
3352}
3353
3354fn abandoned_route_target(
3355 inner: &ForwardingInner,
3356 reservation: &RouteReservation,
3357) -> Option<GoodbyeTarget> {
3358 let module_id = inner
3359 .module_id_by_endpoint
3360 .get(&reservation.module_key.endpoint)?;
3361 let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
3362 (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
3363 connection_id: module.endpoint.connection_id,
3364 sink: module.sink.clone(),
3365 negotiated_ver: module.negotiated_ver,
3366 channel: reservation.module_key.channel,
3367 epoch: reservation.module_epoch,
3368 kind: GoodbyeTargetKind::Module,
3369 module_id: Some(module_id.clone()),
3370 })
3371}
3372
3373fn enqueue_hello_ack_locked(
3378 sink: &FrameSink,
3379 connection_id: ConnectionId,
3380 hello_ack: Option<Frame>,
3381) -> Result<(), ForwardingError> {
3382 let Some(hello_ack) = hello_ack else {
3383 return Ok(());
3384 };
3385 sink.try_send(hello_ack)
3386 .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
3387}
3388
3389fn remove_module_connection_locked(
3390 inner: &mut ForwardingInner,
3391 endpoint: ModuleEndpointId,
3392) -> ConnectionCleanup {
3393 inner.operator_confirms.module_closed(endpoint);
3396 inner.draining_endpoints.remove(&endpoint);
3397 inner.daemon_drain_results.remove(&endpoint);
3398 let module_id = inner.module_id_by_endpoint.remove(&endpoint);
3399 if let Some(module_id) = module_id.as_ref() {
3400 if inner
3401 .modules_by_id
3402 .get(module_id)
3403 .is_some_and(|module| module.endpoint == endpoint)
3404 {
3405 inner.modules_by_id.remove(module_id);
3406 }
3407 if inner
3408 .candidates_by_id
3409 .get(module_id)
3410 .is_some_and(|module| module.endpoint == endpoint)
3411 {
3412 inner.candidates_by_id.remove(module_id);
3413 }
3414 }
3415 inner.superseded_endpoints.remove(&endpoint);
3416 inner.endpoint_by_connection.remove(&endpoint.connection_id);
3417 inner.next_module_channel.remove(&endpoint);
3418 inner.next_control_corr.remove(&endpoint);
3419 inner
3420 .health_probe_tombstones
3421 .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
3422 inner
3423 .module_slot_epochs
3424 .retain(|key, _| key.endpoint != endpoint);
3425 let reserved_module_keys: Vec<ModuleRouteKey> = inner
3426 .reserved_module
3427 .keys()
3428 .filter(|module_key| module_key.endpoint == endpoint)
3429 .copied()
3430 .collect();
3431 for module_key in reserved_module_keys {
3432 if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
3433 release_reserved_route_locked(inner, client_key, module_key);
3434 }
3435 }
3436
3437 let pending_keys: Vec<_> = inner
3438 .pending_relays
3439 .keys()
3440 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3441 .copied()
3442 .collect();
3443 let pending: Vec<_> = pending_keys
3444 .into_iter()
3445 .filter_map(|key| {
3446 inner
3447 .pending_relays
3448 .remove(&key)
3449 .map(|pending| (key, pending))
3450 })
3451 .collect();
3452 let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
3453 for (key, pending) in pending {
3454 if !pending.relay_enqueued {
3458 inner.relays_released_before_enqueue.insert(key);
3459 }
3460 let module_label = module_id.as_deref().unwrap_or("unknown");
3461 let _ = pending
3462 .sender
3463 .send(RouteBindRelayOutcome::ModuleGone(format!(
3464 "module '{module_label}' connection closed during route.bind relay"
3465 )));
3466 }
3467
3468 let pending_control_keys: Vec<_> = inner
3469 .pending_control_rpcs
3470 .keys()
3471 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3472 .copied()
3473 .collect();
3474 let pending_control: Vec<_> = pending_control_keys
3475 .into_iter()
3476 .filter_map(|key| inner.pending_control_rpcs.remove(&key))
3477 .collect();
3478 for pending in pending_control {
3479 let module_label = module_id.as_deref().unwrap_or("unknown");
3480 let _ = pending
3481 .sender
3482 .send(ModuleControlRpcOutcome::ModuleGone(format!(
3483 "module '{module_label}' connection closed during module-control RPC"
3484 )));
3485 }
3486
3487 let module_routes = inner
3488 .module_to_client
3489 .iter()
3490 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3491 .map(|(module_key, route)| (*module_key, route.module_epoch))
3492 .collect::<Vec<_>>();
3493 let mut released = Vec::with_capacity(module_routes.len());
3494 for (module_key, epoch) in module_routes {
3495 if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
3496 {
3497 released.push(target);
3498 }
3499 }
3500 ConnectionCleanup {
3501 released,
3502 abandoned_relays,
3503 }
3504}
3505
3506#[derive(Debug, Clone, Copy)]
3507struct RequestCredit {
3508 subscription: bool,
3509 excluded_from_drain: bool,
3510}
3511
3512#[derive(Debug, Default)]
3513struct CreditLedger {
3514 by_corr: HashMap<u64, Vec<RequestCredit>>,
3515}
3516
3517impl CreditLedger {
3518 fn acquire(&mut self, corr: u64, subscription: bool) {
3519 self.by_corr.entry(corr).or_default().push(RequestCredit {
3520 subscription,
3521 excluded_from_drain: false,
3522 });
3523 }
3524
3525 fn release(&mut self, corr: u64) -> bool {
3526 let Some(credits) = self.by_corr.get_mut(&corr) else {
3527 return false;
3528 };
3529 let released = credits.pop().is_some();
3530 if credits.is_empty() {
3531 self.by_corr.remove(&corr);
3532 }
3533 released
3534 }
3535
3536 fn capture_subscription_exclusions(&mut self) -> u32 {
3537 let mut excluded = 0u32;
3538 for credit in self.by_corr.values_mut().flatten() {
3539 if credit.subscription && !credit.excluded_from_drain {
3540 credit.excluded_from_drain = true;
3541 excluded = excluded.saturating_add(1);
3542 }
3543 }
3544 excluded
3545 }
3546
3547 #[cfg(test)]
3548 fn in_flight(&self) -> usize {
3549 self.by_corr.values().map(Vec::len).sum()
3550 }
3551
3552 fn drain_in_flight(&self) -> usize {
3553 self.by_corr
3554 .values()
3555 .flatten()
3556 .filter(|credit| !credit.excluded_from_drain)
3557 .count()
3558 }
3559
3560 fn drain_held_corrs(&self) -> Vec<u64> {
3563 let mut corrs = self
3564 .by_corr
3565 .iter()
3566 .flat_map(|(corr, credits)| {
3567 credits
3568 .iter()
3569 .filter(|credit| !credit.excluded_from_drain)
3570 .map(move |_| *corr)
3571 })
3572 .collect::<Vec<_>>();
3573 corrs.sort_unstable();
3574 corrs
3575 }
3576}
3577
3578#[derive(Debug, Default)]
3579struct ChannelFlowState {
3580 closed: bool,
3581 credits: CreditLedger,
3582}
3583
3584#[derive(Debug)]
3586pub(crate) struct ChannelFlow {
3587 sem: Semaphore,
3588 window: usize,
3589 state: Mutex<ChannelFlowState>,
3590}
3591
3592impl ChannelFlow {
3593 pub(crate) fn new(window: usize) -> Self {
3594 debug_assert!(window > 0, "flow-control window must be non-zero");
3595 Self {
3596 sem: Semaphore::new(window),
3597 window,
3598 state: Mutex::new(ChannelFlowState::default()),
3599 }
3600 }
3601
3602 #[cfg(test)]
3603 pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
3604 self.acquire_tagged(0, false).await
3605 }
3606
3607 pub(crate) async fn acquire_tagged(
3608 &self,
3609 corr: u64,
3610 subscription: bool,
3611 ) -> Result<(), ChannelFlowClosed> {
3612 let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
3613 let mut state = self
3614 .state
3615 .lock()
3616 .unwrap_or_else(|poisoned| poisoned.into_inner());
3617 if state.closed {
3618 return Err(ChannelFlowClosed);
3619 }
3620 state.credits.acquire(corr, subscription);
3621 permit.forget();
3622 Ok(())
3623 }
3624
3625 #[cfg(test)]
3626 pub(crate) fn release(&self) {
3627 self.release_corr(0);
3628 }
3629
3630 pub(crate) fn release_corr(&self, corr: u64) {
3631 let released = self
3632 .state
3633 .lock()
3634 .unwrap_or_else(|poisoned| poisoned.into_inner())
3635 .credits
3636 .release(corr);
3637 if !released {
3638 warn!(
3642 window = self.window,
3643 available = self.sem.available_permits(),
3644 "flow-control over-release ignored"
3645 );
3646 return;
3647 }
3648 if !self.sem.is_closed() {
3649 self.sem.add_permits(1);
3650 }
3651 }
3652
3653 #[cfg(test)]
3654 pub(crate) fn in_flight(&self) -> usize {
3655 self.state
3656 .lock()
3657 .unwrap_or_else(|poisoned| poisoned.into_inner())
3658 .credits
3659 .in_flight()
3660 }
3661
3662 pub(crate) fn drain_in_flight(&self) -> usize {
3663 self.state
3664 .lock()
3665 .unwrap_or_else(|poisoned| poisoned.into_inner())
3666 .credits
3667 .drain_in_flight()
3668 }
3669
3670 pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
3671 self.state
3672 .lock()
3673 .unwrap_or_else(|poisoned| poisoned.into_inner())
3674 .credits
3675 .drain_held_corrs()
3676 }
3677
3678 #[cfg(test)]
3679 pub(crate) fn available_permits(&self) -> usize {
3680 self.sem.available_permits()
3681 }
3682
3683 pub(crate) fn begin_drain(&self) -> u32 {
3684 let mut state = self
3685 .state
3686 .lock()
3687 .unwrap_or_else(|poisoned| poisoned.into_inner());
3688 state.closed = true;
3689 self.sem.close();
3690 state.credits.capture_subscription_exclusions()
3691 }
3692
3693 pub(crate) fn close(&self) {
3694 self.state
3695 .lock()
3696 .unwrap_or_else(|poisoned| poisoned.into_inner())
3697 .closed = true;
3698 self.sem.close();
3699 }
3700}
3701
3702#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3703pub(crate) struct ChannelFlowClosed;
3704
3705impl fmt::Display for ChannelFlowClosed {
3706 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3707 write!(f, "flow-control window closed")
3708 }
3709}
3710
3711impl Error for ChannelFlowClosed {}
3712
3713fn window_for(concurrency: &Concurrency) -> usize {
3714 match concurrency {
3715 Concurrency::Serial => 1,
3716 Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
3717 Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
3718 }
3719}
3720
3721#[derive(Debug, Clone, PartialEq, Eq)]
3722pub enum ForwardingError {
3723 ConnectionRoleConflict {
3726 connection_id: ConnectionId,
3727 },
3728 NoModuleConnection,
3729 ModuleReloading {
3730 module_id: String,
3731 },
3732 StaleModuleEndpoint,
3733 UnknownReservation {
3734 client_channel: u16,
3735 module_channel: u16,
3736 },
3737 ClientRouteChannelExhausted {
3738 connection_id: ConnectionId,
3739 },
3740 ModuleRouteChannelExhausted {
3741 endpoint: ModuleEndpointId,
3742 },
3743 RelayCorrelationExhausted,
3744 ConnectionClosing {
3745 connection_id: ConnectionId,
3746 },
3747 ClientEgressClosed {
3748 connection_id: ConnectionId,
3749 },
3750 RouteOpenBuild(String),
3751 CandidateSlotOccupied {
3753 module_id: String,
3754 },
3755 ModuleEgressUnavailable {
3758 connection_id: ConnectionId,
3759 },
3760 Poisoned,
3761}
3762
3763impl fmt::Display for ForwardingError {
3764 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3765 match self {
3766 Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
3767 Self::NoModuleConnection => write!(f, "no module connection is registered"),
3768 Self::ModuleReloading { module_id } => {
3769 write!(f, "module_id '{module_id}' is reloading")
3770 }
3771 Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
3772 Self::UnknownReservation {
3773 client_channel,
3774 module_channel,
3775 } => write!(
3776 f,
3777 "route reservation client channel {client_channel} / module channel {module_channel} was not found"
3778 ),
3779 Self::ClientRouteChannelExhausted { connection_id } => write!(
3780 f,
3781 "no client route channels are available for connection {}",
3782 connection_id.get()
3783 ),
3784 Self::ModuleRouteChannelExhausted { endpoint } => write!(
3785 f,
3786 "no module route channels are available for endpoint generation {} on connection {}",
3787 endpoint.generation,
3788 endpoint.connection_id.get()
3789 ),
3790 Self::RelayCorrelationExhausted => {
3791 write!(f, "module control correlation ids are exhausted")
3792 }
3793 Self::ConnectionClosing { connection_id } => write!(
3794 f,
3795 "connection {} is closing and cannot accept route allocation",
3796 connection_id.get()
3797 ),
3798 Self::ClientEgressClosed { connection_id } => write!(
3799 f,
3800 "client connection {} egress is closed",
3801 connection_id.get()
3802 ),
3803 Self::RouteOpenBuild(message) => {
3804 write!(f, "failed to prebuild route.open response: {message}")
3805 }
3806 Self::CandidateSlotOccupied { module_id } => write!(
3807 f,
3808 "module_id '{module_id}' already has a swap candidate registered"
3809 ),
3810 Self::ModuleEgressUnavailable { connection_id } => write!(
3811 f,
3812 "module connection {} egress is unavailable; HELLO_ACK could not be queued",
3813 connection_id.get()
3814 ),
3815 Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
3816 }
3817 }
3818}
3819
3820impl Error for ForwardingError {}
3821
3822#[cfg(test)]
3823mod tests {
3824 use std::time::Duration;
3825
3826 use super::*;
3827 use tokio::sync::mpsc;
3828
3829 #[test]
3830 fn ordinary_long_running_request_is_not_excluded_from_drain() {
3831 let mut ledger = CreditLedger::default();
3832 ledger.acquire(1, false);
3833
3834 assert_eq!(ledger.capture_subscription_exclusions(), 0);
3835 assert_eq!(ledger.drain_in_flight(), 1);
3836 }
3837
3838 #[test]
3839 fn bit_set_subscription_is_excluded_and_counted() {
3840 let mut ledger = CreditLedger::default();
3841 ledger.acquire(1, true);
3842
3843 assert_eq!(ledger.capture_subscription_exclusions(), 1);
3844 assert_eq!(ledger.drain_in_flight(), 0);
3845 }
3846
3847 #[test]
3848 fn subscription_opened_after_drain_snapshot_is_not_excluded() {
3849 let mut ledger = CreditLedger::default();
3850 ledger.acquire(1, true);
3851 assert_eq!(ledger.capture_subscription_exclusions(), 1);
3852
3853 ledger.acquire(2, true);
3854
3855 assert_eq!(ledger.drain_in_flight(), 1);
3856 }
3857
3858 #[test]
3859 fn drain_with_no_subscriptions_reports_zero_excluded() {
3860 let mut ledger = CreditLedger::default();
3861 assert_eq!(ledger.capture_subscription_exclusions(), 0);
3862 }
3863
3864 fn test_hello_ack(corr: u64) -> Frame {
3865 Frame::build(
3866 FrameType::HelloAck,
3867 Flags::new(false, Priority::Passive, false),
3868 0,
3869 0,
3870 corr,
3871 Vec::new(),
3872 )
3873 .unwrap()
3874 }
3875
3876 #[test]
3880 fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
3881 let forwarding = ForwardingTable::default();
3882
3883 let (closed_tx, closed_rx) = mpsc::channel(8);
3884 drop(closed_rx);
3885 let closed = ConnectionId::new(1);
3886 assert_eq!(
3887 forwarding.register_module_connection_acked(
3888 closed,
3889 "closed".to_string(),
3890 2,
3891 Concurrency::ModuleManaged,
3892 FrameSink::new(closed_tx),
3893 test_hello_ack(1),
3894 ),
3895 Err(ForwardingError::ModuleEgressUnavailable {
3896 connection_id: closed
3897 })
3898 );
3899
3900 let (full_tx, _full_rx) = mpsc::channel(1);
3901 let full_sink = FrameSink::new(full_tx);
3902 full_sink.try_send(test_hello_ack(99)).unwrap();
3903 let full = ConnectionId::new(2);
3904 assert_eq!(
3905 forwarding.register_module_connection_acked(
3906 full,
3907 "full".to_string(),
3908 2,
3909 Concurrency::ModuleManaged,
3910 full_sink.clone(),
3911 test_hello_ack(2),
3912 ),
3913 Err(ForwardingError::ModuleEgressUnavailable {
3914 connection_id: full
3915 })
3916 );
3917 assert_eq!(
3918 forwarding.register_candidate_module_connection_acked(
3919 full,
3920 "full".to_string(),
3921 2,
3922 Concurrency::ModuleManaged,
3923 full_sink,
3924 test_hello_ack(3),
3925 ),
3926 Err(ForwardingError::ModuleEgressUnavailable {
3927 connection_id: full
3928 })
3929 );
3930
3931 for (connection, module_id) in [(closed, "closed"), (full, "full")] {
3932 assert_eq!(
3933 forwarding
3934 .module_endpoint_for_connection(connection)
3935 .unwrap(),
3936 None
3937 );
3938 let (client_tx, _client_rx) = mpsc::channel(8);
3939 assert_eq!(
3940 forwarding
3941 .begin_route_bind_relay_for_test(
3942 ConnectionId::new(50),
3943 FrameSink::new(client_tx),
3944 1,
3945 module_id,
3946 )
3947 .err(),
3948 Some(ForwardingError::NoModuleConnection)
3949 );
3950 }
3951 assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
3952 }
3953
3954 #[test]
3957 fn acked_registration_queues_the_hello_ack_first() {
3958 let forwarding = ForwardingTable::default();
3959 let (active_tx, mut active_rx) = mpsc::channel(8);
3960 forwarding
3961 .register_module_connection_acked(
3962 ConnectionId::new(1),
3963 "acked".to_string(),
3964 2,
3965 Concurrency::ModuleManaged,
3966 FrameSink::new(active_tx),
3967 test_hello_ack(11),
3968 )
3969 .unwrap();
3970 let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
3971 forwarding
3972 .register_candidate_module_connection_acked(
3973 ConnectionId::new(2),
3974 "acked".to_string(),
3975 2,
3976 Concurrency::ModuleManaged,
3977 FrameSink::new(candidate_tx),
3978 test_hello_ack(12),
3979 )
3980 .unwrap();
3981
3982 let active_first = active_rx.try_recv().unwrap().frame;
3983 assert_eq!(active_first.header.ty, FrameType::HelloAck);
3984 assert_eq!(active_first.header.corr, 11);
3985 let candidate_first = candidate_rx.try_recv().unwrap().frame;
3986 assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
3987 assert_eq!(candidate_first.header.corr, 12);
3988 }
3989
3990 #[test]
3991 fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
3992 let forwarding = ForwardingTable::default();
3993 let module_connection = ConnectionId::new(10);
3994 let exhausted_client = ConnectionId::new(20);
3995 let second_client = ConnectionId::new(30);
3996 let (module_tx, _module_rx) = mpsc::channel(1);
3997 let endpoint = forwarding
3998 .register_module_connection(
3999 module_connection,
4000 "route-limit-provider".to_string(),
4001 1,
4002 Concurrency::ModuleManaged,
4003 FrameSink::new(module_tx),
4004 )
4005 .unwrap();
4006
4007 {
4008 let mut inner = forwarding.inner.write().unwrap();
4009 for channel in 1..=u16::MAX {
4010 inner.reserved_client.insert(
4011 ClientRouteKey {
4012 connection_id: exhausted_client,
4013 channel,
4014 },
4015 ModuleRouteKey {
4016 endpoint,
4017 channel: 1,
4018 },
4019 );
4020 }
4021 }
4022
4023 let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
4024 let err = forwarding
4025 .begin_route_bind_relay_for_test(
4026 exhausted_client,
4027 FrameSink::new(exhausted_tx),
4028 1,
4029 "route-limit-provider",
4030 )
4031 .unwrap_err();
4032 assert!(matches!(
4033 err,
4034 ForwardingError::ClientRouteChannelExhausted { connection_id }
4035 if connection_id == exhausted_client
4036 ));
4037
4038 let (second_tx, _second_rx) = mpsc::channel(1);
4039 let pending = forwarding
4040 .begin_route_bind_relay_for_test(
4041 second_client,
4042 FrameSink::new(second_tx),
4043 2,
4044 "route-limit-provider",
4045 )
4046 .unwrap();
4047 assert_eq!(pending.client_channel, 1);
4048 }
4049
4050 #[test]
4051 fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
4052 let forwarding = ForwardingTable::default();
4053 let module_connection = ConnectionId::new(40);
4054 let client = ConnectionId::new(50);
4055 let (module_tx, _module_rx) = mpsc::channel(1);
4056 forwarding
4057 .register_module_connection(
4058 module_connection,
4059 "slot-reuse-provider".to_string(),
4060 1,
4061 Concurrency::ModuleManaged,
4062 FrameSink::new(module_tx),
4063 )
4064 .unwrap();
4065
4066 let (client_tx, _client_rx) = mpsc::channel(1);
4067 let client_sink = FrameSink::new(client_tx);
4068 let mut wrapped_channel = None;
4069 for index in 0..=usize::from(u16::MAX) {
4070 let pending = forwarding
4071 .begin_route_bind_relay_for_test(
4072 client,
4073 client_sink.clone(),
4074 index as u64 + 1,
4075 "slot-reuse-provider",
4076 )
4077 .unwrap();
4078 if index == usize::from(u16::MAX) {
4079 wrapped_channel = Some(pending.module_channel);
4080 }
4081 forwarding
4082 .abort_pending_relay(
4083 pending.endpoint,
4084 pending.corr,
4085 RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
4086 )
4087 .unwrap();
4088 }
4089
4090 assert_eq!(wrapped_channel, Some(1));
4091 }
4092
4093 #[test]
4094 fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
4095 let forwarding = ForwardingTable::default();
4096 let client = ConnectionId::new(60);
4097 forwarding
4098 .inner
4099 .write()
4100 .unwrap()
4101 .next_client_channel
4102 .insert(client, 41);
4103
4104 let released = forwarding.cleanup_connection(client).unwrap();
4105
4106 assert!(released.is_empty());
4107 assert!(!forwarding
4108 .inner
4109 .read()
4110 .unwrap()
4111 .next_client_channel
4112 .contains_key(&client));
4113 }
4114
4115 #[test]
4116 fn stale_module_cleanup_preserves_fast_reconnect_successor() {
4117 let forwarding = ForwardingTable::default();
4118 let module_id = "fast-reconnect-provider";
4119 let first_connection = ConnectionId::new(70);
4120 let second_connection = ConnectionId::new(80);
4121 let (first_tx, _first_rx) = mpsc::channel(1);
4122 let first_endpoint = forwarding
4123 .register_module_connection(
4124 first_connection,
4125 module_id.to_string(),
4126 1,
4127 Concurrency::ModuleManaged,
4128 FrameSink::new(first_tx),
4129 )
4130 .unwrap();
4131 let (second_tx, _second_rx) = mpsc::channel(1);
4132 let second_endpoint = forwarding
4133 .register_module_connection(
4134 second_connection,
4135 module_id.to_string(),
4136 1,
4137 Concurrency::ModuleManaged,
4138 FrameSink::new(second_tx),
4139 )
4140 .unwrap();
4141 assert_ne!(first_endpoint, second_endpoint);
4142
4143 let released = forwarding.cleanup_connection(first_connection).unwrap();
4144
4145 assert!(released.is_empty());
4146 assert_eq!(
4147 forwarding
4148 .inner
4149 .read()
4150 .unwrap()
4151 .modules_by_id
4152 .get(module_id)
4153 .map(|module| module.endpoint),
4154 Some(second_endpoint)
4155 );
4156 assert!(forwarding.has_live_module_connection(module_id).unwrap());
4157 let control_rpc = forwarding
4158 .begin_module_control_rpc_for(
4159 module_id,
4160 "health.check",
4161 Instant::now() + Duration::from_secs(1),
4162 )
4163 .unwrap();
4164 assert_eq!(control_rpc.endpoint, second_endpoint);
4165 }
4166
4167 fn route_fixture(
4168 module_id: &str,
4169 ) -> (
4170 ForwardingTable,
4171 ConnectionId,
4172 ModuleEndpointId,
4173 ConnectionId,
4174 FrameSink,
4175 mpsc::Receiver<crate::router::OutboundFrame>,
4176 ) {
4177 let forwarding = ForwardingTable::default();
4178 let module_connection = ConnectionId::new(100);
4179 let client_connection = ConnectionId::new(200);
4180 let (module_tx, _module_rx) = mpsc::channel(8);
4181 let endpoint = forwarding
4182 .register_module_connection(
4183 module_connection,
4184 module_id.to_string(),
4185 2,
4186 Concurrency::ModuleManaged,
4187 FrameSink::new(module_tx),
4188 )
4189 .unwrap();
4190 let (client_tx, client_rx) = mpsc::channel(8);
4191 (
4192 forwarding,
4193 module_connection,
4194 endpoint,
4195 client_connection,
4196 FrameSink::new(client_tx),
4197 client_rx,
4198 )
4199 }
4200
4201 #[test]
4202 #[cfg(unix)]
4203 fn daemon_drain_gates_current_and_racing_provider_registrations() {
4204 let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
4205 assert_eq!(
4206 forwarding.begin_daemon_drain().unwrap(),
4207 [("provider".to_string(), endpoint)]
4208 );
4209 assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
4210 assert!(matches!(
4211 forwarding.register_module_connection(
4212 ConnectionId::new(300),
4213 "late-provider".into(),
4214 2,
4215 Concurrency::ModuleManaged,
4216 sink,
4217 ),
4218 Err(ForwardingError::ConnectionClosing { .. })
4219 ));
4220 }
4221
4222 #[test]
4223 #[cfg(unix)]
4224 fn daemon_drain_results_are_endpoint_fenced_and_removed_on_cleanup() {
4225 let (forwarding, connection, endpoint, _, _, _) = route_fixture("provider");
4226 assert_eq!(
4227 forwarding
4228 .daemon_drain_result_for_connection(connection)
4229 .unwrap(),
4230 None
4231 );
4232 forwarding
4233 .record_daemon_drain_result(endpoint, true)
4234 .unwrap();
4235 forwarding.begin_daemon_drain().unwrap();
4236 assert_eq!(
4237 forwarding
4238 .daemon_drain_result_for_connection(connection)
4239 .unwrap(),
4240 Some(false)
4241 );
4242 let replacement = ModuleEndpointId {
4243 generation: endpoint.generation + 1,
4244 ..endpoint
4245 };
4246 forwarding
4247 .record_daemon_drain_result(replacement, true)
4248 .unwrap();
4249 assert_eq!(
4250 forwarding
4251 .daemon_drain_result_for_connection(connection)
4252 .unwrap(),
4253 Some(false)
4254 );
4255 forwarding
4256 .record_daemon_drain_result(endpoint, true)
4257 .unwrap();
4258 assert_eq!(
4259 forwarding
4260 .daemon_drain_result_for_connection(connection)
4261 .unwrap(),
4262 Some(true)
4263 );
4264 assert_eq!(
4265 forwarding
4266 .daemon_drain_result_for_connection(ConnectionId::new(300))
4267 .unwrap(),
4268 Some(false)
4269 );
4270 forwarding.cleanup_connection(connection).unwrap();
4271 forwarding
4272 .record_daemon_drain_result(endpoint, true)
4273 .unwrap();
4274 assert_eq!(
4275 forwarding
4276 .daemon_drain_result_for_connection(connection)
4277 .unwrap(),
4278 Some(false)
4279 );
4280 assert!(forwarding
4281 .read_inner()
4282 .unwrap()
4283 .daemon_drain_results
4284 .is_empty());
4285 }
4286
4287 #[test]
4288 #[cfg(unix)]
4289 fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
4290 let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
4291 route_fixture("provider");
4292 let mut pending =
4293 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
4294 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
4295 forwarding.begin_daemon_drain().unwrap();
4296 let completion = forwarding
4297 .complete_pending_relay(
4298 module_connection,
4299 pending.corr,
4300 RouteBindRelayOutcome::Accepted,
4301 )
4302 .expect("draining admission is not a fatal module error");
4303 assert!(completion.settled);
4304 let abandoned = completion
4305 .abandoned
4306 .expect("late accepted bind needs a route GOODBYE");
4307 assert_eq!(abandoned.channel, pending.module_channel);
4308 assert!(matches!(pending.receiver.try_recv().unwrap(),
4309 RouteBindRelayOutcome::Rejected(error) if error.code == "module_reloading"));
4310 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4311 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4312 assert!(
4313 rx.try_recv().is_err(),
4314 "no route.open may be published during drain"
4315 );
4316 assert_eq!(
4317 forwarding
4318 .module_endpoint_for_connection(module_connection)
4319 .unwrap(),
4320 Some(endpoint)
4321 );
4322 assert!(
4323 !forwarding
4324 .complete_pending_relay(
4325 module_connection,
4326 pending.corr,
4327 RouteBindRelayOutcome::Accepted,
4328 )
4329 .unwrap()
4330 .settled
4331 );
4332 }
4333
4334 fn test_ping(corr: u64) -> Frame {
4335 Frame::build(
4336 FrameType::Ping,
4337 Flags::new(false, Priority::Passive, false),
4338 0,
4339 0,
4340 corr,
4341 Vec::new(),
4342 )
4343 .unwrap()
4344 }
4345
4346 fn begin_test_route(
4347 forwarding: &ForwardingTable,
4348 client_connection: ConnectionId,
4349 client_sink: FrameSink,
4350 corr: u64,
4351 module_id: &str,
4352 ) -> PendingRouteBindRelay {
4353 forwarding
4354 .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
4355 .unwrap()
4356 }
4357
4358 #[test]
4359 fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
4360 for candidate in [false, true] {
4361 for register_first in [false, true] {
4362 let (forwarding, module_connection, _, client_connection, sink, _rx) =
4363 route_fixture("target");
4364 let register = || {
4365 if candidate {
4366 forwarding.register_candidate_module_connection(
4367 client_connection,
4368 "source".into(),
4369 2,
4370 Concurrency::ModuleManaged,
4371 sink.clone(),
4372 )
4373 } else {
4374 forwarding.register_module_connection(
4375 client_connection,
4376 "source".into(),
4377 2,
4378 Concurrency::ModuleManaged,
4379 sink.clone(),
4380 )
4381 }
4382 };
4383 if register_first {
4384 register().unwrap();
4385 assert!(
4386 forwarding
4387 .begin_route_bind_relay_for_test(
4388 client_connection,
4389 sink.clone(),
4390 1,
4391 "target",
4392 )
4393 .is_err(),
4394 "a deferred route.open cannot reserve after HELLO"
4395 );
4396 assert!(forwarding
4397 .cleanup_connection(client_connection)
4398 .unwrap()
4399 .is_empty());
4400 } else {
4401 let pending =
4402 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
4403 assert!(
4404 register().is_err(),
4405 "HELLO cannot register after a route reservation"
4406 );
4407 forwarding
4408 .complete_pending_relay(
4409 module_connection,
4410 pending.corr,
4411 RouteBindRelayOutcome::Accepted,
4412 )
4413 .unwrap();
4414 assert_eq!(
4415 forwarding
4416 .cleanup_connection(client_connection)
4417 .unwrap()
4418 .len(),
4419 1
4420 );
4421 }
4422 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4423 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4424 }
4425 }
4426 }
4427
4428 #[tokio::test]
4434 async fn pending_route_open_completes_behind_queued_data_frames() {
4435 assert_eq!(
4436 crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
4437 8,
4438 "the per-connection pending route.open limit is its own constant"
4439 );
4440 let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
4441 route_fixture("open-behind-data");
4442 let (sink, mut client_rx) = crate::server::connection_egress();
4443 const DATA_FRAMES: usize = 1_000;
4444 let data = |corr: u64| {
4445 Frame::build(
4446 FrameType::StreamData,
4447 Flags::new(false, Priority::Interactive, false),
4448 9,
4449 1,
4450 corr,
4451 vec![b'x'; 200],
4452 )
4453 .unwrap()
4454 };
4455 for corr in 0..DATA_FRAMES as u64 {
4456 sink.try_send(data(corr)).unwrap();
4457 }
4458 let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
4459 assert_eq!(sink.backlog().queued_bytes, data_bytes);
4460
4461 let pending = tokio::time::timeout(
4462 Duration::from_secs(5),
4463 forwarding.begin_route_bind_relay_for(
4464 client,
4465 sink.clone(),
4466 subc_protocol::PROTOCOL_VERSION,
4467 4_242,
4468 "open-behind-data",
4469 Principal::Direct,
4470 None,
4471 None,
4472 Instant::now() + Duration::from_secs(60),
4473 ),
4474 )
4475 .await
4476 .expect("reserving the route.open slot must not wait behind data frames")
4477 .unwrap();
4478 forwarding
4479 .complete_pending_relay(
4480 module_connection,
4481 pending.corr,
4482 RouteBindRelayOutcome::Accepted,
4483 )
4484 .unwrap();
4485
4486 let backlog = sink.backlog();
4487 assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
4488 assert!(
4489 backlog.queued_bytes > data_bytes,
4490 "the route.open response must be counted in queued bytes: {backlog:?}"
4491 );
4492 for corr in 0..DATA_FRAMES as u64 {
4493 assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
4494 }
4495 let open = client_rx.recv().await.unwrap();
4496 assert_eq!(open.header.corr, 4_242);
4497 assert_eq!(open.header.ty, FrameType::Response);
4498 drop(open);
4499 assert_eq!(sink.backlog().queued_bytes, 0);
4500 assert_eq!(sink.backlog().queued_frames, 0);
4501 }
4502
4503 #[tokio::test]
4508 async fn drain_holdouts_count_held_requests_and_name_the_connection() {
4509 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4510 route_fixture("holdouts");
4511 let mut bound = |corr| {
4512 let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
4513 forwarding
4514 .complete_pending_relay(
4515 module_connection,
4516 route.corr,
4517 RouteBindRelayOutcome::Accepted,
4518 )
4519 .unwrap();
4520 client_rx.try_recv().unwrap();
4521 match forwarding
4522 .lookup_data_route(client, route.client_channel, route.client_epoch)
4523 .unwrap()
4524 {
4525 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4526 other => panic!("expected live route, got {other:?}"),
4527 }
4528 };
4529 let holding = bound(61);
4530 let _idle = bound(62);
4531 holding.flow.acquire_tagged(7, false).await.unwrap();
4532 holding.flow.acquire_tagged(2, false).await.unwrap();
4533 holding.flow.acquire_tagged(3, true).await.unwrap();
4534 forwarding
4535 .begin_module_drain("holdouts", RouteCloseReason::Restart)
4536 .unwrap();
4537
4538 let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
4539 assert_eq!(
4540 holdouts,
4541 DrainHoldouts {
4542 requests: 2,
4543 routes: 1,
4544 total_routes: 2,
4545 top_connections: vec![(client.get(), 2)],
4546 held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
4549 }
4550 );
4551 }
4552
4553 #[test]
4554 fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
4555 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4556 route_fixture("census");
4557 let pending = begin_test_route(&forwarding, client, sink, 1, "census");
4558 forwarding
4559 .complete_pending_relay(
4560 module_connection,
4561 pending.corr,
4562 RouteBindRelayOutcome::Accepted,
4563 )
4564 .unwrap();
4565
4566 let routes = forwarding.endpoint_routes(endpoint).unwrap();
4567 assert_eq!(routes.len(), 1);
4568 assert!(matches!(routes[0].principal, Principal::Direct));
4569 assert_eq!(routes[0].goodbye_target.connection_id, client);
4570 assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
4571 assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
4572 assert!(!routes[0].draining);
4573
4574 forwarding
4575 .begin_module_drain("census", RouteCloseReason::Restart)
4576 .unwrap();
4577 let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
4578 assert_eq!(draining_routes.len(), 1);
4579 assert!(draining_routes[0].draining);
4580 }
4581
4582 #[test]
4583 fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
4584 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
4585 let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
4586 assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
4587 forwarding
4588 .abort_pending_relay(
4589 first.endpoint,
4590 first.corr,
4591 RouteBindRelayOutcome::ModuleGone("abort".into()),
4592 )
4593 .unwrap();
4594 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4595 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4596
4597 let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
4598 assert_eq!(second.client_channel, first.client_channel);
4599 assert_eq!(second.module_channel, first.module_channel);
4600 assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
4601 }
4602
4603 #[test]
4604 fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
4605 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4606 route_fixture("epoch-release");
4607 let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
4608 forwarding
4609 .complete_pending_relay(
4610 module_connection,
4611 first.corr,
4612 RouteBindRelayOutcome::Accepted,
4613 )
4614 .unwrap();
4615 assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
4616 assert!(matches!(
4617 forwarding
4618 .release_client_route(client, first.client_channel, first.client_epoch)
4619 .unwrap(),
4620 RouteRelease::Removed(_)
4621 ));
4622 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4623 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4624
4625 let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
4626 forwarding
4627 .complete_pending_relay(
4628 module_connection,
4629 second.corr,
4630 RouteBindRelayOutcome::Accepted,
4631 )
4632 .unwrap();
4633 assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
4634 assert!(matches!(
4635 forwarding
4636 .release_client_route(client, second.client_channel, first.client_epoch)
4637 .unwrap(),
4638 RouteRelease::Stale
4639 ));
4640 assert!(!forwarding
4641 .cache_status(
4642 endpoint,
4643 second.module_channel,
4644 first.module_epoch,
4645 "stale".into(),
4646 )
4647 .unwrap());
4648 assert!(forwarding
4649 .cache_status(
4650 endpoint,
4651 second.module_channel,
4652 second.module_epoch,
4653 "current".into(),
4654 )
4655 .unwrap());
4656 match forwarding
4657 .route_poll_snapshot(client, second.client_channel, second.client_epoch)
4658 .unwrap()
4659 {
4660 RoutePollSnapshot::Bound { status, .. } => {
4661 assert_eq!(status.as_deref(), Some("current"));
4662 }
4663 RoutePollSnapshot::Absent => panic!("successor binding was removed"),
4664 }
4665 let counters = forwarding.counters().snapshot();
4666 assert_eq!(counters["route_released_epoch_fenced"], 1);
4667 assert_eq!(counters["route_release_stale_skipped"], 1);
4668 }
4669
4670 #[test]
4671 fn max_epoch_reservation_retires_only_that_slot() {
4672 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
4673 forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
4674 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
4675 let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
4676 assert_eq!(
4677 (final_use.client_channel, final_use.client_epoch),
4678 (7, u32::MAX)
4679 );
4680 assert_eq!(
4681 (final_use.module_channel, final_use.module_epoch),
4682 (9, u32::MAX)
4683 );
4684 forwarding
4685 .abort_pending_relay(
4686 endpoint,
4687 final_use.corr,
4688 RouteBindRelayOutcome::ModuleGone("abort".into()),
4689 )
4690 .unwrap();
4691 forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
4692 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
4693 let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
4694 assert_ne!(next.client_channel, 7);
4695 assert_ne!(next.module_channel, 9);
4696 assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
4697 }
4698
4699 #[test]
4700 fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
4701 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4702 route_fixture("corr-shared");
4703 let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
4704 assert_eq!(bind.corr, 1);
4705 forwarding
4706 .abort_pending_relay(
4707 endpoint,
4708 bind.corr,
4709 RouteBindRelayOutcome::ModuleGone("abort".into()),
4710 )
4711 .unwrap();
4712 let rpc = forwarding
4713 .begin_module_control_rpc_for(
4714 "corr-shared",
4715 "health.check",
4716 Instant::now() - Duration::from_millis(1),
4717 )
4718 .unwrap();
4719 assert_eq!(rpc.corr, 2);
4720 assert_eq!(
4721 forwarding
4722 .complete_module_control_rpc(
4723 module_connection,
4724 rpc.corr,
4725 Some("health.check"),
4726 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
4727 status: subc_protocol::session::HealthStatus::Ok,
4728 detail: None,
4729 metrics: None,
4730 }),
4731 )
4732 .unwrap(),
4733 ModuleControlRpcCompletion::Settled
4734 );
4735 assert!(matches!(
4736 rpc.receiver.blocking_recv().unwrap(),
4737 ModuleControlRpcOutcome::DeadlineElapsed
4738 ));
4739 }
4740
4741 #[tokio::test(start_paused = true)]
4742 async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
4743 let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
4744 let probe_started_at = Instant::now();
4745 let rpc = forwarding
4746 .begin_health_probe_rpc_for(
4747 "tombstone-ttl",
4748 "health.check",
4749 probe_started_at,
4750 probe_started_at + Duration::from_secs(5),
4751 )
4752 .unwrap();
4753 assert!(forwarding
4754 .tombstone_health_probe_rpc(endpoint, rpc.corr)
4755 .unwrap());
4756 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
4757
4758 tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
4759 tokio::task::yield_now().await;
4760
4761 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
4762 }
4763
4764 #[test]
4765 fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
4766 let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
4767 let mut close = forwarding.register_connection_close(endpoint.connection_id);
4768 forwarding.inject_control_corr(endpoint, u64::MAX);
4769 let final_rpc = forwarding
4770 .begin_module_control_rpc_for(
4771 "corr-max",
4772 "health.check",
4773 Instant::now() + Duration::from_secs(1),
4774 )
4775 .unwrap();
4776 assert_eq!(final_rpc.corr, u64::MAX);
4777 forwarding
4778 .cancel_module_control_rpc(endpoint, final_rpc.corr)
4779 .unwrap();
4780 assert!(matches!(
4781 forwarding.begin_module_control_rpc_for(
4782 "corr-max",
4783 "health.check",
4784 Instant::now() + Duration::from_secs(1),
4785 ),
4786 Err(ForwardingError::RelayCorrelationExhausted)
4787 ));
4788 assert!(close.try_recv().is_ok());
4789 }
4790
4791 #[test]
4792 fn publication_epoch_controls_delivery_failure_escalation() {
4793 fn setup_successor(
4794 commit_successor: Option<bool>,
4795 ) -> (ForwardingTable, ConnectionId, u16, u32) {
4796 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4797 route_fixture("escalation");
4798 let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
4799 forwarding
4800 .complete_pending_relay(
4801 module_connection,
4802 first.corr,
4803 RouteBindRelayOutcome::Accepted,
4804 )
4805 .unwrap();
4806 client_rx.try_recv().unwrap();
4807 assert!(matches!(
4808 forwarding
4809 .release_client_route(client, first.client_channel, first.client_epoch)
4810 .unwrap(),
4811 RouteRelease::Removed(_)
4812 ));
4813 if let Some(commit_successor) = commit_successor {
4814 forwarding.inject_client_slot_epoch(
4815 client,
4816 first.client_channel,
4817 first.client_epoch,
4818 );
4819 forwarding.inject_module_slot_epoch(
4820 endpoint,
4821 first.module_channel,
4822 first.module_epoch,
4823 );
4824 let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
4825 if commit_successor {
4826 forwarding
4827 .complete_pending_relay(
4828 module_connection,
4829 successor.corr,
4830 RouteBindRelayOutcome::Accepted,
4831 )
4832 .unwrap();
4833 client_rx.try_recv().unwrap();
4834 } else {
4835 forwarding
4836 .abort_pending_relay(
4837 endpoint,
4838 successor.corr,
4839 RouteBindRelayOutcome::ModuleGone("abort".into()),
4840 )
4841 .unwrap();
4842 }
4843 }
4844 (forwarding, client, first.client_channel, first.client_epoch)
4845 }
4846
4847 let probe_sink = FrameSink::new(mpsc::channel(1).0);
4848 let (no_successor, client, channel, epoch) = setup_successor(None);
4849 let mut close = no_successor.register_connection_close(client);
4850 assert!(no_successor
4851 .escalate_client_delivery_failure(
4852 client,
4853 channel,
4854 epoch,
4855 CloseReason::new("delivery", "failed"),
4856 UndeliveredFrame {
4857 module_id: None,
4858 sink: &probe_sink,
4859 },
4860 )
4861 .unwrap());
4862 assert!(close.try_recv().is_ok());
4863
4864 let (aborted, client, channel, epoch) = setup_successor(Some(false));
4865 let mut close = aborted.register_connection_close(client);
4866 assert!(aborted
4867 .escalate_client_delivery_failure(
4868 client,
4869 channel,
4870 epoch,
4871 CloseReason::new("delivery", "failed"),
4872 UndeliveredFrame {
4873 module_id: None,
4874 sink: &probe_sink,
4875 },
4876 )
4877 .unwrap());
4878 assert!(close.try_recv().is_ok());
4879
4880 let (published, client, channel, epoch) = setup_successor(Some(true));
4881 let mut close = published.register_connection_close(client);
4882 assert!(!published
4883 .escalate_client_delivery_failure(
4884 client,
4885 channel,
4886 epoch,
4887 CloseReason::new("delivery", "stale failure"),
4888 UndeliveredFrame {
4889 module_id: None,
4890 sink: &probe_sink,
4891 },
4892 )
4893 .unwrap());
4894 assert!(close.try_recv().is_err());
4895 }
4896
4897 #[test]
4898 fn route_concentration_separates_client_count_from_routes_per_client() {
4899 let (forwarding, module_connection, _, client, sink, _client_rx) =
4903 route_fixture("concentration");
4904 assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
4905
4906 for corr in [70_u64, 71] {
4907 let pending =
4908 begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
4909 forwarding
4910 .complete_pending_relay(
4911 module_connection,
4912 pending.corr,
4913 RouteBindRelayOutcome::Accepted,
4914 )
4915 .unwrap();
4916 }
4917
4918 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
4920 assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
4921 }
4922
4923 #[test]
4924 fn cleanup_and_accepted_resolution_have_one_lock_winner() {
4925 let (forwarding, module_connection, _, client, sink, mut client_rx) =
4926 route_fixture("cleanup-race");
4927 let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
4928 forwarding
4929 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4930 .unwrap();
4931 let released = forwarding.cleanup_connection(client).unwrap();
4932 assert_eq!(released.len(), 1);
4933 let completion = forwarding
4934 .complete_pending_relay(
4935 module_connection,
4936 pending.corr,
4937 RouteBindRelayOutcome::Accepted,
4938 )
4939 .unwrap();
4940 assert!(!completion.settled);
4941 assert!(client_rx.try_recv().is_err());
4942 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4943
4944 let (forwarding, module_connection, _, client, sink, mut client_rx) =
4945 route_fixture("accepted-race");
4946 let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
4947 forwarding
4948 .complete_pending_relay(
4949 module_connection,
4950 pending.corr,
4951 RouteBindRelayOutcome::Accepted,
4952 )
4953 .unwrap();
4954 assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
4955 let released = forwarding.cleanup_connection(client).unwrap();
4956 assert_eq!(released.len(), 1);
4957 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4958 }
4959
4960 #[test]
4961 fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
4962 let (forwarding, module_connection, _, client, sink, mut client_rx) =
4963 route_fixture("drain-gap");
4964 let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
4965 forwarding
4966 .complete_pending_relay(
4967 module_connection,
4968 live.corr,
4969 RouteBindRelayOutcome::Accepted,
4970 )
4971 .unwrap();
4972 client_rx.try_recv().unwrap();
4973 let binding = match forwarding
4974 .lookup_data_route(client, live.client_channel, live.client_epoch)
4975 .unwrap()
4976 {
4977 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4978 other => panic!("expected live route, got {other:?}"),
4979 };
4980
4981 let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
4982 forwarding
4983 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4984 .unwrap();
4985 let control_rpc = forwarding
4986 .begin_module_control_rpc_for(
4987 "drain-gap",
4988 "health.check",
4989 Instant::now() + Duration::from_secs(1),
4990 )
4991 .unwrap();
4992 let target = forwarding
4993 .begin_module_drain("drain-gap", RouteCloseReason::Reload)
4994 .unwrap()
4995 .unwrap();
4996 assert!(matches!(
4997 control_rpc.receiver.blocking_recv().unwrap(),
4998 ModuleControlRpcOutcome::ModuleGone(_)
4999 ));
5000 assert_eq!(target.abandoned_bindings.len(), 1);
5001 assert!(binding.flow.sem.is_closed());
5002 assert!(
5003 !forwarding
5004 .complete_pending_relay(
5005 module_connection,
5006 pending.corr,
5007 RouteBindRelayOutcome::Accepted,
5008 )
5009 .unwrap()
5010 .settled
5011 );
5012 assert!(matches!(
5013 forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
5014 Err(ForwardingError::ModuleReloading { .. })
5015 ));
5016 let released = forwarding
5017 .release_module_endpoint_routes(target.endpoint)
5018 .unwrap();
5019 assert_eq!(released.len(), 1);
5020 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5021 }
5022
5023 #[test]
5031 fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
5032 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5033 route_fixture("closing-client");
5034
5035 let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
5038 forwarding
5039 .complete_pending_relay(
5040 module_connection,
5041 live.corr,
5042 RouteBindRelayOutcome::Accepted,
5043 )
5044 .unwrap();
5045 client_rx.try_recv().unwrap();
5046
5047 let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
5049 forwarding
5050 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5051 .unwrap();
5052
5053 assert!(forwarding
5055 .escalate_client_delivery_failure(
5056 client,
5057 live.client_channel,
5058 live.client_epoch,
5059 CloseReason::new(
5060 "module_to_client_delivery_failed",
5061 "client egress refused a module frame",
5062 ),
5063 UndeliveredFrame {
5064 module_id: None,
5065 sink: &sink,
5066 },
5067 )
5068 .unwrap());
5069 assert!(!sink.is_closed());
5070
5071 let completion = forwarding
5072 .complete_pending_relay(
5073 module_connection,
5074 pending.corr,
5075 RouteBindRelayOutcome::Accepted,
5076 )
5077 .expect("a closing client must not turn a module's ack into an error");
5078
5079 assert!(completion.settled);
5080 let abandoned = completion
5081 .abandoned
5082 .expect("the module must be told to drop the binding it just created");
5083 assert_eq!(abandoned.connection_id, module_connection);
5084 assert_eq!(abandoned.channel, pending.module_channel);
5085 assert_eq!(abandoned.epoch, pending.module_epoch);
5086 assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
5087 assert!(matches!(
5088 pending.receiver.blocking_recv().unwrap(),
5089 RouteBindRelayOutcome::ModuleGone(_)
5090 ));
5091 assert!(client_rx.try_recv().is_err());
5094 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
5095
5096 assert!(forwarding
5099 .has_live_module_connection("closing-client")
5100 .unwrap());
5101 let cotenant = ConnectionId::new(201);
5102 let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
5103 let cotenant_route = begin_test_route(
5104 &forwarding,
5105 cotenant,
5106 FrameSink::new(cotenant_tx),
5107 62,
5108 "closing-client",
5109 );
5110 assert_eq!(cotenant_route.endpoint, endpoint);
5111 forwarding
5112 .complete_pending_relay(
5113 module_connection,
5114 cotenant_route.corr,
5115 RouteBindRelayOutcome::Accepted,
5116 )
5117 .unwrap();
5118 assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
5119 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5120 }
5121
5122 #[test]
5123 fn pending_route_permit_is_released_on_rejection_and_abort() {
5124 let forwarding = ForwardingTable::default();
5125 let module_connection = ConnectionId::new(300);
5126 let client = ConnectionId::new(301);
5127 let (module_tx, _module_rx) = mpsc::channel(1);
5128 let endpoint = forwarding
5129 .register_module_connection(
5130 module_connection,
5131 "permit".into(),
5132 2,
5133 Concurrency::ModuleManaged,
5134 FrameSink::new(module_tx),
5135 )
5136 .unwrap();
5137 let (client_tx, mut client_rx) = mpsc::channel(1);
5138 let sink = FrameSink::new(client_tx);
5139 let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
5140 assert!(sink.try_send(test_ping(999)).is_err());
5141 forwarding
5142 .complete_pending_relay(
5143 module_connection,
5144 rejected.corr,
5145 RouteBindRelayOutcome::Rejected(ErrorBody {
5146 code: "no".into(),
5147 message: "rejected".into(),
5148 detail: None,
5149 }),
5150 )
5151 .unwrap();
5152 sink.try_send(test_ping(1000)).unwrap();
5153 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
5154
5155 let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
5156 assert!(sink.try_send(test_ping(1001)).is_err());
5157 forwarding
5158 .abort_pending_relay(
5159 endpoint,
5160 aborted.corr,
5161 RouteBindRelayOutcome::ModuleGone("abort".into()),
5162 )
5163 .unwrap();
5164 sink.try_send(test_ping(1002)).unwrap();
5165 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
5166
5167 let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
5168 forwarding
5169 .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
5170 .unwrap();
5171 drop(client_rx);
5172 let completion = forwarding
5173 .complete_pending_relay(
5174 module_connection,
5175 receiver_closed.corr,
5176 RouteBindRelayOutcome::Accepted,
5177 )
5178 .unwrap();
5179 assert!(completion.abandoned.is_some());
5180 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5181 }
5182
5183 #[test]
5189 fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
5190 let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
5191 route_fixture("closing-set-leak");
5192
5193 const CONNECTIONS: u64 = 32;
5194 for index in 0..CONNECTIONS {
5195 let client = ConnectionId::new(1000 + index);
5196 let (client_tx, _client_rx) = mpsc::channel(8);
5197 let route = begin_test_route(
5198 &forwarding,
5199 client,
5200 FrameSink::new(client_tx),
5201 index + 1,
5202 "closing-set-leak",
5203 );
5204 forwarding
5205 .complete_pending_relay(
5206 module_connection,
5207 route.corr,
5208 RouteBindRelayOutcome::Accepted,
5209 )
5210 .unwrap();
5211 forwarding.cleanup_connection(client).unwrap();
5212 }
5213 forwarding.cleanup_connection(module_connection).unwrap();
5214
5215 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5216 }
5217
5218 #[test]
5225 fn closing_connection_is_refused_new_work_until_cleanup_completes() {
5226 let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
5227 route_fixture("closing-gate");
5228
5229 let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
5232 forwarding
5233 .complete_pending_relay(
5234 module_connection,
5235 live.corr,
5236 RouteBindRelayOutcome::Accepted,
5237 )
5238 .unwrap();
5239 client_rx.try_recv().unwrap();
5240
5241 assert!(forwarding
5244 .escalate_client_delivery_failure(
5245 client,
5246 live.client_channel,
5247 live.client_epoch,
5248 CloseReason::new(
5249 "module_to_client_delivery_failed",
5250 "client egress refused a module frame",
5251 ),
5252 UndeliveredFrame {
5253 module_id: None,
5254 sink: &sink,
5255 },
5256 )
5257 .unwrap());
5258 assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
5259
5260 assert!(matches!(
5262 forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
5263 Err(ForwardingError::ConnectionClosing { connection_id })
5264 if connection_id == client
5265 ));
5266 let (late_tx, _late_rx) = mpsc::channel(1);
5268 assert!(matches!(
5269 forwarding.register_module_connection(
5270 client,
5271 "late-module".into(),
5272 2,
5273 Concurrency::ModuleManaged,
5274 FrameSink::new(late_tx),
5275 ),
5276 Err(ForwardingError::ConnectionClosing { connection_id })
5277 if connection_id == client
5278 ));
5279
5280 forwarding.cleanup_connection(client).unwrap();
5284 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5285 }
5286}
5287
5288#[cfg(test)]
5291mod swap_slot_tests {
5292 use std::time::Duration;
5293
5294 use super::*;
5295 use tokio::sync::mpsc;
5296
5297 const MODULE_ID: &str = "swapped";
5298
5299 struct SwapFixture {
5300 forwarding: ForwardingTable,
5301 incumbent_connection: ConnectionId,
5302 incumbent: ModuleEndpointId,
5303 candidate_connection: ConnectionId,
5304 candidate: ModuleEndpointId,
5305 _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
5306 }
5307
5308 fn swap_fixture() -> SwapFixture {
5309 let forwarding = ForwardingTable::default();
5310 let incumbent_connection = ConnectionId::new(100);
5311 let candidate_connection = ConnectionId::new(110);
5312 let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
5313 let incumbent = forwarding
5314 .register_module_connection(
5315 incumbent_connection,
5316 MODULE_ID.to_string(),
5317 2,
5318 Concurrency::ModuleManaged,
5319 FrameSink::new(incumbent_tx),
5320 )
5321 .unwrap();
5322 let (candidate_tx, candidate_rx) = mpsc::channel(8);
5323 let candidate = forwarding
5324 .register_candidate_module_connection(
5325 candidate_connection,
5326 MODULE_ID.to_string(),
5327 2,
5328 Concurrency::ModuleManaged,
5329 FrameSink::new(candidate_tx),
5330 )
5331 .unwrap();
5332 SwapFixture {
5333 forwarding,
5334 incumbent_connection,
5335 incumbent,
5336 candidate_connection,
5337 candidate,
5338 _module_rxs: vec![incumbent_rx, candidate_rx],
5339 }
5340 }
5341
5342 fn client(
5343 raw: u64,
5344 ) -> (
5345 ConnectionId,
5346 FrameSink,
5347 mpsc::Receiver<crate::router::OutboundFrame>,
5348 ) {
5349 let (tx, rx) = mpsc::channel(8);
5350 (ConnectionId::new(raw), FrameSink::new(tx), rx)
5351 }
5352
5353 fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
5354 forwarding
5355 .read_inner()
5356 .unwrap()
5357 .client_to_module
5358 .values()
5359 .map(|route| route.module_endpoint)
5360 .collect()
5361 }
5362
5363 #[test]
5364 fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
5365 let fixture = swap_fixture();
5366 let forwarding = &fixture.forwarding;
5367 assert_ne!(fixture.incumbent, fixture.candidate);
5368
5369 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5371 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5372 let (client_connection, client_sink, _client_rx) = client(200);
5373 let pending = forwarding
5374 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5375 .unwrap();
5376 assert_eq!(pending.endpoint, fixture.incumbent);
5377 let rpc = forwarding
5378 .begin_module_control_rpc_for(
5379 MODULE_ID,
5380 "health.check",
5381 Instant::now() + Duration::from_secs(1),
5382 )
5383 .unwrap();
5384 assert_eq!(rpc.endpoint, fixture.incumbent);
5385 let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
5386 assert_eq!(census.len(), 1, "the census lists one endpoint per id");
5387
5388 assert_eq!(
5390 forwarding
5391 .module_endpoint_for_connection(fixture.candidate_connection)
5392 .unwrap(),
5393 Some(fixture.candidate)
5394 );
5395 assert_eq!(
5396 forwarding
5397 .module_id_for_connection(fixture.candidate_connection)
5398 .unwrap()
5399 .as_deref(),
5400 Some(MODULE_ID)
5401 );
5402
5403 let (other_tx, _other_rx) = mpsc::channel(1);
5405 assert_eq!(
5406 forwarding.register_candidate_module_connection(
5407 ConnectionId::new(120),
5408 MODULE_ID.to_string(),
5409 2,
5410 Concurrency::ModuleManaged,
5411 FrameSink::new(other_tx),
5412 ),
5413 Err(ForwardingError::CandidateSlotOccupied {
5414 module_id: MODULE_ID.to_string()
5415 })
5416 );
5417 }
5418
5419 #[test]
5423 fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
5424 let fixture = swap_fixture();
5425 let forwarding = &fixture.forwarding;
5426 let (early_client, early_sink, _early_rx) = client(200);
5427 let mut early = forwarding
5428 .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
5429 .unwrap();
5430 assert_eq!(early.endpoint, fixture.incumbent);
5431 assert_eq!(
5432 forwarding
5433 .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
5434 .unwrap(),
5435 RouteBindRelayMark::Marked
5436 );
5437
5438 let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5439 assert_eq!(
5440 cutover,
5441 ForwardingCutover {
5442 promoted: fixture.candidate,
5443 incumbent: Some(fixture.incumbent),
5444 }
5445 );
5446
5447 let (late_client, late_sink, _late_rx) = client(201);
5449 let late = forwarding
5450 .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
5451 .unwrap();
5452 assert_eq!(
5453 late.endpoint, fixture.candidate,
5454 "a route.open after cutover was reserved on the incumbent"
5455 );
5456
5457 let completion = forwarding
5459 .complete_pending_relay(
5460 fixture.incumbent_connection,
5461 early.corr,
5462 RouteBindRelayOutcome::Accepted,
5463 )
5464 .expect("a superseded endpoint's ack is not an error on its connection");
5465 assert!(completion.settled);
5466 assert!(
5467 !committed_endpoints(forwarding).contains(&fixture.incumbent),
5468 "a relay reserved before cutover committed a route on the incumbent"
5469 );
5470 let goodbye = completion
5471 .abandoned
5472 .expect("the incumbent is told to drop the binding it just created");
5473 assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
5474 assert_eq!(goodbye.channel, early.module_channel);
5475 assert_eq!(goodbye.epoch, early.module_epoch);
5476 assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
5477 match early.receiver.try_recv() {
5478 Ok(RouteBindRelayOutcome::Rejected(body)) => assert_eq!(body.code, "module_reloading"),
5479 other => panic!("expected a retryable module_reloading answer, got {other:?}"),
5480 }
5481 assert!(matches!(
5482 forwarding
5483 .lookup_data_route(early_client, early.client_channel, early.client_epoch)
5484 .unwrap(),
5485 DataRoute::Client(DataRouteState::Absent)
5486 ));
5487
5488 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
5491 forwarding
5492 .complete_pending_relay(
5493 fixture.candidate_connection,
5494 late.corr,
5495 RouteBindRelayOutcome::Accepted,
5496 )
5497 .unwrap();
5498 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5499 assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
5500 }
5501
5502 #[test]
5503 fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
5504 let fixture = swap_fixture();
5505 let forwarding = &fixture.forwarding;
5506 let (bound_client, bound_sink, _bound_rx) = client(200);
5508 let bound = forwarding
5509 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5510 .unwrap();
5511 forwarding
5512 .complete_pending_relay(
5513 fixture.incumbent_connection,
5514 bound.corr,
5515 RouteBindRelayOutcome::Accepted,
5516 )
5517 .unwrap();
5518 let (pending_client, pending_sink, _pending_rx) = client(201);
5519 let mut in_flight = forwarding
5520 .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
5521 .unwrap();
5522 forwarding
5523 .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
5524 .unwrap();
5525
5526 let incumbent = forwarding
5527 .cutover_candidate(MODULE_ID)
5528 .unwrap()
5529 .unwrap()
5530 .incumbent
5531 .unwrap();
5532 let target = forwarding
5533 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
5534 .unwrap()
5535 .expect("the superseded incumbent is still registered");
5536
5537 assert_eq!(target.endpoint, fixture.incumbent);
5538 assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
5539 assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
5540 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5541 assert_eq!(target.abandoned_bindings.len(), 1);
5542 assert_eq!(
5543 target.abandoned_bindings[0].channel,
5544 in_flight.module_channel
5545 );
5546 assert!(matches!(
5547 in_flight.receiver.try_recv(),
5548 Ok(RouteBindRelayOutcome::Rejected(body)) if body.code == "module_reloading"
5549 ));
5550 assert_eq!(
5551 forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
5552 1,
5553 "the incumbent's bound route stays until its drain finishes"
5554 );
5555
5556 let (next_client, next_sink, _next_rx) = client(202);
5557 let next = forwarding
5558 .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
5559 .expect("the promoted candidate keeps accepting routes");
5560 assert_eq!(next.endpoint, fixture.candidate);
5561 }
5562
5563 #[test]
5569 fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
5570 let forwarding = ForwardingTable::default();
5571 let first_connection = ConnectionId::new(70);
5572 let (first_tx, _first_rx) = mpsc::channel(8);
5573 forwarding
5574 .register_module_connection(
5575 first_connection,
5576 MODULE_ID.to_string(),
5577 2,
5578 Concurrency::ModuleManaged,
5579 FrameSink::new(first_tx),
5580 )
5581 .unwrap();
5582 let (client_connection, client_sink, _client_rx) = client(200);
5583 let mut pending = forwarding
5584 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5585 .unwrap();
5586 let (second_tx, _second_rx) = mpsc::channel(8);
5587 forwarding
5588 .register_module_connection(
5589 ConnectionId::new(80),
5590 MODULE_ID.to_string(),
5591 2,
5592 Concurrency::ModuleManaged,
5593 FrameSink::new(second_tx),
5594 )
5595 .unwrap();
5596
5597 assert_eq!(
5598 forwarding
5599 .complete_pending_relay(
5600 first_connection,
5601 pending.corr,
5602 RouteBindRelayOutcome::Accepted
5603 )
5604 .unwrap_err(),
5605 ForwardingError::StaleModuleEndpoint
5606 );
5607 assert!(committed_endpoints(&forwarding).is_empty());
5608 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5609 assert!(matches!(
5610 pending.receiver.try_recv(),
5611 Err(oneshot::error::TryRecvError::Closed)
5612 ));
5613 }
5614
5615 #[test]
5616 fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
5617 let fixture = swap_fixture();
5619 let forwarding = &fixture.forwarding;
5620 assert!(forwarding
5621 .cleanup_connection(fixture.candidate_connection)
5622 .unwrap()
5623 .is_empty());
5624 assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
5625 let (client_connection, client_sink, _client_rx) = client(200);
5626 assert_eq!(
5627 forwarding
5628 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5629 .unwrap()
5630 .endpoint,
5631 fixture.incumbent
5632 );
5633
5634 let fixture = swap_fixture();
5637 let forwarding = &fixture.forwarding;
5638 let (bound_client, bound_sink, _bound_rx) = client(200);
5639 let bound = forwarding
5640 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5641 .unwrap();
5642 forwarding
5643 .complete_pending_relay(
5644 fixture.incumbent_connection,
5645 bound.corr,
5646 RouteBindRelayOutcome::Accepted,
5647 )
5648 .unwrap();
5649 forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5650 let released = forwarding
5651 .cleanup_connection(fixture.incumbent_connection)
5652 .unwrap();
5653 assert_eq!(released.len(), 1);
5654 assert_eq!(released[0].connection_id, bound_client);
5655 assert!(forwarding
5656 .read_inner()
5657 .unwrap()
5658 .superseded_endpoints
5659 .is_empty());
5660 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5661 let (next_client, next_sink, _next_rx) = client(201);
5662 assert_eq!(
5663 forwarding
5664 .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
5665 .unwrap()
5666 .endpoint,
5667 fixture.candidate
5668 );
5669 }
5670}