1use std::{
2 collections::{BTreeMap, HashMap, HashSet},
3 error::Error,
4 fmt,
5 sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
6 time::Duration,
7};
8
9use subc_control::{ClientControlResponse, RouteCloseReason};
10use subc_protocol::{
11 manifest::Concurrency,
12 session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
13 ErrorBody, Flags, FrameType, Principal, Priority,
14};
15use tokio::sync::{oneshot, Semaphore};
16use tokio::time::Instant;
17use tracing::{debug, info, warn};
18
19use crate::{
20 control::{RouteBindBreakers, RouteBindConcurrency},
21 observability::DaemonCounters,
22 registry::ConnectionId,
23 router::FrameSink,
24 scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
25 Frame, ProjectRootId,
26};
27
28const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
30
31const STATELESS_PARALLEL_WINDOW: usize = 1024;
33
34const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
38
39#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
44pub struct ModuleEndpointId {
45 pub connection_id: ConnectionId,
46 pub generation: u64,
47}
48
49#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
51pub(crate) struct ClientRouteKey {
52 pub connection_id: ConnectionId,
53 pub channel: u16,
54}
55
56#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
58pub(crate) struct ModuleRouteKey {
59 pub endpoint: ModuleEndpointId,
60 pub channel: u16,
61}
62
63#[derive(Debug)]
64pub(crate) struct RouteBinding {
65 pub client_connection_id: ConnectionId,
66 pub client_sink: FrameSink,
67 pub client_negotiated_ver: u8,
68 pub client_channel: u16,
69 pub client_epoch: u32,
70 pub module_id: String,
71 pub module_endpoint: ModuleEndpointId,
72 pub module_sink: FrameSink,
73 pub module_negotiated_ver: u8,
74 pub module_channel: u16,
75 pub module_epoch: u32,
76 pub principal: Principal,
77 #[allow(dead_code)]
79 pub harness: String,
80 #[allow(dead_code)]
82 pub build_class: Option<String>,
83 #[allow(dead_code)]
85 pub operator_app: Option<crate::operator_apps::OperatorAppStamp>,
86 pub project_root: Option<ProjectRootId>,
87 pub bound_at: Instant,
88 pub flow: Arc<ChannelFlow>,
89 pub scope: Option<BoundScope>,
92}
93
94#[derive(Debug, Clone)]
95pub(crate) enum DataRoute {
96 Client(DataRouteState),
97 Module(DataRouteState),
98}
99
100#[derive(Debug, Clone)]
101pub(crate) enum DataRouteState {
102 Bound(Arc<RouteBinding>),
103 Reserved,
104 EpochMismatch,
105 Absent,
106}
107
108#[derive(Debug, Clone, Copy, PartialEq, Eq)]
137pub(crate) enum GoodbyeTargetKind {
138 Client,
139 Module,
140}
141
142#[derive(Debug, Clone)]
143pub(crate) struct GoodbyeTarget {
144 pub connection_id: ConnectionId,
145 pub sink: FrameSink,
146 pub negotiated_ver: u8,
147 pub channel: u16,
148 pub epoch: u32,
149 pub kind: GoodbyeTargetKind,
150 pub module_id: Option<String>,
155}
156
157#[derive(Debug, Clone, Copy)]
160pub(crate) struct UndeliveredFrame<'a> {
161 pub module_id: Option<&'a str>,
163 pub sink: &'a FrameSink,
165}
166
167pub(crate) fn principal_label(principal: &Principal) -> String {
169 match principal {
170 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
171 Principal::Direct => "direct".to_string(),
172 other => format!("{other:?}"),
173 }
174}
175
176fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
179 let labels = inner
180 .client_to_module
181 .iter()
182 .filter(|(key, _)| key.connection_id == connection_id)
183 .map(|(_, route)| principal_label(&route.principal))
184 .collect::<std::collections::BTreeSet<_>>();
185 if labels.is_empty() {
186 "none".to_string()
187 } else {
188 labels.into_iter().collect::<Vec<_>>().join(",")
189 }
190}
191
192impl GoodbyeTarget {
193 pub(crate) fn close_on_delivery_failure(&self) -> bool {
196 matches!(self.kind, GoodbyeTargetKind::Client)
197 }
198}
199
200pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
213
214pub(crate) fn send_module_route_goodbye(
226 counters: &DaemonCounters,
227 sink: &FrameSink,
228 frame: Frame,
229 module_id: Option<&str>,
230 context: &'static str,
231) {
232 let channel = frame.header.channel;
233 let epoch = frame.header.epoch;
234 let Err(err) = sink.try_send(frame.clone()) else {
235 return;
236 };
237 let runtime = match tokio::runtime::Handle::try_current() {
240 Ok(runtime) if !sink.is_closed() => runtime,
241 _ => {
242 counters.increment_goodbye_relay_module_dropped(module_id);
243 warn!(
244 module_id = module_id.unwrap_or("unknown"),
245 route_channel = channel,
246 route_epoch = epoch,
247 error = %err,
248 context,
249 "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
250 );
251 return;
252 }
253 };
254 debug!(
255 module_id = module_id.unwrap_or("unknown"),
256 route_channel = channel,
257 route_epoch = epoch,
258 error = %err,
259 context,
260 "module egress queue refused route GOODBYE; delivering it once the module frees room"
261 );
262 let counters = counters.clone();
263 let sink = sink.clone();
264 let module_id = module_id.map(str::to_string);
265 runtime.spawn(async move {
266 let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
267 let why = match outcome {
268 Ok(Ok(())) => {
269 debug!(
270 module_id = module_id.as_deref().unwrap_or("unknown"),
271 route_channel = channel,
272 route_epoch = epoch,
273 context,
274 "late route GOODBYE delivered to module"
275 );
276 return;
277 }
278 Ok(Err(err)) => err.to_string(),
279 Err(_) => format!(
280 "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
281 ),
282 };
283 counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
284 warn!(
285 module_id = module_id.as_deref().unwrap_or("unknown"),
286 route_channel = channel,
287 route_epoch = epoch,
288 error = %why,
289 context,
290 "route GOODBYE to module dropped under backpressure; not closing shared module connection"
291 );
292 });
293}
294
295#[derive(Debug, Clone)]
301pub(crate) struct EndpointRoute {
302 pub goodbye_target: GoodbyeTarget,
303 pub principal: Principal,
304 pub bound_at: Instant,
305 pub draining: bool,
306 pub drain_reason: Option<RouteCloseReason>,
310}
311
312#[derive(Debug, Clone)]
315pub(crate) struct ScopeDrainedRoute {
316 pub reason: RouteCloseReason,
317 pub scope: BoundScope,
318 pub module_id: String,
319 pub client: GoodbyeTarget,
320 pub module: GoodbyeTarget,
321}
322
323#[derive(Debug)]
324pub(crate) struct PendingRouteBindRelay {
325 pub endpoint: ModuleEndpointId,
326 pub module_sink: FrameSink,
327 pub negotiated_ver: u8,
328 pub client_channel: u16,
329 pub client_epoch: u32,
330 pub module_channel: u16,
331 pub module_epoch: u32,
332 pub corr: u64,
333 pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
334}
335
336#[derive(Debug, Clone)]
337pub(crate) struct ModuleDrainTarget {
338 pub endpoint: ModuleEndpointId,
339 pub sink: FrameSink,
340 pub negotiated_ver: u8,
341 pub abandoned_bindings: Vec<GoodbyeTarget>,
342 pub excluded_subscriptions: u32,
343}
344
345#[cfg(unix)]
348#[derive(Debug, Clone)]
349pub(crate) struct ModuleConnectionTarget {
350 pub module_id: String,
351 pub endpoint: ModuleEndpointId,
352 pub sink: FrameSink,
353 pub negotiated_ver: u8,
354}
355
356#[derive(Debug, Clone)]
357pub(crate) enum RouteBindRelayOutcome {
358 Accepted,
359 Rejected(ErrorBody),
364 DaemonRejected(ErrorBody),
372 ModuleGone(String),
373}
374
375#[derive(Debug, Clone, Copy, PartialEq, Eq)]
377pub(crate) struct ForwardingCutover {
378 pub promoted: ModuleEndpointId,
380 pub incumbent: Option<ModuleEndpointId>,
383}
384
385#[derive(Debug)]
387pub(crate) struct ConnectionCleanup {
388 pub released: Vec<GoodbyeTarget>,
390 pub abandoned_relays: u32,
393}
394
395#[derive(Debug, Clone)]
396pub(crate) struct PendingRelayCompletion {
397 pub settled: bool,
398 pub abandoned: Option<GoodbyeTarget>,
399}
400
401#[derive(Debug)]
402pub(crate) struct PendingModuleControlRpc {
403 pub endpoint: ModuleEndpointId,
404 pub module_sink: FrameSink,
405 pub negotiated_ver: u8,
406 pub corr: u64,
407 pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
408}
409
410#[derive(Debug, Clone)]
411pub(crate) enum ModuleControlRpcOutcome {
412 Response(ModuleControlResponse),
413 Rejected(ErrorBody),
414 ModuleGone(String),
415 MalformedResponse(String),
416 UnexpectedOp { expected: String, actual: String },
417 DeadlineElapsed,
418}
419
420#[derive(Debug, Clone, PartialEq, Eq)]
421pub(crate) enum ModuleControlRpcCompletion {
422 Unknown,
423 Settled,
424 LateHealthAnswer {
425 module_id: String,
426 latency: Duration,
427 },
428}
429
430#[derive(Debug)]
431struct PendingModuleControlRpcEntry {
432 expected_op: String,
433 deadline: Instant,
434 health_probe_started_at: Option<Instant>,
435 sender: oneshot::Sender<ModuleControlRpcOutcome>,
436}
437
438#[derive(Debug)]
439struct HealthProbeTombstone {
440 expected_op: String,
441 module_id: String,
442 probe_started_at: Instant,
443 expires_at: Instant,
444}
445
446#[derive(Debug, Clone)]
447struct RouteReservation {
448 client_key: ClientRouteKey,
449 module_key: ModuleRouteKey,
450 client_epoch: u32,
451 module_epoch: u32,
452 project_root: Option<ProjectRootId>,
453}
454
455#[derive(Debug)]
456struct PendingRouteBindRelayEntry {
457 reservation: RouteReservation,
458 client_sink: FrameSink,
459 client_negotiated_ver: u8,
460 client_permit: crate::router::EgressPermit,
461 route_open_frame: Frame,
462 principal: Principal,
463 scope: Option<BoundScope>,
466 harness: String,
467 build_class: Option<String>,
468 operator_app: Option<crate::operator_apps::OperatorAppStamp>,
469 deadline: Instant,
470 relay_enqueued: bool,
481 sender: oneshot::Sender<RouteBindRelayOutcome>,
482}
483
484#[derive(Debug, Clone, Copy, PartialEq, Eq)]
487pub(crate) enum RouteBindRelayMark {
488 Marked,
490 SettledByReply,
494 ReleasedWithoutReply,
498}
499
500#[derive(Debug, Clone)]
501pub(crate) enum RouteRelease {
502 Removed(GoodbyeTarget),
503 Stale,
504 Absent,
505}
506
507#[derive(Debug, Clone)]
508pub(crate) enum RoutePollSnapshot {
509 Bound {
510 module_id: String,
511 status: Option<String>,
512 },
513 Absent,
514}
515
516#[derive(Debug, Clone)]
517struct ModuleConnection {
518 endpoint: ModuleEndpointId,
519 sink: FrameSink,
520 negotiated_ver: u8,
521 concurrency: Concurrency,
522}
523
524#[derive(Debug, Default)]
525struct ForwardingInner {
526 operator_confirms: Arc<crate::operator_confirm::OperatorConfirms>,
527 daemon_draining: bool,
528 daemon_drain_results: HashMap<ModuleEndpointId, bool>,
533 modules_by_id: HashMap<String, ModuleConnection>,
537 candidates_by_id: HashMap<String, ModuleConnection>,
543 superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
549 endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
550 module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
551 draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
555 closing_connections: HashSet<ConnectionId>,
556 next_generation: u64,
557 reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
558 reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
559 next_client_channel: HashMap<ConnectionId, u16>,
560 next_module_channel: HashMap<ModuleEndpointId, u16>,
561 client_slot_epochs: HashMap<ClientRouteKey, u32>,
562 module_slot_epochs: HashMap<ModuleRouteKey, u32>,
563 last_published_epoch: HashMap<ClientRouteKey, u32>,
564 client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
565 module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
566 status: HashMap<(ClientRouteKey, u32), String>,
567 pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
568 relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
573 scope_tags: HashMap<(String, String), ScopeTag>,
578 next_control_corr: HashMap<ModuleEndpointId, u64>,
579 pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
580 health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
581}
582
583#[derive(Debug, Clone)]
584pub(crate) struct CloseReason {
585 code: &'static str,
586 message: String,
587}
588
589impl CloseReason {
590 pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
591 Self {
592 code,
593 message: message.into(),
594 }
595 }
596}
597
598impl fmt::Display for CloseReason {
599 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
600 write!(f, "{}: {}", self.code, self.message)
601 }
602}
603
604pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
605
606#[derive(Debug, Default)]
608pub struct ForwardingTable {
609 inner: Arc<RwLock<ForwardingInner>>,
610 #[cfg(feature = "test-support")]
611 route_closing_hold: Mutex<Option<oneshot::Receiver<()>>>,
612 close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
613 counters: DaemonCounters,
614 route_bind_breakers: RouteBindBreakers,
619 route_bind_concurrency: RouteBindConcurrency,
622 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
626}
627
628impl ForwardingTable {
629 #[cfg(feature = "test-support")]
671 pub fn configure_operator_script(
672 &self,
673 script: &std::path::Path,
674 events: Option<&std::path::Path>,
675 limits: Option<crate::operator_confirm::OperatorLimits>,
676 ) {
677 self.operator_confirms()
678 .configure_script(script, events, limits);
679 }
680
681 #[cfg(feature = "test-support")]
685 pub fn hold_route_closing_for_test(&self) -> oneshot::Sender<()> {
686 let (release, hold) = oneshot::channel();
687 *self.route_closing_hold.lock().unwrap() = Some(hold);
688 release
689 }
690
691 #[cfg(feature = "test-support")]
692 pub(crate) async fn wait_before_route_closing_for_test(&self) {
693 let hold = self.route_closing_hold.lock().unwrap().take();
694 if let Some(hold) = hold {
695 let _ = hold.await;
696 }
697 }
698
699 pub(crate) fn operator_confirms(&self) -> Arc<crate::operator_confirm::OperatorConfirms> {
700 Arc::clone(
701 &self
702 .inner
703 .read()
704 .unwrap_or_else(|p| p.into_inner())
705 .operator_confirms,
706 )
707 }
708
709 #[cfg(test)]
710 pub(crate) fn inject_operator_principal(&self, key: ModuleRouteKey, principal: Principal) {
711 let mut inner = self.write_inner().unwrap();
712 let old = inner.module_to_client.remove(&key).unwrap();
713 let client = ClientRouteKey {
714 connection_id: old.client_connection_id,
715 channel: old.client_channel,
716 };
717 inner.client_to_module.remove(&client);
718 let mut binding = Arc::try_unwrap(old).expect("test binding has no outstanding readers");
719 binding.principal = principal;
720 let binding = Arc::new(binding);
721 inner.client_to_module.insert(client, Arc::clone(&binding));
722 inner.module_to_client.insert(key, binding);
723 }
724
725 pub(crate) fn with_operator_route<T>(
729 &self,
730 connection_id: ConnectionId,
731 channel: u16,
732 epoch: u32,
733 admit: impl FnOnce(Option<&RouteBinding>) -> T,
734 ) -> Result<T, ForwardingError> {
735 let inner = self.read_inner()?;
736 let binding = inner
737 .endpoint_by_connection
738 .get(&connection_id)
739 .and_then(|endpoint| {
740 inner.module_to_client.get(&ModuleRouteKey {
741 endpoint: *endpoint,
742 channel,
743 })
744 })
745 .filter(|binding| binding.module_epoch == epoch);
746 Ok(admit(binding.map(Arc::as_ref)))
747 }
748
749 pub(crate) fn counters(&self) -> DaemonCounters {
750 self.counters.clone()
751 }
752
753 pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
754 self.route_bind_breakers.clone()
755 }
756
757 pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
758 self.route_bind_concurrency.clone()
759 }
760
761 pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
762 Arc::clone(&self.route_outages)
763 }
764
765 pub(crate) fn register_connection_close(
766 &self,
767 connection_id: ConnectionId,
768 ) -> ConnectionCloseReceiver {
769 let (sender, receiver) = oneshot::channel();
770 let replaced = self
771 .lock_close_registry()
772 .insert(connection_id, sender)
773 .is_some();
774 if replaced {
775 warn!(
776 connection_id = connection_id.get(),
777 "replaced existing connection close registration"
778 );
779 }
780 receiver
781 }
782
783 pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
784 self.lock_close_registry().remove(&connection_id);
785 }
786
787 #[cfg(unix)]
796 pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
797 let senders: Vec<_> = self.lock_close_registry().drain().collect();
798 let count = senders.len();
799 for (_, sender) in senders {
800 let _ = sender.send(reason.clone());
801 }
802 count
803 }
804
805 #[cfg(unix)]
809 pub(crate) fn module_connections(
810 &self,
811 ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
812 let inner = self.read_inner()?;
813 let mut seen = HashSet::new();
814 Ok(inner
815 .modules_by_id
816 .values()
817 .chain(inner.candidates_by_id.values())
818 .chain(inner.superseded_endpoints.values())
819 .filter(|module| seen.insert(module.endpoint))
820 .map(|module| ModuleConnectionTarget {
821 module_id: inner
822 .module_id_by_endpoint
823 .get(&module.endpoint)
824 .cloned()
825 .unwrap_or_default(),
826 endpoint: module.endpoint,
827 sink: module.sink.clone(),
828 negotiated_ver: module.negotiated_ver,
829 })
830 .collect())
831 }
832
833 pub(crate) fn request_connection_close(
837 &self,
838 connection_id: ConnectionId,
839 reason: CloseReason,
840 ) -> bool {
841 let sender = self.lock_close_registry().remove(&connection_id);
842 if let Some(sender) = sender {
843 debug!(
844 connection_id = connection_id.get(),
845 close_reason = %reason,
846 "requesting connection close"
847 );
848 let _ = sender.send(reason);
849 true
850 } else {
851 debug!(
852 connection_id = connection_id.get(),
853 close_reason = %reason,
854 "connection close request ignored for inactive connection"
855 );
856 false
857 }
858 }
859
860 pub fn register_module_connection(
861 &self,
862 connection_id: ConnectionId,
863 module_id: String,
864 negotiated_ver: u8,
865 concurrency: Concurrency,
866 sink: FrameSink,
867 ) -> Result<ModuleEndpointId, ForwardingError> {
868 self.register_module_connection_inner(
869 connection_id,
870 module_id,
871 negotiated_ver,
872 concurrency,
873 sink,
874 None,
875 )
876 }
877
878 pub(crate) fn register_module_connection_acked(
891 &self,
892 connection_id: ConnectionId,
893 module_id: String,
894 negotiated_ver: u8,
895 concurrency: Concurrency,
896 sink: FrameSink,
897 hello_ack: Frame,
898 ) -> Result<ModuleEndpointId, ForwardingError> {
899 self.register_module_connection_inner(
900 connection_id,
901 module_id,
902 negotiated_ver,
903 concurrency,
904 sink,
905 Some(hello_ack),
906 )
907 }
908
909 fn register_module_connection_inner(
910 &self,
911 connection_id: ConnectionId,
912 module_id: String,
913 negotiated_ver: u8,
914 concurrency: Concurrency,
915 sink: FrameSink,
916 hello_ack: Option<Frame>,
917 ) -> Result<ModuleEndpointId, ForwardingError> {
918 let mut inner = self.write_inner()?;
919 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
920 return Err(ForwardingError::ConnectionClosing { connection_id });
921 }
922 check_module_connection_role_locked(&inner, connection_id)?;
923 enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
926
927 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
928 let endpoint = ModuleEndpointId {
929 connection_id,
930 generation: inner.next_generation,
931 };
932 inner.endpoint_by_connection.insert(connection_id, endpoint);
933 inner
934 .module_id_by_endpoint
935 .insert(endpoint, module_id.clone());
936 inner.next_module_channel.insert(endpoint, 1);
937 inner.next_control_corr.insert(endpoint, 1);
938 inner.modules_by_id.insert(
939 module_id.clone(),
940 ModuleConnection {
941 endpoint,
942 sink,
943 negotiated_ver,
944 concurrency,
945 },
946 );
947 drop(inner);
948
949 if let Some(discarded) = self
962 .route_bind_breakers
963 .reset_for_new_module_connection(&module_id)
964 {
965 info!(
966 module_id = %module_id,
967 discarded_consecutive_timeouts = discarded,
968 "route.bind breaker state discarded: a new module connection replaced the process it described"
969 );
970 }
971 Ok(endpoint)
972 }
973
974 #[cfg(test)]
988 pub(crate) fn register_candidate_module_connection(
989 &self,
990 connection_id: ConnectionId,
991 module_id: String,
992 negotiated_ver: u8,
993 concurrency: Concurrency,
994 sink: FrameSink,
995 ) -> Result<ModuleEndpointId, ForwardingError> {
996 self.register_candidate_module_connection_inner(
997 connection_id,
998 module_id,
999 negotiated_ver,
1000 concurrency,
1001 sink,
1002 None,
1003 )
1004 }
1005
1006 pub(crate) fn register_candidate_module_connection_acked(
1013 &self,
1014 connection_id: ConnectionId,
1015 module_id: String,
1016 negotiated_ver: u8,
1017 concurrency: Concurrency,
1018 sink: FrameSink,
1019 hello_ack: Frame,
1020 ) -> Result<ModuleEndpointId, ForwardingError> {
1021 self.register_candidate_module_connection_inner(
1022 connection_id,
1023 module_id,
1024 negotiated_ver,
1025 concurrency,
1026 sink,
1027 Some(hello_ack),
1028 )
1029 }
1030
1031 fn register_candidate_module_connection_inner(
1032 &self,
1033 connection_id: ConnectionId,
1034 module_id: String,
1035 negotiated_ver: u8,
1036 concurrency: Concurrency,
1037 sink: FrameSink,
1038 hello_ack: Option<Frame>,
1039 ) -> Result<ModuleEndpointId, ForwardingError> {
1040 let mut inner = self.write_inner()?;
1041 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1042 return Err(ForwardingError::ConnectionClosing { connection_id });
1043 }
1044 check_module_connection_role_locked(&inner, connection_id)?;
1045 if inner.candidates_by_id.contains_key(&module_id) {
1046 return Err(ForwardingError::CandidateSlotOccupied { module_id });
1047 }
1048 enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1051
1052 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1053 let endpoint = ModuleEndpointId {
1054 connection_id,
1055 generation: inner.next_generation,
1056 };
1057 inner.endpoint_by_connection.insert(connection_id, endpoint);
1058 inner
1059 .module_id_by_endpoint
1060 .insert(endpoint, module_id.clone());
1061 inner.next_module_channel.insert(endpoint, 1);
1062 inner.next_control_corr.insert(endpoint, 1);
1063 inner.candidates_by_id.insert(
1064 module_id,
1065 ModuleConnection {
1066 endpoint,
1067 sink,
1068 negotiated_ver,
1069 concurrency,
1070 },
1071 );
1072 Ok(endpoint)
1073 }
1074
1075 pub(crate) fn cutover_candidate(
1093 &self,
1094 module_id: &str,
1095 ) -> Result<Option<ForwardingCutover>, ForwardingError> {
1096 let mut inner = self.write_inner()?;
1097 if inner.daemon_draining {
1098 return Err(ForwardingError::ModuleReloading {
1099 module_id: module_id.to_string(),
1100 });
1101 }
1102 let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
1103 return Ok(None);
1104 };
1105 let promoted = candidate.endpoint;
1106 let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
1107 let incumbent = incumbent.map(|incumbent| {
1108 let endpoint = incumbent.endpoint;
1109 inner.superseded_endpoints.insert(endpoint, incumbent);
1110 endpoint
1111 });
1112 drop(inner);
1113
1114 if let Some(discarded) = self
1118 .route_bind_breakers
1119 .reset_for_new_module_connection(module_id)
1120 {
1121 info!(
1122 module_id = %module_id,
1123 discarded_consecutive_timeouts = discarded,
1124 "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
1125 );
1126 }
1127 Ok(Some(ForwardingCutover {
1128 promoted,
1129 incumbent,
1130 }))
1131 }
1132
1133 #[allow(clippy::too_many_arguments)]
1134 pub(crate) async fn begin_route_bind_relay_for(
1135 &self,
1136 client_connection_id: ConnectionId,
1137 client_sink: FrameSink,
1138 client_negotiated_ver: u8,
1139 client_corr: u64,
1140 module_id: &str,
1141 principal: Principal,
1142 scope: Option<BoundScope>,
1143 project_root: Option<ProjectRootId>,
1144 harness: String,
1145 build_class: Option<String>,
1146 operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1147 deadline: Instant,
1148 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1149 let client_permit =
1153 client_sink
1154 .reserve_owned()
1155 .await
1156 .map_err(|_| ForwardingError::ClientEgressClosed {
1157 connection_id: client_connection_id,
1158 })?;
1159 self.begin_route_bind_relay_inner(
1160 client_connection_id,
1161 client_sink,
1162 client_negotiated_ver,
1163 client_corr,
1164 module_id,
1165 principal,
1166 scope,
1167 project_root,
1168 harness,
1169 build_class,
1170 operator_app,
1171 deadline,
1172 client_permit,
1173 )
1174 }
1175
1176 #[cfg(test)]
1177 pub(crate) fn begin_route_bind_relay_for_test(
1178 &self,
1179 client_connection_id: ConnectionId,
1180 client_sink: FrameSink,
1181 client_corr: u64,
1182 module_id: &str,
1183 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1184 let permit =
1185 client_sink
1186 .try_reserve_owned()
1187 .map_err(|_| ForwardingError::ClientEgressClosed {
1188 connection_id: client_connection_id,
1189 })?;
1190 self.begin_route_bind_relay_inner(
1191 client_connection_id,
1192 client_sink,
1193 subc_protocol::PROTOCOL_VERSION,
1194 client_corr,
1195 module_id,
1196 Principal::Direct,
1197 None,
1198 None,
1199 String::new(),
1200 None,
1201 None,
1202 Instant::now() + std::time::Duration::from_secs(60),
1203 permit,
1204 )
1205 }
1206
1207 pub(crate) fn begin_module_control_rpc_for(
1208 &self,
1209 module_id: &str,
1210 expected_op: &str,
1211 deadline: Instant,
1212 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1213 self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1214 }
1215
1216 pub(crate) fn begin_health_probe_rpc_for(
1217 &self,
1218 module_id: &str,
1219 expected_op: &str,
1220 probe_started_at: Instant,
1221 deadline: Instant,
1222 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1223 self.begin_module_control_rpc_inner(
1224 module_id,
1225 expected_op,
1226 deadline,
1227 Some(probe_started_at),
1228 false,
1229 )
1230 }
1231
1232 pub(crate) fn begin_drain_health_probe_rpc_for(
1233 &self,
1234 module_id: &str,
1235 expected_op: &str,
1236 probe_started_at: Instant,
1237 deadline: Instant,
1238 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1239 self.begin_module_control_rpc_inner(
1240 module_id,
1241 expected_op,
1242 deadline,
1243 Some(probe_started_at),
1244 true,
1245 )
1246 }
1247
1248 pub(crate) fn begin_endpoint_health_probe_rpc_for(
1256 &self,
1257 endpoint: ModuleEndpointId,
1258 expected_op: &str,
1259 probe_started_at: Instant,
1260 deadline: Instant,
1261 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1262 let inner = self.write_inner()?;
1263 let module = module_connection_for_endpoint_locked(&inner, endpoint)
1264 .cloned()
1265 .ok_or(ForwardingError::NoModuleConnection)?;
1266 let module_id = inner
1267 .module_id_by_endpoint
1268 .get(&endpoint)
1269 .cloned()
1270 .unwrap_or_default();
1271 self.begin_control_rpc_locked(
1274 inner,
1275 &module_id,
1276 module,
1277 expected_op,
1278 deadline,
1279 Some(probe_started_at),
1280 true,
1281 )
1282 }
1283
1284 fn begin_module_control_rpc_inner(
1285 &self,
1286 module_id: &str,
1287 expected_op: &str,
1288 deadline: Instant,
1289 health_probe_started_at: Option<Instant>,
1290 allow_draining: bool,
1291 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1292 let inner = self.write_inner()?;
1293 let module = inner
1294 .modules_by_id
1295 .get(module_id)
1296 .cloned()
1297 .ok_or(ForwardingError::NoModuleConnection)?;
1298 self.begin_control_rpc_locked(
1299 inner,
1300 module_id,
1301 module,
1302 expected_op,
1303 deadline,
1304 health_probe_started_at,
1305 allow_draining,
1306 )
1307 }
1308
1309 #[allow(clippy::too_many_arguments)]
1310 fn begin_control_rpc_locked(
1311 &self,
1312 mut inner: RwLockWriteGuard<'_, ForwardingInner>,
1313 module_id: &str,
1314 module: ModuleConnection,
1315 expected_op: &str,
1316 deadline: Instant,
1317 health_probe_started_at: Option<Instant>,
1318 allow_draining: bool,
1319 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1320 if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1321 return Err(ForwardingError::ModuleReloading {
1322 module_id: module_id.to_string(),
1323 });
1324 }
1325 if inner
1326 .closing_connections
1327 .contains(&module.endpoint.connection_id)
1328 {
1329 return Err(ForwardingError::ConnectionClosing {
1330 connection_id: module.endpoint.connection_id,
1331 });
1332 }
1333 if health_probe_started_at.is_some() {
1334 inner
1338 .health_probe_tombstones
1339 .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1340 }
1341 let corr = match inner.allocate_control_corr(module.endpoint) {
1342 Ok(corr) => corr,
1343 Err(err) => {
1344 drop(inner);
1345 self.request_connection_close(
1346 module.endpoint.connection_id,
1347 CloseReason::new(
1348 "control_correlation_exhausted",
1349 "daemon-originated channel-0 correlation space exhausted",
1350 ),
1351 );
1352 return Err(err);
1353 }
1354 };
1355 let (sender, receiver) = oneshot::channel();
1356 inner.pending_control_rpcs.insert(
1357 (module.endpoint, corr),
1358 PendingModuleControlRpcEntry {
1359 expected_op: expected_op.to_string(),
1360 deadline,
1361 health_probe_started_at,
1362 sender,
1363 },
1364 );
1365
1366 Ok(PendingModuleControlRpc {
1367 endpoint: module.endpoint,
1368 module_sink: module.sink,
1369 negotiated_ver: module.negotiated_ver,
1370 corr,
1371 receiver,
1372 })
1373 }
1374
1375 #[allow(clippy::too_many_arguments)]
1376 fn begin_route_bind_relay_inner(
1377 &self,
1378 client_connection_id: ConnectionId,
1379 client_sink: FrameSink,
1380 client_negotiated_ver: u8,
1381 client_corr: u64,
1382 expected_module_id: &str,
1383 principal: Principal,
1384 scope: Option<BoundScope>,
1385 project_root: Option<ProjectRootId>,
1386 harness: String,
1387 build_class: Option<String>,
1388 operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1389 deadline: Instant,
1390 client_permit: crate::router::EgressPermit,
1391 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1392 let mut inner = self.write_inner()?;
1393 if inner
1397 .endpoint_by_connection
1398 .contains_key(&client_connection_id)
1399 {
1400 return Err(ForwardingError::ConnectionRoleConflict {
1401 connection_id: client_connection_id,
1402 });
1403 }
1404 if inner.closing_connections.contains(&client_connection_id) {
1405 return Err(ForwardingError::ConnectionClosing {
1406 connection_id: client_connection_id,
1407 });
1408 }
1409 let module = inner
1410 .modules_by_id
1411 .get(expected_module_id)
1412 .cloned()
1413 .ok_or(ForwardingError::NoModuleConnection)?;
1414 if inner.draining_endpoints.contains_key(&module.endpoint) {
1415 return Err(ForwardingError::ModuleReloading {
1416 module_id: expected_module_id.to_string(),
1417 });
1418 }
1419 if inner
1420 .closing_connections
1421 .contains(&module.endpoint.connection_id)
1422 {
1423 return Err(ForwardingError::ConnectionClosing {
1424 connection_id: module.endpoint.connection_id,
1425 });
1426 }
1427
1428 let corr = match inner.allocate_control_corr(module.endpoint) {
1429 Ok(corr) => corr,
1430 Err(err) => {
1431 drop(inner);
1432 self.request_connection_close(
1433 module.endpoint.connection_id,
1434 CloseReason::new(
1435 "control_correlation_exhausted",
1436 "daemon-originated channel-0 correlation space exhausted",
1437 ),
1438 );
1439 return Err(err);
1440 }
1441 };
1442 let (client_channel, client_epoch, module_channel, module_epoch) =
1443 inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1444 let client_key = ClientRouteKey {
1445 connection_id: client_connection_id,
1446 channel: client_channel,
1447 };
1448 let module_key = ModuleRouteKey {
1449 endpoint: module.endpoint,
1450 channel: module_channel,
1451 };
1452 let reservation = RouteReservation {
1453 client_key,
1454 module_key,
1455 client_epoch,
1456 module_epoch,
1457 project_root,
1458 };
1459 let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
1460 route_channel: client_channel,
1461 route_epoch: client_epoch,
1462 })
1463 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1464 let route_open_frame = Frame::build_with_version(
1465 client_negotiated_ver,
1466 FrameType::Response,
1467 Flags::new(false, Priority::Passive, false),
1468 0,
1469 0,
1470 client_corr,
1471 response_body,
1472 )
1473 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1474 let (sender, receiver) = oneshot::channel();
1475 inner.reserved_client.insert(client_key, module_key);
1476 inner.reserved_module.insert(module_key, client_key);
1477 inner.pending_relays.insert(
1478 (module.endpoint, corr),
1479 PendingRouteBindRelayEntry {
1480 reservation,
1481 client_sink,
1482 client_negotiated_ver,
1483 client_permit,
1484 route_open_frame,
1485 principal,
1486 scope,
1487 harness,
1488 build_class,
1489 operator_app,
1490 deadline,
1491 relay_enqueued: false,
1492 sender,
1493 },
1494 );
1495
1496 Ok(PendingRouteBindRelay {
1497 endpoint: module.endpoint,
1498 module_sink: module.sink,
1499 negotiated_ver: module.negotiated_ver,
1500 client_channel,
1501 client_epoch,
1502 module_channel,
1503 module_epoch,
1504 corr,
1505 receiver,
1506 })
1507 }
1508
1509 pub(crate) fn mark_route_bind_relay_enqueued(
1519 &self,
1520 endpoint: ModuleEndpointId,
1521 corr: u64,
1522 ) -> Result<RouteBindRelayMark, ForwardingError> {
1523 let mut inner = self.write_inner()?;
1524 if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
1525 pending.relay_enqueued = true;
1526 return Ok(RouteBindRelayMark::Marked);
1527 }
1528 if inner
1529 .relays_released_before_enqueue
1530 .remove(&(endpoint, corr))
1531 {
1532 return Ok(RouteBindRelayMark::ReleasedWithoutReply);
1533 }
1534 Ok(RouteBindRelayMark::SettledByReply)
1535 }
1536
1537 pub(crate) fn release_client_route(
1538 &self,
1539 client_connection_id: ConnectionId,
1540 client_channel: u16,
1541 expected_epoch: u32,
1542 ) -> Result<RouteRelease, ForwardingError> {
1543 let mut inner = self.write_inner()?;
1544 let release = release_client_route_locked(
1545 &mut inner,
1546 ClientRouteKey {
1547 connection_id: client_connection_id,
1548 channel: client_channel,
1549 },
1550 expected_epoch,
1551 );
1552 self.record_route_release(&release);
1553 Ok(release)
1554 }
1555
1556 pub(crate) fn release_module_route(
1557 &self,
1558 module_connection_id: ConnectionId,
1559 module_channel: u16,
1560 expected_epoch: u32,
1561 ) -> Result<RouteRelease, ForwardingError> {
1562 let mut inner = self.write_inner()?;
1563 let Some(endpoint) = inner
1564 .endpoint_by_connection
1565 .get(&module_connection_id)
1566 .copied()
1567 else {
1568 return Ok(RouteRelease::Absent);
1569 };
1570 let release = release_module_route_locked(
1571 &mut inner,
1572 ModuleRouteKey {
1573 endpoint,
1574 channel: module_channel,
1575 },
1576 expected_epoch,
1577 );
1578 self.record_route_release(&release);
1579 Ok(release)
1580 }
1581
1582 pub(crate) fn abort_pending_relay(
1583 &self,
1584 endpoint: ModuleEndpointId,
1585 corr: u64,
1586 outcome: RouteBindRelayOutcome,
1587 ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
1588 let mut inner = self.write_inner()?;
1589 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1590 inner
1595 .relays_released_before_enqueue
1596 .remove(&(endpoint, corr));
1597 return Ok(None);
1598 };
1599 release_reserved_route_locked(
1600 &mut inner,
1601 pending.reservation.client_key,
1602 pending.reservation.module_key,
1603 );
1604 let target = pending
1605 .relay_enqueued
1606 .then(|| abandoned_route_target(&inner, &pending.reservation));
1607 let _ = pending.sender.send(outcome);
1608 Ok(target.flatten())
1609 }
1610
1611 pub(crate) fn cancel_module_control_rpc(
1612 &self,
1613 endpoint: ModuleEndpointId,
1614 corr: u64,
1615 ) -> Result<(), ForwardingError> {
1616 self.write_inner()?
1617 .pending_control_rpcs
1618 .remove(&(endpoint, corr));
1619 Ok(())
1620 }
1621
1622 pub(crate) fn tombstone_health_probe_rpc(
1623 &self,
1624 endpoint: ModuleEndpointId,
1625 corr: u64,
1626 ) -> Result<bool, ForwardingError> {
1627 let key = (endpoint, corr);
1628 let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
1629 {
1630 let mut inner = self.write_inner()?;
1631 let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
1632 return Ok(false);
1633 };
1634 let Some(probe_started_at) = pending.health_probe_started_at else {
1635 inner.pending_control_rpcs.insert(key, pending);
1636 return Ok(false);
1637 };
1638 let module_id = inner
1639 .module_id_by_endpoint
1640 .get(&endpoint)
1641 .cloned()
1642 .unwrap_or_else(|| "unknown".to_string());
1643 inner.health_probe_tombstones.insert(
1644 key,
1645 HealthProbeTombstone {
1646 expected_op: pending.expected_op,
1647 module_id,
1648 probe_started_at,
1649 expires_at,
1650 },
1651 );
1652 }
1653 self.schedule_health_probe_tombstone_expiration(key, expires_at);
1654 Ok(true)
1655 }
1656
1657 fn schedule_health_probe_tombstone_expiration(
1658 &self,
1659 key: (ModuleEndpointId, u64),
1660 expires_at: Instant,
1661 ) {
1662 let inner = Arc::downgrade(&self.inner);
1663 tokio::spawn(async move {
1664 tokio::time::sleep_until(expires_at).await;
1665 let Some(inner) = inner.upgrade() else {
1666 return;
1667 };
1668 let Ok(mut inner) = inner.write() else {
1669 return;
1670 };
1671 let expired = inner
1672 .health_probe_tombstones
1673 .get(&key)
1674 .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
1675 if expired {
1676 inner.health_probe_tombstones.remove(&key);
1677 }
1678 });
1679 }
1680
1681 pub(crate) fn complete_pending_relay(
1682 &self,
1683 connection_id: ConnectionId,
1684 corr: u64,
1685 outcome: RouteBindRelayOutcome,
1686 ) -> Result<PendingRelayCompletion, ForwardingError> {
1687 let mut inner = self.write_inner()?;
1688 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1689 return Ok(PendingRelayCompletion {
1690 settled: false,
1691 abandoned: None,
1692 });
1693 };
1694 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1695 return Ok(PendingRelayCompletion {
1696 settled: false,
1697 abandoned: None,
1698 });
1699 };
1700
1701 if Instant::now() >= pending.deadline {
1702 release_reserved_route_locked(
1703 &mut inner,
1704 pending.reservation.client_key,
1705 pending.reservation.module_key,
1706 );
1707 let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
1708 .then(|| abandoned_route_target(&inner, &pending.reservation))
1709 .flatten();
1710 let _ = pending
1711 .sender
1712 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody {
1713 code: "module_timeout".to_string(),
1714 message: "route.bind response arrived after its daemon deadline".to_string(),
1715 detail: None,
1716 }));
1717 return Ok(PendingRelayCompletion {
1718 settled: true,
1719 abandoned,
1720 });
1721 }
1722
1723 match outcome {
1724 RouteBindRelayOutcome::Accepted
1743 if pending.client_sink.is_closed()
1744 || inner
1745 .closing_connections
1746 .contains(&pending.reservation.client_key.connection_id) =>
1747 {
1748 let reason = if pending.client_sink.is_closed() {
1749 "client egress closed before route publication"
1750 } else {
1751 "client connection is closing before route publication"
1752 };
1753 release_reserved_route_locked(
1754 &mut inner,
1755 pending.reservation.client_key,
1756 pending.reservation.module_key,
1757 );
1758 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1762 let _ = pending
1763 .sender
1764 .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
1765 return Ok(PendingRelayCompletion {
1766 settled: true,
1767 abandoned,
1768 });
1769 }
1770 RouteBindRelayOutcome::Accepted
1793 if inner.superseded_endpoints.contains_key(&endpoint)
1794 || inner.draining_endpoints.contains_key(&endpoint) =>
1795 {
1796 release_reserved_route_locked(
1797 &mut inner,
1798 pending.reservation.client_key,
1799 pending.reservation.module_key,
1800 );
1801 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1804 let module_id = inner
1805 .module_id_by_endpoint
1806 .get(&endpoint)
1807 .cloned()
1808 .unwrap_or_else(|| "unknown".to_string());
1809 let _ = pending
1810 .sender
1811 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
1812 "module_reloading",
1813 format!("module_id '{module_id}' is reloading"),
1814 )));
1815 return Ok(PendingRelayCompletion {
1816 settled: true,
1817 abandoned,
1818 });
1819 }
1820 RouteBindRelayOutcome::Accepted
1829 if pending
1830 .scope
1831 .as_ref()
1832 .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
1833 {
1834 let refusal = pending
1835 .scope
1836 .as_ref()
1837 .and_then(|scope| scope_refusal_locked(&inner, scope))
1838 .expect("guard matched a refusal under the same lock");
1839 release_reserved_route_locked(
1840 &mut inner,
1841 pending.reservation.client_key,
1842 pending.reservation.module_key,
1843 );
1844 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1845 let _ = pending
1846 .sender
1847 .send(RouteBindRelayOutcome::DaemonRejected(refusal));
1848 return Ok(PendingRelayCompletion {
1849 settled: true,
1850 abandoned,
1851 });
1852 }
1853 RouteBindRelayOutcome::Accepted => {
1854 let abandoned = commit_route_locked(&mut inner, pending)?;
1855 return Ok(PendingRelayCompletion {
1856 settled: true,
1857 abandoned,
1858 });
1859 }
1860 terminal => {
1861 release_reserved_route_locked(
1862 &mut inner,
1863 pending.reservation.client_key,
1864 pending.reservation.module_key,
1865 );
1866 let _ = pending.sender.send(terminal);
1867 }
1868 }
1869 Ok(PendingRelayCompletion {
1870 settled: true,
1871 abandoned: None,
1872 })
1873 }
1874
1875 pub(crate) fn pending_module_control_op(
1876 &self,
1877 connection_id: ConnectionId,
1878 corr: u64,
1879 ) -> Result<Option<String>, ForwardingError> {
1880 let inner = self.read_inner()?;
1881 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1882 return Ok(None);
1883 };
1884 let key = (endpoint, corr);
1885 Ok(inner
1886 .pending_control_rpcs
1887 .get(&key)
1888 .map(|pending| pending.expected_op.clone())
1889 .or_else(|| {
1890 inner
1891 .health_probe_tombstones
1892 .get(&key)
1893 .filter(|tombstone| tombstone.expires_at > Instant::now())
1894 .map(|tombstone| tombstone.expected_op.clone())
1895 }))
1896 }
1897
1898 pub(crate) fn complete_module_control_rpc(
1899 &self,
1900 connection_id: ConnectionId,
1901 corr: u64,
1902 actual_op: Option<&str>,
1903 outcome: ModuleControlRpcOutcome,
1904 ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
1905 let now = Instant::now();
1906 let mut inner = self.write_inner()?;
1907 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1908 return Ok(ModuleControlRpcCompletion::Unknown);
1909 };
1910 let key = (endpoint, corr);
1911 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
1912 if now >= pending.deadline {
1913 let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
1914 ModuleControlRpcCompletion::LateHealthAnswer {
1915 module_id: inner
1916 .module_id_by_endpoint
1917 .get(&endpoint)
1918 .cloned()
1919 .unwrap_or_else(|| "unknown".to_string()),
1920 latency: now.saturating_duration_since(probe_started_at),
1921 }
1922 });
1923 let _ = pending
1924 .sender
1925 .send(ModuleControlRpcOutcome::DeadlineElapsed);
1926 return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
1927 }
1928 let outcome = match actual_op {
1929 Some(actual) if actual != pending.expected_op => {
1930 ModuleControlRpcOutcome::UnexpectedOp {
1931 expected: pending.expected_op,
1932 actual: actual.to_string(),
1933 }
1934 }
1935 _ => outcome,
1936 };
1937 let _ = pending.sender.send(outcome);
1938 return Ok(ModuleControlRpcCompletion::Settled);
1939 }
1940
1941 let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
1942 return Ok(ModuleControlRpcCompletion::Unknown);
1943 };
1944 if tombstone.expires_at <= now {
1945 return Ok(ModuleControlRpcCompletion::Unknown);
1946 }
1947 Ok(ModuleControlRpcCompletion::LateHealthAnswer {
1948 module_id: tombstone.module_id,
1949 latency: now.saturating_duration_since(tombstone.probe_started_at),
1950 })
1951 }
1952
1953 #[cfg(test)]
1954 pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
1955 Ok(self.read_inner()?.health_probe_tombstones.len())
1956 }
1957
1958 #[cfg(test)]
1959 pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
1960 Ok(self.read_inner()?.closing_connections.len())
1961 }
1962
1963 #[cfg(test)]
1966 pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
1967 let inner = self.read_inner()?;
1968 Ok((inner.reserved_client.len(), inner.reserved_module.len()))
1969 }
1970
1971 pub(crate) fn module_endpoint_for_connection(
1972 &self,
1973 connection_id: ConnectionId,
1974 ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
1975 Ok(self
1976 .read_inner()?
1977 .endpoint_by_connection
1978 .get(&connection_id)
1979 .copied())
1980 }
1981
1982 pub(crate) fn module_id_for_connection(
1985 &self,
1986 connection_id: ConnectionId,
1987 ) -> Result<Option<String>, ForwardingError> {
1988 let inner = self.read_inner()?;
1989 Ok(inner
1990 .endpoint_by_connection
1991 .get(&connection_id)
1992 .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
1993 .cloned())
1994 }
1995
1996 pub(crate) fn module_route_epoch_was_allocated(
2003 &self,
2004 connection_id: ConnectionId,
2005 channel: u16,
2006 epoch: u32,
2007 ) -> Result<bool, ForwardingError> {
2008 let inner = self.read_inner()?;
2009 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2010 return Ok(false);
2011 };
2012 Ok(inner
2013 .module_slot_epochs
2014 .get(&ModuleRouteKey { endpoint, channel })
2015 .is_some_and(|last| epoch != 0 && epoch <= *last))
2016 }
2017
2018 pub(crate) fn has_live_module_connection(
2019 &self,
2020 module_id: &str,
2021 ) -> Result<bool, ForwardingError> {
2022 Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
2023 }
2024
2025 pub(crate) fn lookup_data_route(
2026 &self,
2027 connection_id: ConnectionId,
2028 channel: u16,
2029 epoch: u32,
2030 ) -> Result<DataRoute, ForwardingError> {
2031 let inner = self.read_inner()?;
2032 let state = if let Some(endpoint) =
2033 inner.endpoint_by_connection.get(&connection_id).copied()
2034 {
2035 let key = ModuleRouteKey { endpoint, channel };
2036 match inner.module_to_client.get(&key) {
2037 Some(route) if route.module_epoch == epoch => {
2038 DataRouteState::Bound(Arc::clone(route))
2039 }
2040 Some(_) => DataRouteState::EpochMismatch,
2041 None if inner.reserved_module.contains_key(&key)
2042 && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
2043 {
2044 DataRouteState::Reserved
2045 }
2046 None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
2047 None => DataRouteState::Absent,
2048 }
2049 } else {
2050 let key = ClientRouteKey {
2051 connection_id,
2052 channel,
2053 };
2054 match inner.client_to_module.get(&key) {
2055 Some(route) if route.client_epoch == epoch => {
2056 DataRouteState::Bound(Arc::clone(route))
2057 }
2058 Some(_) => DataRouteState::EpochMismatch,
2059 None if inner.reserved_client.contains_key(&key)
2060 && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
2061 {
2062 DataRouteState::Reserved
2063 }
2064 None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
2065 None => DataRouteState::Absent,
2066 }
2067 };
2068 Ok(
2069 if inner.endpoint_by_connection.contains_key(&connection_id) {
2070 DataRoute::Module(state)
2071 } else {
2072 DataRoute::Client(state)
2073 },
2074 )
2075 }
2076
2077 #[cfg(test)]
2078 pub(crate) fn inject_client_slot_epoch(
2079 &self,
2080 connection_id: ConnectionId,
2081 channel: u16,
2082 last_epoch: u32,
2083 ) {
2084 let mut inner = self.write_inner().expect("forwarding lock");
2085 inner.client_slot_epochs.insert(
2086 ClientRouteKey {
2087 connection_id,
2088 channel,
2089 },
2090 last_epoch,
2091 );
2092 inner.next_client_channel.insert(connection_id, channel);
2093 }
2094
2095 #[cfg(test)]
2096 pub(crate) fn inject_module_slot_epoch(
2097 &self,
2098 endpoint: ModuleEndpointId,
2099 channel: u16,
2100 last_epoch: u32,
2101 ) {
2102 let mut inner = self.write_inner().expect("forwarding lock");
2103 inner
2104 .module_slot_epochs
2105 .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
2106 inner.next_module_channel.insert(endpoint, channel);
2107 }
2108
2109 #[cfg(test)]
2110 pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
2111 self.write_inner()
2112 .expect("forwarding lock")
2113 .next_control_corr
2114 .insert(endpoint, next_corr);
2115 }
2116
2117 pub(crate) fn cache_status(
2118 &self,
2119 endpoint: ModuleEndpointId,
2120 module_channel: u16,
2121 module_epoch: u32,
2122 status: String,
2123 ) -> Result<bool, ForwardingError> {
2124 let mut inner = self.write_inner()?;
2125 if !inner.module_id_by_endpoint.contains_key(&endpoint) {
2126 return Err(ForwardingError::StaleModuleEndpoint);
2127 }
2128
2129 let module_key = ModuleRouteKey {
2130 endpoint,
2131 channel: module_channel,
2132 };
2133 let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
2134 (route.module_epoch == module_epoch).then_some((
2135 ClientRouteKey {
2136 connection_id: route.client_connection_id,
2137 channel: route.client_channel,
2138 },
2139 route.client_epoch,
2140 ))
2141 } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
2142 (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
2143 client_key,
2144 inner
2145 .client_slot_epochs
2146 .get(&client_key)
2147 .copied()
2148 .unwrap_or(0),
2149 ))
2150 } else {
2151 None
2152 };
2153
2154 if let Some(handle) = handle {
2155 inner.status.insert(handle, status);
2156 Ok(true)
2157 } else {
2158 debug!(
2159 module_channel,
2160 module_epoch,
2161 generation = endpoint.generation,
2162 connection_id = endpoint.connection_id.get(),
2163 "dropping stale status update for module route handle"
2164 );
2165 Ok(false)
2166 }
2167 }
2168
2169 pub(crate) fn route_poll_snapshot(
2170 &self,
2171 client_connection_id: ConnectionId,
2172 client_channel: u16,
2173 client_epoch: u32,
2174 ) -> Result<RoutePollSnapshot, ForwardingError> {
2175 let inner = self.read_inner()?;
2176 let client_key = ClientRouteKey {
2177 connection_id: client_connection_id,
2178 channel: client_channel,
2179 };
2180 let Some(route) = inner.client_to_module.get(&client_key) else {
2181 return Ok(RoutePollSnapshot::Absent);
2182 };
2183 if route.client_epoch != client_epoch
2184 || !inner
2185 .module_id_by_endpoint
2186 .contains_key(&route.module_endpoint)
2187 {
2188 return Ok(RoutePollSnapshot::Absent);
2189 }
2190 Ok(RoutePollSnapshot::Bound {
2191 module_id: route.module_id.clone(),
2192 status: inner.status.get(&(client_key, client_epoch)).cloned(),
2193 })
2194 }
2195
2196 pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2197 Ok(self.read_inner()?.client_to_module.len())
2198 }
2199
2200 pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2210 let inner = self.read_inner()?;
2211 let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2212 for key in inner.client_to_module.keys() {
2213 *per_connection.entry(key.connection_id).or_insert(0) += 1;
2214 }
2215 let max = per_connection.values().copied().max().unwrap_or(0);
2216 Ok((per_connection.len(), max))
2217 }
2218
2219 pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2220 let inner = self.read_inner()?;
2221 Ok(inner
2222 .client_to_module
2223 .keys()
2224 .any(|key| key.channel == route_channel))
2225 }
2226
2227 pub(crate) fn daemon_drain_result_for_connection(
2231 &self,
2232 connection_id: ConnectionId,
2233 ) -> Result<Option<bool>, ForwardingError> {
2234 let inner = self.read_inner()?;
2235 Ok(inner.daemon_draining.then(|| {
2236 inner
2237 .endpoint_by_connection
2238 .get(&connection_id)
2239 .and_then(|endpoint| inner.daemon_drain_results.get(endpoint))
2240 .copied()
2241 .unwrap_or(false)
2242 }))
2243 }
2244
2245 #[cfg(unix)]
2246 pub(crate) fn record_daemon_drain_result(
2247 &self,
2248 endpoint: ModuleEndpointId,
2249 drained: bool,
2250 ) -> Result<(), ForwardingError> {
2251 let mut inner = self.write_inner()?;
2252 if let Some(result) = inner.daemon_drain_results.get_mut(&endpoint) {
2256 *result = drained;
2257 }
2258 Ok(())
2259 }
2260
2261 #[cfg(unix)]
2264 pub(crate) fn begin_daemon_drain(
2265 &self,
2266 ) -> Result<Vec<(String, ModuleEndpointId)>, ForwardingError> {
2267 let mut inner = self.write_inner()?;
2268 if inner.daemon_draining {
2269 return Ok(Vec::new());
2270 }
2271 inner.daemon_draining = true;
2272 let modules = inner
2273 .modules_by_id
2274 .iter()
2275 .map(|(id, module)| (id.clone(), module.endpoint))
2276 .collect::<Vec<_>>();
2277 for (_, endpoint) in &modules {
2278 inner.daemon_drain_results.insert(*endpoint, false);
2279 inner
2280 .draining_endpoints
2281 .insert(*endpoint, RouteCloseReason::Restart);
2282 }
2283 let off_slot_endpoints = inner
2288 .candidates_by_id
2289 .values()
2290 .map(|module| module.endpoint)
2291 .chain(inner.superseded_endpoints.keys().copied())
2292 .collect::<Vec<_>>();
2293 for endpoint in off_slot_endpoints {
2294 inner
2295 .draining_endpoints
2296 .insert(endpoint, RouteCloseReason::Restart);
2297 }
2298 Ok(modules)
2299 }
2300
2301 pub(crate) fn begin_module_drain(
2308 &self,
2309 module_id: &str,
2310 reason: RouteCloseReason,
2311 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2312 let mut inner = self.write_inner()?;
2313 let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2314 return Ok(None);
2315 };
2316 Ok(Some(begin_drain_locked(
2317 &mut inner, module_id, module, reason,
2318 )))
2319 }
2320
2321 pub(crate) fn begin_endpoint_drain(
2328 &self,
2329 endpoint: ModuleEndpointId,
2330 reason: RouteCloseReason,
2331 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2332 let mut inner = self.write_inner()?;
2333 let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2334 return Ok(None);
2335 };
2336 let module_id = inner
2337 .module_id_by_endpoint
2338 .get(&endpoint)
2339 .cloned()
2340 .expect("an endpoint resolved to a module connection has a module id");
2341 Ok(Some(begin_drain_locked(
2342 &mut inner, &module_id, module, reason,
2343 )))
2344 }
2345}
2346
2347fn begin_drain_locked(
2351 inner: &mut ForwardingInner,
2352 module_id: &str,
2353 module: ModuleConnection,
2354 reason: RouteCloseReason,
2355) -> ModuleDrainTarget {
2356 {
2357 let endpoint = module.endpoint;
2358 inner.draining_endpoints.insert(endpoint, reason);
2359
2360 let flows = inner
2361 .client_to_module
2362 .values()
2363 .filter(|route| route.module_endpoint == endpoint)
2364 .map(|route| Arc::clone(&route.flow))
2365 .collect::<Vec<_>>();
2366 let excluded_subscriptions = flows
2367 .into_iter()
2368 .map(|flow| flow.begin_drain())
2369 .fold(0u32, u32::saturating_add);
2370
2371 let pending_keys = inner
2372 .pending_relays
2373 .keys()
2374 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2375 .copied()
2376 .collect::<Vec<_>>();
2377 let mut abandoned_bindings = Vec::new();
2378 for key in pending_keys {
2379 let Some(pending) = inner.pending_relays.remove(&key) else {
2380 continue;
2381 };
2382 release_reserved_route_locked(
2383 inner,
2384 pending.reservation.client_key,
2385 pending.reservation.module_key,
2386 );
2387 if pending.relay_enqueued {
2388 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2389 abandoned_bindings.push(target);
2390 }
2391 } else {
2392 inner.relays_released_before_enqueue.insert(key);
2396 }
2397 let _ = pending
2398 .sender
2399 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
2400 "module_reloading",
2401 format!("module_id '{module_id}' is reloading"),
2402 )));
2403 }
2404
2405 let pending_control_keys = inner
2406 .pending_control_rpcs
2407 .keys()
2408 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2409 .copied()
2410 .collect::<Vec<_>>();
2411 for key in pending_control_keys {
2412 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2413 let _ = pending
2414 .sender
2415 .send(ModuleControlRpcOutcome::ModuleGone(format!(
2416 "module '{module_id}' began draining during module-control RPC"
2417 )));
2418 }
2419 }
2420
2421 ModuleDrainTarget {
2422 endpoint,
2423 sink: module.sink,
2424 negotiated_ver: module.negotiated_ver,
2425 abandoned_bindings,
2426 excluded_subscriptions,
2427 }
2428 }
2429}
2430
2431#[derive(Debug, Default, PartialEq, Eq)]
2434pub(crate) struct DrainHoldouts {
2435 pub(crate) requests: usize,
2438 pub(crate) routes: usize,
2440 pub(crate) total_routes: usize,
2442 pub(crate) top_connections: Vec<(u64, usize)>,
2445 pub(crate) held: Vec<(u16, u64)>,
2454}
2455
2456pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
2458
2459impl ForwardingTable {
2460 pub(crate) fn endpoint_drain_holdouts(
2462 &self,
2463 endpoint: ModuleEndpointId,
2464 ) -> Result<DrainHoldouts, ForwardingError> {
2465 let inner = self.read_inner()?;
2466 let mut holdouts = DrainHoldouts::default();
2467 let mut by_connection: HashMap<u64, usize> = HashMap::new();
2468 for (key, route) in &inner.client_to_module {
2469 if route.module_endpoint != endpoint {
2470 continue;
2471 }
2472 holdouts.total_routes += 1;
2473 let held = route.flow.drain_in_flight();
2474 if held == 0 {
2475 continue;
2476 }
2477 holdouts.requests += held;
2478 holdouts.routes += 1;
2479 *by_connection.entry(key.connection_id.get()).or_default() += held;
2480 holdouts.held.extend(
2481 route
2482 .flow
2483 .drain_held_corrs()
2484 .into_iter()
2485 .map(|corr| (route.module_channel, corr)),
2486 );
2487 }
2488 holdouts.held.sort_unstable();
2489 holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
2490 let mut connections = by_connection.into_iter().collect::<Vec<_>>();
2491 connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
2492 connections.truncate(3);
2493 holdouts.top_connections = connections;
2494 Ok(holdouts)
2495 }
2496
2497 pub(crate) fn endpoint_in_flight_count(
2498 &self,
2499 endpoint: ModuleEndpointId,
2500 ) -> Result<usize, ForwardingError> {
2501 let inner = self.read_inner()?;
2502 Ok(inner
2503 .client_to_module
2504 .values()
2505 .filter(|route| route.module_endpoint == endpoint)
2506 .map(|route| route.flow.drain_in_flight())
2507 .sum())
2508 }
2509
2510 pub(crate) fn endpoint_is_draining(
2511 &self,
2512 endpoint: ModuleEndpointId,
2513 ) -> Result<bool, ForwardingError> {
2514 Ok(self
2515 .read_inner()?
2516 .draining_endpoints
2517 .contains_key(&endpoint))
2518 }
2519
2520 pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
2521 let inner = self.read_inner()?;
2522 Ok(inner
2523 .modules_by_id
2524 .get(module_id)
2525 .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
2526 }
2527
2528 pub(crate) fn release_module_endpoint_routes(
2529 &self,
2530 endpoint: ModuleEndpointId,
2531 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2532 let mut inner = self.write_inner()?;
2533 let routes = inner
2534 .module_to_client
2535 .iter()
2536 .filter(|(module_key, _)| module_key.endpoint == endpoint)
2537 .map(|(module_key, route)| (*module_key, route.module_epoch))
2538 .collect::<Vec<_>>();
2539 let mut released = Vec::with_capacity(routes.len());
2540 for (module_key, epoch) in routes {
2541 if let RouteRelease::Removed(target) =
2542 release_module_route_locked(&mut inner, module_key, epoch)
2543 {
2544 released.push(target);
2545 }
2546 }
2547 Ok(released)
2548 }
2549
2550 pub(crate) fn endpoint_routes(
2556 &self,
2557 endpoint: ModuleEndpointId,
2558 ) -> Result<Vec<EndpointRoute>, ForwardingError> {
2559 let inner = self.read_inner()?;
2560 Ok(endpoint_routes_locked(&inner, endpoint))
2561 }
2562
2563 pub(crate) fn route_census(
2565 &self,
2566 module_id: Option<&str>,
2567 ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
2568 let inner = self.read_inner()?;
2569 let mut endpoints = inner
2570 .modules_by_id
2571 .iter()
2572 .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
2573 .map(|(id, module)| (id.clone(), module.endpoint))
2574 .collect::<Vec<_>>();
2575 endpoints.sort_by(|left, right| left.0.cmp(&right.0));
2576 Ok(endpoints
2577 .into_iter()
2578 .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
2579 .collect())
2580 }
2581
2582 pub(crate) fn live_roots(
2584 &self,
2585 module_id: &str,
2586 ) -> Result<ModuleControlResponseToModule, ForwardingError> {
2587 let inner = self.read_inner()?;
2588 let endpoint = inner
2589 .modules_by_id
2590 .get(module_id)
2591 .map(|module| module.endpoint);
2592 let mut roots = BTreeMap::new();
2593 let mut unknown_root_bindings = 0;
2594 let mut total_bindings = 0;
2595 if let Some(endpoint) = endpoint {
2596 for binding in inner
2597 .module_to_client
2598 .values()
2599 .filter(|binding| binding.module_endpoint == endpoint)
2600 {
2601 total_bindings += 1;
2602 let workdir = binding
2603 .scope
2604 .as_ref()
2605 .and_then(|scope| scope.workdir.as_deref())
2606 .map(std::path::Path::new);
2607 if let Some(root) = &binding.project_root {
2608 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2609 entry.0 += 1;
2610 } else if workdir.is_none() {
2611 unknown_root_bindings += 1;
2612 }
2613 if let Some(workdir) = workdir.filter(|workdir| {
2614 binding.project_root.as_ref().map(ProjectRootId::as_path) != Some(*workdir)
2615 }) {
2616 roots.entry(workdir.to_path_buf()).or_insert((0, 0)).0 += 1;
2617 }
2618 }
2619 for pending in inner
2620 .pending_relays
2621 .values()
2622 .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
2623 {
2624 total_bindings += 1;
2625 if let Some(root) = &pending.reservation.project_root {
2626 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2627 entry.1 += 1;
2628 } else {
2629 unknown_root_bindings += 1;
2630 }
2631 }
2632 }
2633 Ok(ModuleControlResponseToModule::LiveRoots {
2634 roots: roots
2635 .into_iter()
2636 .map(|(project_root, (bound, pending))| LiveRoot {
2637 project_root,
2638 bound,
2639 pending,
2640 })
2641 .collect(),
2642 unknown_root_bindings,
2643 total_bindings,
2644 })
2645 }
2646
2647 pub(crate) fn connection_has_client_routes(
2653 &self,
2654 connection_id: ConnectionId,
2655 ) -> Result<bool, ForwardingError> {
2656 let inner = self.read_inner()?;
2657 Ok(connection_has_client_routes_locked(&inner, connection_id))
2658 }
2659
2660 pub(crate) fn cleanup_connection(
2661 &self,
2662 connection_id: ConnectionId,
2663 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2664 self.cleanup_connection_counted(connection_id)
2665 .map(|cleanup| cleanup.released)
2666 }
2667
2668 pub(crate) fn cleanup_connection_counted(
2673 &self,
2674 connection_id: ConnectionId,
2675 ) -> Result<ConnectionCleanup, ForwardingError> {
2676 let mut inner = self.write_inner()?;
2677 inner.closing_connections.insert(connection_id);
2678 let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
2679 remove_module_connection_locked(&mut inner, endpoint)
2680 } else {
2681 ConnectionCleanup {
2682 released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
2683 abandoned_relays: 0,
2684 }
2685 };
2686 inner.closing_connections.remove(&connection_id);
2695 Ok(cleanup)
2696 }
2697
2698 fn cleanup_client_connection_locked(
2699 inner: &mut ForwardingInner,
2700 connection_id: ConnectionId,
2701 ) -> Vec<GoodbyeTarget> {
2702 let routes = inner
2703 .client_to_module
2704 .iter()
2705 .filter(|(key, _)| key.connection_id == connection_id)
2706 .map(|(key, route)| (*key, route.client_epoch))
2707 .collect::<Vec<_>>();
2708 let mut released = Vec::with_capacity(routes.len());
2709 for (client_key, epoch) in routes {
2710 if let RouteRelease::Removed(target) =
2711 release_client_route_locked(inner, client_key, epoch)
2712 {
2713 released.push(target);
2714 }
2715 }
2716
2717 let pending_keys = inner
2718 .pending_relays
2719 .iter()
2720 .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
2721 .map(|(key, _)| *key)
2722 .collect::<Vec<_>>();
2723 for key in pending_keys {
2724 let Some(pending) = inner.pending_relays.remove(&key) else {
2725 continue;
2726 };
2727 release_reserved_route_locked(
2728 inner,
2729 pending.reservation.client_key,
2730 pending.reservation.module_key,
2731 );
2732 if pending.relay_enqueued {
2733 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2734 released.push(target);
2735 }
2736 } else {
2737 inner.relays_released_before_enqueue.insert(key);
2741 }
2742 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
2743 "client connection closed during route.bind relay".to_string(),
2744 ));
2745 }
2746
2747 let orphaned = inner
2748 .reserved_client
2749 .iter()
2750 .filter(|(key, _)| key.connection_id == connection_id)
2751 .map(|(client, module)| (*client, *module))
2752 .collect::<Vec<_>>();
2753 for (client_key, module_key) in orphaned {
2754 release_reserved_route_locked(inner, client_key, module_key);
2755 }
2756 inner.next_client_channel.remove(&connection_id);
2757 inner
2758 .client_slot_epochs
2759 .retain(|key, _| key.connection_id != connection_id);
2760 inner
2761 .last_published_epoch
2762 .retain(|key, _| key.connection_id != connection_id);
2763 inner
2764 .status
2765 .retain(|(key, _), _| key.connection_id != connection_id);
2766
2767 released
2768 }
2769
2770 pub(crate) fn escalate_client_delivery_failure(
2779 &self,
2780 connection_id: ConnectionId,
2781 channel: u16,
2782 expected_epoch: u32,
2783 reason: CloseReason,
2784 undelivered: UndeliveredFrame<'_>,
2785 ) -> Result<bool, ForwardingError> {
2786 self.escalate_client_delivery_failure_diagnosed(
2787 connection_id,
2788 channel,
2789 expected_epoch,
2790 reason,
2791 undelivered,
2792 None,
2793 )
2794 }
2795
2796 pub(crate) fn escalate_client_delivery_failure_diagnosed(
2797 &self,
2798 connection_id: ConnectionId,
2799 channel: u16,
2800 expected_epoch: u32,
2801 reason: CloseReason,
2802 undelivered: UndeliveredFrame<'_>,
2803 diagnosis: Option<&crate::router::ClientDeliveryDiagnosis>,
2804 ) -> Result<bool, ForwardingError> {
2805 let principals = {
2806 let mut inner = self.write_inner()?;
2807 let key = ClientRouteKey {
2808 connection_id,
2809 channel,
2810 };
2811 if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
2812 None
2813 } else {
2814 inner.closing_connections.insert(connection_id);
2815 Some(connection_principals_locked(&inner, connection_id))
2816 }
2817 };
2818 let Some(principals) = principals else {
2819 return Ok(false);
2820 };
2821 let backlog = undelivered.sink.backlog();
2822 let close_reason = reason.to_string();
2823 if self.request_connection_close(connection_id, reason) {
2824 warn!(
2825 connection_id = connection_id.get(),
2826 principals = %principals,
2827 module_id = undelivered.module_id.unwrap_or("unknown"),
2828 client_channel = channel,
2829 queued_bytes = backlog.queued_bytes,
2830 queued_frames = backlog.queued_frames,
2831 frame_type = ?diagnosis.map(|failure| failure.frame_type),
2832 frame_flags = ?diagnosis.map(|failure| failure.flags.0),
2833 close_cause = diagnosis.map(|failure| failure.cause).unwrap_or("delivery_failed"),
2834 oldest_queued_ms = backlog
2835 .oldest_age
2836 .map(|age| age.as_millis() as u64)
2837 .unwrap_or(0),
2838 close_reason = %close_reason,
2839 "closing client connection: its egress queue could not take a frame"
2840 );
2841 }
2842 Ok(true)
2843 }
2844
2845 pub(crate) fn publish_scope_changes(
2859 &self,
2860 changes: &[ScopeTagChange],
2861 ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
2862 let mut inner = self.write_inner()?;
2863 let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
2864 for change in changes {
2865 let key = (change.owner.clone(), change.scope_ref.clone());
2866 match change.after {
2867 Some(tag) => {
2868 inner.scope_tags.insert(key, tag);
2869 }
2870 None => {
2871 inner.scope_tags.remove(&key);
2872 }
2873 }
2874 by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
2875 }
2876 let mut selected = Vec::new();
2877 for (client_key, route) in &inner.client_to_module {
2878 let Some(scope) = &route.scope else {
2879 continue;
2880 };
2881 let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
2882 else {
2883 continue;
2884 };
2885 if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
2886 continue;
2887 }
2888 let reason = match &change.drain {
2889 ScopeDrain::Nothing => continue,
2890 ScopeDrain::All(reason) => *reason,
2891 ScopeDrain::Carriers(narrowed) => {
2892 let owner = Principal::Reserved {
2893 module_id: scope.owner.clone(),
2894 };
2895 let hit = route.principal != owner
2896 && narrowed.iter().any(|(principal, allowed)| {
2897 *principal == route.principal
2898 && allowed
2899 .as_ref()
2900 .is_none_or(|targets| !targets.contains(&route.module_id))
2901 });
2902 if !hit {
2903 continue;
2904 }
2905 RouteCloseReason::ScopeCarrierRemoved
2906 }
2907 };
2908 selected.push((*client_key, route.client_epoch, reason));
2909 }
2910 let mut drained = Vec::new();
2911 for (client_key, client_epoch, reason) in selected {
2912 let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
2913 continue;
2914 };
2915 let release = release_client_route_locked(&mut inner, client_key, client_epoch);
2916 self.record_route_release(&release);
2917 if let RouteRelease::Removed(module) = release {
2918 drained.push(ScopeDrainedRoute {
2919 reason,
2920 scope: route
2921 .scope
2922 .clone()
2923 .expect("selected scope drain has a bound scope"),
2924 module_id: route.module_id.clone(),
2925 client: GoodbyeTarget {
2926 connection_id: route.client_connection_id,
2927 sink: route.client_sink.clone(),
2928 negotiated_ver: route.client_negotiated_ver,
2929 channel: route.client_channel,
2930 epoch: route.client_epoch,
2931 kind: GoodbyeTargetKind::Client,
2932 module_id: Some(route.module_id.clone()),
2933 },
2934 module,
2935 });
2936 }
2937 }
2938 Ok(drained)
2939 }
2940
2941 #[cfg(test)]
2943 pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
2944 self.read_inner()
2945 .ok()?
2946 .scope_tags
2947 .get(&(owner.to_string(), scope_ref.to_string()))
2948 .copied()
2949 }
2950
2951 fn record_route_release(&self, release: &RouteRelease) {
2952 match release {
2953 RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
2954 RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
2955 RouteRelease::Absent => {}
2956 }
2957 }
2958
2959 fn read_inner(&self) -> Result<RwLockReadGuard<'_, ForwardingInner>, ForwardingError> {
2960 self.inner.read().map_err(|_| ForwardingError::Poisoned)
2961 }
2962
2963 fn write_inner(&self) -> Result<RwLockWriteGuard<'_, ForwardingInner>, ForwardingError> {
2964 self.inner.write().map_err(|_| ForwardingError::Poisoned)
2965 }
2966
2967 fn lock_close_registry(
2968 &self,
2969 ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
2970 self.close_registry
2971 .lock()
2972 .unwrap_or_else(|poisoned| poisoned.into_inner())
2973 }
2974}
2975
2976impl ForwardingInner {
2977 fn allocate_route_slots(
2978 &mut self,
2979 connection_id: ConnectionId,
2980 endpoint: ModuleEndpointId,
2981 ) -> Result<(u16, u32, u16, u32), ForwardingError> {
2982 let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
2983 let mut client_channel = client_start;
2984 let client_channel = loop {
2985 let key = ClientRouteKey {
2986 connection_id,
2987 channel: client_channel,
2988 };
2989 let eligible = !self.client_to_module.contains_key(&key)
2990 && !self.reserved_client.contains_key(&key)
2991 && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2992 if eligible {
2993 break client_channel;
2994 }
2995 client_channel = next_channel(client_channel);
2996 if client_channel == client_start {
2997 return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
2998 }
2999 };
3000
3001 let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
3002 let mut module_channel = module_start;
3003 let module_channel = loop {
3004 let key = ModuleRouteKey {
3005 endpoint,
3006 channel: module_channel,
3007 };
3008 let eligible = !self.module_to_client.contains_key(&key)
3009 && !self.reserved_module.contains_key(&key)
3010 && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3011 if eligible {
3012 break module_channel;
3013 }
3014 module_channel = next_channel(module_channel);
3015 if module_channel == module_start {
3016 return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
3017 }
3018 };
3019
3020 let client_key = ClientRouteKey {
3021 connection_id,
3022 channel: client_channel,
3023 };
3024 let module_key = ModuleRouteKey {
3025 endpoint,
3026 channel: module_channel,
3027 };
3028 let client_epoch = self
3029 .client_slot_epochs
3030 .get(&client_key)
3031 .copied()
3032 .unwrap_or(0)
3033 + 1;
3034 let module_epoch = self
3035 .module_slot_epochs
3036 .get(&module_key)
3037 .copied()
3038 .unwrap_or(0)
3039 + 1;
3040 self.client_slot_epochs.insert(client_key, client_epoch);
3041 self.module_slot_epochs.insert(module_key, module_epoch);
3042 self.next_client_channel
3043 .insert(connection_id, next_channel(client_channel));
3044 self.next_module_channel
3045 .insert(endpoint, next_channel(module_channel));
3046 Ok((client_channel, client_epoch, module_channel, module_epoch))
3047 }
3048
3049 fn allocate_control_corr(
3050 &mut self,
3051 endpoint: ModuleEndpointId,
3052 ) -> Result<u64, ForwardingError> {
3053 let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
3054 if candidate == 0 {
3055 self.closing_connections.insert(endpoint.connection_id);
3056 return Err(ForwardingError::RelayCorrelationExhausted);
3057 }
3058 self.next_control_corr.insert(
3059 endpoint,
3060 if candidate == u64::MAX {
3061 0
3062 } else {
3063 candidate + 1
3064 },
3065 );
3066 Ok(candidate)
3067 }
3068}
3069
3070fn connection_has_client_routes_locked(
3071 inner: &ForwardingInner,
3072 connection_id: ConnectionId,
3073) -> bool {
3074 inner
3075 .client_to_module
3076 .keys()
3077 .any(|key| key.connection_id == connection_id)
3078 || inner
3079 .reserved_client
3080 .keys()
3081 .any(|key| key.connection_id == connection_id)
3082}
3083
3084fn check_module_connection_role_locked(
3085 inner: &ForwardingInner,
3086 connection_id: ConnectionId,
3087) -> Result<(), ForwardingError> {
3088 if inner.endpoint_by_connection.contains_key(&connection_id)
3089 || connection_has_client_routes_locked(inner, connection_id)
3090 {
3091 return Err(ForwardingError::ConnectionRoleConflict { connection_id });
3092 }
3093 Ok(())
3094}
3095
3096fn next_channel(channel: u16) -> u16 {
3097 let next = channel.wrapping_add(1);
3098 if next == 0 {
3099 1
3100 } else {
3101 next
3102 }
3103}
3104
3105fn endpoint_routes_locked(
3106 inner: &ForwardingInner,
3107 endpoint: ModuleEndpointId,
3108) -> Vec<EndpointRoute> {
3109 let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
3110 let draining = drain_reason.is_some();
3111 let mut routes = inner
3112 .module_to_client
3113 .iter()
3114 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3115 .map(|(_, route)| EndpointRoute {
3116 goodbye_target: GoodbyeTarget {
3117 connection_id: route.client_connection_id,
3118 sink: route.client_sink.clone(),
3119 negotiated_ver: route.client_negotiated_ver,
3120 channel: route.client_channel,
3121 epoch: route.client_epoch,
3122 kind: GoodbyeTargetKind::Client,
3123 module_id: Some(route.module_id.clone()),
3124 },
3125 principal: route.principal.clone(),
3126 bound_at: route.bound_at,
3127 draining,
3128 drain_reason,
3129 })
3130 .collect::<Vec<_>>();
3131 routes.sort_by_key(|route| {
3132 (
3133 route.goodbye_target.connection_id.get(),
3134 route.goodbye_target.channel,
3135 route.goodbye_target.epoch,
3136 )
3137 });
3138 routes
3139}
3140
3141fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
3145 let key = (scope.owner.clone(), scope.scope_ref.clone());
3146 match inner.scope_tags.get(&key) {
3147 Some(current) if *current == scope.tag => None,
3148 Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
3149 subc_protocol::error_codes::SCOPE_CHANGED,
3150 format!(
3151 "scope '{}' of {} changed while the route was being bound; re-open it",
3152 scope.scope_ref, scope.owner
3153 ),
3154 )),
3155 _ => Some(ErrorBody::new(
3156 subc_protocol::error_codes::SCOPE_ENDED,
3157 format!(
3158 "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
3159 scope.scope_ref, scope.owner, scope.tag.scope_epoch
3160 ),
3161 )),
3162 }
3163}
3164
3165fn release_reserved_route_locked(
3166 inner: &mut ForwardingInner,
3167 client_key: ClientRouteKey,
3168 module_key: ModuleRouteKey,
3169) {
3170 if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
3171 inner.reserved_client.remove(&client_key);
3172 }
3173 if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
3174 inner.reserved_module.remove(&module_key);
3175 }
3176 inner.status.retain(|(key, _), _| *key != client_key);
3177}
3178
3179fn release_client_route_locked(
3180 inner: &mut ForwardingInner,
3181 client_key: ClientRouteKey,
3182 expected_epoch: u32,
3183) -> RouteRelease {
3184 let Some(route) = inner.client_to_module.get(&client_key) else {
3185 return RouteRelease::Absent;
3186 };
3187 if route.client_epoch != expected_epoch {
3188 return RouteRelease::Stale;
3189 }
3190 let route = inner
3191 .client_to_module
3192 .remove(&client_key)
3193 .expect("route checked under the same forwarding lock");
3194 route.flow.close();
3195 if let Some(scope) = &route.scope {
3196 scope.release_bind();
3197 }
3198 inner.module_to_client.remove(&ModuleRouteKey {
3199 endpoint: route.module_endpoint,
3200 channel: route.module_channel,
3201 });
3202 inner.operator_confirms.route_closed(ModuleRouteKey {
3203 endpoint: route.module_endpoint,
3204 channel: route.module_channel,
3205 });
3206 inner.status.remove(&(client_key, expected_epoch));
3207 RouteRelease::Removed(GoodbyeTarget {
3208 connection_id: route.module_endpoint.connection_id,
3209 sink: route.module_sink.clone(),
3210 negotiated_ver: route.module_negotiated_ver,
3211 channel: route.module_channel,
3212 epoch: route.module_epoch,
3213 kind: GoodbyeTargetKind::Module,
3214 module_id: Some(route.module_id.clone()),
3215 })
3216}
3217
3218fn release_module_route_locked(
3219 inner: &mut ForwardingInner,
3220 module_key: ModuleRouteKey,
3221 expected_epoch: u32,
3222) -> RouteRelease {
3223 let Some(route) = inner.module_to_client.get(&module_key) else {
3224 return RouteRelease::Absent;
3225 };
3226 if route.module_epoch != expected_epoch {
3227 return RouteRelease::Stale;
3228 }
3229 let route = inner
3230 .module_to_client
3231 .remove(&module_key)
3232 .expect("route checked under the same forwarding lock");
3233 inner.operator_confirms.route_closed(module_key);
3234 route.flow.close();
3235 if let Some(scope) = &route.scope {
3236 scope.release_bind();
3237 }
3238 let client_key = ClientRouteKey {
3239 connection_id: route.client_connection_id,
3240 channel: route.client_channel,
3241 };
3242 inner.client_to_module.remove(&client_key);
3243 inner.status.remove(&(client_key, route.client_epoch));
3244 RouteRelease::Removed(GoodbyeTarget {
3245 connection_id: route.client_connection_id,
3246 sink: route.client_sink.clone(),
3247 negotiated_ver: route.client_negotiated_ver,
3248 channel: route.client_channel,
3249 epoch: route.client_epoch,
3250 kind: GoodbyeTargetKind::Client,
3251 module_id: Some(route.module_id.clone()),
3252 })
3253}
3254
3255fn commit_route_locked(
3256 inner: &mut ForwardingInner,
3257 pending: PendingRouteBindRelayEntry,
3258) -> Result<Option<GoodbyeTarget>, ForwardingError> {
3259 let reservation = pending.reservation;
3260 if inner
3261 .closing_connections
3262 .contains(&reservation.client_key.connection_id)
3263 {
3264 return Err(ForwardingError::ConnectionClosing {
3265 connection_id: reservation.client_key.connection_id,
3266 });
3267 }
3268 let module_id = inner
3269 .module_id_by_endpoint
3270 .get(&reservation.module_key.endpoint)
3271 .cloned()
3272 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3273 if inner
3274 .draining_endpoints
3275 .contains_key(&reservation.module_key.endpoint)
3276 {
3277 return Err(ForwardingError::ModuleReloading { module_id });
3278 }
3279 if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
3280 || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
3281 {
3282 return Err(ForwardingError::UnknownReservation {
3283 client_channel: reservation.client_key.channel,
3284 module_channel: reservation.module_key.channel,
3285 });
3286 }
3287 let module = inner
3288 .modules_by_id
3289 .get(&module_id)
3290 .filter(|module| module.endpoint == reservation.module_key.endpoint)
3291 .cloned()
3292 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3293 let binding = Arc::new(RouteBinding {
3294 client_connection_id: reservation.client_key.connection_id,
3295 client_sink: pending.client_sink,
3296 client_negotiated_ver: pending.client_negotiated_ver,
3297 client_channel: reservation.client_key.channel,
3298 client_epoch: reservation.client_epoch,
3299 module_id,
3300 module_endpoint: reservation.module_key.endpoint,
3301 module_sink: module.sink,
3302 module_negotiated_ver: module.negotiated_ver,
3303 module_channel: reservation.module_key.channel,
3304 module_epoch: reservation.module_epoch,
3305 principal: pending.principal,
3306 harness: pending.harness,
3307 build_class: pending.build_class,
3308 operator_app: pending.operator_app,
3309 project_root: reservation.project_root.clone(),
3310 bound_at: Instant::now(),
3311 flow: Arc::new(ChannelFlow::new(window_for(&module.concurrency))),
3312 scope: pending.scope,
3313 });
3314 inner
3315 .client_to_module
3316 .insert(reservation.client_key, Arc::clone(&binding));
3317 inner
3318 .module_to_client
3319 .insert(reservation.module_key, Arc::clone(&binding));
3320 let previous_published = inner
3321 .last_published_epoch
3322 .insert(reservation.client_key, reservation.client_epoch);
3323
3324 if let Some(scope) = &binding.scope {
3328 scope.record_served(&binding.module_id);
3329 }
3330
3331 let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
3336 if client_writer_closed {
3337 let abandoned = abandoned_route_target(inner, &reservation);
3341 if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
3342 route.flow.close();
3343 }
3344 inner.module_to_client.remove(&reservation.module_key);
3345 inner
3346 .status
3347 .remove(&(reservation.client_key, reservation.client_epoch));
3348 match previous_published {
3349 Some(epoch) => {
3350 inner
3351 .last_published_epoch
3352 .insert(reservation.client_key, epoch);
3353 }
3354 None => {
3355 inner.last_published_epoch.remove(&reservation.client_key);
3356 }
3357 }
3358 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3359 "client egress closed during route publication".to_string(),
3360 ));
3361 return Ok(abandoned);
3362 }
3363
3364 if let Some(scope) = &binding.scope {
3365 scope.commit_presence();
3366 }
3367 let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
3368 Ok(None)
3369}
3370
3371fn module_connection_for_endpoint_locked(
3379 inner: &ForwardingInner,
3380 endpoint: ModuleEndpointId,
3381) -> Option<&ModuleConnection> {
3382 let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
3383 inner
3384 .modules_by_id
3385 .get(module_id)
3386 .filter(|module| module.endpoint == endpoint)
3387 .or_else(|| {
3388 inner
3389 .candidates_by_id
3390 .get(module_id)
3391 .filter(|module| module.endpoint == endpoint)
3392 })
3393 .or_else(|| inner.superseded_endpoints.get(&endpoint))
3394}
3395
3396fn abandoned_route_target(
3397 inner: &ForwardingInner,
3398 reservation: &RouteReservation,
3399) -> Option<GoodbyeTarget> {
3400 let module_id = inner
3401 .module_id_by_endpoint
3402 .get(&reservation.module_key.endpoint)?;
3403 let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
3404 (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
3405 connection_id: module.endpoint.connection_id,
3406 sink: module.sink.clone(),
3407 negotiated_ver: module.negotiated_ver,
3408 channel: reservation.module_key.channel,
3409 epoch: reservation.module_epoch,
3410 kind: GoodbyeTargetKind::Module,
3411 module_id: Some(module_id.clone()),
3412 })
3413}
3414
3415fn enqueue_hello_ack_locked(
3420 sink: &FrameSink,
3421 connection_id: ConnectionId,
3422 hello_ack: Option<Frame>,
3423) -> Result<(), ForwardingError> {
3424 let Some(hello_ack) = hello_ack else {
3425 return Ok(());
3426 };
3427 sink.try_send(hello_ack)
3428 .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
3429}
3430
3431fn remove_module_connection_locked(
3432 inner: &mut ForwardingInner,
3433 endpoint: ModuleEndpointId,
3434) -> ConnectionCleanup {
3435 inner.operator_confirms.module_closed(endpoint);
3438 inner.draining_endpoints.remove(&endpoint);
3439 inner.daemon_drain_results.remove(&endpoint);
3440 let module_id = inner.module_id_by_endpoint.remove(&endpoint);
3441 if let Some(module_id) = module_id.as_ref() {
3442 if inner
3443 .modules_by_id
3444 .get(module_id)
3445 .is_some_and(|module| module.endpoint == endpoint)
3446 {
3447 inner.modules_by_id.remove(module_id);
3448 }
3449 if inner
3450 .candidates_by_id
3451 .get(module_id)
3452 .is_some_and(|module| module.endpoint == endpoint)
3453 {
3454 inner.candidates_by_id.remove(module_id);
3455 }
3456 }
3457 inner.superseded_endpoints.remove(&endpoint);
3458 inner.endpoint_by_connection.remove(&endpoint.connection_id);
3459 inner.next_module_channel.remove(&endpoint);
3460 inner.next_control_corr.remove(&endpoint);
3461 inner
3462 .health_probe_tombstones
3463 .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
3464 inner
3465 .module_slot_epochs
3466 .retain(|key, _| key.endpoint != endpoint);
3467 let reserved_module_keys: Vec<ModuleRouteKey> = inner
3468 .reserved_module
3469 .keys()
3470 .filter(|module_key| module_key.endpoint == endpoint)
3471 .copied()
3472 .collect();
3473 for module_key in reserved_module_keys {
3474 if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
3475 release_reserved_route_locked(inner, client_key, module_key);
3476 }
3477 }
3478
3479 let pending_keys: Vec<_> = inner
3480 .pending_relays
3481 .keys()
3482 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3483 .copied()
3484 .collect();
3485 let pending: Vec<_> = pending_keys
3486 .into_iter()
3487 .filter_map(|key| {
3488 inner
3489 .pending_relays
3490 .remove(&key)
3491 .map(|pending| (key, pending))
3492 })
3493 .collect();
3494 let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
3495 for (key, pending) in pending {
3496 if !pending.relay_enqueued {
3500 inner.relays_released_before_enqueue.insert(key);
3501 }
3502 let module_label = module_id.as_deref().unwrap_or("unknown");
3503 let _ = pending
3504 .sender
3505 .send(RouteBindRelayOutcome::ModuleGone(format!(
3506 "module '{module_label}' connection closed during route.bind relay"
3507 )));
3508 }
3509
3510 let pending_control_keys: Vec<_> = inner
3511 .pending_control_rpcs
3512 .keys()
3513 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3514 .copied()
3515 .collect();
3516 let pending_control: Vec<_> = pending_control_keys
3517 .into_iter()
3518 .filter_map(|key| inner.pending_control_rpcs.remove(&key))
3519 .collect();
3520 for pending in pending_control {
3521 let module_label = module_id.as_deref().unwrap_or("unknown");
3522 let _ = pending
3523 .sender
3524 .send(ModuleControlRpcOutcome::ModuleGone(format!(
3525 "module '{module_label}' connection closed during module-control RPC"
3526 )));
3527 }
3528
3529 let module_routes = inner
3530 .module_to_client
3531 .iter()
3532 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3533 .map(|(module_key, route)| (*module_key, route.module_epoch))
3534 .collect::<Vec<_>>();
3535 let mut released = Vec::with_capacity(module_routes.len());
3536 for (module_key, epoch) in module_routes {
3537 if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
3538 {
3539 released.push(target);
3540 }
3541 }
3542 ConnectionCleanup {
3543 released,
3544 abandoned_relays,
3545 }
3546}
3547
3548#[derive(Debug, Clone, Copy)]
3549struct RequestCredit {
3550 subscription: bool,
3551 excluded_from_drain: bool,
3552}
3553
3554#[derive(Debug, Default)]
3555struct CreditLedger {
3556 by_corr: HashMap<u64, Vec<RequestCredit>>,
3557}
3558
3559impl CreditLedger {
3560 fn acquire(&mut self, corr: u64, subscription: bool) {
3561 self.by_corr.entry(corr).or_default().push(RequestCredit {
3562 subscription,
3563 excluded_from_drain: false,
3564 });
3565 }
3566
3567 fn release(&mut self, corr: u64) -> bool {
3568 let Some(credits) = self.by_corr.get_mut(&corr) else {
3569 return false;
3570 };
3571 let released = credits.pop().is_some();
3572 if credits.is_empty() {
3573 self.by_corr.remove(&corr);
3574 }
3575 released
3576 }
3577
3578 fn capture_subscription_exclusions(&mut self) -> u32 {
3579 let mut excluded = 0u32;
3580 for credit in self.by_corr.values_mut().flatten() {
3581 if credit.subscription && !credit.excluded_from_drain {
3582 credit.excluded_from_drain = true;
3583 excluded = excluded.saturating_add(1);
3584 }
3585 }
3586 excluded
3587 }
3588
3589 #[cfg(test)]
3590 fn in_flight(&self) -> usize {
3591 self.by_corr.values().map(Vec::len).sum()
3592 }
3593
3594 fn drain_in_flight(&self) -> usize {
3595 self.by_corr
3596 .values()
3597 .flatten()
3598 .filter(|credit| !credit.excluded_from_drain)
3599 .count()
3600 }
3601
3602 fn drain_held_corrs(&self) -> Vec<u64> {
3605 let mut corrs = self
3606 .by_corr
3607 .iter()
3608 .flat_map(|(corr, credits)| {
3609 credits
3610 .iter()
3611 .filter(|credit| !credit.excluded_from_drain)
3612 .map(move |_| *corr)
3613 })
3614 .collect::<Vec<_>>();
3615 corrs.sort_unstable();
3616 corrs
3617 }
3618}
3619
3620#[derive(Debug, Default)]
3621struct ChannelFlowState {
3622 closed: bool,
3623 credits: CreditLedger,
3624}
3625
3626#[derive(Debug)]
3628pub(crate) struct ChannelFlow {
3629 sem: Semaphore,
3630 window: usize,
3631 state: Mutex<ChannelFlowState>,
3632}
3633
3634impl ChannelFlow {
3635 pub(crate) fn new(window: usize) -> Self {
3636 debug_assert!(window > 0, "flow-control window must be non-zero");
3637 Self {
3638 sem: Semaphore::new(window),
3639 window,
3640 state: Mutex::new(ChannelFlowState::default()),
3641 }
3642 }
3643
3644 #[cfg(test)]
3645 pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
3646 self.acquire_tagged(0, false).await
3647 }
3648
3649 pub(crate) async fn acquire_tagged(
3650 &self,
3651 corr: u64,
3652 subscription: bool,
3653 ) -> Result<(), ChannelFlowClosed> {
3654 let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
3655 let mut state = self
3656 .state
3657 .lock()
3658 .unwrap_or_else(|poisoned| poisoned.into_inner());
3659 if state.closed {
3660 return Err(ChannelFlowClosed);
3661 }
3662 state.credits.acquire(corr, subscription);
3663 permit.forget();
3664 Ok(())
3665 }
3666
3667 #[cfg(test)]
3668 pub(crate) fn release(&self) {
3669 self.release_corr(0);
3670 }
3671
3672 pub(crate) fn release_corr(&self, corr: u64) {
3673 let released = self
3674 .state
3675 .lock()
3676 .unwrap_or_else(|poisoned| poisoned.into_inner())
3677 .credits
3678 .release(corr);
3679 if !released {
3680 warn!(
3684 window = self.window,
3685 available = self.sem.available_permits(),
3686 "flow-control over-release ignored"
3687 );
3688 return;
3689 }
3690 if !self.sem.is_closed() {
3691 self.sem.add_permits(1);
3692 }
3693 }
3694
3695 #[cfg(test)]
3696 pub(crate) fn in_flight(&self) -> usize {
3697 self.state
3698 .lock()
3699 .unwrap_or_else(|poisoned| poisoned.into_inner())
3700 .credits
3701 .in_flight()
3702 }
3703
3704 pub(crate) fn drain_in_flight(&self) -> usize {
3705 self.state
3706 .lock()
3707 .unwrap_or_else(|poisoned| poisoned.into_inner())
3708 .credits
3709 .drain_in_flight()
3710 }
3711
3712 pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
3713 self.state
3714 .lock()
3715 .unwrap_or_else(|poisoned| poisoned.into_inner())
3716 .credits
3717 .drain_held_corrs()
3718 }
3719
3720 #[cfg(test)]
3721 pub(crate) fn available_permits(&self) -> usize {
3722 self.sem.available_permits()
3723 }
3724
3725 pub(crate) fn begin_drain(&self) -> u32 {
3726 let mut state = self
3727 .state
3728 .lock()
3729 .unwrap_or_else(|poisoned| poisoned.into_inner());
3730 state.closed = true;
3731 self.sem.close();
3732 state.credits.capture_subscription_exclusions()
3733 }
3734
3735 pub(crate) fn close(&self) {
3736 self.state
3737 .lock()
3738 .unwrap_or_else(|poisoned| poisoned.into_inner())
3739 .closed = true;
3740 self.sem.close();
3741 }
3742}
3743
3744#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3745pub(crate) struct ChannelFlowClosed;
3746
3747impl fmt::Display for ChannelFlowClosed {
3748 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3749 write!(f, "flow-control window closed")
3750 }
3751}
3752
3753impl Error for ChannelFlowClosed {}
3754
3755fn window_for(concurrency: &Concurrency) -> usize {
3756 match concurrency {
3757 Concurrency::Serial => 1,
3758 Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
3759 Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
3760 }
3761}
3762
3763#[derive(Debug, Clone, PartialEq, Eq)]
3764pub enum ForwardingError {
3765 ConnectionRoleConflict {
3768 connection_id: ConnectionId,
3769 },
3770 NoModuleConnection,
3771 ModuleReloading {
3772 module_id: String,
3773 },
3774 StaleModuleEndpoint,
3775 UnknownReservation {
3776 client_channel: u16,
3777 module_channel: u16,
3778 },
3779 ClientRouteChannelExhausted {
3780 connection_id: ConnectionId,
3781 },
3782 ModuleRouteChannelExhausted {
3783 endpoint: ModuleEndpointId,
3784 },
3785 RelayCorrelationExhausted,
3786 ConnectionClosing {
3787 connection_id: ConnectionId,
3788 },
3789 ClientEgressClosed {
3790 connection_id: ConnectionId,
3791 },
3792 RouteOpenBuild(String),
3793 CandidateSlotOccupied {
3795 module_id: String,
3796 },
3797 ModuleEgressUnavailable {
3800 connection_id: ConnectionId,
3801 },
3802 Poisoned,
3803}
3804
3805impl fmt::Display for ForwardingError {
3806 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3807 match self {
3808 Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
3809 Self::NoModuleConnection => write!(f, "no module connection is registered"),
3810 Self::ModuleReloading { module_id } => {
3811 write!(f, "module_id '{module_id}' is reloading")
3812 }
3813 Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
3814 Self::UnknownReservation {
3815 client_channel,
3816 module_channel,
3817 } => write!(
3818 f,
3819 "route reservation client channel {client_channel} / module channel {module_channel} was not found"
3820 ),
3821 Self::ClientRouteChannelExhausted { connection_id } => write!(
3822 f,
3823 "no client route channels are available for connection {}",
3824 connection_id.get()
3825 ),
3826 Self::ModuleRouteChannelExhausted { endpoint } => write!(
3827 f,
3828 "no module route channels are available for endpoint generation {} on connection {}",
3829 endpoint.generation,
3830 endpoint.connection_id.get()
3831 ),
3832 Self::RelayCorrelationExhausted => {
3833 write!(f, "module control correlation ids are exhausted")
3834 }
3835 Self::ConnectionClosing { connection_id } => write!(
3836 f,
3837 "connection {} is closing and cannot accept route allocation",
3838 connection_id.get()
3839 ),
3840 Self::ClientEgressClosed { connection_id } => write!(
3841 f,
3842 "client connection {} egress is closed",
3843 connection_id.get()
3844 ),
3845 Self::RouteOpenBuild(message) => {
3846 write!(f, "failed to prebuild route.open response: {message}")
3847 }
3848 Self::CandidateSlotOccupied { module_id } => write!(
3849 f,
3850 "module_id '{module_id}' already has a swap candidate registered"
3851 ),
3852 Self::ModuleEgressUnavailable { connection_id } => write!(
3853 f,
3854 "module connection {} egress is unavailable; HELLO_ACK could not be queued",
3855 connection_id.get()
3856 ),
3857 Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
3858 }
3859 }
3860}
3861
3862impl Error for ForwardingError {}
3863
3864#[cfg(test)]
3865mod tests {
3866 use std::time::Duration;
3867
3868 use super::*;
3869 use tokio::sync::mpsc;
3870
3871 #[test]
3872 fn ordinary_long_running_request_is_not_excluded_from_drain() {
3873 let mut ledger = CreditLedger::default();
3874 ledger.acquire(1, false);
3875
3876 assert_eq!(ledger.capture_subscription_exclusions(), 0);
3877 assert_eq!(ledger.drain_in_flight(), 1);
3878 }
3879
3880 #[test]
3881 fn bit_set_subscription_is_excluded_and_counted() {
3882 let mut ledger = CreditLedger::default();
3883 ledger.acquire(1, true);
3884
3885 assert_eq!(ledger.capture_subscription_exclusions(), 1);
3886 assert_eq!(ledger.drain_in_flight(), 0);
3887 }
3888
3889 #[test]
3890 fn subscription_opened_after_drain_snapshot_is_not_excluded() {
3891 let mut ledger = CreditLedger::default();
3892 ledger.acquire(1, true);
3893 assert_eq!(ledger.capture_subscription_exclusions(), 1);
3894
3895 ledger.acquire(2, true);
3896
3897 assert_eq!(ledger.drain_in_flight(), 1);
3898 }
3899
3900 #[test]
3901 fn drain_with_no_subscriptions_reports_zero_excluded() {
3902 let mut ledger = CreditLedger::default();
3903 assert_eq!(ledger.capture_subscription_exclusions(), 0);
3904 }
3905
3906 fn test_hello_ack(corr: u64) -> Frame {
3907 Frame::build(
3908 FrameType::HelloAck,
3909 Flags::new(false, Priority::Passive, false),
3910 0,
3911 0,
3912 corr,
3913 Vec::new(),
3914 )
3915 .unwrap()
3916 }
3917
3918 #[test]
3922 fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
3923 let forwarding = ForwardingTable::default();
3924
3925 let (closed_tx, closed_rx) = mpsc::channel(8);
3926 drop(closed_rx);
3927 let closed = ConnectionId::new(1);
3928 assert_eq!(
3929 forwarding.register_module_connection_acked(
3930 closed,
3931 "closed".to_string(),
3932 2,
3933 Concurrency::ModuleManaged,
3934 FrameSink::new(closed_tx),
3935 test_hello_ack(1),
3936 ),
3937 Err(ForwardingError::ModuleEgressUnavailable {
3938 connection_id: closed
3939 })
3940 );
3941
3942 let (full_tx, _full_rx) = mpsc::channel(1);
3943 let full_sink = FrameSink::new(full_tx);
3944 full_sink.try_send(test_hello_ack(99)).unwrap();
3945 let full = ConnectionId::new(2);
3946 assert_eq!(
3947 forwarding.register_module_connection_acked(
3948 full,
3949 "full".to_string(),
3950 2,
3951 Concurrency::ModuleManaged,
3952 full_sink.clone(),
3953 test_hello_ack(2),
3954 ),
3955 Err(ForwardingError::ModuleEgressUnavailable {
3956 connection_id: full
3957 })
3958 );
3959 assert_eq!(
3960 forwarding.register_candidate_module_connection_acked(
3961 full,
3962 "full".to_string(),
3963 2,
3964 Concurrency::ModuleManaged,
3965 full_sink,
3966 test_hello_ack(3),
3967 ),
3968 Err(ForwardingError::ModuleEgressUnavailable {
3969 connection_id: full
3970 })
3971 );
3972
3973 for (connection, module_id) in [(closed, "closed"), (full, "full")] {
3974 assert_eq!(
3975 forwarding
3976 .module_endpoint_for_connection(connection)
3977 .unwrap(),
3978 None
3979 );
3980 let (client_tx, _client_rx) = mpsc::channel(8);
3981 assert_eq!(
3982 forwarding
3983 .begin_route_bind_relay_for_test(
3984 ConnectionId::new(50),
3985 FrameSink::new(client_tx),
3986 1,
3987 module_id,
3988 )
3989 .err(),
3990 Some(ForwardingError::NoModuleConnection)
3991 );
3992 }
3993 assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
3994 }
3995
3996 #[test]
3999 fn acked_registration_queues_the_hello_ack_first() {
4000 let forwarding = ForwardingTable::default();
4001 let (active_tx, mut active_rx) = mpsc::channel(8);
4002 forwarding
4003 .register_module_connection_acked(
4004 ConnectionId::new(1),
4005 "acked".to_string(),
4006 2,
4007 Concurrency::ModuleManaged,
4008 FrameSink::new(active_tx),
4009 test_hello_ack(11),
4010 )
4011 .unwrap();
4012 let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
4013 forwarding
4014 .register_candidate_module_connection_acked(
4015 ConnectionId::new(2),
4016 "acked".to_string(),
4017 2,
4018 Concurrency::ModuleManaged,
4019 FrameSink::new(candidate_tx),
4020 test_hello_ack(12),
4021 )
4022 .unwrap();
4023
4024 let active_first = active_rx.try_recv().unwrap().frame;
4025 assert_eq!(active_first.header.ty, FrameType::HelloAck);
4026 assert_eq!(active_first.header.corr, 11);
4027 let candidate_first = candidate_rx.try_recv().unwrap().frame;
4028 assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
4029 assert_eq!(candidate_first.header.corr, 12);
4030 }
4031
4032 #[test]
4033 fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
4034 let forwarding = ForwardingTable::default();
4035 let module_connection = ConnectionId::new(10);
4036 let exhausted_client = ConnectionId::new(20);
4037 let second_client = ConnectionId::new(30);
4038 let (module_tx, _module_rx) = mpsc::channel(1);
4039 let endpoint = forwarding
4040 .register_module_connection(
4041 module_connection,
4042 "route-limit-provider".to_string(),
4043 1,
4044 Concurrency::ModuleManaged,
4045 FrameSink::new(module_tx),
4046 )
4047 .unwrap();
4048
4049 {
4050 let mut inner = forwarding.inner.write().unwrap();
4051 for channel in 1..=u16::MAX {
4052 inner.reserved_client.insert(
4053 ClientRouteKey {
4054 connection_id: exhausted_client,
4055 channel,
4056 },
4057 ModuleRouteKey {
4058 endpoint,
4059 channel: 1,
4060 },
4061 );
4062 }
4063 }
4064
4065 let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
4066 let err = forwarding
4067 .begin_route_bind_relay_for_test(
4068 exhausted_client,
4069 FrameSink::new(exhausted_tx),
4070 1,
4071 "route-limit-provider",
4072 )
4073 .unwrap_err();
4074 assert!(matches!(
4075 err,
4076 ForwardingError::ClientRouteChannelExhausted { connection_id }
4077 if connection_id == exhausted_client
4078 ));
4079
4080 let (second_tx, _second_rx) = mpsc::channel(1);
4081 let pending = forwarding
4082 .begin_route_bind_relay_for_test(
4083 second_client,
4084 FrameSink::new(second_tx),
4085 2,
4086 "route-limit-provider",
4087 )
4088 .unwrap();
4089 assert_eq!(pending.client_channel, 1);
4090 }
4091
4092 #[test]
4093 fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
4094 let forwarding = ForwardingTable::default();
4095 let module_connection = ConnectionId::new(40);
4096 let client = ConnectionId::new(50);
4097 let (module_tx, _module_rx) = mpsc::channel(1);
4098 forwarding
4099 .register_module_connection(
4100 module_connection,
4101 "slot-reuse-provider".to_string(),
4102 1,
4103 Concurrency::ModuleManaged,
4104 FrameSink::new(module_tx),
4105 )
4106 .unwrap();
4107
4108 let (client_tx, _client_rx) = mpsc::channel(1);
4109 let client_sink = FrameSink::new(client_tx);
4110 let mut wrapped_channel = None;
4111 for index in 0..=usize::from(u16::MAX) {
4112 let pending = forwarding
4113 .begin_route_bind_relay_for_test(
4114 client,
4115 client_sink.clone(),
4116 index as u64 + 1,
4117 "slot-reuse-provider",
4118 )
4119 .unwrap();
4120 if index == usize::from(u16::MAX) {
4121 wrapped_channel = Some(pending.module_channel);
4122 }
4123 forwarding
4124 .abort_pending_relay(
4125 pending.endpoint,
4126 pending.corr,
4127 RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
4128 )
4129 .unwrap();
4130 }
4131
4132 assert_eq!(wrapped_channel, Some(1));
4133 }
4134
4135 #[test]
4136 fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
4137 let forwarding = ForwardingTable::default();
4138 let client = ConnectionId::new(60);
4139 forwarding
4140 .inner
4141 .write()
4142 .unwrap()
4143 .next_client_channel
4144 .insert(client, 41);
4145
4146 let released = forwarding.cleanup_connection(client).unwrap();
4147
4148 assert!(released.is_empty());
4149 assert!(!forwarding
4150 .inner
4151 .read()
4152 .unwrap()
4153 .next_client_channel
4154 .contains_key(&client));
4155 }
4156
4157 #[test]
4158 fn stale_module_cleanup_preserves_fast_reconnect_successor() {
4159 let forwarding = ForwardingTable::default();
4160 let module_id = "fast-reconnect-provider";
4161 let first_connection = ConnectionId::new(70);
4162 let second_connection = ConnectionId::new(80);
4163 let (first_tx, _first_rx) = mpsc::channel(1);
4164 let first_endpoint = forwarding
4165 .register_module_connection(
4166 first_connection,
4167 module_id.to_string(),
4168 1,
4169 Concurrency::ModuleManaged,
4170 FrameSink::new(first_tx),
4171 )
4172 .unwrap();
4173 let (second_tx, _second_rx) = mpsc::channel(1);
4174 let second_endpoint = forwarding
4175 .register_module_connection(
4176 second_connection,
4177 module_id.to_string(),
4178 1,
4179 Concurrency::ModuleManaged,
4180 FrameSink::new(second_tx),
4181 )
4182 .unwrap();
4183 assert_ne!(first_endpoint, second_endpoint);
4184
4185 let released = forwarding.cleanup_connection(first_connection).unwrap();
4186
4187 assert!(released.is_empty());
4188 assert_eq!(
4189 forwarding
4190 .inner
4191 .read()
4192 .unwrap()
4193 .modules_by_id
4194 .get(module_id)
4195 .map(|module| module.endpoint),
4196 Some(second_endpoint)
4197 );
4198 assert!(forwarding.has_live_module_connection(module_id).unwrap());
4199 let control_rpc = forwarding
4200 .begin_module_control_rpc_for(
4201 module_id,
4202 "health.check",
4203 Instant::now() + Duration::from_secs(1),
4204 )
4205 .unwrap();
4206 assert_eq!(control_rpc.endpoint, second_endpoint);
4207 }
4208
4209 fn route_fixture(
4210 module_id: &str,
4211 ) -> (
4212 ForwardingTable,
4213 ConnectionId,
4214 ModuleEndpointId,
4215 ConnectionId,
4216 FrameSink,
4217 mpsc::Receiver<crate::router::OutboundFrame>,
4218 ) {
4219 let forwarding = ForwardingTable::default();
4220 let module_connection = ConnectionId::new(100);
4221 let client_connection = ConnectionId::new(200);
4222 let (module_tx, _module_rx) = mpsc::channel(8);
4223 let endpoint = forwarding
4224 .register_module_connection(
4225 module_connection,
4226 module_id.to_string(),
4227 2,
4228 Concurrency::ModuleManaged,
4229 FrameSink::new(module_tx),
4230 )
4231 .unwrap();
4232 let (client_tx, client_rx) = mpsc::channel(8);
4233 (
4234 forwarding,
4235 module_connection,
4236 endpoint,
4237 client_connection,
4238 FrameSink::new(client_tx),
4239 client_rx,
4240 )
4241 }
4242
4243 #[test]
4244 #[cfg(unix)]
4245 fn daemon_drain_gates_current_and_racing_provider_registrations() {
4246 let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
4247 assert_eq!(
4248 forwarding.begin_daemon_drain().unwrap(),
4249 [("provider".to_string(), endpoint)]
4250 );
4251 assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
4252 assert!(matches!(
4253 forwarding.register_module_connection(
4254 ConnectionId::new(300),
4255 "late-provider".into(),
4256 2,
4257 Concurrency::ModuleManaged,
4258 sink,
4259 ),
4260 Err(ForwardingError::ConnectionClosing { .. })
4261 ));
4262 }
4263
4264 #[test]
4265 #[cfg(unix)]
4266 fn daemon_drain_results_are_endpoint_fenced_and_removed_on_cleanup() {
4267 let (forwarding, connection, endpoint, _, _, _) = route_fixture("provider");
4268 assert_eq!(
4269 forwarding
4270 .daemon_drain_result_for_connection(connection)
4271 .unwrap(),
4272 None
4273 );
4274 forwarding
4275 .record_daemon_drain_result(endpoint, true)
4276 .unwrap();
4277 forwarding.begin_daemon_drain().unwrap();
4278 assert_eq!(
4279 forwarding
4280 .daemon_drain_result_for_connection(connection)
4281 .unwrap(),
4282 Some(false)
4283 );
4284 let replacement = ModuleEndpointId {
4285 generation: endpoint.generation + 1,
4286 ..endpoint
4287 };
4288 forwarding
4289 .record_daemon_drain_result(replacement, true)
4290 .unwrap();
4291 assert_eq!(
4292 forwarding
4293 .daemon_drain_result_for_connection(connection)
4294 .unwrap(),
4295 Some(false)
4296 );
4297 forwarding
4298 .record_daemon_drain_result(endpoint, true)
4299 .unwrap();
4300 assert_eq!(
4301 forwarding
4302 .daemon_drain_result_for_connection(connection)
4303 .unwrap(),
4304 Some(true)
4305 );
4306 assert_eq!(
4307 forwarding
4308 .daemon_drain_result_for_connection(ConnectionId::new(300))
4309 .unwrap(),
4310 Some(false)
4311 );
4312 forwarding.cleanup_connection(connection).unwrap();
4313 forwarding
4314 .record_daemon_drain_result(endpoint, true)
4315 .unwrap();
4316 assert_eq!(
4317 forwarding
4318 .daemon_drain_result_for_connection(connection)
4319 .unwrap(),
4320 Some(false)
4321 );
4322 assert!(forwarding
4323 .read_inner()
4324 .unwrap()
4325 .daemon_drain_results
4326 .is_empty());
4327 }
4328
4329 #[test]
4330 #[cfg(unix)]
4331 fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
4332 let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
4333 route_fixture("provider");
4334 let mut pending =
4335 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
4336 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
4337 forwarding.begin_daemon_drain().unwrap();
4338 let completion = forwarding
4339 .complete_pending_relay(
4340 module_connection,
4341 pending.corr,
4342 RouteBindRelayOutcome::Accepted,
4343 )
4344 .expect("draining admission is not a fatal module error");
4345 assert!(completion.settled);
4346 let abandoned = completion
4347 .abandoned
4348 .expect("late accepted bind needs a route GOODBYE");
4349 assert_eq!(abandoned.channel, pending.module_channel);
4350 assert!(matches!(pending.receiver.try_recv().unwrap(),
4351 RouteBindRelayOutcome::DaemonRejected(error) if error.code == "module_reloading"));
4352 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4353 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4354 assert!(
4355 rx.try_recv().is_err(),
4356 "no route.open may be published during drain"
4357 );
4358 assert_eq!(
4359 forwarding
4360 .module_endpoint_for_connection(module_connection)
4361 .unwrap(),
4362 Some(endpoint)
4363 );
4364 assert!(
4365 !forwarding
4366 .complete_pending_relay(
4367 module_connection,
4368 pending.corr,
4369 RouteBindRelayOutcome::Accepted,
4370 )
4371 .unwrap()
4372 .settled
4373 );
4374 }
4375
4376 fn test_ping(corr: u64) -> Frame {
4377 Frame::build(
4378 FrameType::Ping,
4379 Flags::new(false, Priority::Passive, false),
4380 0,
4381 0,
4382 corr,
4383 Vec::new(),
4384 )
4385 .unwrap()
4386 }
4387
4388 fn begin_test_route(
4389 forwarding: &ForwardingTable,
4390 client_connection: ConnectionId,
4391 client_sink: FrameSink,
4392 corr: u64,
4393 module_id: &str,
4394 ) -> PendingRouteBindRelay {
4395 forwarding
4396 .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
4397 .unwrap()
4398 }
4399
4400 #[test]
4401 fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
4402 for candidate in [false, true] {
4403 for register_first in [false, true] {
4404 let (forwarding, module_connection, _, client_connection, sink, _rx) =
4405 route_fixture("target");
4406 let register = || {
4407 if candidate {
4408 forwarding.register_candidate_module_connection(
4409 client_connection,
4410 "source".into(),
4411 2,
4412 Concurrency::ModuleManaged,
4413 sink.clone(),
4414 )
4415 } else {
4416 forwarding.register_module_connection(
4417 client_connection,
4418 "source".into(),
4419 2,
4420 Concurrency::ModuleManaged,
4421 sink.clone(),
4422 )
4423 }
4424 };
4425 if register_first {
4426 register().unwrap();
4427 assert!(
4428 forwarding
4429 .begin_route_bind_relay_for_test(
4430 client_connection,
4431 sink.clone(),
4432 1,
4433 "target",
4434 )
4435 .is_err(),
4436 "a deferred route.open cannot reserve after HELLO"
4437 );
4438 assert!(forwarding
4439 .cleanup_connection(client_connection)
4440 .unwrap()
4441 .is_empty());
4442 } else {
4443 let pending =
4444 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
4445 assert!(
4446 register().is_err(),
4447 "HELLO cannot register after a route reservation"
4448 );
4449 forwarding
4450 .complete_pending_relay(
4451 module_connection,
4452 pending.corr,
4453 RouteBindRelayOutcome::Accepted,
4454 )
4455 .unwrap();
4456 assert_eq!(
4457 forwarding
4458 .cleanup_connection(client_connection)
4459 .unwrap()
4460 .len(),
4461 1
4462 );
4463 }
4464 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4465 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4466 }
4467 }
4468 }
4469
4470 #[tokio::test]
4476 async fn pending_route_open_completes_behind_queued_data_frames() {
4477 assert_eq!(
4478 crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
4479 8,
4480 "the per-connection pending route.open limit is its own constant"
4481 );
4482 let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
4483 route_fixture("open-behind-data");
4484 let (sink, mut client_rx) = crate::server::connection_egress();
4485 const DATA_FRAMES: usize = 1_000;
4486 let data = |corr: u64| {
4487 Frame::build(
4488 FrameType::StreamData,
4489 Flags::new(false, Priority::Interactive, false),
4490 9,
4491 1,
4492 corr,
4493 vec![b'x'; 200],
4494 )
4495 .unwrap()
4496 };
4497 for corr in 0..DATA_FRAMES as u64 {
4498 sink.try_send(data(corr)).unwrap();
4499 }
4500 let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
4501 assert_eq!(sink.backlog().queued_bytes, data_bytes);
4502
4503 let pending = tokio::time::timeout(
4504 Duration::from_secs(5),
4505 forwarding.begin_route_bind_relay_for(
4506 client,
4507 sink.clone(),
4508 subc_protocol::PROTOCOL_VERSION,
4509 4_242,
4510 "open-behind-data",
4511 Principal::Direct,
4512 None,
4513 None,
4514 "fixture".into(),
4515 None,
4516 None,
4517 Instant::now() + Duration::from_secs(60),
4518 ),
4519 )
4520 .await
4521 .expect("reserving the route.open slot must not wait behind data frames")
4522 .unwrap();
4523 forwarding
4524 .complete_pending_relay(
4525 module_connection,
4526 pending.corr,
4527 RouteBindRelayOutcome::Accepted,
4528 )
4529 .unwrap();
4530
4531 let backlog = sink.backlog();
4532 assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
4533 assert!(
4534 backlog.queued_bytes > data_bytes,
4535 "the route.open response must be counted in queued bytes: {backlog:?}"
4536 );
4537 for corr in 0..DATA_FRAMES as u64 {
4538 assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
4539 }
4540 let open = client_rx.recv().await.unwrap();
4541 assert_eq!(open.header.corr, 4_242);
4542 assert_eq!(open.header.ty, FrameType::Response);
4543 drop(open);
4544 assert_eq!(sink.backlog().queued_bytes, 0);
4545 assert_eq!(sink.backlog().queued_frames, 0);
4546 }
4547
4548 #[tokio::test]
4553 async fn drain_holdouts_count_held_requests_and_name_the_connection() {
4554 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4555 route_fixture("holdouts");
4556 let mut bound = |corr| {
4557 let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
4558 forwarding
4559 .complete_pending_relay(
4560 module_connection,
4561 route.corr,
4562 RouteBindRelayOutcome::Accepted,
4563 )
4564 .unwrap();
4565 client_rx.try_recv().unwrap();
4566 match forwarding
4567 .lookup_data_route(client, route.client_channel, route.client_epoch)
4568 .unwrap()
4569 {
4570 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4571 other => panic!("expected live route, got {other:?}"),
4572 }
4573 };
4574 let holding = bound(61);
4575 let _idle = bound(62);
4576 holding.flow.acquire_tagged(7, false).await.unwrap();
4577 holding.flow.acquire_tagged(2, false).await.unwrap();
4578 holding.flow.acquire_tagged(3, true).await.unwrap();
4579 forwarding
4580 .begin_module_drain("holdouts", RouteCloseReason::Restart)
4581 .unwrap();
4582
4583 let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
4584 assert_eq!(
4585 holdouts,
4586 DrainHoldouts {
4587 requests: 2,
4588 routes: 1,
4589 total_routes: 2,
4590 top_connections: vec![(client.get(), 2)],
4591 held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
4594 }
4595 );
4596 }
4597
4598 #[test]
4599 fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
4600 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4601 route_fixture("census");
4602 let pending = begin_test_route(&forwarding, client, sink, 1, "census");
4603 forwarding
4604 .complete_pending_relay(
4605 module_connection,
4606 pending.corr,
4607 RouteBindRelayOutcome::Accepted,
4608 )
4609 .unwrap();
4610
4611 let routes = forwarding.endpoint_routes(endpoint).unwrap();
4612 assert_eq!(routes.len(), 1);
4613 assert!(matches!(routes[0].principal, Principal::Direct));
4614 assert_eq!(routes[0].goodbye_target.connection_id, client);
4615 assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
4616 assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
4617 assert!(!routes[0].draining);
4618
4619 forwarding
4620 .begin_module_drain("census", RouteCloseReason::Restart)
4621 .unwrap();
4622 let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
4623 assert_eq!(draining_routes.len(), 1);
4624 assert!(draining_routes[0].draining);
4625 }
4626
4627 #[test]
4628 fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
4629 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
4630 let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
4631 assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
4632 forwarding
4633 .abort_pending_relay(
4634 first.endpoint,
4635 first.corr,
4636 RouteBindRelayOutcome::ModuleGone("abort".into()),
4637 )
4638 .unwrap();
4639 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4640 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4641
4642 let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
4643 assert_eq!(second.client_channel, first.client_channel);
4644 assert_eq!(second.module_channel, first.module_channel);
4645 assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
4646 }
4647
4648 #[test]
4649 fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
4650 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4651 route_fixture("epoch-release");
4652 let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
4653 forwarding
4654 .complete_pending_relay(
4655 module_connection,
4656 first.corr,
4657 RouteBindRelayOutcome::Accepted,
4658 )
4659 .unwrap();
4660 assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
4661 assert!(matches!(
4662 forwarding
4663 .release_client_route(client, first.client_channel, first.client_epoch)
4664 .unwrap(),
4665 RouteRelease::Removed(_)
4666 ));
4667 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4668 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4669
4670 let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
4671 forwarding
4672 .complete_pending_relay(
4673 module_connection,
4674 second.corr,
4675 RouteBindRelayOutcome::Accepted,
4676 )
4677 .unwrap();
4678 assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
4679 assert!(matches!(
4680 forwarding
4681 .release_client_route(client, second.client_channel, first.client_epoch)
4682 .unwrap(),
4683 RouteRelease::Stale
4684 ));
4685 assert!(!forwarding
4686 .cache_status(
4687 endpoint,
4688 second.module_channel,
4689 first.module_epoch,
4690 "stale".into(),
4691 )
4692 .unwrap());
4693 assert!(forwarding
4694 .cache_status(
4695 endpoint,
4696 second.module_channel,
4697 second.module_epoch,
4698 "current".into(),
4699 )
4700 .unwrap());
4701 match forwarding
4702 .route_poll_snapshot(client, second.client_channel, second.client_epoch)
4703 .unwrap()
4704 {
4705 RoutePollSnapshot::Bound { status, .. } => {
4706 assert_eq!(status.as_deref(), Some("current"));
4707 }
4708 RoutePollSnapshot::Absent => panic!("successor binding was removed"),
4709 }
4710 let counters = forwarding.counters().snapshot();
4711 assert_eq!(counters["route_released_epoch_fenced"], 1);
4712 assert_eq!(counters["route_release_stale_skipped"], 1);
4713 }
4714
4715 #[test]
4716 fn max_epoch_reservation_retires_only_that_slot() {
4717 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
4718 forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
4719 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
4720 let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
4721 assert_eq!(
4722 (final_use.client_channel, final_use.client_epoch),
4723 (7, u32::MAX)
4724 );
4725 assert_eq!(
4726 (final_use.module_channel, final_use.module_epoch),
4727 (9, u32::MAX)
4728 );
4729 forwarding
4730 .abort_pending_relay(
4731 endpoint,
4732 final_use.corr,
4733 RouteBindRelayOutcome::ModuleGone("abort".into()),
4734 )
4735 .unwrap();
4736 forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
4737 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
4738 let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
4739 assert_ne!(next.client_channel, 7);
4740 assert_ne!(next.module_channel, 9);
4741 assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
4742 }
4743
4744 #[test]
4745 fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
4746 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4747 route_fixture("corr-shared");
4748 let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
4749 assert_eq!(bind.corr, 1);
4750 forwarding
4751 .abort_pending_relay(
4752 endpoint,
4753 bind.corr,
4754 RouteBindRelayOutcome::ModuleGone("abort".into()),
4755 )
4756 .unwrap();
4757 let rpc = forwarding
4758 .begin_module_control_rpc_for(
4759 "corr-shared",
4760 "health.check",
4761 Instant::now() - Duration::from_millis(1),
4762 )
4763 .unwrap();
4764 assert_eq!(rpc.corr, 2);
4765 assert_eq!(
4766 forwarding
4767 .complete_module_control_rpc(
4768 module_connection,
4769 rpc.corr,
4770 Some("health.check"),
4771 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
4772 status: subc_protocol::session::HealthStatus::Ok,
4773 detail: None,
4774 metrics: None,
4775 }),
4776 )
4777 .unwrap(),
4778 ModuleControlRpcCompletion::Settled
4779 );
4780 assert!(matches!(
4781 rpc.receiver.blocking_recv().unwrap(),
4782 ModuleControlRpcOutcome::DeadlineElapsed
4783 ));
4784 }
4785
4786 #[tokio::test(start_paused = true)]
4787 async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
4788 let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
4789 let probe_started_at = Instant::now();
4790 let rpc = forwarding
4791 .begin_health_probe_rpc_for(
4792 "tombstone-ttl",
4793 "health.check",
4794 probe_started_at,
4795 probe_started_at + Duration::from_secs(5),
4796 )
4797 .unwrap();
4798 assert!(forwarding
4799 .tombstone_health_probe_rpc(endpoint, rpc.corr)
4800 .unwrap());
4801 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
4802
4803 tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
4804 tokio::task::yield_now().await;
4805
4806 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
4807 }
4808
4809 #[test]
4810 fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
4811 let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
4812 let mut close = forwarding.register_connection_close(endpoint.connection_id);
4813 forwarding.inject_control_corr(endpoint, u64::MAX);
4814 let final_rpc = forwarding
4815 .begin_module_control_rpc_for(
4816 "corr-max",
4817 "health.check",
4818 Instant::now() + Duration::from_secs(1),
4819 )
4820 .unwrap();
4821 assert_eq!(final_rpc.corr, u64::MAX);
4822 forwarding
4823 .cancel_module_control_rpc(endpoint, final_rpc.corr)
4824 .unwrap();
4825 assert!(matches!(
4826 forwarding.begin_module_control_rpc_for(
4827 "corr-max",
4828 "health.check",
4829 Instant::now() + Duration::from_secs(1),
4830 ),
4831 Err(ForwardingError::RelayCorrelationExhausted)
4832 ));
4833 assert!(close.try_recv().is_ok());
4834 }
4835
4836 #[test]
4837 fn publication_epoch_controls_delivery_failure_escalation() {
4838 fn setup_successor(
4839 commit_successor: Option<bool>,
4840 ) -> (ForwardingTable, ConnectionId, u16, u32) {
4841 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4842 route_fixture("escalation");
4843 let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
4844 forwarding
4845 .complete_pending_relay(
4846 module_connection,
4847 first.corr,
4848 RouteBindRelayOutcome::Accepted,
4849 )
4850 .unwrap();
4851 client_rx.try_recv().unwrap();
4852 assert!(matches!(
4853 forwarding
4854 .release_client_route(client, first.client_channel, first.client_epoch)
4855 .unwrap(),
4856 RouteRelease::Removed(_)
4857 ));
4858 if let Some(commit_successor) = commit_successor {
4859 forwarding.inject_client_slot_epoch(
4860 client,
4861 first.client_channel,
4862 first.client_epoch,
4863 );
4864 forwarding.inject_module_slot_epoch(
4865 endpoint,
4866 first.module_channel,
4867 first.module_epoch,
4868 );
4869 let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
4870 if commit_successor {
4871 forwarding
4872 .complete_pending_relay(
4873 module_connection,
4874 successor.corr,
4875 RouteBindRelayOutcome::Accepted,
4876 )
4877 .unwrap();
4878 client_rx.try_recv().unwrap();
4879 } else {
4880 forwarding
4881 .abort_pending_relay(
4882 endpoint,
4883 successor.corr,
4884 RouteBindRelayOutcome::ModuleGone("abort".into()),
4885 )
4886 .unwrap();
4887 }
4888 }
4889 (forwarding, client, first.client_channel, first.client_epoch)
4890 }
4891
4892 let probe_sink = FrameSink::new(mpsc::channel(1).0);
4893 let (no_successor, client, channel, epoch) = setup_successor(None);
4894 let mut close = no_successor.register_connection_close(client);
4895 assert!(no_successor
4896 .escalate_client_delivery_failure(
4897 client,
4898 channel,
4899 epoch,
4900 CloseReason::new("delivery", "failed"),
4901 UndeliveredFrame {
4902 module_id: None,
4903 sink: &probe_sink,
4904 },
4905 )
4906 .unwrap());
4907 assert!(close.try_recv().is_ok());
4908
4909 let (aborted, client, channel, epoch) = setup_successor(Some(false));
4910 let mut close = aborted.register_connection_close(client);
4911 assert!(aborted
4912 .escalate_client_delivery_failure(
4913 client,
4914 channel,
4915 epoch,
4916 CloseReason::new("delivery", "failed"),
4917 UndeliveredFrame {
4918 module_id: None,
4919 sink: &probe_sink,
4920 },
4921 )
4922 .unwrap());
4923 assert!(close.try_recv().is_ok());
4924
4925 let (published, client, channel, epoch) = setup_successor(Some(true));
4926 let mut close = published.register_connection_close(client);
4927 assert!(!published
4928 .escalate_client_delivery_failure(
4929 client,
4930 channel,
4931 epoch,
4932 CloseReason::new("delivery", "stale failure"),
4933 UndeliveredFrame {
4934 module_id: None,
4935 sink: &probe_sink,
4936 },
4937 )
4938 .unwrap());
4939 assert!(close.try_recv().is_err());
4940 }
4941
4942 #[test]
4943 fn route_concentration_separates_client_count_from_routes_per_client() {
4944 let (forwarding, module_connection, _, client, sink, _client_rx) =
4948 route_fixture("concentration");
4949 assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
4950
4951 for corr in [70_u64, 71] {
4952 let pending =
4953 begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
4954 forwarding
4955 .complete_pending_relay(
4956 module_connection,
4957 pending.corr,
4958 RouteBindRelayOutcome::Accepted,
4959 )
4960 .unwrap();
4961 }
4962
4963 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
4965 assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
4966 }
4967
4968 #[test]
4969 fn cleanup_and_accepted_resolution_have_one_lock_winner() {
4970 let (forwarding, module_connection, _, client, sink, mut client_rx) =
4971 route_fixture("cleanup-race");
4972 let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
4973 forwarding
4974 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4975 .unwrap();
4976 let released = forwarding.cleanup_connection(client).unwrap();
4977 assert_eq!(released.len(), 1);
4978 let completion = forwarding
4979 .complete_pending_relay(
4980 module_connection,
4981 pending.corr,
4982 RouteBindRelayOutcome::Accepted,
4983 )
4984 .unwrap();
4985 assert!(!completion.settled);
4986 assert!(client_rx.try_recv().is_err());
4987 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4988
4989 let (forwarding, module_connection, _, client, sink, mut client_rx) =
4990 route_fixture("accepted-race");
4991 let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
4992 forwarding
4993 .complete_pending_relay(
4994 module_connection,
4995 pending.corr,
4996 RouteBindRelayOutcome::Accepted,
4997 )
4998 .unwrap();
4999 assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
5000 let released = forwarding.cleanup_connection(client).unwrap();
5001 assert_eq!(released.len(), 1);
5002 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5003 }
5004
5005 #[test]
5006 fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
5007 let (forwarding, module_connection, _, client, sink, mut client_rx) =
5008 route_fixture("drain-gap");
5009 let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
5010 forwarding
5011 .complete_pending_relay(
5012 module_connection,
5013 live.corr,
5014 RouteBindRelayOutcome::Accepted,
5015 )
5016 .unwrap();
5017 client_rx.try_recv().unwrap();
5018 let binding = match forwarding
5019 .lookup_data_route(client, live.client_channel, live.client_epoch)
5020 .unwrap()
5021 {
5022 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
5023 other => panic!("expected live route, got {other:?}"),
5024 };
5025
5026 let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
5027 forwarding
5028 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5029 .unwrap();
5030 let control_rpc = forwarding
5031 .begin_module_control_rpc_for(
5032 "drain-gap",
5033 "health.check",
5034 Instant::now() + Duration::from_secs(1),
5035 )
5036 .unwrap();
5037 let target = forwarding
5038 .begin_module_drain("drain-gap", RouteCloseReason::Reload)
5039 .unwrap()
5040 .unwrap();
5041 assert!(matches!(
5042 control_rpc.receiver.blocking_recv().unwrap(),
5043 ModuleControlRpcOutcome::ModuleGone(_)
5044 ));
5045 assert_eq!(target.abandoned_bindings.len(), 1);
5046 assert!(binding.flow.sem.is_closed());
5047 assert!(
5048 !forwarding
5049 .complete_pending_relay(
5050 module_connection,
5051 pending.corr,
5052 RouteBindRelayOutcome::Accepted,
5053 )
5054 .unwrap()
5055 .settled
5056 );
5057 assert!(matches!(
5058 forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
5059 Err(ForwardingError::ModuleReloading { .. })
5060 ));
5061 let released = forwarding
5062 .release_module_endpoint_routes(target.endpoint)
5063 .unwrap();
5064 assert_eq!(released.len(), 1);
5065 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5066 }
5067
5068 #[test]
5076 fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
5077 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5078 route_fixture("closing-client");
5079
5080 let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
5083 forwarding
5084 .complete_pending_relay(
5085 module_connection,
5086 live.corr,
5087 RouteBindRelayOutcome::Accepted,
5088 )
5089 .unwrap();
5090 client_rx.try_recv().unwrap();
5091
5092 let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
5094 forwarding
5095 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5096 .unwrap();
5097
5098 assert!(forwarding
5100 .escalate_client_delivery_failure(
5101 client,
5102 live.client_channel,
5103 live.client_epoch,
5104 CloseReason::new(
5105 "module_to_client_delivery_failed",
5106 "client egress refused a module frame",
5107 ),
5108 UndeliveredFrame {
5109 module_id: None,
5110 sink: &sink,
5111 },
5112 )
5113 .unwrap());
5114 assert!(!sink.is_closed());
5115
5116 let completion = forwarding
5117 .complete_pending_relay(
5118 module_connection,
5119 pending.corr,
5120 RouteBindRelayOutcome::Accepted,
5121 )
5122 .expect("a closing client must not turn a module's ack into an error");
5123
5124 assert!(completion.settled);
5125 let abandoned = completion
5126 .abandoned
5127 .expect("the module must be told to drop the binding it just created");
5128 assert_eq!(abandoned.connection_id, module_connection);
5129 assert_eq!(abandoned.channel, pending.module_channel);
5130 assert_eq!(abandoned.epoch, pending.module_epoch);
5131 assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
5132 assert!(matches!(
5133 pending.receiver.blocking_recv().unwrap(),
5134 RouteBindRelayOutcome::ModuleGone(_)
5135 ));
5136 assert!(client_rx.try_recv().is_err());
5139 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
5140
5141 assert!(forwarding
5144 .has_live_module_connection("closing-client")
5145 .unwrap());
5146 let cotenant = ConnectionId::new(201);
5147 let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
5148 let cotenant_route = begin_test_route(
5149 &forwarding,
5150 cotenant,
5151 FrameSink::new(cotenant_tx),
5152 62,
5153 "closing-client",
5154 );
5155 assert_eq!(cotenant_route.endpoint, endpoint);
5156 forwarding
5157 .complete_pending_relay(
5158 module_connection,
5159 cotenant_route.corr,
5160 RouteBindRelayOutcome::Accepted,
5161 )
5162 .unwrap();
5163 assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
5164 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5165 }
5166
5167 #[test]
5168 fn pending_route_permit_is_released_on_rejection_and_abort() {
5169 let forwarding = ForwardingTable::default();
5170 let module_connection = ConnectionId::new(300);
5171 let client = ConnectionId::new(301);
5172 let (module_tx, _module_rx) = mpsc::channel(1);
5173 let endpoint = forwarding
5174 .register_module_connection(
5175 module_connection,
5176 "permit".into(),
5177 2,
5178 Concurrency::ModuleManaged,
5179 FrameSink::new(module_tx),
5180 )
5181 .unwrap();
5182 let (client_tx, mut client_rx) = mpsc::channel(1);
5183 let sink = FrameSink::new(client_tx);
5184 let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
5185 assert!(sink.try_send(test_ping(999)).is_err());
5186 forwarding
5187 .complete_pending_relay(
5188 module_connection,
5189 rejected.corr,
5190 RouteBindRelayOutcome::Rejected(ErrorBody {
5191 code: "no".into(),
5192 message: "rejected".into(),
5193 detail: None,
5194 }),
5195 )
5196 .unwrap();
5197 sink.try_send(test_ping(1000)).unwrap();
5198 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
5199
5200 let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
5201 assert!(sink.try_send(test_ping(1001)).is_err());
5202 forwarding
5203 .abort_pending_relay(
5204 endpoint,
5205 aborted.corr,
5206 RouteBindRelayOutcome::ModuleGone("abort".into()),
5207 )
5208 .unwrap();
5209 sink.try_send(test_ping(1002)).unwrap();
5210 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
5211
5212 let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
5213 forwarding
5214 .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
5215 .unwrap();
5216 drop(client_rx);
5217 let completion = forwarding
5218 .complete_pending_relay(
5219 module_connection,
5220 receiver_closed.corr,
5221 RouteBindRelayOutcome::Accepted,
5222 )
5223 .unwrap();
5224 assert!(completion.abandoned.is_some());
5225 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5226 }
5227
5228 #[test]
5234 fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
5235 let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
5236 route_fixture("closing-set-leak");
5237
5238 const CONNECTIONS: u64 = 32;
5239 for index in 0..CONNECTIONS {
5240 let client = ConnectionId::new(1000 + index);
5241 let (client_tx, _client_rx) = mpsc::channel(8);
5242 let route = begin_test_route(
5243 &forwarding,
5244 client,
5245 FrameSink::new(client_tx),
5246 index + 1,
5247 "closing-set-leak",
5248 );
5249 forwarding
5250 .complete_pending_relay(
5251 module_connection,
5252 route.corr,
5253 RouteBindRelayOutcome::Accepted,
5254 )
5255 .unwrap();
5256 forwarding.cleanup_connection(client).unwrap();
5257 }
5258 forwarding.cleanup_connection(module_connection).unwrap();
5259
5260 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5261 }
5262
5263 #[test]
5270 fn closing_connection_is_refused_new_work_until_cleanup_completes() {
5271 let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
5272 route_fixture("closing-gate");
5273
5274 let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
5277 forwarding
5278 .complete_pending_relay(
5279 module_connection,
5280 live.corr,
5281 RouteBindRelayOutcome::Accepted,
5282 )
5283 .unwrap();
5284 client_rx.try_recv().unwrap();
5285
5286 assert!(forwarding
5289 .escalate_client_delivery_failure(
5290 client,
5291 live.client_channel,
5292 live.client_epoch,
5293 CloseReason::new(
5294 "module_to_client_delivery_failed",
5295 "client egress refused a module frame",
5296 ),
5297 UndeliveredFrame {
5298 module_id: None,
5299 sink: &sink,
5300 },
5301 )
5302 .unwrap());
5303 assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
5304
5305 assert!(matches!(
5307 forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
5308 Err(ForwardingError::ConnectionClosing { connection_id })
5309 if connection_id == client
5310 ));
5311 let (late_tx, _late_rx) = mpsc::channel(1);
5313 assert!(matches!(
5314 forwarding.register_module_connection(
5315 client,
5316 "late-module".into(),
5317 2,
5318 Concurrency::ModuleManaged,
5319 FrameSink::new(late_tx),
5320 ),
5321 Err(ForwardingError::ConnectionClosing { connection_id })
5322 if connection_id == client
5323 ));
5324
5325 forwarding.cleanup_connection(client).unwrap();
5329 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5330 }
5331}
5332
5333#[cfg(test)]
5336mod swap_slot_tests {
5337 use std::time::Duration;
5338
5339 use super::*;
5340 use tokio::sync::mpsc;
5341
5342 const MODULE_ID: &str = "swapped";
5343
5344 struct SwapFixture {
5345 forwarding: ForwardingTable,
5346 incumbent_connection: ConnectionId,
5347 incumbent: ModuleEndpointId,
5348 candidate_connection: ConnectionId,
5349 candidate: ModuleEndpointId,
5350 _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
5351 }
5352
5353 fn swap_fixture() -> SwapFixture {
5354 let forwarding = ForwardingTable::default();
5355 let incumbent_connection = ConnectionId::new(100);
5356 let candidate_connection = ConnectionId::new(110);
5357 let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
5358 let incumbent = forwarding
5359 .register_module_connection(
5360 incumbent_connection,
5361 MODULE_ID.to_string(),
5362 2,
5363 Concurrency::ModuleManaged,
5364 FrameSink::new(incumbent_tx),
5365 )
5366 .unwrap();
5367 let (candidate_tx, candidate_rx) = mpsc::channel(8);
5368 let candidate = forwarding
5369 .register_candidate_module_connection(
5370 candidate_connection,
5371 MODULE_ID.to_string(),
5372 2,
5373 Concurrency::ModuleManaged,
5374 FrameSink::new(candidate_tx),
5375 )
5376 .unwrap();
5377 SwapFixture {
5378 forwarding,
5379 incumbent_connection,
5380 incumbent,
5381 candidate_connection,
5382 candidate,
5383 _module_rxs: vec![incumbent_rx, candidate_rx],
5384 }
5385 }
5386
5387 fn client(
5388 raw: u64,
5389 ) -> (
5390 ConnectionId,
5391 FrameSink,
5392 mpsc::Receiver<crate::router::OutboundFrame>,
5393 ) {
5394 let (tx, rx) = mpsc::channel(8);
5395 (ConnectionId::new(raw), FrameSink::new(tx), rx)
5396 }
5397
5398 fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
5399 forwarding
5400 .read_inner()
5401 .unwrap()
5402 .client_to_module
5403 .values()
5404 .map(|route| route.module_endpoint)
5405 .collect()
5406 }
5407
5408 #[test]
5409 fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
5410 let fixture = swap_fixture();
5411 let forwarding = &fixture.forwarding;
5412 assert_ne!(fixture.incumbent, fixture.candidate);
5413
5414 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5416 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5417 let (client_connection, client_sink, _client_rx) = client(200);
5418 let pending = forwarding
5419 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5420 .unwrap();
5421 assert_eq!(pending.endpoint, fixture.incumbent);
5422 let rpc = forwarding
5423 .begin_module_control_rpc_for(
5424 MODULE_ID,
5425 "health.check",
5426 Instant::now() + Duration::from_secs(1),
5427 )
5428 .unwrap();
5429 assert_eq!(rpc.endpoint, fixture.incumbent);
5430 let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
5431 assert_eq!(census.len(), 1, "the census lists one endpoint per id");
5432
5433 assert_eq!(
5435 forwarding
5436 .module_endpoint_for_connection(fixture.candidate_connection)
5437 .unwrap(),
5438 Some(fixture.candidate)
5439 );
5440 assert_eq!(
5441 forwarding
5442 .module_id_for_connection(fixture.candidate_connection)
5443 .unwrap()
5444 .as_deref(),
5445 Some(MODULE_ID)
5446 );
5447
5448 let (other_tx, _other_rx) = mpsc::channel(1);
5450 assert_eq!(
5451 forwarding.register_candidate_module_connection(
5452 ConnectionId::new(120),
5453 MODULE_ID.to_string(),
5454 2,
5455 Concurrency::ModuleManaged,
5456 FrameSink::new(other_tx),
5457 ),
5458 Err(ForwardingError::CandidateSlotOccupied {
5459 module_id: MODULE_ID.to_string()
5460 })
5461 );
5462 }
5463
5464 #[test]
5468 fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
5469 let fixture = swap_fixture();
5470 let forwarding = &fixture.forwarding;
5471 let (early_client, early_sink, _early_rx) = client(200);
5472 let mut early = forwarding
5473 .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
5474 .unwrap();
5475 assert_eq!(early.endpoint, fixture.incumbent);
5476 assert_eq!(
5477 forwarding
5478 .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
5479 .unwrap(),
5480 RouteBindRelayMark::Marked
5481 );
5482
5483 let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5484 assert_eq!(
5485 cutover,
5486 ForwardingCutover {
5487 promoted: fixture.candidate,
5488 incumbent: Some(fixture.incumbent),
5489 }
5490 );
5491
5492 let (late_client, late_sink, _late_rx) = client(201);
5494 let late = forwarding
5495 .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
5496 .unwrap();
5497 assert_eq!(
5498 late.endpoint, fixture.candidate,
5499 "a route.open after cutover was reserved on the incumbent"
5500 );
5501
5502 let completion = forwarding
5504 .complete_pending_relay(
5505 fixture.incumbent_connection,
5506 early.corr,
5507 RouteBindRelayOutcome::Accepted,
5508 )
5509 .expect("a superseded endpoint's ack is not an error on its connection");
5510 assert!(completion.settled);
5511 assert!(
5512 !committed_endpoints(forwarding).contains(&fixture.incumbent),
5513 "a relay reserved before cutover committed a route on the incumbent"
5514 );
5515 let goodbye = completion
5516 .abandoned
5517 .expect("the incumbent is told to drop the binding it just created");
5518 assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
5519 assert_eq!(goodbye.channel, early.module_channel);
5520 assert_eq!(goodbye.epoch, early.module_epoch);
5521 assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
5522 match early.receiver.try_recv() {
5523 Ok(RouteBindRelayOutcome::DaemonRejected(body)) => {
5524 assert_eq!(body.code, "module_reloading")
5525 }
5526 other => panic!("expected a retryable module_reloading answer, got {other:?}"),
5527 }
5528 assert!(matches!(
5529 forwarding
5530 .lookup_data_route(early_client, early.client_channel, early.client_epoch)
5531 .unwrap(),
5532 DataRoute::Client(DataRouteState::Absent)
5533 ));
5534
5535 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
5538 forwarding
5539 .complete_pending_relay(
5540 fixture.candidate_connection,
5541 late.corr,
5542 RouteBindRelayOutcome::Accepted,
5543 )
5544 .unwrap();
5545 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5546 assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
5547 }
5548
5549 #[test]
5550 fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
5551 let fixture = swap_fixture();
5552 let forwarding = &fixture.forwarding;
5553 let (bound_client, bound_sink, _bound_rx) = client(200);
5555 let bound = forwarding
5556 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5557 .unwrap();
5558 forwarding
5559 .complete_pending_relay(
5560 fixture.incumbent_connection,
5561 bound.corr,
5562 RouteBindRelayOutcome::Accepted,
5563 )
5564 .unwrap();
5565 let (pending_client, pending_sink, _pending_rx) = client(201);
5566 let mut in_flight = forwarding
5567 .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
5568 .unwrap();
5569 forwarding
5570 .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
5571 .unwrap();
5572
5573 let incumbent = forwarding
5574 .cutover_candidate(MODULE_ID)
5575 .unwrap()
5576 .unwrap()
5577 .incumbent
5578 .unwrap();
5579 let target = forwarding
5580 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
5581 .unwrap()
5582 .expect("the superseded incumbent is still registered");
5583
5584 assert_eq!(target.endpoint, fixture.incumbent);
5585 assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
5586 assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
5587 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5588 assert_eq!(target.abandoned_bindings.len(), 1);
5589 assert_eq!(
5590 target.abandoned_bindings[0].channel,
5591 in_flight.module_channel
5592 );
5593 assert!(matches!(
5594 in_flight.receiver.try_recv(),
5595 Ok(RouteBindRelayOutcome::DaemonRejected(body)) if body.code == "module_reloading"
5596 ));
5597 assert_eq!(
5598 forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
5599 1,
5600 "the incumbent's bound route stays until its drain finishes"
5601 );
5602
5603 let (next_client, next_sink, _next_rx) = client(202);
5604 let next = forwarding
5605 .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
5606 .expect("the promoted candidate keeps accepting routes");
5607 assert_eq!(next.endpoint, fixture.candidate);
5608 }
5609
5610 #[test]
5616 fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
5617 let forwarding = ForwardingTable::default();
5618 let first_connection = ConnectionId::new(70);
5619 let (first_tx, _first_rx) = mpsc::channel(8);
5620 forwarding
5621 .register_module_connection(
5622 first_connection,
5623 MODULE_ID.to_string(),
5624 2,
5625 Concurrency::ModuleManaged,
5626 FrameSink::new(first_tx),
5627 )
5628 .unwrap();
5629 let (client_connection, client_sink, _client_rx) = client(200);
5630 let mut pending = forwarding
5631 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5632 .unwrap();
5633 let (second_tx, _second_rx) = mpsc::channel(8);
5634 forwarding
5635 .register_module_connection(
5636 ConnectionId::new(80),
5637 MODULE_ID.to_string(),
5638 2,
5639 Concurrency::ModuleManaged,
5640 FrameSink::new(second_tx),
5641 )
5642 .unwrap();
5643
5644 assert_eq!(
5645 forwarding
5646 .complete_pending_relay(
5647 first_connection,
5648 pending.corr,
5649 RouteBindRelayOutcome::Accepted
5650 )
5651 .unwrap_err(),
5652 ForwardingError::StaleModuleEndpoint
5653 );
5654 assert!(committed_endpoints(&forwarding).is_empty());
5655 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5656 assert!(matches!(
5657 pending.receiver.try_recv(),
5658 Err(oneshot::error::TryRecvError::Closed)
5659 ));
5660 }
5661
5662 #[test]
5663 fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
5664 let fixture = swap_fixture();
5666 let forwarding = &fixture.forwarding;
5667 assert!(forwarding
5668 .cleanup_connection(fixture.candidate_connection)
5669 .unwrap()
5670 .is_empty());
5671 assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
5672 let (client_connection, client_sink, _client_rx) = client(200);
5673 assert_eq!(
5674 forwarding
5675 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5676 .unwrap()
5677 .endpoint,
5678 fixture.incumbent
5679 );
5680
5681 let fixture = swap_fixture();
5684 let forwarding = &fixture.forwarding;
5685 let (bound_client, bound_sink, _bound_rx) = client(200);
5686 let bound = forwarding
5687 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5688 .unwrap();
5689 forwarding
5690 .complete_pending_relay(
5691 fixture.incumbent_connection,
5692 bound.corr,
5693 RouteBindRelayOutcome::Accepted,
5694 )
5695 .unwrap();
5696 forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5697 let released = forwarding
5698 .cleanup_connection(fixture.incumbent_connection)
5699 .unwrap();
5700 assert_eq!(released.len(), 1);
5701 assert_eq!(released[0].connection_id, bound_client);
5702 assert!(forwarding
5703 .read_inner()
5704 .unwrap()
5705 .superseded_endpoints
5706 .is_empty());
5707 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5708 let (next_client, next_sink, _next_rx) = client(201);
5709 assert_eq!(
5710 forwarding
5711 .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
5712 .unwrap()
5713 .endpoint,
5714 fixture.candidate
5715 );
5716 }
5717}