1use std::{
2 collections::{BTreeMap, HashMap, HashSet},
3 error::Error,
4 fmt,
5 sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
6 time::Duration,
7};
8
9use subc_control::{ClientControlResponse, RouteCloseReason};
10use subc_protocol::{
11 manifest::Concurrency,
12 session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
13 ErrorBody, Flags, FrameType, Principal, Priority,
14};
15use tokio::sync::{oneshot, Semaphore};
16use tokio::time::Instant;
17use tracing::{debug, info, warn};
18
19use crate::{
20 control::{RouteBindBreakers, RouteBindConcurrency},
21 observability::DaemonCounters,
22 registry::ConnectionId,
23 router::FrameSink,
24 scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
25 Frame, ProjectRootId,
26};
27
28const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
30
31const STATELESS_PARALLEL_WINDOW: usize = 1024;
33
34const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
38
39#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
44pub struct ModuleEndpointId {
45 pub connection_id: ConnectionId,
46 pub generation: u64,
47}
48
49#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
51pub(crate) struct ClientRouteKey {
52 pub connection_id: ConnectionId,
53 pub channel: u16,
54}
55
56#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
58pub(crate) struct ModuleRouteKey {
59 pub endpoint: ModuleEndpointId,
60 pub channel: u16,
61}
62
63#[derive(Debug)]
64pub(crate) struct RouteBinding {
65 pub client_connection_id: ConnectionId,
66 pub client_sink: FrameSink,
67 pub client_negotiated_ver: u8,
68 pub client_channel: u16,
69 pub client_epoch: u32,
70 pub module_id: String,
71 pub module_endpoint: ModuleEndpointId,
72 pub module_sink: FrameSink,
73 pub module_negotiated_ver: u8,
74 pub module_channel: u16,
75 pub module_epoch: u32,
76 pub principal: Principal,
77 #[allow(dead_code)]
79 pub harness: String,
80 #[allow(dead_code)]
82 pub build_class: Option<String>,
83 #[allow(dead_code)]
85 pub operator_app: Option<crate::operator_apps::OperatorAppStamp>,
86 pub project_root: Option<ProjectRootId>,
87 pub bound_at: Instant,
88 pub flow: Arc<ChannelFlow>,
89 pub scope: Option<BoundScope>,
92}
93
94#[derive(Debug, Clone)]
95pub(crate) enum DataRoute {
96 Client(DataRouteState),
97 Module(DataRouteState),
98}
99
100#[derive(Debug, Clone)]
101pub(crate) enum DataRouteState {
102 Bound(Arc<RouteBinding>),
103 Reserved,
104 EpochMismatch,
105 Absent,
106}
107
108#[derive(Debug, Clone, Copy, PartialEq, Eq)]
137pub(crate) enum GoodbyeTargetKind {
138 Client,
139 Module,
140}
141
142#[derive(Debug, Clone)]
143pub(crate) struct GoodbyeTarget {
144 pub connection_id: ConnectionId,
145 pub sink: FrameSink,
146 pub negotiated_ver: u8,
147 pub channel: u16,
148 pub epoch: u32,
149 pub kind: GoodbyeTargetKind,
150 pub module_id: Option<String>,
155}
156
157#[derive(Debug, Clone, Copy)]
160pub(crate) struct UndeliveredFrame<'a> {
161 pub module_id: Option<&'a str>,
163 pub sink: &'a FrameSink,
165}
166
167pub(crate) fn principal_label(principal: &Principal) -> String {
169 match principal {
170 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
171 Principal::Direct => "direct".to_string(),
172 other => format!("{other:?}"),
173 }
174}
175
176fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
179 let labels = inner
180 .client_to_module
181 .iter()
182 .filter(|(key, _)| key.connection_id == connection_id)
183 .map(|(_, route)| principal_label(&route.principal))
184 .collect::<std::collections::BTreeSet<_>>();
185 if labels.is_empty() {
186 "none".to_string()
187 } else {
188 labels.into_iter().collect::<Vec<_>>().join(",")
189 }
190}
191
192impl GoodbyeTarget {
193 pub(crate) fn close_on_delivery_failure(&self) -> bool {
196 matches!(self.kind, GoodbyeTargetKind::Client)
197 }
198}
199
200pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
213
214pub(crate) fn send_module_route_goodbye(
226 counters: &DaemonCounters,
227 sink: &FrameSink,
228 frame: Frame,
229 module_id: Option<&str>,
230 context: &'static str,
231) {
232 let channel = frame.header.channel;
233 let epoch = frame.header.epoch;
234 let Err(err) = sink.try_send(frame.clone()) else {
235 return;
236 };
237 let runtime = match tokio::runtime::Handle::try_current() {
240 Ok(runtime) if !sink.is_closed() => runtime,
241 _ => {
242 counters.increment_goodbye_relay_module_dropped(module_id);
243 warn!(
244 module_id = module_id.unwrap_or("unknown"),
245 route_channel = channel,
246 route_epoch = epoch,
247 error = %err,
248 context,
249 "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
250 );
251 return;
252 }
253 };
254 debug!(
255 module_id = module_id.unwrap_or("unknown"),
256 route_channel = channel,
257 route_epoch = epoch,
258 error = %err,
259 context,
260 "module egress queue refused route GOODBYE; delivering it once the module frees room"
261 );
262 let counters = counters.clone();
263 let sink = sink.clone();
264 let module_id = module_id.map(str::to_string);
265 runtime.spawn(async move {
266 let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
267 let why = match outcome {
268 Ok(Ok(())) => {
269 debug!(
270 module_id = module_id.as_deref().unwrap_or("unknown"),
271 route_channel = channel,
272 route_epoch = epoch,
273 context,
274 "late route GOODBYE delivered to module"
275 );
276 return;
277 }
278 Ok(Err(err)) => err.to_string(),
279 Err(_) => format!(
280 "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
281 ),
282 };
283 counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
284 warn!(
285 module_id = module_id.as_deref().unwrap_or("unknown"),
286 route_channel = channel,
287 route_epoch = epoch,
288 error = %why,
289 context,
290 "route GOODBYE to module dropped under backpressure; not closing shared module connection"
291 );
292 });
293}
294
295#[derive(Debug, Clone)]
301pub(crate) struct EndpointRoute {
302 pub goodbye_target: GoodbyeTarget,
303 pub principal: Principal,
304 pub bound_at: Instant,
305 pub draining: bool,
306 pub drain_reason: Option<RouteCloseReason>,
310}
311
312#[derive(Debug, Clone)]
315pub(crate) struct ScopeDrainedRoute {
316 pub reason: RouteCloseReason,
317 pub scope: BoundScope,
318 pub module_id: String,
319 pub client: GoodbyeTarget,
320 pub module: GoodbyeTarget,
321}
322
323#[derive(Debug)]
324pub(crate) struct PendingRouteBindRelay {
325 pub endpoint: ModuleEndpointId,
326 pub module_sink: FrameSink,
327 pub negotiated_ver: u8,
328 pub client_channel: u16,
329 pub client_epoch: u32,
330 pub module_channel: u16,
331 pub module_epoch: u32,
332 pub corr: u64,
333 pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
334}
335
336#[derive(Debug, Clone)]
337pub(crate) struct ModuleDrainTarget {
338 pub endpoint: ModuleEndpointId,
339 pub sink: FrameSink,
340 pub negotiated_ver: u8,
341 pub abandoned_bindings: Vec<GoodbyeTarget>,
342 pub excluded_subscriptions: u32,
343}
344
345#[cfg(unix)]
348#[derive(Debug, Clone)]
349pub(crate) struct ModuleConnectionTarget {
350 pub module_id: String,
351 pub endpoint: ModuleEndpointId,
352 pub sink: FrameSink,
353 pub negotiated_ver: u8,
354}
355
356#[derive(Debug, Clone)]
357pub(crate) enum RouteBindRelayOutcome {
358 Accepted,
359 Rejected(ErrorBody),
364 DaemonRejected(ErrorBody),
372 ModuleGone(String),
373}
374
375#[derive(Debug, Clone, Copy, PartialEq, Eq)]
377pub(crate) struct ForwardingCutover {
378 pub promoted: ModuleEndpointId,
380 pub incumbent: Option<ModuleEndpointId>,
383}
384
385#[derive(Debug)]
387pub(crate) struct ConnectionCleanup {
388 pub released: Vec<GoodbyeTarget>,
390 pub abandoned_relays: u32,
393}
394
395#[derive(Debug, Clone)]
396pub(crate) struct PendingRelayCompletion {
397 pub settled: bool,
398 pub abandoned: Option<GoodbyeTarget>,
399}
400
401#[derive(Debug)]
402pub(crate) struct PendingModuleControlRpc {
403 pub endpoint: ModuleEndpointId,
404 pub module_sink: FrameSink,
405 pub negotiated_ver: u8,
406 pub corr: u64,
407 pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
408}
409
410#[derive(Debug, Clone)]
411pub(crate) enum ModuleControlRpcOutcome {
412 Response(ModuleControlResponse),
413 Rejected(ErrorBody),
414 ModuleGone(String),
415 MalformedResponse(String),
416 UnexpectedOp { expected: String, actual: String },
417 DeadlineElapsed,
418}
419
420#[derive(Debug, Clone, PartialEq, Eq)]
421pub(crate) enum ModuleControlRpcCompletion {
422 Unknown,
423 Settled,
424 LateHealthAnswer {
425 module_id: String,
426 latency: Duration,
427 },
428}
429
430#[derive(Debug)]
431struct PendingModuleControlRpcEntry {
432 expected_op: String,
433 deadline: Instant,
434 health_probe_started_at: Option<Instant>,
435 sender: oneshot::Sender<ModuleControlRpcOutcome>,
436}
437
438#[derive(Debug)]
439struct HealthProbeTombstone {
440 expected_op: String,
441 module_id: String,
442 probe_started_at: Instant,
443 expires_at: Instant,
444}
445
446#[derive(Debug, Clone)]
447struct RouteReservation {
448 client_key: ClientRouteKey,
449 module_key: ModuleRouteKey,
450 client_epoch: u32,
451 module_epoch: u32,
452 project_root: Option<ProjectRootId>,
453}
454
455#[derive(Debug)]
456struct PendingRouteBindRelayEntry {
457 reservation: RouteReservation,
458 client_sink: FrameSink,
459 client_negotiated_ver: u8,
460 client_permit: crate::router::EgressPermit,
461 route_open_frame: Frame,
462 principal: Principal,
463 scope: Option<BoundScope>,
466 harness: String,
467 build_class: Option<String>,
468 operator_app: Option<crate::operator_apps::OperatorAppStamp>,
469 deadline: Instant,
470 relay_enqueued: bool,
481 sender: oneshot::Sender<RouteBindRelayOutcome>,
482}
483
484#[derive(Debug, Clone, Copy, PartialEq, Eq)]
487pub(crate) enum RouteBindRelayMark {
488 Marked,
490 SettledByReply,
494 ReleasedWithoutReply,
498}
499
500#[derive(Debug, Clone)]
501pub(crate) enum RouteRelease {
502 Removed(GoodbyeTarget),
503 Stale,
504 Absent,
505}
506
507#[derive(Debug, Clone)]
508pub(crate) enum RoutePollSnapshot {
509 Bound {
510 module_id: String,
511 status: Option<String>,
512 },
513 Absent,
514}
515
516#[derive(Debug, Clone)]
517struct ModuleConnection {
518 endpoint: ModuleEndpointId,
519 sink: FrameSink,
520 negotiated_ver: u8,
521 concurrency: Concurrency,
522}
523
524#[derive(Debug, Default)]
525struct ForwardingInner {
526 operator_confirms: Arc<crate::operator_confirm::OperatorConfirms>,
527 daemon_draining: bool,
528 daemon_drain_results: HashMap<ModuleEndpointId, bool>,
533 modules_by_id: HashMap<String, ModuleConnection>,
537 candidates_by_id: HashMap<String, ModuleConnection>,
543 superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
549 endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
550 module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
551 draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
555 closing_connections: HashSet<ConnectionId>,
556 next_generation: u64,
557 reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
558 reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
559 next_client_channel: HashMap<ConnectionId, u16>,
560 next_module_channel: HashMap<ModuleEndpointId, u16>,
561 client_slot_epochs: HashMap<ClientRouteKey, u32>,
562 module_slot_epochs: HashMap<ModuleRouteKey, u32>,
563 last_published_epoch: HashMap<ClientRouteKey, u32>,
564 client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
565 module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
566 status: HashMap<(ClientRouteKey, u32), String>,
567 pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
568 relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
573 scope_tags: HashMap<(String, String), ScopeTag>,
578 next_control_corr: HashMap<ModuleEndpointId, u64>,
579 pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
580 health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
581}
582
583#[derive(Debug, Clone)]
584pub(crate) struct CloseReason {
585 code: &'static str,
586 message: String,
587}
588
589impl CloseReason {
590 pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
591 Self {
592 code,
593 message: message.into(),
594 }
595 }
596}
597
598impl fmt::Display for CloseReason {
599 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
600 write!(f, "{}: {}", self.code, self.message)
601 }
602}
603
604pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
605
606#[derive(Debug, Default)]
608pub struct ForwardingTable {
609 inner: Arc<RwLock<ForwardingInner>>,
610 frame_drop_warnings: Mutex<HashMap<ConnectionId, ChannelDropWarnings>>,
611 #[cfg(feature = "test-support")]
612 route_closing_hold: Mutex<Option<oneshot::Receiver<()>>>,
613 close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
614 counters: DaemonCounters,
615 route_bind_breakers: RouteBindBreakers,
620 route_bind_concurrency: RouteBindConcurrency,
623 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
627}
628
629#[derive(Debug)]
630struct FrameDropWarning {
631 last_warned: tokio::time::Instant,
632 suppressed: u64,
633}
634
635type ChannelDropWarnings = HashMap<(u16, &'static str), FrameDropWarning>;
636
637impl ForwardingTable {
638 pub(crate) fn warn_frame_drop(
641 &self,
642 connection_id: ConnectionId,
643 channel: u16,
644 epoch: u32,
645 direction: &'static str,
646 reason: &'static str,
647 ) {
648 let now = tokio::time::Instant::now();
649 let mut warnings = self
650 .frame_drop_warnings
651 .lock()
652 .unwrap_or_else(std::sync::PoisonError::into_inner);
653 let entry = warnings
654 .entry(connection_id)
655 .or_default()
656 .entry((channel, reason));
657 let suppressed = match entry {
658 std::collections::hash_map::Entry::Occupied(mut entry) => {
659 let state = entry.get_mut();
660 if now.saturating_duration_since(state.last_warned) < Duration::from_secs(60) {
661 state.suppressed = state.suppressed.saturating_add(1);
662 return;
663 }
664 state.last_warned = now;
665 std::mem::take(&mut state.suppressed)
666 }
667 std::collections::hash_map::Entry::Vacant(entry) => {
668 entry.insert(FrameDropWarning {
669 last_warned: now,
670 suppressed: 0,
671 });
672 0
673 }
674 };
675 warn!(
676 connection_id = connection_id.get(),
677 channel, epoch, direction, reason, suppressed, "dropping data-plane frame"
678 );
679 }
680
681 #[cfg(feature = "test-support")]
723 pub fn configure_operator_script(
724 &self,
725 script: &std::path::Path,
726 events: Option<&std::path::Path>,
727 limits: Option<crate::operator_confirm::OperatorLimits>,
728 ) {
729 self.operator_confirms()
730 .configure_script(script, events, limits);
731 }
732
733 #[cfg(feature = "test-support")]
737 pub fn hold_route_closing_for_test(&self) -> oneshot::Sender<()> {
738 let (release, hold) = oneshot::channel();
739 *self.route_closing_hold.lock().unwrap() = Some(hold);
740 release
741 }
742
743 #[cfg(feature = "test-support")]
744 pub(crate) async fn wait_before_route_closing_for_test(&self) {
745 let hold = self.route_closing_hold.lock().unwrap().take();
746 if let Some(hold) = hold {
747 let _ = hold.await;
748 }
749 }
750
751 pub(crate) fn operator_confirms(&self) -> Arc<crate::operator_confirm::OperatorConfirms> {
752 Arc::clone(
753 &self
754 .inner
755 .read()
756 .unwrap_or_else(|p| p.into_inner())
757 .operator_confirms,
758 )
759 }
760
761 #[cfg(test)]
762 pub(crate) fn inject_operator_principal(&self, key: ModuleRouteKey, principal: Principal) {
763 self.alter_operator_binding_for_test(key, |binding| binding.principal = principal);
764 }
765
766 #[cfg(test)]
767 pub(crate) fn alter_operator_binding_for_test(
768 &self,
769 key: ModuleRouteKey,
770 alter: impl FnOnce(&mut RouteBinding),
771 ) {
772 let mut inner = self.write_inner().unwrap();
773 let old = inner.module_to_client.remove(&key).unwrap();
774 let client = ClientRouteKey {
775 connection_id: old.client_connection_id,
776 channel: old.client_channel,
777 };
778 inner.client_to_module.remove(&client);
779 let mut binding = Arc::try_unwrap(old).expect("test binding has no outstanding readers");
780 alter(&mut binding);
781 let binding = Arc::new(binding);
782 inner.client_to_module.insert(client, Arc::clone(&binding));
783 inner.module_to_client.insert(key, binding);
784 }
785
786 #[cfg(test)]
789 pub(crate) fn invalidate_operator_route_for_test(&self, key: ModuleRouteKey, stale: bool) {
790 let mut inner = self.write_inner().unwrap();
791 let old = inner.module_to_client.remove(&key).unwrap();
792 let client = ClientRouteKey {
793 connection_id: old.client_connection_id,
794 channel: old.client_channel,
795 };
796 inner.client_to_module.remove(&client);
797 if stale {
798 let mut binding = Arc::try_unwrap(old).expect("no binding readers at the test pause");
799 binding.module_epoch += 1;
800 let binding = Arc::new(binding);
801 inner.client_to_module.insert(client, binding.clone());
802 inner.module_to_client.insert(key, binding);
803 }
804 }
805
806 pub(crate) fn with_operator_route<T>(
810 &self,
811 connection_id: ConnectionId,
812 channel: u16,
813 epoch: u32,
814 admit: impl FnOnce(Option<&RouteBinding>) -> T,
815 ) -> Result<T, ForwardingError> {
816 self.with_operator_route_epoch(connection_id, channel, |binding| {
817 admit(binding.filter(|binding| binding.module_epoch == epoch))
818 })
819 }
820
821 pub(crate) fn with_operator_route_epoch<T>(
824 &self,
825 connection_id: ConnectionId,
826 channel: u16,
827 decide: impl FnOnce(Option<&RouteBinding>) -> T,
828 ) -> Result<T, ForwardingError> {
829 let inner = self.read_inner()?;
830 let binding = inner
831 .endpoint_by_connection
832 .get(&connection_id)
833 .and_then(|endpoint| {
834 inner.module_to_client.get(&ModuleRouteKey {
835 endpoint: *endpoint,
836 channel,
837 })
838 });
839 Ok(decide(binding.map(Arc::as_ref)))
840 }
841
842 pub(crate) fn counters(&self) -> DaemonCounters {
843 self.counters.clone()
844 }
845
846 pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
847 self.route_bind_breakers.clone()
848 }
849
850 pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
851 self.route_bind_concurrency.clone()
852 }
853
854 pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
855 Arc::clone(&self.route_outages)
856 }
857
858 pub(crate) fn register_connection_close(
859 &self,
860 connection_id: ConnectionId,
861 ) -> ConnectionCloseReceiver {
862 let (sender, receiver) = oneshot::channel();
863 let replaced = self
864 .lock_close_registry()
865 .insert(connection_id, sender)
866 .is_some();
867 if replaced {
868 warn!(
869 connection_id = connection_id.get(),
870 "replaced existing connection close registration"
871 );
872 }
873 receiver
874 }
875
876 pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
877 self.lock_close_registry().remove(&connection_id);
878 }
879
880 #[cfg(unix)]
889 pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
890 let senders: Vec<_> = self.lock_close_registry().drain().collect();
891 let count = senders.len();
892 for (_, sender) in senders {
893 let _ = sender.send(reason.clone());
894 }
895 count
896 }
897
898 #[cfg(unix)]
902 pub(crate) fn module_connections(
903 &self,
904 ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
905 let inner = self.read_inner()?;
906 let mut seen = HashSet::new();
907 Ok(inner
908 .modules_by_id
909 .values()
910 .chain(inner.candidates_by_id.values())
911 .chain(inner.superseded_endpoints.values())
912 .filter(|module| seen.insert(module.endpoint))
913 .map(|module| ModuleConnectionTarget {
914 module_id: inner
915 .module_id_by_endpoint
916 .get(&module.endpoint)
917 .cloned()
918 .unwrap_or_default(),
919 endpoint: module.endpoint,
920 sink: module.sink.clone(),
921 negotiated_ver: module.negotiated_ver,
922 })
923 .collect())
924 }
925
926 pub(crate) fn request_connection_close(
930 &self,
931 connection_id: ConnectionId,
932 reason: CloseReason,
933 ) -> bool {
934 let sender = self.lock_close_registry().remove(&connection_id);
935 if let Some(sender) = sender {
936 debug!(
937 connection_id = connection_id.get(),
938 close_reason = %reason,
939 "requesting connection close"
940 );
941 let _ = sender.send(reason);
942 true
943 } else {
944 debug!(
945 connection_id = connection_id.get(),
946 close_reason = %reason,
947 "connection close request ignored for inactive connection"
948 );
949 false
950 }
951 }
952
953 pub fn register_module_connection(
954 &self,
955 connection_id: ConnectionId,
956 module_id: String,
957 negotiated_ver: u8,
958 concurrency: Concurrency,
959 sink: FrameSink,
960 ) -> Result<ModuleEndpointId, ForwardingError> {
961 self.register_module_connection_inner(
962 connection_id,
963 module_id,
964 negotiated_ver,
965 concurrency,
966 sink,
967 None,
968 )
969 }
970
971 pub(crate) fn register_module_connection_acked(
984 &self,
985 connection_id: ConnectionId,
986 module_id: String,
987 negotiated_ver: u8,
988 concurrency: Concurrency,
989 sink: FrameSink,
990 hello_ack: Frame,
991 ) -> Result<ModuleEndpointId, ForwardingError> {
992 self.register_module_connection_inner(
993 connection_id,
994 module_id,
995 negotiated_ver,
996 concurrency,
997 sink,
998 Some(hello_ack),
999 )
1000 }
1001
1002 fn register_module_connection_inner(
1003 &self,
1004 connection_id: ConnectionId,
1005 module_id: String,
1006 negotiated_ver: u8,
1007 concurrency: Concurrency,
1008 sink: FrameSink,
1009 hello_ack: Option<Frame>,
1010 ) -> Result<ModuleEndpointId, ForwardingError> {
1011 let mut inner = self.write_inner()?;
1012 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1013 return Err(ForwardingError::ConnectionClosing { connection_id });
1014 }
1015 check_module_connection_role_locked(&inner, connection_id)?;
1016 enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1019
1020 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1021 let endpoint = ModuleEndpointId {
1022 connection_id,
1023 generation: inner.next_generation,
1024 };
1025 inner.endpoint_by_connection.insert(connection_id, endpoint);
1026 inner
1027 .module_id_by_endpoint
1028 .insert(endpoint, module_id.clone());
1029 inner.next_module_channel.insert(endpoint, 1);
1030 inner.next_control_corr.insert(endpoint, 1);
1031 inner.modules_by_id.insert(
1032 module_id.clone(),
1033 ModuleConnection {
1034 endpoint,
1035 sink,
1036 negotiated_ver,
1037 concurrency,
1038 },
1039 );
1040 drop(inner);
1041
1042 if let Some(discarded) = self
1055 .route_bind_breakers
1056 .reset_for_new_module_connection(&module_id)
1057 {
1058 info!(
1059 module_id = %module_id,
1060 discarded_consecutive_timeouts = discarded,
1061 "route.bind breaker state discarded: a new module connection replaced the process it described"
1062 );
1063 }
1064 Ok(endpoint)
1065 }
1066
1067 #[cfg(test)]
1081 pub(crate) fn register_candidate_module_connection(
1082 &self,
1083 connection_id: ConnectionId,
1084 module_id: String,
1085 negotiated_ver: u8,
1086 concurrency: Concurrency,
1087 sink: FrameSink,
1088 ) -> Result<ModuleEndpointId, ForwardingError> {
1089 self.register_candidate_module_connection_inner(
1090 connection_id,
1091 module_id,
1092 negotiated_ver,
1093 concurrency,
1094 sink,
1095 None,
1096 )
1097 }
1098
1099 pub(crate) fn register_candidate_module_connection_acked(
1106 &self,
1107 connection_id: ConnectionId,
1108 module_id: String,
1109 negotiated_ver: u8,
1110 concurrency: Concurrency,
1111 sink: FrameSink,
1112 hello_ack: Frame,
1113 ) -> Result<ModuleEndpointId, ForwardingError> {
1114 self.register_candidate_module_connection_inner(
1115 connection_id,
1116 module_id,
1117 negotiated_ver,
1118 concurrency,
1119 sink,
1120 Some(hello_ack),
1121 )
1122 }
1123
1124 fn register_candidate_module_connection_inner(
1125 &self,
1126 connection_id: ConnectionId,
1127 module_id: String,
1128 negotiated_ver: u8,
1129 concurrency: Concurrency,
1130 sink: FrameSink,
1131 hello_ack: Option<Frame>,
1132 ) -> Result<ModuleEndpointId, ForwardingError> {
1133 let mut inner = self.write_inner()?;
1134 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1135 return Err(ForwardingError::ConnectionClosing { connection_id });
1136 }
1137 check_module_connection_role_locked(&inner, connection_id)?;
1138 if inner.candidates_by_id.contains_key(&module_id) {
1139 return Err(ForwardingError::CandidateSlotOccupied { module_id });
1140 }
1141 enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1144
1145 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1146 let endpoint = ModuleEndpointId {
1147 connection_id,
1148 generation: inner.next_generation,
1149 };
1150 inner.endpoint_by_connection.insert(connection_id, endpoint);
1151 inner
1152 .module_id_by_endpoint
1153 .insert(endpoint, module_id.clone());
1154 inner.next_module_channel.insert(endpoint, 1);
1155 inner.next_control_corr.insert(endpoint, 1);
1156 inner.candidates_by_id.insert(
1157 module_id,
1158 ModuleConnection {
1159 endpoint,
1160 sink,
1161 negotiated_ver,
1162 concurrency,
1163 },
1164 );
1165 Ok(endpoint)
1166 }
1167
1168 pub(crate) fn cutover_candidate(
1186 &self,
1187 module_id: &str,
1188 ) -> Result<Option<ForwardingCutover>, ForwardingError> {
1189 let mut inner = self.write_inner()?;
1190 if inner.daemon_draining {
1191 return Err(ForwardingError::ModuleReloading {
1192 module_id: module_id.to_string(),
1193 });
1194 }
1195 let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
1196 return Ok(None);
1197 };
1198 let promoted = candidate.endpoint;
1199 let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
1200 let incumbent = incumbent.map(|incumbent| {
1201 let endpoint = incumbent.endpoint;
1202 inner.superseded_endpoints.insert(endpoint, incumbent);
1203 endpoint
1204 });
1205 drop(inner);
1206
1207 if let Some(discarded) = self
1211 .route_bind_breakers
1212 .reset_for_new_module_connection(module_id)
1213 {
1214 info!(
1215 module_id = %module_id,
1216 discarded_consecutive_timeouts = discarded,
1217 "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
1218 );
1219 }
1220 Ok(Some(ForwardingCutover {
1221 promoted,
1222 incumbent,
1223 }))
1224 }
1225
1226 #[allow(clippy::too_many_arguments)]
1227 pub(crate) async fn begin_route_bind_relay_for(
1228 &self,
1229 client_connection_id: ConnectionId,
1230 client_sink: FrameSink,
1231 client_negotiated_ver: u8,
1232 client_corr: u64,
1233 module_id: &str,
1234 principal: Principal,
1235 scope: Option<BoundScope>,
1236 project_root: Option<ProjectRootId>,
1237 harness: String,
1238 build_class: Option<String>,
1239 operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1240 deadline: Instant,
1241 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1242 let client_permit =
1246 client_sink
1247 .reserve_owned()
1248 .await
1249 .map_err(|_| ForwardingError::ClientEgressClosed {
1250 connection_id: client_connection_id,
1251 })?;
1252 self.begin_route_bind_relay_inner(
1253 client_connection_id,
1254 client_sink,
1255 client_negotiated_ver,
1256 client_corr,
1257 module_id,
1258 principal,
1259 scope,
1260 project_root,
1261 harness,
1262 build_class,
1263 operator_app,
1264 deadline,
1265 client_permit,
1266 )
1267 }
1268
1269 #[cfg(test)]
1270 pub(crate) fn begin_route_bind_relay_for_test(
1271 &self,
1272 client_connection_id: ConnectionId,
1273 client_sink: FrameSink,
1274 client_corr: u64,
1275 module_id: &str,
1276 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1277 let permit =
1278 client_sink
1279 .try_reserve_owned()
1280 .map_err(|_| ForwardingError::ClientEgressClosed {
1281 connection_id: client_connection_id,
1282 })?;
1283 self.begin_route_bind_relay_inner(
1284 client_connection_id,
1285 client_sink,
1286 subc_protocol::PROTOCOL_VERSION,
1287 client_corr,
1288 module_id,
1289 Principal::Direct,
1290 None,
1291 None,
1292 String::new(),
1293 None,
1294 None,
1295 Instant::now() + std::time::Duration::from_secs(60),
1296 permit,
1297 )
1298 }
1299
1300 pub(crate) fn begin_module_control_rpc_for(
1301 &self,
1302 module_id: &str,
1303 expected_op: &str,
1304 deadline: Instant,
1305 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1306 self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1307 }
1308
1309 pub(crate) fn begin_health_probe_rpc_for(
1310 &self,
1311 module_id: &str,
1312 expected_op: &str,
1313 probe_started_at: Instant,
1314 deadline: Instant,
1315 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1316 self.begin_module_control_rpc_inner(
1317 module_id,
1318 expected_op,
1319 deadline,
1320 Some(probe_started_at),
1321 false,
1322 )
1323 }
1324
1325 pub(crate) fn begin_drain_health_probe_rpc_for(
1326 &self,
1327 module_id: &str,
1328 expected_op: &str,
1329 probe_started_at: Instant,
1330 deadline: Instant,
1331 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1332 self.begin_module_control_rpc_inner(
1333 module_id,
1334 expected_op,
1335 deadline,
1336 Some(probe_started_at),
1337 true,
1338 )
1339 }
1340
1341 pub(crate) fn begin_endpoint_health_probe_rpc_for(
1349 &self,
1350 endpoint: ModuleEndpointId,
1351 expected_op: &str,
1352 probe_started_at: Instant,
1353 deadline: Instant,
1354 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1355 let inner = self.write_inner()?;
1356 let module = module_connection_for_endpoint_locked(&inner, endpoint)
1357 .cloned()
1358 .ok_or(ForwardingError::NoModuleConnection)?;
1359 let module_id = inner
1360 .module_id_by_endpoint
1361 .get(&endpoint)
1362 .cloned()
1363 .unwrap_or_default();
1364 self.begin_control_rpc_locked(
1367 inner,
1368 &module_id,
1369 module,
1370 expected_op,
1371 deadline,
1372 Some(probe_started_at),
1373 true,
1374 )
1375 }
1376
1377 fn begin_module_control_rpc_inner(
1378 &self,
1379 module_id: &str,
1380 expected_op: &str,
1381 deadline: Instant,
1382 health_probe_started_at: Option<Instant>,
1383 allow_draining: bool,
1384 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1385 let inner = self.write_inner()?;
1386 let module = inner
1387 .modules_by_id
1388 .get(module_id)
1389 .cloned()
1390 .ok_or(ForwardingError::NoModuleConnection)?;
1391 self.begin_control_rpc_locked(
1392 inner,
1393 module_id,
1394 module,
1395 expected_op,
1396 deadline,
1397 health_probe_started_at,
1398 allow_draining,
1399 )
1400 }
1401
1402 #[allow(clippy::too_many_arguments)]
1403 fn begin_control_rpc_locked(
1404 &self,
1405 mut inner: RwLockWriteGuard<'_, ForwardingInner>,
1406 module_id: &str,
1407 module: ModuleConnection,
1408 expected_op: &str,
1409 deadline: Instant,
1410 health_probe_started_at: Option<Instant>,
1411 allow_draining: bool,
1412 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1413 if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1414 return Err(ForwardingError::ModuleReloading {
1415 module_id: module_id.to_string(),
1416 });
1417 }
1418 if inner
1419 .closing_connections
1420 .contains(&module.endpoint.connection_id)
1421 {
1422 return Err(ForwardingError::ConnectionClosing {
1423 connection_id: module.endpoint.connection_id,
1424 });
1425 }
1426 if health_probe_started_at.is_some() {
1427 inner
1431 .health_probe_tombstones
1432 .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1433 }
1434 let corr = match inner.allocate_control_corr(module.endpoint) {
1435 Ok(corr) => corr,
1436 Err(err) => {
1437 drop(inner);
1438 self.request_connection_close(
1439 module.endpoint.connection_id,
1440 CloseReason::new(
1441 "control_correlation_exhausted",
1442 "daemon-originated channel-0 correlation space exhausted",
1443 ),
1444 );
1445 return Err(err);
1446 }
1447 };
1448 let (sender, receiver) = oneshot::channel();
1449 inner.pending_control_rpcs.insert(
1450 (module.endpoint, corr),
1451 PendingModuleControlRpcEntry {
1452 expected_op: expected_op.to_string(),
1453 deadline,
1454 health_probe_started_at,
1455 sender,
1456 },
1457 );
1458
1459 Ok(PendingModuleControlRpc {
1460 endpoint: module.endpoint,
1461 module_sink: module.sink,
1462 negotiated_ver: module.negotiated_ver,
1463 corr,
1464 receiver,
1465 })
1466 }
1467
1468 #[allow(clippy::too_many_arguments)]
1469 fn begin_route_bind_relay_inner(
1470 &self,
1471 client_connection_id: ConnectionId,
1472 client_sink: FrameSink,
1473 client_negotiated_ver: u8,
1474 client_corr: u64,
1475 expected_module_id: &str,
1476 principal: Principal,
1477 scope: Option<BoundScope>,
1478 project_root: Option<ProjectRootId>,
1479 harness: String,
1480 build_class: Option<String>,
1481 operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1482 deadline: Instant,
1483 client_permit: crate::router::EgressPermit,
1484 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1485 let mut inner = self.write_inner()?;
1486 if inner
1490 .endpoint_by_connection
1491 .contains_key(&client_connection_id)
1492 {
1493 return Err(ForwardingError::ConnectionRoleConflict {
1494 connection_id: client_connection_id,
1495 });
1496 }
1497 if inner.closing_connections.contains(&client_connection_id) {
1498 return Err(ForwardingError::ConnectionClosing {
1499 connection_id: client_connection_id,
1500 });
1501 }
1502 let module = inner
1503 .modules_by_id
1504 .get(expected_module_id)
1505 .cloned()
1506 .ok_or(ForwardingError::NoModuleConnection)?;
1507 if inner.draining_endpoints.contains_key(&module.endpoint) {
1508 return Err(ForwardingError::ModuleReloading {
1509 module_id: expected_module_id.to_string(),
1510 });
1511 }
1512 if inner
1513 .closing_connections
1514 .contains(&module.endpoint.connection_id)
1515 {
1516 return Err(ForwardingError::ConnectionClosing {
1517 connection_id: module.endpoint.connection_id,
1518 });
1519 }
1520
1521 let corr = match inner.allocate_control_corr(module.endpoint) {
1522 Ok(corr) => corr,
1523 Err(err) => {
1524 drop(inner);
1525 self.request_connection_close(
1526 module.endpoint.connection_id,
1527 CloseReason::new(
1528 "control_correlation_exhausted",
1529 "daemon-originated channel-0 correlation space exhausted",
1530 ),
1531 );
1532 return Err(err);
1533 }
1534 };
1535 let (client_channel, client_epoch, module_channel, module_epoch) =
1536 inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1537 let client_key = ClientRouteKey {
1538 connection_id: client_connection_id,
1539 channel: client_channel,
1540 };
1541 let module_key = ModuleRouteKey {
1542 endpoint: module.endpoint,
1543 channel: module_channel,
1544 };
1545 let reservation = RouteReservation {
1546 client_key,
1547 module_key,
1548 client_epoch,
1549 module_epoch,
1550 project_root,
1551 };
1552 let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
1553 route_channel: client_channel,
1554 route_epoch: client_epoch,
1555 })
1556 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1557 let route_open_frame = Frame::build_with_version(
1558 client_negotiated_ver,
1559 FrameType::Response,
1560 Flags::new(false, Priority::Passive, false),
1561 0,
1562 0,
1563 client_corr,
1564 response_body,
1565 )
1566 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1567 let (sender, receiver) = oneshot::channel();
1568 inner.reserved_client.insert(client_key, module_key);
1569 inner.reserved_module.insert(module_key, client_key);
1570 inner.pending_relays.insert(
1571 (module.endpoint, corr),
1572 PendingRouteBindRelayEntry {
1573 reservation,
1574 client_sink,
1575 client_negotiated_ver,
1576 client_permit,
1577 route_open_frame,
1578 principal,
1579 scope,
1580 harness,
1581 build_class,
1582 operator_app,
1583 deadline,
1584 relay_enqueued: false,
1585 sender,
1586 },
1587 );
1588
1589 Ok(PendingRouteBindRelay {
1590 endpoint: module.endpoint,
1591 module_sink: module.sink,
1592 negotiated_ver: module.negotiated_ver,
1593 client_channel,
1594 client_epoch,
1595 module_channel,
1596 module_epoch,
1597 corr,
1598 receiver,
1599 })
1600 }
1601
1602 pub(crate) fn mark_route_bind_relay_enqueued(
1612 &self,
1613 endpoint: ModuleEndpointId,
1614 corr: u64,
1615 ) -> Result<RouteBindRelayMark, ForwardingError> {
1616 let mut inner = self.write_inner()?;
1617 if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
1618 pending.relay_enqueued = true;
1619 return Ok(RouteBindRelayMark::Marked);
1620 }
1621 if inner
1622 .relays_released_before_enqueue
1623 .remove(&(endpoint, corr))
1624 {
1625 return Ok(RouteBindRelayMark::ReleasedWithoutReply);
1626 }
1627 Ok(RouteBindRelayMark::SettledByReply)
1628 }
1629
1630 pub(crate) fn release_client_route(
1631 &self,
1632 client_connection_id: ConnectionId,
1633 client_channel: u16,
1634 expected_epoch: u32,
1635 ) -> Result<RouteRelease, ForwardingError> {
1636 let mut inner = self.write_inner()?;
1637 let release = release_client_route_locked(
1638 &mut inner,
1639 ClientRouteKey {
1640 connection_id: client_connection_id,
1641 channel: client_channel,
1642 },
1643 expected_epoch,
1644 );
1645 self.record_route_release(&release);
1646 Ok(release)
1647 }
1648
1649 pub(crate) fn release_module_route(
1650 &self,
1651 module_connection_id: ConnectionId,
1652 module_channel: u16,
1653 expected_epoch: u32,
1654 ) -> Result<RouteRelease, ForwardingError> {
1655 let mut inner = self.write_inner()?;
1656 let Some(endpoint) = inner
1657 .endpoint_by_connection
1658 .get(&module_connection_id)
1659 .copied()
1660 else {
1661 return Ok(RouteRelease::Absent);
1662 };
1663 let release = release_module_route_locked(
1664 &mut inner,
1665 ModuleRouteKey {
1666 endpoint,
1667 channel: module_channel,
1668 },
1669 expected_epoch,
1670 );
1671 self.record_route_release(&release);
1672 Ok(release)
1673 }
1674
1675 pub(crate) fn abort_pending_relay(
1676 &self,
1677 endpoint: ModuleEndpointId,
1678 corr: u64,
1679 outcome: RouteBindRelayOutcome,
1680 ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
1681 let mut inner = self.write_inner()?;
1682 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1683 inner
1688 .relays_released_before_enqueue
1689 .remove(&(endpoint, corr));
1690 return Ok(None);
1691 };
1692 release_reserved_route_locked(
1693 &mut inner,
1694 pending.reservation.client_key,
1695 pending.reservation.module_key,
1696 );
1697 let target = pending
1698 .relay_enqueued
1699 .then(|| abandoned_route_target(&inner, &pending.reservation));
1700 let _ = pending.sender.send(outcome);
1701 Ok(target.flatten())
1702 }
1703
1704 pub(crate) fn cancel_module_control_rpc(
1705 &self,
1706 endpoint: ModuleEndpointId,
1707 corr: u64,
1708 ) -> Result<(), ForwardingError> {
1709 self.write_inner()?
1710 .pending_control_rpcs
1711 .remove(&(endpoint, corr));
1712 Ok(())
1713 }
1714
1715 pub(crate) fn tombstone_health_probe_rpc(
1716 &self,
1717 endpoint: ModuleEndpointId,
1718 corr: u64,
1719 ) -> Result<bool, ForwardingError> {
1720 let key = (endpoint, corr);
1721 let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
1722 {
1723 let mut inner = self.write_inner()?;
1724 let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
1725 return Ok(false);
1726 };
1727 let Some(probe_started_at) = pending.health_probe_started_at else {
1728 inner.pending_control_rpcs.insert(key, pending);
1729 return Ok(false);
1730 };
1731 let module_id = inner
1732 .module_id_by_endpoint
1733 .get(&endpoint)
1734 .cloned()
1735 .unwrap_or_else(|| "unknown".to_string());
1736 inner.health_probe_tombstones.insert(
1737 key,
1738 HealthProbeTombstone {
1739 expected_op: pending.expected_op,
1740 module_id,
1741 probe_started_at,
1742 expires_at,
1743 },
1744 );
1745 }
1746 self.schedule_health_probe_tombstone_expiration(key, expires_at);
1747 Ok(true)
1748 }
1749
1750 fn schedule_health_probe_tombstone_expiration(
1751 &self,
1752 key: (ModuleEndpointId, u64),
1753 expires_at: Instant,
1754 ) {
1755 let inner = Arc::downgrade(&self.inner);
1756 tokio::spawn(async move {
1757 tokio::time::sleep_until(expires_at).await;
1758 let Some(inner) = inner.upgrade() else {
1759 return;
1760 };
1761 let Ok(mut inner) = inner.write() else {
1762 return;
1763 };
1764 let expired = inner
1765 .health_probe_tombstones
1766 .get(&key)
1767 .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
1768 if expired {
1769 inner.health_probe_tombstones.remove(&key);
1770 }
1771 });
1772 }
1773
1774 pub(crate) fn complete_pending_relay(
1775 &self,
1776 connection_id: ConnectionId,
1777 corr: u64,
1778 outcome: RouteBindRelayOutcome,
1779 ) -> Result<PendingRelayCompletion, ForwardingError> {
1780 let mut inner = self.write_inner()?;
1781 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1782 return Ok(PendingRelayCompletion {
1783 settled: false,
1784 abandoned: None,
1785 });
1786 };
1787 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1788 return Ok(PendingRelayCompletion {
1789 settled: false,
1790 abandoned: None,
1791 });
1792 };
1793
1794 if Instant::now() >= pending.deadline {
1795 release_reserved_route_locked(
1796 &mut inner,
1797 pending.reservation.client_key,
1798 pending.reservation.module_key,
1799 );
1800 let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
1801 .then(|| abandoned_route_target(&inner, &pending.reservation))
1802 .flatten();
1803 let _ = pending
1804 .sender
1805 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody {
1806 code: "module_timeout".to_string(),
1807 message: "route.bind response arrived after its daemon deadline".to_string(),
1808 detail: None,
1809 }));
1810 return Ok(PendingRelayCompletion {
1811 settled: true,
1812 abandoned,
1813 });
1814 }
1815
1816 match outcome {
1817 RouteBindRelayOutcome::Accepted
1836 if pending.client_sink.is_closed()
1837 || inner
1838 .closing_connections
1839 .contains(&pending.reservation.client_key.connection_id) =>
1840 {
1841 let reason = if pending.client_sink.is_closed() {
1842 "client egress closed before route publication"
1843 } else {
1844 "client connection is closing before route publication"
1845 };
1846 release_reserved_route_locked(
1847 &mut inner,
1848 pending.reservation.client_key,
1849 pending.reservation.module_key,
1850 );
1851 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1855 let _ = pending
1856 .sender
1857 .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
1858 return Ok(PendingRelayCompletion {
1859 settled: true,
1860 abandoned,
1861 });
1862 }
1863 RouteBindRelayOutcome::Accepted
1886 if inner.superseded_endpoints.contains_key(&endpoint)
1887 || inner.draining_endpoints.contains_key(&endpoint) =>
1888 {
1889 release_reserved_route_locked(
1890 &mut inner,
1891 pending.reservation.client_key,
1892 pending.reservation.module_key,
1893 );
1894 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1897 let module_id = inner
1898 .module_id_by_endpoint
1899 .get(&endpoint)
1900 .cloned()
1901 .unwrap_or_else(|| "unknown".to_string());
1902 let _ = pending
1903 .sender
1904 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
1905 "module_reloading",
1906 format!("module_id '{module_id}' is reloading"),
1907 )));
1908 return Ok(PendingRelayCompletion {
1909 settled: true,
1910 abandoned,
1911 });
1912 }
1913 RouteBindRelayOutcome::Accepted
1922 if pending
1923 .scope
1924 .as_ref()
1925 .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
1926 {
1927 let refusal = pending
1928 .scope
1929 .as_ref()
1930 .and_then(|scope| scope_refusal_locked(&inner, scope))
1931 .expect("guard matched a refusal under the same lock");
1932 release_reserved_route_locked(
1933 &mut inner,
1934 pending.reservation.client_key,
1935 pending.reservation.module_key,
1936 );
1937 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1938 let _ = pending
1939 .sender
1940 .send(RouteBindRelayOutcome::DaemonRejected(refusal));
1941 return Ok(PendingRelayCompletion {
1942 settled: true,
1943 abandoned,
1944 });
1945 }
1946 RouteBindRelayOutcome::Accepted => {
1947 let abandoned = commit_route_locked(&mut inner, pending)?;
1948 return Ok(PendingRelayCompletion {
1949 settled: true,
1950 abandoned,
1951 });
1952 }
1953 terminal => {
1954 release_reserved_route_locked(
1955 &mut inner,
1956 pending.reservation.client_key,
1957 pending.reservation.module_key,
1958 );
1959 let _ = pending.sender.send(terminal);
1960 }
1961 }
1962 Ok(PendingRelayCompletion {
1963 settled: true,
1964 abandoned: None,
1965 })
1966 }
1967
1968 pub(crate) fn pending_module_control_op(
1969 &self,
1970 connection_id: ConnectionId,
1971 corr: u64,
1972 ) -> Result<Option<String>, ForwardingError> {
1973 let inner = self.read_inner()?;
1974 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1975 return Ok(None);
1976 };
1977 let key = (endpoint, corr);
1978 Ok(inner
1979 .pending_control_rpcs
1980 .get(&key)
1981 .map(|pending| pending.expected_op.clone())
1982 .or_else(|| {
1983 inner
1984 .health_probe_tombstones
1985 .get(&key)
1986 .filter(|tombstone| tombstone.expires_at > Instant::now())
1987 .map(|tombstone| tombstone.expected_op.clone())
1988 }))
1989 }
1990
1991 pub(crate) fn complete_module_control_rpc(
1992 &self,
1993 connection_id: ConnectionId,
1994 corr: u64,
1995 actual_op: Option<&str>,
1996 outcome: ModuleControlRpcOutcome,
1997 ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
1998 let now = Instant::now();
1999 let mut inner = self.write_inner()?;
2000 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2001 return Ok(ModuleControlRpcCompletion::Unknown);
2002 };
2003 let key = (endpoint, corr);
2004 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2005 if now >= pending.deadline {
2006 let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
2007 ModuleControlRpcCompletion::LateHealthAnswer {
2008 module_id: inner
2009 .module_id_by_endpoint
2010 .get(&endpoint)
2011 .cloned()
2012 .unwrap_or_else(|| "unknown".to_string()),
2013 latency: now.saturating_duration_since(probe_started_at),
2014 }
2015 });
2016 let _ = pending
2017 .sender
2018 .send(ModuleControlRpcOutcome::DeadlineElapsed);
2019 return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
2020 }
2021 let outcome = match actual_op {
2022 Some(actual) if actual != pending.expected_op => {
2023 ModuleControlRpcOutcome::UnexpectedOp {
2024 expected: pending.expected_op,
2025 actual: actual.to_string(),
2026 }
2027 }
2028 _ => outcome,
2029 };
2030 let _ = pending.sender.send(outcome);
2031 return Ok(ModuleControlRpcCompletion::Settled);
2032 }
2033
2034 let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
2035 return Ok(ModuleControlRpcCompletion::Unknown);
2036 };
2037 if tombstone.expires_at <= now {
2038 return Ok(ModuleControlRpcCompletion::Unknown);
2039 }
2040 Ok(ModuleControlRpcCompletion::LateHealthAnswer {
2041 module_id: tombstone.module_id,
2042 latency: now.saturating_duration_since(tombstone.probe_started_at),
2043 })
2044 }
2045
2046 #[cfg(test)]
2047 pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
2048 Ok(self.read_inner()?.health_probe_tombstones.len())
2049 }
2050
2051 #[cfg(test)]
2052 pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
2053 Ok(self.read_inner()?.closing_connections.len())
2054 }
2055
2056 #[cfg(test)]
2059 pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
2060 let inner = self.read_inner()?;
2061 Ok((inner.reserved_client.len(), inner.reserved_module.len()))
2062 }
2063
2064 pub(crate) fn module_endpoint_for_connection(
2065 &self,
2066 connection_id: ConnectionId,
2067 ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
2068 Ok(self
2069 .read_inner()?
2070 .endpoint_by_connection
2071 .get(&connection_id)
2072 .copied())
2073 }
2074
2075 pub(crate) fn module_id_for_connection(
2078 &self,
2079 connection_id: ConnectionId,
2080 ) -> Result<Option<String>, ForwardingError> {
2081 let inner = self.read_inner()?;
2082 Ok(inner
2083 .endpoint_by_connection
2084 .get(&connection_id)
2085 .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
2086 .cloned())
2087 }
2088
2089 pub(crate) fn module_route_epoch_was_allocated(
2096 &self,
2097 connection_id: ConnectionId,
2098 channel: u16,
2099 epoch: u32,
2100 ) -> Result<bool, ForwardingError> {
2101 let inner = self.read_inner()?;
2102 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2103 return Ok(false);
2104 };
2105 Ok(inner
2106 .module_slot_epochs
2107 .get(&ModuleRouteKey { endpoint, channel })
2108 .is_some_and(|last| epoch != 0 && epoch <= *last))
2109 }
2110
2111 pub(crate) fn has_live_module_connection(
2112 &self,
2113 module_id: &str,
2114 ) -> Result<bool, ForwardingError> {
2115 Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
2116 }
2117
2118 pub(crate) fn lookup_data_route(
2119 &self,
2120 connection_id: ConnectionId,
2121 channel: u16,
2122 epoch: u32,
2123 ) -> Result<DataRoute, ForwardingError> {
2124 let inner = self.read_inner()?;
2125 let state = if let Some(endpoint) =
2126 inner.endpoint_by_connection.get(&connection_id).copied()
2127 {
2128 let key = ModuleRouteKey { endpoint, channel };
2129 match inner.module_to_client.get(&key) {
2130 Some(route) if route.module_epoch == epoch => {
2131 DataRouteState::Bound(Arc::clone(route))
2132 }
2133 Some(_) => DataRouteState::EpochMismatch,
2134 None if inner.reserved_module.contains_key(&key)
2135 && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
2136 {
2137 DataRouteState::Reserved
2138 }
2139 None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
2140 None => DataRouteState::Absent,
2141 }
2142 } else {
2143 let key = ClientRouteKey {
2144 connection_id,
2145 channel,
2146 };
2147 match inner.client_to_module.get(&key) {
2148 Some(route) if route.client_epoch == epoch => {
2149 DataRouteState::Bound(Arc::clone(route))
2150 }
2151 Some(_) => DataRouteState::EpochMismatch,
2152 None if inner.reserved_client.contains_key(&key)
2153 && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
2154 {
2155 DataRouteState::Reserved
2156 }
2157 None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
2158 None => DataRouteState::Absent,
2159 }
2160 };
2161 Ok(
2162 if inner.endpoint_by_connection.contains_key(&connection_id) {
2163 DataRoute::Module(state)
2164 } else {
2165 DataRoute::Client(state)
2166 },
2167 )
2168 }
2169
2170 #[cfg(test)]
2171 pub(crate) fn inject_client_slot_epoch(
2172 &self,
2173 connection_id: ConnectionId,
2174 channel: u16,
2175 last_epoch: u32,
2176 ) {
2177 let mut inner = self.write_inner().expect("forwarding lock");
2178 inner.client_slot_epochs.insert(
2179 ClientRouteKey {
2180 connection_id,
2181 channel,
2182 },
2183 last_epoch,
2184 );
2185 inner.next_client_channel.insert(connection_id, channel);
2186 }
2187
2188 #[cfg(test)]
2189 pub(crate) fn inject_module_slot_epoch(
2190 &self,
2191 endpoint: ModuleEndpointId,
2192 channel: u16,
2193 last_epoch: u32,
2194 ) {
2195 let mut inner = self.write_inner().expect("forwarding lock");
2196 inner
2197 .module_slot_epochs
2198 .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
2199 inner.next_module_channel.insert(endpoint, channel);
2200 }
2201
2202 #[cfg(test)]
2203 pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
2204 self.write_inner()
2205 .expect("forwarding lock")
2206 .next_control_corr
2207 .insert(endpoint, next_corr);
2208 }
2209
2210 pub(crate) fn cache_status(
2211 &self,
2212 endpoint: ModuleEndpointId,
2213 module_channel: u16,
2214 module_epoch: u32,
2215 status: String,
2216 ) -> Result<bool, ForwardingError> {
2217 let mut inner = self.write_inner()?;
2218 if !inner.module_id_by_endpoint.contains_key(&endpoint) {
2219 return Err(ForwardingError::StaleModuleEndpoint);
2220 }
2221
2222 let module_key = ModuleRouteKey {
2223 endpoint,
2224 channel: module_channel,
2225 };
2226 let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
2227 (route.module_epoch == module_epoch).then_some((
2228 ClientRouteKey {
2229 connection_id: route.client_connection_id,
2230 channel: route.client_channel,
2231 },
2232 route.client_epoch,
2233 ))
2234 } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
2235 (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
2236 client_key,
2237 inner
2238 .client_slot_epochs
2239 .get(&client_key)
2240 .copied()
2241 .unwrap_or(0),
2242 ))
2243 } else {
2244 None
2245 };
2246
2247 if let Some(handle) = handle {
2248 inner.status.insert(handle, status);
2249 Ok(true)
2250 } else {
2251 debug!(
2252 module_channel,
2253 module_epoch,
2254 generation = endpoint.generation,
2255 connection_id = endpoint.connection_id.get(),
2256 "dropping stale status update for module route handle"
2257 );
2258 Ok(false)
2259 }
2260 }
2261
2262 pub(crate) fn route_poll_snapshot(
2263 &self,
2264 client_connection_id: ConnectionId,
2265 client_channel: u16,
2266 client_epoch: u32,
2267 ) -> Result<RoutePollSnapshot, ForwardingError> {
2268 let inner = self.read_inner()?;
2269 let client_key = ClientRouteKey {
2270 connection_id: client_connection_id,
2271 channel: client_channel,
2272 };
2273 let Some(route) = inner.client_to_module.get(&client_key) else {
2274 return Ok(RoutePollSnapshot::Absent);
2275 };
2276 if route.client_epoch != client_epoch
2277 || !inner
2278 .module_id_by_endpoint
2279 .contains_key(&route.module_endpoint)
2280 {
2281 return Ok(RoutePollSnapshot::Absent);
2282 }
2283 Ok(RoutePollSnapshot::Bound {
2284 module_id: route.module_id.clone(),
2285 status: inner.status.get(&(client_key, client_epoch)).cloned(),
2286 })
2287 }
2288
2289 pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2290 Ok(self.read_inner()?.client_to_module.len())
2291 }
2292
2293 pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2303 let inner = self.read_inner()?;
2304 let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2305 for key in inner.client_to_module.keys() {
2306 *per_connection.entry(key.connection_id).or_insert(0) += 1;
2307 }
2308 let max = per_connection.values().copied().max().unwrap_or(0);
2309 Ok((per_connection.len(), max))
2310 }
2311
2312 pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2313 let inner = self.read_inner()?;
2314 Ok(inner
2315 .client_to_module
2316 .keys()
2317 .any(|key| key.channel == route_channel))
2318 }
2319
2320 pub(crate) fn daemon_drain_result_for_connection(
2324 &self,
2325 connection_id: ConnectionId,
2326 ) -> Result<Option<bool>, ForwardingError> {
2327 let inner = self.read_inner()?;
2328 Ok(inner.daemon_draining.then(|| {
2329 inner
2330 .endpoint_by_connection
2331 .get(&connection_id)
2332 .and_then(|endpoint| inner.daemon_drain_results.get(endpoint))
2333 .copied()
2334 .unwrap_or(false)
2335 }))
2336 }
2337
2338 #[cfg(unix)]
2339 pub(crate) fn record_daemon_drain_result(
2340 &self,
2341 endpoint: ModuleEndpointId,
2342 drained: bool,
2343 ) -> Result<(), ForwardingError> {
2344 let mut inner = self.write_inner()?;
2345 if let Some(result) = inner.daemon_drain_results.get_mut(&endpoint) {
2349 *result = drained;
2350 }
2351 Ok(())
2352 }
2353
2354 #[cfg(unix)]
2357 pub(crate) fn begin_daemon_drain(
2358 &self,
2359 ) -> Result<Vec<(String, ModuleEndpointId)>, ForwardingError> {
2360 let mut inner = self.write_inner()?;
2361 if inner.daemon_draining {
2362 return Ok(Vec::new());
2363 }
2364 inner.daemon_draining = true;
2365 let modules = inner
2366 .modules_by_id
2367 .iter()
2368 .map(|(id, module)| (id.clone(), module.endpoint))
2369 .collect::<Vec<_>>();
2370 for (_, endpoint) in &modules {
2371 inner.daemon_drain_results.insert(*endpoint, false);
2372 inner
2373 .draining_endpoints
2374 .insert(*endpoint, RouteCloseReason::Restart);
2375 }
2376 let off_slot_endpoints = inner
2381 .candidates_by_id
2382 .values()
2383 .map(|module| module.endpoint)
2384 .chain(inner.superseded_endpoints.keys().copied())
2385 .collect::<Vec<_>>();
2386 for endpoint in off_slot_endpoints {
2387 inner
2388 .draining_endpoints
2389 .insert(endpoint, RouteCloseReason::Restart);
2390 }
2391 Ok(modules)
2392 }
2393
2394 pub(crate) fn begin_module_drain(
2401 &self,
2402 module_id: &str,
2403 reason: RouteCloseReason,
2404 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2405 let mut inner = self.write_inner()?;
2406 let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2407 return Ok(None);
2408 };
2409 Ok(Some(begin_drain_locked(
2410 &mut inner, module_id, module, reason,
2411 )))
2412 }
2413
2414 pub(crate) fn begin_endpoint_drain(
2421 &self,
2422 endpoint: ModuleEndpointId,
2423 reason: RouteCloseReason,
2424 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2425 let mut inner = self.write_inner()?;
2426 let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2427 return Ok(None);
2428 };
2429 let module_id = inner
2430 .module_id_by_endpoint
2431 .get(&endpoint)
2432 .cloned()
2433 .expect("an endpoint resolved to a module connection has a module id");
2434 Ok(Some(begin_drain_locked(
2435 &mut inner, &module_id, module, reason,
2436 )))
2437 }
2438}
2439
2440fn begin_drain_locked(
2444 inner: &mut ForwardingInner,
2445 module_id: &str,
2446 module: ModuleConnection,
2447 reason: RouteCloseReason,
2448) -> ModuleDrainTarget {
2449 {
2450 let endpoint = module.endpoint;
2451 inner.draining_endpoints.insert(endpoint, reason);
2452
2453 let flows = inner
2454 .client_to_module
2455 .values()
2456 .filter(|route| route.module_endpoint == endpoint)
2457 .map(|route| Arc::clone(&route.flow))
2458 .collect::<Vec<_>>();
2459 let excluded_subscriptions = flows
2460 .into_iter()
2461 .map(|flow| flow.begin_drain())
2462 .fold(0u32, u32::saturating_add);
2463
2464 let pending_keys = inner
2465 .pending_relays
2466 .keys()
2467 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2468 .copied()
2469 .collect::<Vec<_>>();
2470 let mut abandoned_bindings = Vec::new();
2471 for key in pending_keys {
2472 let Some(pending) = inner.pending_relays.remove(&key) else {
2473 continue;
2474 };
2475 release_reserved_route_locked(
2476 inner,
2477 pending.reservation.client_key,
2478 pending.reservation.module_key,
2479 );
2480 if pending.relay_enqueued {
2481 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2482 abandoned_bindings.push(target);
2483 }
2484 } else {
2485 inner.relays_released_before_enqueue.insert(key);
2489 }
2490 let _ = pending
2491 .sender
2492 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
2493 "module_reloading",
2494 format!("module_id '{module_id}' is reloading"),
2495 )));
2496 }
2497
2498 let pending_control_keys = inner
2499 .pending_control_rpcs
2500 .keys()
2501 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2502 .copied()
2503 .collect::<Vec<_>>();
2504 for key in pending_control_keys {
2505 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2506 let _ = pending
2507 .sender
2508 .send(ModuleControlRpcOutcome::ModuleGone(format!(
2509 "module '{module_id}' began draining during module-control RPC"
2510 )));
2511 }
2512 }
2513
2514 ModuleDrainTarget {
2515 endpoint,
2516 sink: module.sink,
2517 negotiated_ver: module.negotiated_ver,
2518 abandoned_bindings,
2519 excluded_subscriptions,
2520 }
2521 }
2522}
2523
2524#[derive(Debug, Default, PartialEq, Eq)]
2527pub(crate) struct DrainHoldouts {
2528 pub(crate) requests: usize,
2531 pub(crate) routes: usize,
2533 pub(crate) total_routes: usize,
2535 pub(crate) top_connections: Vec<(u64, usize)>,
2538 pub(crate) held: Vec<(u16, u64)>,
2547}
2548
2549pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
2551
2552impl ForwardingTable {
2553 pub(crate) fn endpoint_drain_holdouts(
2555 &self,
2556 endpoint: ModuleEndpointId,
2557 ) -> Result<DrainHoldouts, ForwardingError> {
2558 let inner = self.read_inner()?;
2559 let mut holdouts = DrainHoldouts::default();
2560 let mut by_connection: HashMap<u64, usize> = HashMap::new();
2561 for (key, route) in &inner.client_to_module {
2562 if route.module_endpoint != endpoint {
2563 continue;
2564 }
2565 holdouts.total_routes += 1;
2566 let held = route.flow.drain_in_flight();
2567 if held == 0 {
2568 continue;
2569 }
2570 holdouts.requests += held;
2571 holdouts.routes += 1;
2572 *by_connection.entry(key.connection_id.get()).or_default() += held;
2573 holdouts.held.extend(
2574 route
2575 .flow
2576 .drain_held_corrs()
2577 .into_iter()
2578 .map(|corr| (route.module_channel, corr)),
2579 );
2580 }
2581 holdouts.held.sort_unstable();
2582 holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
2583 let mut connections = by_connection.into_iter().collect::<Vec<_>>();
2584 connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
2585 connections.truncate(3);
2586 holdouts.top_connections = connections;
2587 Ok(holdouts)
2588 }
2589
2590 pub(crate) fn endpoint_in_flight_count(
2591 &self,
2592 endpoint: ModuleEndpointId,
2593 ) -> Result<usize, ForwardingError> {
2594 let inner = self.read_inner()?;
2595 Ok(inner
2596 .client_to_module
2597 .values()
2598 .filter(|route| route.module_endpoint == endpoint)
2599 .map(|route| route.flow.drain_in_flight())
2600 .sum())
2601 }
2602
2603 pub(crate) fn endpoint_is_draining(
2604 &self,
2605 endpoint: ModuleEndpointId,
2606 ) -> Result<bool, ForwardingError> {
2607 Ok(self
2608 .read_inner()?
2609 .draining_endpoints
2610 .contains_key(&endpoint))
2611 }
2612
2613 pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
2614 let inner = self.read_inner()?;
2615 Ok(inner
2616 .modules_by_id
2617 .get(module_id)
2618 .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
2619 }
2620
2621 pub(crate) fn release_module_endpoint_routes(
2622 &self,
2623 endpoint: ModuleEndpointId,
2624 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2625 let mut inner = self.write_inner()?;
2626 let routes = inner
2627 .module_to_client
2628 .iter()
2629 .filter(|(module_key, _)| module_key.endpoint == endpoint)
2630 .map(|(module_key, route)| (*module_key, route.module_epoch))
2631 .collect::<Vec<_>>();
2632 let mut released = Vec::with_capacity(routes.len());
2633 for (module_key, epoch) in routes {
2634 if let RouteRelease::Removed(target) =
2635 release_module_route_locked(&mut inner, module_key, epoch)
2636 {
2637 released.push(target);
2638 }
2639 }
2640 Ok(released)
2641 }
2642
2643 pub(crate) fn endpoint_routes(
2649 &self,
2650 endpoint: ModuleEndpointId,
2651 ) -> Result<Vec<EndpointRoute>, ForwardingError> {
2652 let inner = self.read_inner()?;
2653 Ok(endpoint_routes_locked(&inner, endpoint))
2654 }
2655
2656 pub(crate) fn route_census(
2658 &self,
2659 module_id: Option<&str>,
2660 ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
2661 let inner = self.read_inner()?;
2662 let mut endpoints = inner
2663 .modules_by_id
2664 .iter()
2665 .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
2666 .map(|(id, module)| (id.clone(), module.endpoint))
2667 .collect::<Vec<_>>();
2668 endpoints.sort_by(|left, right| left.0.cmp(&right.0));
2669 Ok(endpoints
2670 .into_iter()
2671 .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
2672 .collect())
2673 }
2674
2675 pub(crate) fn live_roots(
2677 &self,
2678 module_id: &str,
2679 ) -> Result<ModuleControlResponseToModule, ForwardingError> {
2680 let inner = self.read_inner()?;
2681 let endpoint = inner
2682 .modules_by_id
2683 .get(module_id)
2684 .map(|module| module.endpoint);
2685 let mut roots = BTreeMap::new();
2686 let mut unknown_root_bindings = 0;
2687 let mut total_bindings = 0;
2688 if let Some(endpoint) = endpoint {
2689 for binding in inner
2690 .module_to_client
2691 .values()
2692 .filter(|binding| binding.module_endpoint == endpoint)
2693 {
2694 total_bindings += 1;
2695 let workdir = binding
2696 .scope
2697 .as_ref()
2698 .and_then(|scope| scope.workdir.as_deref())
2699 .map(std::path::Path::new);
2700 if let Some(root) = &binding.project_root {
2701 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2702 entry.0 += 1;
2703 } else if workdir.is_none() {
2704 unknown_root_bindings += 1;
2705 }
2706 if let Some(workdir) = workdir.filter(|workdir| {
2707 binding.project_root.as_ref().map(ProjectRootId::as_path) != Some(*workdir)
2708 }) {
2709 roots.entry(workdir.to_path_buf()).or_insert((0, 0)).0 += 1;
2710 }
2711 }
2712 for pending in inner
2713 .pending_relays
2714 .values()
2715 .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
2716 {
2717 total_bindings += 1;
2718 if let Some(root) = &pending.reservation.project_root {
2719 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2720 entry.1 += 1;
2721 } else {
2722 unknown_root_bindings += 1;
2723 }
2724 }
2725 }
2726 Ok(ModuleControlResponseToModule::LiveRoots {
2727 roots: roots
2728 .into_iter()
2729 .map(|(project_root, (bound, pending))| LiveRoot {
2730 project_root,
2731 bound,
2732 pending,
2733 })
2734 .collect(),
2735 unknown_root_bindings,
2736 total_bindings,
2737 })
2738 }
2739
2740 pub(crate) fn connection_has_client_routes(
2746 &self,
2747 connection_id: ConnectionId,
2748 ) -> Result<bool, ForwardingError> {
2749 let inner = self.read_inner()?;
2750 Ok(connection_has_client_routes_locked(&inner, connection_id))
2751 }
2752
2753 pub(crate) fn cleanup_connection(
2754 &self,
2755 connection_id: ConnectionId,
2756 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2757 self.cleanup_connection_counted(connection_id)
2758 .map(|cleanup| cleanup.released)
2759 }
2760
2761 pub(crate) fn cleanup_connection_counted(
2766 &self,
2767 connection_id: ConnectionId,
2768 ) -> Result<ConnectionCleanup, ForwardingError> {
2769 self.frame_drop_warnings
2770 .lock()
2771 .unwrap_or_else(std::sync::PoisonError::into_inner)
2772 .remove(&connection_id);
2773 let mut inner = self.write_inner()?;
2774 inner.closing_connections.insert(connection_id);
2775 let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
2776 remove_module_connection_locked(&mut inner, endpoint)
2777 } else {
2778 ConnectionCleanup {
2779 released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
2780 abandoned_relays: 0,
2781 }
2782 };
2783 inner.closing_connections.remove(&connection_id);
2792 Ok(cleanup)
2793 }
2794
2795 fn cleanup_client_connection_locked(
2796 inner: &mut ForwardingInner,
2797 connection_id: ConnectionId,
2798 ) -> Vec<GoodbyeTarget> {
2799 let routes = inner
2800 .client_to_module
2801 .iter()
2802 .filter(|(key, _)| key.connection_id == connection_id)
2803 .map(|(key, route)| (*key, route.client_epoch))
2804 .collect::<Vec<_>>();
2805 let mut released = Vec::with_capacity(routes.len());
2806 for (client_key, epoch) in routes {
2807 if let RouteRelease::Removed(target) =
2808 release_client_route_locked(inner, client_key, epoch)
2809 {
2810 released.push(target);
2811 }
2812 }
2813
2814 let pending_keys = inner
2815 .pending_relays
2816 .iter()
2817 .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
2818 .map(|(key, _)| *key)
2819 .collect::<Vec<_>>();
2820 for key in pending_keys {
2821 let Some(pending) = inner.pending_relays.remove(&key) else {
2822 continue;
2823 };
2824 release_reserved_route_locked(
2825 inner,
2826 pending.reservation.client_key,
2827 pending.reservation.module_key,
2828 );
2829 if pending.relay_enqueued {
2830 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2831 released.push(target);
2832 }
2833 } else {
2834 inner.relays_released_before_enqueue.insert(key);
2838 }
2839 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
2840 "client connection closed during route.bind relay".to_string(),
2841 ));
2842 }
2843
2844 let orphaned = inner
2845 .reserved_client
2846 .iter()
2847 .filter(|(key, _)| key.connection_id == connection_id)
2848 .map(|(client, module)| (*client, *module))
2849 .collect::<Vec<_>>();
2850 for (client_key, module_key) in orphaned {
2851 release_reserved_route_locked(inner, client_key, module_key);
2852 }
2853 inner.next_client_channel.remove(&connection_id);
2854 inner
2855 .client_slot_epochs
2856 .retain(|key, _| key.connection_id != connection_id);
2857 inner
2858 .last_published_epoch
2859 .retain(|key, _| key.connection_id != connection_id);
2860 inner
2861 .status
2862 .retain(|(key, _), _| key.connection_id != connection_id);
2863
2864 released
2865 }
2866
2867 pub(crate) fn escalate_client_delivery_failure(
2876 &self,
2877 connection_id: ConnectionId,
2878 channel: u16,
2879 expected_epoch: u32,
2880 reason: CloseReason,
2881 undelivered: UndeliveredFrame<'_>,
2882 ) -> Result<bool, ForwardingError> {
2883 self.escalate_client_delivery_failure_diagnosed(
2884 connection_id,
2885 channel,
2886 expected_epoch,
2887 reason,
2888 undelivered,
2889 None,
2890 )
2891 }
2892
2893 pub(crate) fn escalate_client_delivery_failure_diagnosed(
2894 &self,
2895 connection_id: ConnectionId,
2896 channel: u16,
2897 expected_epoch: u32,
2898 reason: CloseReason,
2899 undelivered: UndeliveredFrame<'_>,
2900 diagnosis: Option<&crate::router::ClientDeliveryDiagnosis>,
2901 ) -> Result<bool, ForwardingError> {
2902 let principals = {
2903 let mut inner = self.write_inner()?;
2904 let key = ClientRouteKey {
2905 connection_id,
2906 channel,
2907 };
2908 if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
2909 None
2910 } else {
2911 inner.closing_connections.insert(connection_id);
2912 Some(connection_principals_locked(&inner, connection_id))
2913 }
2914 };
2915 let Some(principals) = principals else {
2916 self.warn_frame_drop(
2920 connection_id,
2921 channel,
2922 expected_epoch,
2923 "module_to_client",
2924 diagnosis
2925 .map(|failure| failure.cause)
2926 .unwrap_or("delivery_failed"),
2927 );
2928 return Ok(false);
2929 };
2930 let backlog = undelivered.sink.backlog();
2931 let close_reason = reason.to_string();
2932 if self.request_connection_close(connection_id, reason) {
2933 warn!(
2934 connection_id = connection_id.get(),
2935 principals = %principals,
2936 module_id = undelivered.module_id.unwrap_or("unknown"),
2937 client_channel = channel,
2938 queued_bytes = backlog.queued_bytes,
2939 queued_frames = backlog.queued_frames,
2940 frame_type = ?diagnosis.map(|failure| failure.frame_type),
2941 frame_flags = ?diagnosis.map(|failure| failure.flags.0),
2942 close_cause = diagnosis.map(|failure| failure.cause).unwrap_or("delivery_failed"),
2943 oldest_queued_ms = backlog
2944 .oldest_age
2945 .map(|age| age.as_millis() as u64)
2946 .unwrap_or(0),
2947 close_reason = %close_reason,
2948 "closing client connection: its egress queue could not take a frame"
2949 );
2950 }
2951 Ok(true)
2952 }
2953
2954 pub(crate) fn publish_scope_changes(
2968 &self,
2969 changes: &[ScopeTagChange],
2970 ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
2971 let mut inner = self.write_inner()?;
2972 let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
2973 for change in changes {
2974 let key = (change.owner.clone(), change.scope_ref.clone());
2975 match change.after {
2976 Some(tag) => {
2977 inner.scope_tags.insert(key, tag);
2978 }
2979 None => {
2980 inner.scope_tags.remove(&key);
2981 }
2982 }
2983 by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
2984 }
2985 let mut selected = Vec::new();
2986 for (client_key, route) in &inner.client_to_module {
2987 let Some(scope) = &route.scope else {
2988 continue;
2989 };
2990 let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
2991 else {
2992 continue;
2993 };
2994 if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
2995 continue;
2996 }
2997 let reason = match &change.drain {
2998 ScopeDrain::Nothing => continue,
2999 ScopeDrain::All(reason) => *reason,
3000 ScopeDrain::Carriers(narrowed) => {
3001 let owner = Principal::Reserved {
3002 module_id: scope.owner.clone(),
3003 };
3004 let hit = route.principal != owner
3005 && narrowed.iter().any(|(principal, allowed)| {
3006 *principal == route.principal
3007 && allowed
3008 .as_ref()
3009 .is_none_or(|targets| !targets.contains(&route.module_id))
3010 });
3011 if !hit {
3012 continue;
3013 }
3014 RouteCloseReason::ScopeCarrierRemoved
3015 }
3016 };
3017 selected.push((*client_key, route.client_epoch, reason));
3018 }
3019 let mut drained = Vec::new();
3020 for (client_key, client_epoch, reason) in selected {
3021 let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
3022 continue;
3023 };
3024 let release = release_client_route_locked(&mut inner, client_key, client_epoch);
3025 self.record_route_release(&release);
3026 if let RouteRelease::Removed(module) = release {
3027 drained.push(ScopeDrainedRoute {
3028 reason,
3029 scope: route
3030 .scope
3031 .clone()
3032 .expect("selected scope drain has a bound scope"),
3033 module_id: route.module_id.clone(),
3034 client: GoodbyeTarget {
3035 connection_id: route.client_connection_id,
3036 sink: route.client_sink.clone(),
3037 negotiated_ver: route.client_negotiated_ver,
3038 channel: route.client_channel,
3039 epoch: route.client_epoch,
3040 kind: GoodbyeTargetKind::Client,
3041 module_id: Some(route.module_id.clone()),
3042 },
3043 module,
3044 });
3045 }
3046 }
3047 Ok(drained)
3048 }
3049
3050 #[cfg(test)]
3052 pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
3053 self.read_inner()
3054 .ok()?
3055 .scope_tags
3056 .get(&(owner.to_string(), scope_ref.to_string()))
3057 .copied()
3058 }
3059
3060 fn record_route_release(&self, release: &RouteRelease) {
3061 match release {
3062 RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
3063 RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
3064 RouteRelease::Absent => {}
3065 }
3066 }
3067
3068 fn read_inner(&self) -> Result<RwLockReadGuard<'_, ForwardingInner>, ForwardingError> {
3069 self.inner.read().map_err(|_| ForwardingError::Poisoned)
3070 }
3071
3072 fn write_inner(&self) -> Result<RwLockWriteGuard<'_, ForwardingInner>, ForwardingError> {
3073 self.inner.write().map_err(|_| ForwardingError::Poisoned)
3074 }
3075
3076 fn lock_close_registry(
3077 &self,
3078 ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
3079 self.close_registry
3080 .lock()
3081 .unwrap_or_else(|poisoned| poisoned.into_inner())
3082 }
3083}
3084
3085impl ForwardingInner {
3086 fn allocate_route_slots(
3087 &mut self,
3088 connection_id: ConnectionId,
3089 endpoint: ModuleEndpointId,
3090 ) -> Result<(u16, u32, u16, u32), ForwardingError> {
3091 let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
3092 let mut client_channel = client_start;
3093 let client_channel = loop {
3094 let key = ClientRouteKey {
3095 connection_id,
3096 channel: client_channel,
3097 };
3098 let eligible = !self.client_to_module.contains_key(&key)
3099 && !self.reserved_client.contains_key(&key)
3100 && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3101 if eligible {
3102 break client_channel;
3103 }
3104 client_channel = next_channel(client_channel);
3105 if client_channel == client_start {
3106 return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
3107 }
3108 };
3109
3110 let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
3111 let mut module_channel = module_start;
3112 let module_channel = loop {
3113 let key = ModuleRouteKey {
3114 endpoint,
3115 channel: module_channel,
3116 };
3117 let eligible = !self.module_to_client.contains_key(&key)
3118 && !self.reserved_module.contains_key(&key)
3119 && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3120 if eligible {
3121 break module_channel;
3122 }
3123 module_channel = next_channel(module_channel);
3124 if module_channel == module_start {
3125 return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
3126 }
3127 };
3128
3129 let client_key = ClientRouteKey {
3130 connection_id,
3131 channel: client_channel,
3132 };
3133 let module_key = ModuleRouteKey {
3134 endpoint,
3135 channel: module_channel,
3136 };
3137 let client_epoch = self
3138 .client_slot_epochs
3139 .get(&client_key)
3140 .copied()
3141 .unwrap_or(0)
3142 + 1;
3143 let module_epoch = self
3144 .module_slot_epochs
3145 .get(&module_key)
3146 .copied()
3147 .unwrap_or(0)
3148 + 1;
3149 self.client_slot_epochs.insert(client_key, client_epoch);
3150 self.module_slot_epochs.insert(module_key, module_epoch);
3151 self.next_client_channel
3152 .insert(connection_id, next_channel(client_channel));
3153 self.next_module_channel
3154 .insert(endpoint, next_channel(module_channel));
3155 Ok((client_channel, client_epoch, module_channel, module_epoch))
3156 }
3157
3158 fn allocate_control_corr(
3159 &mut self,
3160 endpoint: ModuleEndpointId,
3161 ) -> Result<u64, ForwardingError> {
3162 let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
3163 if candidate == 0 {
3164 self.closing_connections.insert(endpoint.connection_id);
3165 return Err(ForwardingError::RelayCorrelationExhausted);
3166 }
3167 self.next_control_corr.insert(
3168 endpoint,
3169 if candidate == u64::MAX {
3170 0
3171 } else {
3172 candidate + 1
3173 },
3174 );
3175 Ok(candidate)
3176 }
3177}
3178
3179fn connection_has_client_routes_locked(
3180 inner: &ForwardingInner,
3181 connection_id: ConnectionId,
3182) -> bool {
3183 inner
3184 .client_to_module
3185 .keys()
3186 .any(|key| key.connection_id == connection_id)
3187 || inner
3188 .reserved_client
3189 .keys()
3190 .any(|key| key.connection_id == connection_id)
3191}
3192
3193fn check_module_connection_role_locked(
3194 inner: &ForwardingInner,
3195 connection_id: ConnectionId,
3196) -> Result<(), ForwardingError> {
3197 if inner.endpoint_by_connection.contains_key(&connection_id)
3198 || connection_has_client_routes_locked(inner, connection_id)
3199 {
3200 return Err(ForwardingError::ConnectionRoleConflict { connection_id });
3201 }
3202 Ok(())
3203}
3204
3205fn next_channel(channel: u16) -> u16 {
3206 let next = channel.wrapping_add(1);
3207 if next == 0 {
3208 1
3209 } else {
3210 next
3211 }
3212}
3213
3214fn endpoint_routes_locked(
3215 inner: &ForwardingInner,
3216 endpoint: ModuleEndpointId,
3217) -> Vec<EndpointRoute> {
3218 let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
3219 let draining = drain_reason.is_some();
3220 let mut routes = inner
3221 .module_to_client
3222 .iter()
3223 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3224 .map(|(_, route)| EndpointRoute {
3225 goodbye_target: GoodbyeTarget {
3226 connection_id: route.client_connection_id,
3227 sink: route.client_sink.clone(),
3228 negotiated_ver: route.client_negotiated_ver,
3229 channel: route.client_channel,
3230 epoch: route.client_epoch,
3231 kind: GoodbyeTargetKind::Client,
3232 module_id: Some(route.module_id.clone()),
3233 },
3234 principal: route.principal.clone(),
3235 bound_at: route.bound_at,
3236 draining,
3237 drain_reason,
3238 })
3239 .collect::<Vec<_>>();
3240 routes.sort_by_key(|route| {
3241 (
3242 route.goodbye_target.connection_id.get(),
3243 route.goodbye_target.channel,
3244 route.goodbye_target.epoch,
3245 )
3246 });
3247 routes
3248}
3249
3250fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
3254 let key = (scope.owner.clone(), scope.scope_ref.clone());
3255 match inner.scope_tags.get(&key) {
3256 Some(current) if *current == scope.tag => None,
3257 Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
3258 subc_protocol::error_codes::SCOPE_CHANGED,
3259 format!(
3260 "scope '{}' of {} changed while the route was being bound; re-open it",
3261 scope.scope_ref, scope.owner
3262 ),
3263 )),
3264 _ => Some(ErrorBody::new(
3265 subc_protocol::error_codes::SCOPE_ENDED,
3266 format!(
3267 "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
3268 scope.scope_ref, scope.owner, scope.tag.scope_epoch
3269 ),
3270 )),
3271 }
3272}
3273
3274fn release_reserved_route_locked(
3275 inner: &mut ForwardingInner,
3276 client_key: ClientRouteKey,
3277 module_key: ModuleRouteKey,
3278) {
3279 if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
3280 inner.reserved_client.remove(&client_key);
3281 }
3282 if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
3283 inner.reserved_module.remove(&module_key);
3284 }
3285 inner.status.retain(|(key, _), _| *key != client_key);
3286}
3287
3288fn release_client_route_locked(
3289 inner: &mut ForwardingInner,
3290 client_key: ClientRouteKey,
3291 expected_epoch: u32,
3292) -> RouteRelease {
3293 let Some(route) = inner.client_to_module.get(&client_key) else {
3294 return RouteRelease::Absent;
3295 };
3296 if route.client_epoch != expected_epoch {
3297 return RouteRelease::Stale;
3298 }
3299 let route = inner
3300 .client_to_module
3301 .remove(&client_key)
3302 .expect("route checked under the same forwarding lock");
3303 route.flow.close();
3304 if let Some(scope) = &route.scope {
3305 scope.release_bind();
3306 }
3307 inner.module_to_client.remove(&ModuleRouteKey {
3308 endpoint: route.module_endpoint,
3309 channel: route.module_channel,
3310 });
3311 inner.operator_confirms.route_closed(ModuleRouteKey {
3312 endpoint: route.module_endpoint,
3313 channel: route.module_channel,
3314 });
3315 inner.status.remove(&(client_key, expected_epoch));
3316 RouteRelease::Removed(GoodbyeTarget {
3317 connection_id: route.module_endpoint.connection_id,
3318 sink: route.module_sink.clone(),
3319 negotiated_ver: route.module_negotiated_ver,
3320 channel: route.module_channel,
3321 epoch: route.module_epoch,
3322 kind: GoodbyeTargetKind::Module,
3323 module_id: Some(route.module_id.clone()),
3324 })
3325}
3326
3327fn release_module_route_locked(
3328 inner: &mut ForwardingInner,
3329 module_key: ModuleRouteKey,
3330 expected_epoch: u32,
3331) -> RouteRelease {
3332 let Some(route) = inner.module_to_client.get(&module_key) else {
3333 return RouteRelease::Absent;
3334 };
3335 if route.module_epoch != expected_epoch {
3336 return RouteRelease::Stale;
3337 }
3338 let route = inner
3339 .module_to_client
3340 .remove(&module_key)
3341 .expect("route checked under the same forwarding lock");
3342 inner.operator_confirms.route_closed(module_key);
3343 route.flow.close();
3344 if let Some(scope) = &route.scope {
3345 scope.release_bind();
3346 }
3347 let client_key = ClientRouteKey {
3348 connection_id: route.client_connection_id,
3349 channel: route.client_channel,
3350 };
3351 inner.client_to_module.remove(&client_key);
3352 inner.status.remove(&(client_key, route.client_epoch));
3353 RouteRelease::Removed(GoodbyeTarget {
3354 connection_id: route.client_connection_id,
3355 sink: route.client_sink.clone(),
3356 negotiated_ver: route.client_negotiated_ver,
3357 channel: route.client_channel,
3358 epoch: route.client_epoch,
3359 kind: GoodbyeTargetKind::Client,
3360 module_id: Some(route.module_id.clone()),
3361 })
3362}
3363
3364fn commit_route_locked(
3365 inner: &mut ForwardingInner,
3366 pending: PendingRouteBindRelayEntry,
3367) -> Result<Option<GoodbyeTarget>, ForwardingError> {
3368 let reservation = pending.reservation;
3369 if inner
3370 .closing_connections
3371 .contains(&reservation.client_key.connection_id)
3372 {
3373 return Err(ForwardingError::ConnectionClosing {
3374 connection_id: reservation.client_key.connection_id,
3375 });
3376 }
3377 let module_id = inner
3378 .module_id_by_endpoint
3379 .get(&reservation.module_key.endpoint)
3380 .cloned()
3381 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3382 if inner
3383 .draining_endpoints
3384 .contains_key(&reservation.module_key.endpoint)
3385 {
3386 return Err(ForwardingError::ModuleReloading { module_id });
3387 }
3388 if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
3389 || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
3390 {
3391 return Err(ForwardingError::UnknownReservation {
3392 client_channel: reservation.client_key.channel,
3393 module_channel: reservation.module_key.channel,
3394 });
3395 }
3396 let module = inner
3397 .modules_by_id
3398 .get(&module_id)
3399 .filter(|module| module.endpoint == reservation.module_key.endpoint)
3400 .cloned()
3401 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3402 let binding = Arc::new(RouteBinding {
3403 client_connection_id: reservation.client_key.connection_id,
3404 client_sink: pending.client_sink,
3405 client_negotiated_ver: pending.client_negotiated_ver,
3406 client_channel: reservation.client_key.channel,
3407 client_epoch: reservation.client_epoch,
3408 module_id,
3409 module_endpoint: reservation.module_key.endpoint,
3410 module_sink: module.sink,
3411 module_negotiated_ver: module.negotiated_ver,
3412 module_channel: reservation.module_key.channel,
3413 module_epoch: reservation.module_epoch,
3414 principal: pending.principal,
3415 harness: pending.harness,
3416 build_class: pending.build_class,
3417 operator_app: pending.operator_app,
3418 project_root: reservation.project_root.clone(),
3419 bound_at: Instant::now(),
3420 flow: Arc::new(ChannelFlow::new(window_for(&module.concurrency))),
3421 scope: pending.scope,
3422 });
3423 inner
3424 .client_to_module
3425 .insert(reservation.client_key, Arc::clone(&binding));
3426 inner
3427 .module_to_client
3428 .insert(reservation.module_key, Arc::clone(&binding));
3429 let previous_published = inner
3430 .last_published_epoch
3431 .insert(reservation.client_key, reservation.client_epoch);
3432
3433 if let Some(scope) = &binding.scope {
3437 scope.record_served(&binding.module_id);
3438 }
3439
3440 let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
3445 if client_writer_closed {
3446 let abandoned = abandoned_route_target(inner, &reservation);
3450 if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
3451 route.flow.close();
3452 }
3453 inner.module_to_client.remove(&reservation.module_key);
3454 inner
3455 .status
3456 .remove(&(reservation.client_key, reservation.client_epoch));
3457 match previous_published {
3458 Some(epoch) => {
3459 inner
3460 .last_published_epoch
3461 .insert(reservation.client_key, epoch);
3462 }
3463 None => {
3464 inner.last_published_epoch.remove(&reservation.client_key);
3465 }
3466 }
3467 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3468 "client egress closed during route publication".to_string(),
3469 ));
3470 return Ok(abandoned);
3471 }
3472
3473 if let Some(scope) = &binding.scope {
3474 scope.commit_presence();
3475 }
3476 let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
3477 Ok(None)
3478}
3479
3480fn module_connection_for_endpoint_locked(
3488 inner: &ForwardingInner,
3489 endpoint: ModuleEndpointId,
3490) -> Option<&ModuleConnection> {
3491 let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
3492 inner
3493 .modules_by_id
3494 .get(module_id)
3495 .filter(|module| module.endpoint == endpoint)
3496 .or_else(|| {
3497 inner
3498 .candidates_by_id
3499 .get(module_id)
3500 .filter(|module| module.endpoint == endpoint)
3501 })
3502 .or_else(|| inner.superseded_endpoints.get(&endpoint))
3503}
3504
3505fn abandoned_route_target(
3506 inner: &ForwardingInner,
3507 reservation: &RouteReservation,
3508) -> Option<GoodbyeTarget> {
3509 let module_id = inner
3510 .module_id_by_endpoint
3511 .get(&reservation.module_key.endpoint)?;
3512 let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
3513 (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
3514 connection_id: module.endpoint.connection_id,
3515 sink: module.sink.clone(),
3516 negotiated_ver: module.negotiated_ver,
3517 channel: reservation.module_key.channel,
3518 epoch: reservation.module_epoch,
3519 kind: GoodbyeTargetKind::Module,
3520 module_id: Some(module_id.clone()),
3521 })
3522}
3523
3524fn enqueue_hello_ack_locked(
3529 sink: &FrameSink,
3530 connection_id: ConnectionId,
3531 hello_ack: Option<Frame>,
3532) -> Result<(), ForwardingError> {
3533 let Some(hello_ack) = hello_ack else {
3534 return Ok(());
3535 };
3536 sink.try_send(hello_ack)
3537 .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
3538}
3539
3540fn remove_module_connection_locked(
3541 inner: &mut ForwardingInner,
3542 endpoint: ModuleEndpointId,
3543) -> ConnectionCleanup {
3544 inner.operator_confirms.module_closed(endpoint);
3547 inner.draining_endpoints.remove(&endpoint);
3548 inner.daemon_drain_results.remove(&endpoint);
3549 let module_id = inner.module_id_by_endpoint.remove(&endpoint);
3550 if let Some(module_id) = module_id.as_ref() {
3551 if inner
3552 .modules_by_id
3553 .get(module_id)
3554 .is_some_and(|module| module.endpoint == endpoint)
3555 {
3556 inner.modules_by_id.remove(module_id);
3557 }
3558 if inner
3559 .candidates_by_id
3560 .get(module_id)
3561 .is_some_and(|module| module.endpoint == endpoint)
3562 {
3563 inner.candidates_by_id.remove(module_id);
3564 }
3565 }
3566 inner.superseded_endpoints.remove(&endpoint);
3567 inner.endpoint_by_connection.remove(&endpoint.connection_id);
3568 inner.next_module_channel.remove(&endpoint);
3569 inner.next_control_corr.remove(&endpoint);
3570 inner
3571 .health_probe_tombstones
3572 .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
3573 inner
3574 .module_slot_epochs
3575 .retain(|key, _| key.endpoint != endpoint);
3576 let reserved_module_keys: Vec<ModuleRouteKey> = inner
3577 .reserved_module
3578 .keys()
3579 .filter(|module_key| module_key.endpoint == endpoint)
3580 .copied()
3581 .collect();
3582 for module_key in reserved_module_keys {
3583 if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
3584 release_reserved_route_locked(inner, client_key, module_key);
3585 }
3586 }
3587
3588 let pending_keys: Vec<_> = inner
3589 .pending_relays
3590 .keys()
3591 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3592 .copied()
3593 .collect();
3594 let pending: Vec<_> = pending_keys
3595 .into_iter()
3596 .filter_map(|key| {
3597 inner
3598 .pending_relays
3599 .remove(&key)
3600 .map(|pending| (key, pending))
3601 })
3602 .collect();
3603 let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
3604 for (key, pending) in pending {
3605 if !pending.relay_enqueued {
3609 inner.relays_released_before_enqueue.insert(key);
3610 }
3611 let module_label = module_id.as_deref().unwrap_or("unknown");
3612 let _ = pending
3613 .sender
3614 .send(RouteBindRelayOutcome::ModuleGone(format!(
3615 "module '{module_label}' connection closed during route.bind relay"
3616 )));
3617 }
3618
3619 let pending_control_keys: Vec<_> = inner
3620 .pending_control_rpcs
3621 .keys()
3622 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3623 .copied()
3624 .collect();
3625 let pending_control: Vec<_> = pending_control_keys
3626 .into_iter()
3627 .filter_map(|key| inner.pending_control_rpcs.remove(&key))
3628 .collect();
3629 for pending in pending_control {
3630 let module_label = module_id.as_deref().unwrap_or("unknown");
3631 let _ = pending
3632 .sender
3633 .send(ModuleControlRpcOutcome::ModuleGone(format!(
3634 "module '{module_label}' connection closed during module-control RPC"
3635 )));
3636 }
3637
3638 let module_routes = inner
3639 .module_to_client
3640 .iter()
3641 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3642 .map(|(module_key, route)| (*module_key, route.module_epoch))
3643 .collect::<Vec<_>>();
3644 let mut released = Vec::with_capacity(module_routes.len());
3645 for (module_key, epoch) in module_routes {
3646 if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
3647 {
3648 released.push(target);
3649 }
3650 }
3651 ConnectionCleanup {
3652 released,
3653 abandoned_relays,
3654 }
3655}
3656
3657#[derive(Debug, Clone, Copy)]
3658struct RequestCredit {
3659 subscription: bool,
3660 excluded_from_drain: bool,
3661}
3662
3663#[derive(Debug, Default)]
3664struct CreditLedger {
3665 by_corr: HashMap<u64, Vec<RequestCredit>>,
3666}
3667
3668impl CreditLedger {
3669 fn acquire(&mut self, corr: u64, subscription: bool) {
3670 self.by_corr.entry(corr).or_default().push(RequestCredit {
3671 subscription,
3672 excluded_from_drain: false,
3673 });
3674 }
3675
3676 fn release(&mut self, corr: u64) -> bool {
3677 let Some(credits) = self.by_corr.get_mut(&corr) else {
3678 return false;
3679 };
3680 let released = credits.pop().is_some();
3681 if credits.is_empty() {
3682 self.by_corr.remove(&corr);
3683 }
3684 released
3685 }
3686
3687 fn capture_subscription_exclusions(&mut self) -> u32 {
3688 let mut excluded = 0u32;
3689 for credit in self.by_corr.values_mut().flatten() {
3690 if credit.subscription && !credit.excluded_from_drain {
3691 credit.excluded_from_drain = true;
3692 excluded = excluded.saturating_add(1);
3693 }
3694 }
3695 excluded
3696 }
3697
3698 #[cfg(test)]
3699 fn in_flight(&self) -> usize {
3700 self.by_corr.values().map(Vec::len).sum()
3701 }
3702
3703 fn drain_in_flight(&self) -> usize {
3704 self.by_corr
3705 .values()
3706 .flatten()
3707 .filter(|credit| !credit.excluded_from_drain)
3708 .count()
3709 }
3710
3711 fn drain_held_corrs(&self) -> Vec<u64> {
3714 let mut corrs = self
3715 .by_corr
3716 .iter()
3717 .flat_map(|(corr, credits)| {
3718 credits
3719 .iter()
3720 .filter(|credit| !credit.excluded_from_drain)
3721 .map(move |_| *corr)
3722 })
3723 .collect::<Vec<_>>();
3724 corrs.sort_unstable();
3725 corrs
3726 }
3727}
3728
3729#[derive(Debug, Default)]
3730struct ChannelFlowState {
3731 closed: bool,
3732 credits: CreditLedger,
3733}
3734
3735#[derive(Debug)]
3737pub(crate) struct ChannelFlow {
3738 sem: Semaphore,
3739 window: usize,
3740 state: Mutex<ChannelFlowState>,
3741}
3742
3743impl ChannelFlow {
3744 pub(crate) fn new(window: usize) -> Self {
3745 debug_assert!(window > 0, "flow-control window must be non-zero");
3746 Self {
3747 sem: Semaphore::new(window),
3748 window,
3749 state: Mutex::new(ChannelFlowState::default()),
3750 }
3751 }
3752
3753 #[cfg(test)]
3754 pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
3755 self.acquire_tagged(0, false).await
3756 }
3757
3758 pub(crate) async fn acquire_tagged(
3759 &self,
3760 corr: u64,
3761 subscription: bool,
3762 ) -> Result<(), ChannelFlowClosed> {
3763 let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
3764 let mut state = self
3765 .state
3766 .lock()
3767 .unwrap_or_else(|poisoned| poisoned.into_inner());
3768 if state.closed {
3769 return Err(ChannelFlowClosed);
3770 }
3771 state.credits.acquire(corr, subscription);
3772 permit.forget();
3773 Ok(())
3774 }
3775
3776 #[cfg(test)]
3777 pub(crate) fn release(&self) {
3778 self.release_corr(0);
3779 }
3780
3781 pub(crate) fn release_corr(&self, corr: u64) {
3782 let released = self
3783 .state
3784 .lock()
3785 .unwrap_or_else(|poisoned| poisoned.into_inner())
3786 .credits
3787 .release(corr);
3788 if !released {
3789 warn!(
3793 window = self.window,
3794 available = self.sem.available_permits(),
3795 "flow-control over-release ignored"
3796 );
3797 return;
3798 }
3799 if !self.sem.is_closed() {
3800 self.sem.add_permits(1);
3801 }
3802 }
3803
3804 #[cfg(test)]
3805 pub(crate) fn in_flight(&self) -> usize {
3806 self.state
3807 .lock()
3808 .unwrap_or_else(|poisoned| poisoned.into_inner())
3809 .credits
3810 .in_flight()
3811 }
3812
3813 pub(crate) fn drain_in_flight(&self) -> usize {
3814 self.state
3815 .lock()
3816 .unwrap_or_else(|poisoned| poisoned.into_inner())
3817 .credits
3818 .drain_in_flight()
3819 }
3820
3821 pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
3822 self.state
3823 .lock()
3824 .unwrap_or_else(|poisoned| poisoned.into_inner())
3825 .credits
3826 .drain_held_corrs()
3827 }
3828
3829 #[cfg(test)]
3830 pub(crate) fn available_permits(&self) -> usize {
3831 self.sem.available_permits()
3832 }
3833
3834 pub(crate) fn begin_drain(&self) -> u32 {
3835 let mut state = self
3836 .state
3837 .lock()
3838 .unwrap_or_else(|poisoned| poisoned.into_inner());
3839 state.closed = true;
3840 self.sem.close();
3841 state.credits.capture_subscription_exclusions()
3842 }
3843
3844 pub(crate) fn close(&self) {
3845 self.state
3846 .lock()
3847 .unwrap_or_else(|poisoned| poisoned.into_inner())
3848 .closed = true;
3849 self.sem.close();
3850 }
3851}
3852
3853#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3854pub(crate) struct ChannelFlowClosed;
3855
3856impl fmt::Display for ChannelFlowClosed {
3857 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3858 write!(f, "flow-control window closed")
3859 }
3860}
3861
3862impl Error for ChannelFlowClosed {}
3863
3864fn window_for(concurrency: &Concurrency) -> usize {
3865 match concurrency {
3866 Concurrency::Serial => 1,
3867 Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
3868 Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
3869 }
3870}
3871
3872#[derive(Debug, Clone, PartialEq, Eq)]
3873pub enum ForwardingError {
3874 ConnectionRoleConflict {
3877 connection_id: ConnectionId,
3878 },
3879 NoModuleConnection,
3880 ModuleReloading {
3881 module_id: String,
3882 },
3883 StaleModuleEndpoint,
3884 UnknownReservation {
3885 client_channel: u16,
3886 module_channel: u16,
3887 },
3888 ClientRouteChannelExhausted {
3889 connection_id: ConnectionId,
3890 },
3891 ModuleRouteChannelExhausted {
3892 endpoint: ModuleEndpointId,
3893 },
3894 RelayCorrelationExhausted,
3895 ConnectionClosing {
3896 connection_id: ConnectionId,
3897 },
3898 ClientEgressClosed {
3899 connection_id: ConnectionId,
3900 },
3901 RouteOpenBuild(String),
3902 CandidateSlotOccupied {
3904 module_id: String,
3905 },
3906 ModuleEgressUnavailable {
3909 connection_id: ConnectionId,
3910 },
3911 Poisoned,
3912}
3913
3914impl fmt::Display for ForwardingError {
3915 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3916 match self {
3917 Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
3918 Self::NoModuleConnection => write!(f, "no module connection is registered"),
3919 Self::ModuleReloading { module_id } => {
3920 write!(f, "module_id '{module_id}' is reloading")
3921 }
3922 Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
3923 Self::UnknownReservation {
3924 client_channel,
3925 module_channel,
3926 } => write!(
3927 f,
3928 "route reservation client channel {client_channel} / module channel {module_channel} was not found"
3929 ),
3930 Self::ClientRouteChannelExhausted { connection_id } => write!(
3931 f,
3932 "no client route channels are available for connection {}",
3933 connection_id.get()
3934 ),
3935 Self::ModuleRouteChannelExhausted { endpoint } => write!(
3936 f,
3937 "no module route channels are available for endpoint generation {} on connection {}",
3938 endpoint.generation,
3939 endpoint.connection_id.get()
3940 ),
3941 Self::RelayCorrelationExhausted => {
3942 write!(f, "module control correlation ids are exhausted")
3943 }
3944 Self::ConnectionClosing { connection_id } => write!(
3945 f,
3946 "connection {} is closing and cannot accept route allocation",
3947 connection_id.get()
3948 ),
3949 Self::ClientEgressClosed { connection_id } => write!(
3950 f,
3951 "client connection {} egress is closed",
3952 connection_id.get()
3953 ),
3954 Self::RouteOpenBuild(message) => {
3955 write!(f, "failed to prebuild route.open response: {message}")
3956 }
3957 Self::CandidateSlotOccupied { module_id } => write!(
3958 f,
3959 "module_id '{module_id}' already has a swap candidate registered"
3960 ),
3961 Self::ModuleEgressUnavailable { connection_id } => write!(
3962 f,
3963 "module connection {} egress is unavailable; HELLO_ACK could not be queued",
3964 connection_id.get()
3965 ),
3966 Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
3967 }
3968 }
3969}
3970
3971impl Error for ForwardingError {}
3972
3973#[cfg(test)]
3974mod tests {
3975 use std::time::Duration;
3976
3977 use super::*;
3978 use tokio::sync::mpsc;
3979
3980 #[test]
3981 fn ordinary_long_running_request_is_not_excluded_from_drain() {
3982 let mut ledger = CreditLedger::default();
3983 ledger.acquire(1, false);
3984
3985 assert_eq!(ledger.capture_subscription_exclusions(), 0);
3986 assert_eq!(ledger.drain_in_flight(), 1);
3987 }
3988
3989 #[test]
3990 fn bit_set_subscription_is_excluded_and_counted() {
3991 let mut ledger = CreditLedger::default();
3992 ledger.acquire(1, true);
3993
3994 assert_eq!(ledger.capture_subscription_exclusions(), 1);
3995 assert_eq!(ledger.drain_in_flight(), 0);
3996 }
3997
3998 #[test]
3999 fn subscription_opened_after_drain_snapshot_is_not_excluded() {
4000 let mut ledger = CreditLedger::default();
4001 ledger.acquire(1, true);
4002 assert_eq!(ledger.capture_subscription_exclusions(), 1);
4003
4004 ledger.acquire(2, true);
4005
4006 assert_eq!(ledger.drain_in_flight(), 1);
4007 }
4008
4009 #[test]
4010 fn drain_with_no_subscriptions_reports_zero_excluded() {
4011 let mut ledger = CreditLedger::default();
4012 assert_eq!(ledger.capture_subscription_exclusions(), 0);
4013 }
4014
4015 fn test_hello_ack(corr: u64) -> Frame {
4016 Frame::build(
4017 FrameType::HelloAck,
4018 Flags::new(false, Priority::Passive, false),
4019 0,
4020 0,
4021 corr,
4022 Vec::new(),
4023 )
4024 .unwrap()
4025 }
4026
4027 #[test]
4031 fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
4032 let forwarding = ForwardingTable::default();
4033
4034 let (closed_tx, closed_rx) = mpsc::channel(8);
4035 drop(closed_rx);
4036 let closed = ConnectionId::new(1);
4037 assert_eq!(
4038 forwarding.register_module_connection_acked(
4039 closed,
4040 "closed".to_string(),
4041 2,
4042 Concurrency::ModuleManaged,
4043 FrameSink::new(closed_tx),
4044 test_hello_ack(1),
4045 ),
4046 Err(ForwardingError::ModuleEgressUnavailable {
4047 connection_id: closed
4048 })
4049 );
4050
4051 let (full_tx, _full_rx) = mpsc::channel(1);
4052 let full_sink = FrameSink::new(full_tx);
4053 full_sink.try_send(test_hello_ack(99)).unwrap();
4054 let full = ConnectionId::new(2);
4055 assert_eq!(
4056 forwarding.register_module_connection_acked(
4057 full,
4058 "full".to_string(),
4059 2,
4060 Concurrency::ModuleManaged,
4061 full_sink.clone(),
4062 test_hello_ack(2),
4063 ),
4064 Err(ForwardingError::ModuleEgressUnavailable {
4065 connection_id: full
4066 })
4067 );
4068 assert_eq!(
4069 forwarding.register_candidate_module_connection_acked(
4070 full,
4071 "full".to_string(),
4072 2,
4073 Concurrency::ModuleManaged,
4074 full_sink,
4075 test_hello_ack(3),
4076 ),
4077 Err(ForwardingError::ModuleEgressUnavailable {
4078 connection_id: full
4079 })
4080 );
4081
4082 for (connection, module_id) in [(closed, "closed"), (full, "full")] {
4083 assert_eq!(
4084 forwarding
4085 .module_endpoint_for_connection(connection)
4086 .unwrap(),
4087 None
4088 );
4089 let (client_tx, _client_rx) = mpsc::channel(8);
4090 assert_eq!(
4091 forwarding
4092 .begin_route_bind_relay_for_test(
4093 ConnectionId::new(50),
4094 FrameSink::new(client_tx),
4095 1,
4096 module_id,
4097 )
4098 .err(),
4099 Some(ForwardingError::NoModuleConnection)
4100 );
4101 }
4102 assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
4103 }
4104
4105 #[test]
4108 fn acked_registration_queues_the_hello_ack_first() {
4109 let forwarding = ForwardingTable::default();
4110 let (active_tx, mut active_rx) = mpsc::channel(8);
4111 forwarding
4112 .register_module_connection_acked(
4113 ConnectionId::new(1),
4114 "acked".to_string(),
4115 2,
4116 Concurrency::ModuleManaged,
4117 FrameSink::new(active_tx),
4118 test_hello_ack(11),
4119 )
4120 .unwrap();
4121 let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
4122 forwarding
4123 .register_candidate_module_connection_acked(
4124 ConnectionId::new(2),
4125 "acked".to_string(),
4126 2,
4127 Concurrency::ModuleManaged,
4128 FrameSink::new(candidate_tx),
4129 test_hello_ack(12),
4130 )
4131 .unwrap();
4132
4133 let active_first = active_rx.try_recv().unwrap().frame;
4134 assert_eq!(active_first.header.ty, FrameType::HelloAck);
4135 assert_eq!(active_first.header.corr, 11);
4136 let candidate_first = candidate_rx.try_recv().unwrap().frame;
4137 assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
4138 assert_eq!(candidate_first.header.corr, 12);
4139 }
4140
4141 #[test]
4142 fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
4143 let forwarding = ForwardingTable::default();
4144 let module_connection = ConnectionId::new(10);
4145 let exhausted_client = ConnectionId::new(20);
4146 let second_client = ConnectionId::new(30);
4147 let (module_tx, _module_rx) = mpsc::channel(1);
4148 let endpoint = forwarding
4149 .register_module_connection(
4150 module_connection,
4151 "route-limit-provider".to_string(),
4152 1,
4153 Concurrency::ModuleManaged,
4154 FrameSink::new(module_tx),
4155 )
4156 .unwrap();
4157
4158 {
4159 let mut inner = forwarding.inner.write().unwrap();
4160 for channel in 1..=u16::MAX {
4161 inner.reserved_client.insert(
4162 ClientRouteKey {
4163 connection_id: exhausted_client,
4164 channel,
4165 },
4166 ModuleRouteKey {
4167 endpoint,
4168 channel: 1,
4169 },
4170 );
4171 }
4172 }
4173
4174 let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
4175 let err = forwarding
4176 .begin_route_bind_relay_for_test(
4177 exhausted_client,
4178 FrameSink::new(exhausted_tx),
4179 1,
4180 "route-limit-provider",
4181 )
4182 .unwrap_err();
4183 assert!(matches!(
4184 err,
4185 ForwardingError::ClientRouteChannelExhausted { connection_id }
4186 if connection_id == exhausted_client
4187 ));
4188
4189 let (second_tx, _second_rx) = mpsc::channel(1);
4190 let pending = forwarding
4191 .begin_route_bind_relay_for_test(
4192 second_client,
4193 FrameSink::new(second_tx),
4194 2,
4195 "route-limit-provider",
4196 )
4197 .unwrap();
4198 assert_eq!(pending.client_channel, 1);
4199 }
4200
4201 #[test]
4202 fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
4203 let forwarding = ForwardingTable::default();
4204 let module_connection = ConnectionId::new(40);
4205 let client = ConnectionId::new(50);
4206 let (module_tx, _module_rx) = mpsc::channel(1);
4207 forwarding
4208 .register_module_connection(
4209 module_connection,
4210 "slot-reuse-provider".to_string(),
4211 1,
4212 Concurrency::ModuleManaged,
4213 FrameSink::new(module_tx),
4214 )
4215 .unwrap();
4216
4217 let (client_tx, _client_rx) = mpsc::channel(1);
4218 let client_sink = FrameSink::new(client_tx);
4219 let mut wrapped_channel = None;
4220 for index in 0..=usize::from(u16::MAX) {
4221 let pending = forwarding
4222 .begin_route_bind_relay_for_test(
4223 client,
4224 client_sink.clone(),
4225 index as u64 + 1,
4226 "slot-reuse-provider",
4227 )
4228 .unwrap();
4229 if index == usize::from(u16::MAX) {
4230 wrapped_channel = Some(pending.module_channel);
4231 }
4232 forwarding
4233 .abort_pending_relay(
4234 pending.endpoint,
4235 pending.corr,
4236 RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
4237 )
4238 .unwrap();
4239 }
4240
4241 assert_eq!(wrapped_channel, Some(1));
4242 }
4243
4244 #[test]
4245 fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
4246 let forwarding = ForwardingTable::default();
4247 let client = ConnectionId::new(60);
4248 forwarding
4249 .inner
4250 .write()
4251 .unwrap()
4252 .next_client_channel
4253 .insert(client, 41);
4254
4255 let released = forwarding.cleanup_connection(client).unwrap();
4256
4257 assert!(released.is_empty());
4258 assert!(!forwarding
4259 .inner
4260 .read()
4261 .unwrap()
4262 .next_client_channel
4263 .contains_key(&client));
4264 }
4265
4266 #[test]
4267 fn stale_module_cleanup_preserves_fast_reconnect_successor() {
4268 let forwarding = ForwardingTable::default();
4269 let module_id = "fast-reconnect-provider";
4270 let first_connection = ConnectionId::new(70);
4271 let second_connection = ConnectionId::new(80);
4272 let (first_tx, _first_rx) = mpsc::channel(1);
4273 let first_endpoint = forwarding
4274 .register_module_connection(
4275 first_connection,
4276 module_id.to_string(),
4277 1,
4278 Concurrency::ModuleManaged,
4279 FrameSink::new(first_tx),
4280 )
4281 .unwrap();
4282 let (second_tx, _second_rx) = mpsc::channel(1);
4283 let second_endpoint = forwarding
4284 .register_module_connection(
4285 second_connection,
4286 module_id.to_string(),
4287 1,
4288 Concurrency::ModuleManaged,
4289 FrameSink::new(second_tx),
4290 )
4291 .unwrap();
4292 assert_ne!(first_endpoint, second_endpoint);
4293
4294 let released = forwarding.cleanup_connection(first_connection).unwrap();
4295
4296 assert!(released.is_empty());
4297 assert_eq!(
4298 forwarding
4299 .inner
4300 .read()
4301 .unwrap()
4302 .modules_by_id
4303 .get(module_id)
4304 .map(|module| module.endpoint),
4305 Some(second_endpoint)
4306 );
4307 assert!(forwarding.has_live_module_connection(module_id).unwrap());
4308 let control_rpc = forwarding
4309 .begin_module_control_rpc_for(
4310 module_id,
4311 "health.check",
4312 Instant::now() + Duration::from_secs(1),
4313 )
4314 .unwrap();
4315 assert_eq!(control_rpc.endpoint, second_endpoint);
4316 }
4317
4318 fn route_fixture(
4319 module_id: &str,
4320 ) -> (
4321 ForwardingTable,
4322 ConnectionId,
4323 ModuleEndpointId,
4324 ConnectionId,
4325 FrameSink,
4326 mpsc::Receiver<crate::router::OutboundFrame>,
4327 ) {
4328 let forwarding = ForwardingTable::default();
4329 let module_connection = ConnectionId::new(100);
4330 let client_connection = ConnectionId::new(200);
4331 let (module_tx, _module_rx) = mpsc::channel(8);
4332 let endpoint = forwarding
4333 .register_module_connection(
4334 module_connection,
4335 module_id.to_string(),
4336 2,
4337 Concurrency::ModuleManaged,
4338 FrameSink::new(module_tx),
4339 )
4340 .unwrap();
4341 let (client_tx, client_rx) = mpsc::channel(8);
4342 (
4343 forwarding,
4344 module_connection,
4345 endpoint,
4346 client_connection,
4347 FrameSink::new(client_tx),
4348 client_rx,
4349 )
4350 }
4351
4352 #[test]
4353 #[cfg(unix)]
4354 fn daemon_drain_gates_current_and_racing_provider_registrations() {
4355 let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
4356 assert_eq!(
4357 forwarding.begin_daemon_drain().unwrap(),
4358 [("provider".to_string(), endpoint)]
4359 );
4360 assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
4361 assert!(matches!(
4362 forwarding.register_module_connection(
4363 ConnectionId::new(300),
4364 "late-provider".into(),
4365 2,
4366 Concurrency::ModuleManaged,
4367 sink,
4368 ),
4369 Err(ForwardingError::ConnectionClosing { .. })
4370 ));
4371 }
4372
4373 #[test]
4374 #[cfg(unix)]
4375 fn daemon_drain_results_are_endpoint_fenced_and_removed_on_cleanup() {
4376 let (forwarding, connection, endpoint, _, _, _) = route_fixture("provider");
4377 assert_eq!(
4378 forwarding
4379 .daemon_drain_result_for_connection(connection)
4380 .unwrap(),
4381 None
4382 );
4383 forwarding
4384 .record_daemon_drain_result(endpoint, true)
4385 .unwrap();
4386 forwarding.begin_daemon_drain().unwrap();
4387 assert_eq!(
4388 forwarding
4389 .daemon_drain_result_for_connection(connection)
4390 .unwrap(),
4391 Some(false)
4392 );
4393 let replacement = ModuleEndpointId {
4394 generation: endpoint.generation + 1,
4395 ..endpoint
4396 };
4397 forwarding
4398 .record_daemon_drain_result(replacement, true)
4399 .unwrap();
4400 assert_eq!(
4401 forwarding
4402 .daemon_drain_result_for_connection(connection)
4403 .unwrap(),
4404 Some(false)
4405 );
4406 forwarding
4407 .record_daemon_drain_result(endpoint, true)
4408 .unwrap();
4409 assert_eq!(
4410 forwarding
4411 .daemon_drain_result_for_connection(connection)
4412 .unwrap(),
4413 Some(true)
4414 );
4415 assert_eq!(
4416 forwarding
4417 .daemon_drain_result_for_connection(ConnectionId::new(300))
4418 .unwrap(),
4419 Some(false)
4420 );
4421 forwarding.cleanup_connection(connection).unwrap();
4422 forwarding
4423 .record_daemon_drain_result(endpoint, true)
4424 .unwrap();
4425 assert_eq!(
4426 forwarding
4427 .daemon_drain_result_for_connection(connection)
4428 .unwrap(),
4429 Some(false)
4430 );
4431 assert!(forwarding
4432 .read_inner()
4433 .unwrap()
4434 .daemon_drain_results
4435 .is_empty());
4436 }
4437
4438 #[test]
4439 #[cfg(unix)]
4440 fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
4441 let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
4442 route_fixture("provider");
4443 let mut pending =
4444 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
4445 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
4446 forwarding.begin_daemon_drain().unwrap();
4447 let completion = forwarding
4448 .complete_pending_relay(
4449 module_connection,
4450 pending.corr,
4451 RouteBindRelayOutcome::Accepted,
4452 )
4453 .expect("draining admission is not a fatal module error");
4454 assert!(completion.settled);
4455 let abandoned = completion
4456 .abandoned
4457 .expect("late accepted bind needs a route GOODBYE");
4458 assert_eq!(abandoned.channel, pending.module_channel);
4459 assert!(matches!(pending.receiver.try_recv().unwrap(),
4460 RouteBindRelayOutcome::DaemonRejected(error) if error.code == "module_reloading"));
4461 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4462 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4463 assert!(
4464 rx.try_recv().is_err(),
4465 "no route.open may be published during drain"
4466 );
4467 assert_eq!(
4468 forwarding
4469 .module_endpoint_for_connection(module_connection)
4470 .unwrap(),
4471 Some(endpoint)
4472 );
4473 assert!(
4474 !forwarding
4475 .complete_pending_relay(
4476 module_connection,
4477 pending.corr,
4478 RouteBindRelayOutcome::Accepted,
4479 )
4480 .unwrap()
4481 .settled
4482 );
4483 }
4484
4485 fn test_ping(corr: u64) -> Frame {
4486 Frame::build(
4487 FrameType::Ping,
4488 Flags::new(false, Priority::Passive, false),
4489 0,
4490 0,
4491 corr,
4492 Vec::new(),
4493 )
4494 .unwrap()
4495 }
4496
4497 fn begin_test_route(
4498 forwarding: &ForwardingTable,
4499 client_connection: ConnectionId,
4500 client_sink: FrameSink,
4501 corr: u64,
4502 module_id: &str,
4503 ) -> PendingRouteBindRelay {
4504 forwarding
4505 .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
4506 .unwrap()
4507 }
4508
4509 #[test]
4510 fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
4511 for candidate in [false, true] {
4512 for register_first in [false, true] {
4513 let (forwarding, module_connection, _, client_connection, sink, _rx) =
4514 route_fixture("target");
4515 let register = || {
4516 if candidate {
4517 forwarding.register_candidate_module_connection(
4518 client_connection,
4519 "source".into(),
4520 2,
4521 Concurrency::ModuleManaged,
4522 sink.clone(),
4523 )
4524 } else {
4525 forwarding.register_module_connection(
4526 client_connection,
4527 "source".into(),
4528 2,
4529 Concurrency::ModuleManaged,
4530 sink.clone(),
4531 )
4532 }
4533 };
4534 if register_first {
4535 register().unwrap();
4536 assert!(
4537 forwarding
4538 .begin_route_bind_relay_for_test(
4539 client_connection,
4540 sink.clone(),
4541 1,
4542 "target",
4543 )
4544 .is_err(),
4545 "a deferred route.open cannot reserve after HELLO"
4546 );
4547 assert!(forwarding
4548 .cleanup_connection(client_connection)
4549 .unwrap()
4550 .is_empty());
4551 } else {
4552 let pending =
4553 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
4554 assert!(
4555 register().is_err(),
4556 "HELLO cannot register after a route reservation"
4557 );
4558 forwarding
4559 .complete_pending_relay(
4560 module_connection,
4561 pending.corr,
4562 RouteBindRelayOutcome::Accepted,
4563 )
4564 .unwrap();
4565 assert_eq!(
4566 forwarding
4567 .cleanup_connection(client_connection)
4568 .unwrap()
4569 .len(),
4570 1
4571 );
4572 }
4573 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4574 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4575 }
4576 }
4577 }
4578
4579 #[tokio::test]
4585 async fn pending_route_open_completes_behind_queued_data_frames() {
4586 assert_eq!(
4587 crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
4588 8,
4589 "the per-connection pending route.open limit is its own constant"
4590 );
4591 let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
4592 route_fixture("open-behind-data");
4593 let (sink, mut client_rx) = crate::server::connection_egress();
4594 const DATA_FRAMES: usize = 1_000;
4595 let data = |corr: u64| {
4596 Frame::build(
4597 FrameType::StreamData,
4598 Flags::new(false, Priority::Interactive, false),
4599 9,
4600 1,
4601 corr,
4602 vec![b'x'; 200],
4603 )
4604 .unwrap()
4605 };
4606 for corr in 0..DATA_FRAMES as u64 {
4607 sink.try_send(data(corr)).unwrap();
4608 }
4609 let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
4610 assert_eq!(sink.backlog().queued_bytes, data_bytes);
4611
4612 let pending = tokio::time::timeout(
4613 Duration::from_secs(5),
4614 forwarding.begin_route_bind_relay_for(
4615 client,
4616 sink.clone(),
4617 subc_protocol::PROTOCOL_VERSION,
4618 4_242,
4619 "open-behind-data",
4620 Principal::Direct,
4621 None,
4622 None,
4623 "fixture".into(),
4624 None,
4625 None,
4626 Instant::now() + Duration::from_secs(60),
4627 ),
4628 )
4629 .await
4630 .expect("reserving the route.open slot must not wait behind data frames")
4631 .unwrap();
4632 forwarding
4633 .complete_pending_relay(
4634 module_connection,
4635 pending.corr,
4636 RouteBindRelayOutcome::Accepted,
4637 )
4638 .unwrap();
4639
4640 let backlog = sink.backlog();
4641 assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
4642 assert!(
4643 backlog.queued_bytes > data_bytes,
4644 "the route.open response must be counted in queued bytes: {backlog:?}"
4645 );
4646 for corr in 0..DATA_FRAMES as u64 {
4647 assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
4648 }
4649 let open = client_rx.recv().await.unwrap();
4650 assert_eq!(open.header.corr, 4_242);
4651 assert_eq!(open.header.ty, FrameType::Response);
4652 drop(open);
4653 assert_eq!(sink.backlog().queued_bytes, 0);
4654 assert_eq!(sink.backlog().queued_frames, 0);
4655 }
4656
4657 #[tokio::test]
4662 async fn drain_holdouts_count_held_requests_and_name_the_connection() {
4663 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4664 route_fixture("holdouts");
4665 let mut bound = |corr| {
4666 let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
4667 forwarding
4668 .complete_pending_relay(
4669 module_connection,
4670 route.corr,
4671 RouteBindRelayOutcome::Accepted,
4672 )
4673 .unwrap();
4674 client_rx.try_recv().unwrap();
4675 match forwarding
4676 .lookup_data_route(client, route.client_channel, route.client_epoch)
4677 .unwrap()
4678 {
4679 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4680 other => panic!("expected live route, got {other:?}"),
4681 }
4682 };
4683 let holding = bound(61);
4684 let _idle = bound(62);
4685 holding.flow.acquire_tagged(7, false).await.unwrap();
4686 holding.flow.acquire_tagged(2, false).await.unwrap();
4687 holding.flow.acquire_tagged(3, true).await.unwrap();
4688 forwarding
4689 .begin_module_drain("holdouts", RouteCloseReason::Restart)
4690 .unwrap();
4691
4692 let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
4693 assert_eq!(
4694 holdouts,
4695 DrainHoldouts {
4696 requests: 2,
4697 routes: 1,
4698 total_routes: 2,
4699 top_connections: vec![(client.get(), 2)],
4700 held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
4703 }
4704 );
4705 }
4706
4707 #[test]
4708 fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
4709 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4710 route_fixture("census");
4711 let pending = begin_test_route(&forwarding, client, sink, 1, "census");
4712 forwarding
4713 .complete_pending_relay(
4714 module_connection,
4715 pending.corr,
4716 RouteBindRelayOutcome::Accepted,
4717 )
4718 .unwrap();
4719
4720 let routes = forwarding.endpoint_routes(endpoint).unwrap();
4721 assert_eq!(routes.len(), 1);
4722 assert!(matches!(routes[0].principal, Principal::Direct));
4723 assert_eq!(routes[0].goodbye_target.connection_id, client);
4724 assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
4725 assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
4726 assert!(!routes[0].draining);
4727
4728 forwarding
4729 .begin_module_drain("census", RouteCloseReason::Restart)
4730 .unwrap();
4731 let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
4732 assert_eq!(draining_routes.len(), 1);
4733 assert!(draining_routes[0].draining);
4734 }
4735
4736 #[test]
4737 fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
4738 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
4739 let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
4740 assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
4741 forwarding
4742 .abort_pending_relay(
4743 first.endpoint,
4744 first.corr,
4745 RouteBindRelayOutcome::ModuleGone("abort".into()),
4746 )
4747 .unwrap();
4748 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4749 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4750
4751 let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
4752 assert_eq!(second.client_channel, first.client_channel);
4753 assert_eq!(second.module_channel, first.module_channel);
4754 assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
4755 }
4756
4757 #[test]
4758 fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
4759 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4760 route_fixture("epoch-release");
4761 let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
4762 forwarding
4763 .complete_pending_relay(
4764 module_connection,
4765 first.corr,
4766 RouteBindRelayOutcome::Accepted,
4767 )
4768 .unwrap();
4769 assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
4770 assert!(matches!(
4771 forwarding
4772 .release_client_route(client, first.client_channel, first.client_epoch)
4773 .unwrap(),
4774 RouteRelease::Removed(_)
4775 ));
4776 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4777 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4778
4779 let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
4780 forwarding
4781 .complete_pending_relay(
4782 module_connection,
4783 second.corr,
4784 RouteBindRelayOutcome::Accepted,
4785 )
4786 .unwrap();
4787 assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
4788 assert!(matches!(
4789 forwarding
4790 .release_client_route(client, second.client_channel, first.client_epoch)
4791 .unwrap(),
4792 RouteRelease::Stale
4793 ));
4794 assert!(!forwarding
4795 .cache_status(
4796 endpoint,
4797 second.module_channel,
4798 first.module_epoch,
4799 "stale".into(),
4800 )
4801 .unwrap());
4802 assert!(forwarding
4803 .cache_status(
4804 endpoint,
4805 second.module_channel,
4806 second.module_epoch,
4807 "current".into(),
4808 )
4809 .unwrap());
4810 match forwarding
4811 .route_poll_snapshot(client, second.client_channel, second.client_epoch)
4812 .unwrap()
4813 {
4814 RoutePollSnapshot::Bound { status, .. } => {
4815 assert_eq!(status.as_deref(), Some("current"));
4816 }
4817 RoutePollSnapshot::Absent => panic!("successor binding was removed"),
4818 }
4819 let counters = forwarding.counters().snapshot();
4820 assert_eq!(counters["route_released_epoch_fenced"], 1);
4821 assert_eq!(counters["route_release_stale_skipped"], 1);
4822 }
4823
4824 #[test]
4825 fn max_epoch_reservation_retires_only_that_slot() {
4826 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
4827 forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
4828 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
4829 let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
4830 assert_eq!(
4831 (final_use.client_channel, final_use.client_epoch),
4832 (7, u32::MAX)
4833 );
4834 assert_eq!(
4835 (final_use.module_channel, final_use.module_epoch),
4836 (9, u32::MAX)
4837 );
4838 forwarding
4839 .abort_pending_relay(
4840 endpoint,
4841 final_use.corr,
4842 RouteBindRelayOutcome::ModuleGone("abort".into()),
4843 )
4844 .unwrap();
4845 forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
4846 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
4847 let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
4848 assert_ne!(next.client_channel, 7);
4849 assert_ne!(next.module_channel, 9);
4850 assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
4851 }
4852
4853 #[test]
4854 fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
4855 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4856 route_fixture("corr-shared");
4857 let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
4858 assert_eq!(bind.corr, 1);
4859 forwarding
4860 .abort_pending_relay(
4861 endpoint,
4862 bind.corr,
4863 RouteBindRelayOutcome::ModuleGone("abort".into()),
4864 )
4865 .unwrap();
4866 let rpc = forwarding
4867 .begin_module_control_rpc_for(
4868 "corr-shared",
4869 "health.check",
4870 Instant::now() - Duration::from_millis(1),
4871 )
4872 .unwrap();
4873 assert_eq!(rpc.corr, 2);
4874 assert_eq!(
4875 forwarding
4876 .complete_module_control_rpc(
4877 module_connection,
4878 rpc.corr,
4879 Some("health.check"),
4880 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
4881 status: subc_protocol::session::HealthStatus::Ok,
4882 detail: None,
4883 metrics: None,
4884 }),
4885 )
4886 .unwrap(),
4887 ModuleControlRpcCompletion::Settled
4888 );
4889 assert!(matches!(
4890 rpc.receiver.blocking_recv().unwrap(),
4891 ModuleControlRpcOutcome::DeadlineElapsed
4892 ));
4893 }
4894
4895 #[tokio::test(start_paused = true)]
4896 async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
4897 let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
4898 let probe_started_at = Instant::now();
4899 let rpc = forwarding
4900 .begin_health_probe_rpc_for(
4901 "tombstone-ttl",
4902 "health.check",
4903 probe_started_at,
4904 probe_started_at + Duration::from_secs(5),
4905 )
4906 .unwrap();
4907 assert!(forwarding
4908 .tombstone_health_probe_rpc(endpoint, rpc.corr)
4909 .unwrap());
4910 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
4911
4912 tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
4913 tokio::task::yield_now().await;
4914
4915 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
4916 }
4917
4918 #[test]
4919 fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
4920 let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
4921 let mut close = forwarding.register_connection_close(endpoint.connection_id);
4922 forwarding.inject_control_corr(endpoint, u64::MAX);
4923 let final_rpc = forwarding
4924 .begin_module_control_rpc_for(
4925 "corr-max",
4926 "health.check",
4927 Instant::now() + Duration::from_secs(1),
4928 )
4929 .unwrap();
4930 assert_eq!(final_rpc.corr, u64::MAX);
4931 forwarding
4932 .cancel_module_control_rpc(endpoint, final_rpc.corr)
4933 .unwrap();
4934 assert!(matches!(
4935 forwarding.begin_module_control_rpc_for(
4936 "corr-max",
4937 "health.check",
4938 Instant::now() + Duration::from_secs(1),
4939 ),
4940 Err(ForwardingError::RelayCorrelationExhausted)
4941 ));
4942 assert!(close.try_recv().is_ok());
4943 }
4944
4945 #[test]
4946 fn publication_epoch_controls_delivery_failure_escalation() {
4947 fn setup_successor(
4948 commit_successor: Option<bool>,
4949 ) -> (ForwardingTable, ConnectionId, u16, u32) {
4950 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4951 route_fixture("escalation");
4952 let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
4953 forwarding
4954 .complete_pending_relay(
4955 module_connection,
4956 first.corr,
4957 RouteBindRelayOutcome::Accepted,
4958 )
4959 .unwrap();
4960 client_rx.try_recv().unwrap();
4961 assert!(matches!(
4962 forwarding
4963 .release_client_route(client, first.client_channel, first.client_epoch)
4964 .unwrap(),
4965 RouteRelease::Removed(_)
4966 ));
4967 if let Some(commit_successor) = commit_successor {
4968 forwarding.inject_client_slot_epoch(
4969 client,
4970 first.client_channel,
4971 first.client_epoch,
4972 );
4973 forwarding.inject_module_slot_epoch(
4974 endpoint,
4975 first.module_channel,
4976 first.module_epoch,
4977 );
4978 let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
4979 if commit_successor {
4980 forwarding
4981 .complete_pending_relay(
4982 module_connection,
4983 successor.corr,
4984 RouteBindRelayOutcome::Accepted,
4985 )
4986 .unwrap();
4987 client_rx.try_recv().unwrap();
4988 } else {
4989 forwarding
4990 .abort_pending_relay(
4991 endpoint,
4992 successor.corr,
4993 RouteBindRelayOutcome::ModuleGone("abort".into()),
4994 )
4995 .unwrap();
4996 }
4997 }
4998 (forwarding, client, first.client_channel, first.client_epoch)
4999 }
5000
5001 let probe_sink = FrameSink::new(mpsc::channel(1).0);
5002 let (no_successor, client, channel, epoch) = setup_successor(None);
5003 let mut close = no_successor.register_connection_close(client);
5004 assert!(no_successor
5005 .escalate_client_delivery_failure(
5006 client,
5007 channel,
5008 epoch,
5009 CloseReason::new("delivery", "failed"),
5010 UndeliveredFrame {
5011 module_id: None,
5012 sink: &probe_sink,
5013 },
5014 )
5015 .unwrap());
5016 assert!(close.try_recv().is_ok());
5017
5018 let (aborted, client, channel, epoch) = setup_successor(Some(false));
5019 let mut close = aborted.register_connection_close(client);
5020 assert!(aborted
5021 .escalate_client_delivery_failure(
5022 client,
5023 channel,
5024 epoch,
5025 CloseReason::new("delivery", "failed"),
5026 UndeliveredFrame {
5027 module_id: None,
5028 sink: &probe_sink,
5029 },
5030 )
5031 .unwrap());
5032 assert!(close.try_recv().is_ok());
5033
5034 let (published, client, channel, epoch) = setup_successor(Some(true));
5035 let mut close = published.register_connection_close(client);
5036 assert!(!published
5037 .escalate_client_delivery_failure(
5038 client,
5039 channel,
5040 epoch,
5041 CloseReason::new("delivery", "stale failure"),
5042 UndeliveredFrame {
5043 module_id: None,
5044 sink: &probe_sink,
5045 },
5046 )
5047 .unwrap());
5048 assert!(close.try_recv().is_err());
5049 }
5050
5051 #[test]
5052 fn route_concentration_separates_client_count_from_routes_per_client() {
5053 let (forwarding, module_connection, _, client, sink, _client_rx) =
5057 route_fixture("concentration");
5058 assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
5059
5060 for corr in [70_u64, 71] {
5061 let pending =
5062 begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
5063 forwarding
5064 .complete_pending_relay(
5065 module_connection,
5066 pending.corr,
5067 RouteBindRelayOutcome::Accepted,
5068 )
5069 .unwrap();
5070 }
5071
5072 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5074 assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
5075 }
5076
5077 #[test]
5078 fn cleanup_and_accepted_resolution_have_one_lock_winner() {
5079 let (forwarding, module_connection, _, client, sink, mut client_rx) =
5080 route_fixture("cleanup-race");
5081 let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
5082 forwarding
5083 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5084 .unwrap();
5085 let released = forwarding.cleanup_connection(client).unwrap();
5086 assert_eq!(released.len(), 1);
5087 let completion = forwarding
5088 .complete_pending_relay(
5089 module_connection,
5090 pending.corr,
5091 RouteBindRelayOutcome::Accepted,
5092 )
5093 .unwrap();
5094 assert!(!completion.settled);
5095 assert!(client_rx.try_recv().is_err());
5096 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5097
5098 let (forwarding, module_connection, _, client, sink, mut client_rx) =
5099 route_fixture("accepted-race");
5100 let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
5101 forwarding
5102 .complete_pending_relay(
5103 module_connection,
5104 pending.corr,
5105 RouteBindRelayOutcome::Accepted,
5106 )
5107 .unwrap();
5108 assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
5109 let released = forwarding.cleanup_connection(client).unwrap();
5110 assert_eq!(released.len(), 1);
5111 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5112 }
5113
5114 #[test]
5115 fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
5116 let (forwarding, module_connection, _, client, sink, mut client_rx) =
5117 route_fixture("drain-gap");
5118 let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
5119 forwarding
5120 .complete_pending_relay(
5121 module_connection,
5122 live.corr,
5123 RouteBindRelayOutcome::Accepted,
5124 )
5125 .unwrap();
5126 client_rx.try_recv().unwrap();
5127 let binding = match forwarding
5128 .lookup_data_route(client, live.client_channel, live.client_epoch)
5129 .unwrap()
5130 {
5131 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
5132 other => panic!("expected live route, got {other:?}"),
5133 };
5134
5135 let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
5136 forwarding
5137 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5138 .unwrap();
5139 let control_rpc = forwarding
5140 .begin_module_control_rpc_for(
5141 "drain-gap",
5142 "health.check",
5143 Instant::now() + Duration::from_secs(1),
5144 )
5145 .unwrap();
5146 let target = forwarding
5147 .begin_module_drain("drain-gap", RouteCloseReason::Reload)
5148 .unwrap()
5149 .unwrap();
5150 assert!(matches!(
5151 control_rpc.receiver.blocking_recv().unwrap(),
5152 ModuleControlRpcOutcome::ModuleGone(_)
5153 ));
5154 assert_eq!(target.abandoned_bindings.len(), 1);
5155 assert!(binding.flow.sem.is_closed());
5156 assert!(
5157 !forwarding
5158 .complete_pending_relay(
5159 module_connection,
5160 pending.corr,
5161 RouteBindRelayOutcome::Accepted,
5162 )
5163 .unwrap()
5164 .settled
5165 );
5166 assert!(matches!(
5167 forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
5168 Err(ForwardingError::ModuleReloading { .. })
5169 ));
5170 let released = forwarding
5171 .release_module_endpoint_routes(target.endpoint)
5172 .unwrap();
5173 assert_eq!(released.len(), 1);
5174 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5175 }
5176
5177 #[test]
5185 fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
5186 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5187 route_fixture("closing-client");
5188
5189 let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
5192 forwarding
5193 .complete_pending_relay(
5194 module_connection,
5195 live.corr,
5196 RouteBindRelayOutcome::Accepted,
5197 )
5198 .unwrap();
5199 client_rx.try_recv().unwrap();
5200
5201 let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
5203 forwarding
5204 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5205 .unwrap();
5206
5207 assert!(forwarding
5209 .escalate_client_delivery_failure(
5210 client,
5211 live.client_channel,
5212 live.client_epoch,
5213 CloseReason::new(
5214 "module_to_client_delivery_failed",
5215 "client egress refused a module frame",
5216 ),
5217 UndeliveredFrame {
5218 module_id: None,
5219 sink: &sink,
5220 },
5221 )
5222 .unwrap());
5223 assert!(!sink.is_closed());
5224
5225 let completion = forwarding
5226 .complete_pending_relay(
5227 module_connection,
5228 pending.corr,
5229 RouteBindRelayOutcome::Accepted,
5230 )
5231 .expect("a closing client must not turn a module's ack into an error");
5232
5233 assert!(completion.settled);
5234 let abandoned = completion
5235 .abandoned
5236 .expect("the module must be told to drop the binding it just created");
5237 assert_eq!(abandoned.connection_id, module_connection);
5238 assert_eq!(abandoned.channel, pending.module_channel);
5239 assert_eq!(abandoned.epoch, pending.module_epoch);
5240 assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
5241 assert!(matches!(
5242 pending.receiver.blocking_recv().unwrap(),
5243 RouteBindRelayOutcome::ModuleGone(_)
5244 ));
5245 assert!(client_rx.try_recv().is_err());
5248 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
5249
5250 assert!(forwarding
5253 .has_live_module_connection("closing-client")
5254 .unwrap());
5255 let cotenant = ConnectionId::new(201);
5256 let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
5257 let cotenant_route = begin_test_route(
5258 &forwarding,
5259 cotenant,
5260 FrameSink::new(cotenant_tx),
5261 62,
5262 "closing-client",
5263 );
5264 assert_eq!(cotenant_route.endpoint, endpoint);
5265 forwarding
5266 .complete_pending_relay(
5267 module_connection,
5268 cotenant_route.corr,
5269 RouteBindRelayOutcome::Accepted,
5270 )
5271 .unwrap();
5272 assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
5273 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5274 }
5275
5276 #[test]
5277 fn pending_route_permit_is_released_on_rejection_and_abort() {
5278 let forwarding = ForwardingTable::default();
5279 let module_connection = ConnectionId::new(300);
5280 let client = ConnectionId::new(301);
5281 let (module_tx, _module_rx) = mpsc::channel(1);
5282 let endpoint = forwarding
5283 .register_module_connection(
5284 module_connection,
5285 "permit".into(),
5286 2,
5287 Concurrency::ModuleManaged,
5288 FrameSink::new(module_tx),
5289 )
5290 .unwrap();
5291 let (client_tx, mut client_rx) = mpsc::channel(1);
5292 let sink = FrameSink::new(client_tx);
5293 let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
5294 assert!(sink.try_send(test_ping(999)).is_err());
5295 forwarding
5296 .complete_pending_relay(
5297 module_connection,
5298 rejected.corr,
5299 RouteBindRelayOutcome::Rejected(ErrorBody {
5300 code: "no".into(),
5301 message: "rejected".into(),
5302 detail: None,
5303 }),
5304 )
5305 .unwrap();
5306 sink.try_send(test_ping(1000)).unwrap();
5307 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
5308
5309 let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
5310 assert!(sink.try_send(test_ping(1001)).is_err());
5311 forwarding
5312 .abort_pending_relay(
5313 endpoint,
5314 aborted.corr,
5315 RouteBindRelayOutcome::ModuleGone("abort".into()),
5316 )
5317 .unwrap();
5318 sink.try_send(test_ping(1002)).unwrap();
5319 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
5320
5321 let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
5322 forwarding
5323 .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
5324 .unwrap();
5325 drop(client_rx);
5326 let completion = forwarding
5327 .complete_pending_relay(
5328 module_connection,
5329 receiver_closed.corr,
5330 RouteBindRelayOutcome::Accepted,
5331 )
5332 .unwrap();
5333 assert!(completion.abandoned.is_some());
5334 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5335 }
5336
5337 #[test]
5343 fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
5344 let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
5345 route_fixture("closing-set-leak");
5346
5347 const CONNECTIONS: u64 = 32;
5348 for index in 0..CONNECTIONS {
5349 let client = ConnectionId::new(1000 + index);
5350 let (client_tx, _client_rx) = mpsc::channel(8);
5351 let route = begin_test_route(
5352 &forwarding,
5353 client,
5354 FrameSink::new(client_tx),
5355 index + 1,
5356 "closing-set-leak",
5357 );
5358 forwarding
5359 .complete_pending_relay(
5360 module_connection,
5361 route.corr,
5362 RouteBindRelayOutcome::Accepted,
5363 )
5364 .unwrap();
5365 forwarding.cleanup_connection(client).unwrap();
5366 }
5367 forwarding.cleanup_connection(module_connection).unwrap();
5368
5369 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5370 }
5371
5372 #[test]
5379 fn closing_connection_is_refused_new_work_until_cleanup_completes() {
5380 let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
5381 route_fixture("closing-gate");
5382
5383 let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
5386 forwarding
5387 .complete_pending_relay(
5388 module_connection,
5389 live.corr,
5390 RouteBindRelayOutcome::Accepted,
5391 )
5392 .unwrap();
5393 client_rx.try_recv().unwrap();
5394
5395 assert!(forwarding
5398 .escalate_client_delivery_failure(
5399 client,
5400 live.client_channel,
5401 live.client_epoch,
5402 CloseReason::new(
5403 "module_to_client_delivery_failed",
5404 "client egress refused a module frame",
5405 ),
5406 UndeliveredFrame {
5407 module_id: None,
5408 sink: &sink,
5409 },
5410 )
5411 .unwrap());
5412 assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
5413
5414 assert!(matches!(
5416 forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
5417 Err(ForwardingError::ConnectionClosing { connection_id })
5418 if connection_id == client
5419 ));
5420 let (late_tx, _late_rx) = mpsc::channel(1);
5422 assert!(matches!(
5423 forwarding.register_module_connection(
5424 client,
5425 "late-module".into(),
5426 2,
5427 Concurrency::ModuleManaged,
5428 FrameSink::new(late_tx),
5429 ),
5430 Err(ForwardingError::ConnectionClosing { connection_id })
5431 if connection_id == client
5432 ));
5433
5434 forwarding.cleanup_connection(client).unwrap();
5438 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5439 }
5440}
5441
5442#[cfg(test)]
5445mod swap_slot_tests {
5446 use std::time::Duration;
5447
5448 use super::*;
5449 use tokio::sync::mpsc;
5450
5451 const MODULE_ID: &str = "swapped";
5452
5453 struct SwapFixture {
5454 forwarding: ForwardingTable,
5455 incumbent_connection: ConnectionId,
5456 incumbent: ModuleEndpointId,
5457 candidate_connection: ConnectionId,
5458 candidate: ModuleEndpointId,
5459 _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
5460 }
5461
5462 fn swap_fixture() -> SwapFixture {
5463 let forwarding = ForwardingTable::default();
5464 let incumbent_connection = ConnectionId::new(100);
5465 let candidate_connection = ConnectionId::new(110);
5466 let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
5467 let incumbent = forwarding
5468 .register_module_connection(
5469 incumbent_connection,
5470 MODULE_ID.to_string(),
5471 2,
5472 Concurrency::ModuleManaged,
5473 FrameSink::new(incumbent_tx),
5474 )
5475 .unwrap();
5476 let (candidate_tx, candidate_rx) = mpsc::channel(8);
5477 let candidate = forwarding
5478 .register_candidate_module_connection(
5479 candidate_connection,
5480 MODULE_ID.to_string(),
5481 2,
5482 Concurrency::ModuleManaged,
5483 FrameSink::new(candidate_tx),
5484 )
5485 .unwrap();
5486 SwapFixture {
5487 forwarding,
5488 incumbent_connection,
5489 incumbent,
5490 candidate_connection,
5491 candidate,
5492 _module_rxs: vec![incumbent_rx, candidate_rx],
5493 }
5494 }
5495
5496 fn client(
5497 raw: u64,
5498 ) -> (
5499 ConnectionId,
5500 FrameSink,
5501 mpsc::Receiver<crate::router::OutboundFrame>,
5502 ) {
5503 let (tx, rx) = mpsc::channel(8);
5504 (ConnectionId::new(raw), FrameSink::new(tx), rx)
5505 }
5506
5507 fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
5508 forwarding
5509 .read_inner()
5510 .unwrap()
5511 .client_to_module
5512 .values()
5513 .map(|route| route.module_endpoint)
5514 .collect()
5515 }
5516
5517 #[test]
5518 fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
5519 let fixture = swap_fixture();
5520 let forwarding = &fixture.forwarding;
5521 assert_ne!(fixture.incumbent, fixture.candidate);
5522
5523 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5525 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5526 let (client_connection, client_sink, _client_rx) = client(200);
5527 let pending = forwarding
5528 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5529 .unwrap();
5530 assert_eq!(pending.endpoint, fixture.incumbent);
5531 let rpc = forwarding
5532 .begin_module_control_rpc_for(
5533 MODULE_ID,
5534 "health.check",
5535 Instant::now() + Duration::from_secs(1),
5536 )
5537 .unwrap();
5538 assert_eq!(rpc.endpoint, fixture.incumbent);
5539 let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
5540 assert_eq!(census.len(), 1, "the census lists one endpoint per id");
5541
5542 assert_eq!(
5544 forwarding
5545 .module_endpoint_for_connection(fixture.candidate_connection)
5546 .unwrap(),
5547 Some(fixture.candidate)
5548 );
5549 assert_eq!(
5550 forwarding
5551 .module_id_for_connection(fixture.candidate_connection)
5552 .unwrap()
5553 .as_deref(),
5554 Some(MODULE_ID)
5555 );
5556
5557 let (other_tx, _other_rx) = mpsc::channel(1);
5559 assert_eq!(
5560 forwarding.register_candidate_module_connection(
5561 ConnectionId::new(120),
5562 MODULE_ID.to_string(),
5563 2,
5564 Concurrency::ModuleManaged,
5565 FrameSink::new(other_tx),
5566 ),
5567 Err(ForwardingError::CandidateSlotOccupied {
5568 module_id: MODULE_ID.to_string()
5569 })
5570 );
5571 }
5572
5573 #[test]
5577 fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
5578 let fixture = swap_fixture();
5579 let forwarding = &fixture.forwarding;
5580 let (early_client, early_sink, _early_rx) = client(200);
5581 let mut early = forwarding
5582 .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
5583 .unwrap();
5584 assert_eq!(early.endpoint, fixture.incumbent);
5585 assert_eq!(
5586 forwarding
5587 .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
5588 .unwrap(),
5589 RouteBindRelayMark::Marked
5590 );
5591
5592 let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5593 assert_eq!(
5594 cutover,
5595 ForwardingCutover {
5596 promoted: fixture.candidate,
5597 incumbent: Some(fixture.incumbent),
5598 }
5599 );
5600
5601 let (late_client, late_sink, _late_rx) = client(201);
5603 let late = forwarding
5604 .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
5605 .unwrap();
5606 assert_eq!(
5607 late.endpoint, fixture.candidate,
5608 "a route.open after cutover was reserved on the incumbent"
5609 );
5610
5611 let completion = forwarding
5613 .complete_pending_relay(
5614 fixture.incumbent_connection,
5615 early.corr,
5616 RouteBindRelayOutcome::Accepted,
5617 )
5618 .expect("a superseded endpoint's ack is not an error on its connection");
5619 assert!(completion.settled);
5620 assert!(
5621 !committed_endpoints(forwarding).contains(&fixture.incumbent),
5622 "a relay reserved before cutover committed a route on the incumbent"
5623 );
5624 let goodbye = completion
5625 .abandoned
5626 .expect("the incumbent is told to drop the binding it just created");
5627 assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
5628 assert_eq!(goodbye.channel, early.module_channel);
5629 assert_eq!(goodbye.epoch, early.module_epoch);
5630 assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
5631 match early.receiver.try_recv() {
5632 Ok(RouteBindRelayOutcome::DaemonRejected(body)) => {
5633 assert_eq!(body.code, "module_reloading")
5634 }
5635 other => panic!("expected a retryable module_reloading answer, got {other:?}"),
5636 }
5637 assert!(matches!(
5638 forwarding
5639 .lookup_data_route(early_client, early.client_channel, early.client_epoch)
5640 .unwrap(),
5641 DataRoute::Client(DataRouteState::Absent)
5642 ));
5643
5644 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
5647 forwarding
5648 .complete_pending_relay(
5649 fixture.candidate_connection,
5650 late.corr,
5651 RouteBindRelayOutcome::Accepted,
5652 )
5653 .unwrap();
5654 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5655 assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
5656 }
5657
5658 #[test]
5659 fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
5660 let fixture = swap_fixture();
5661 let forwarding = &fixture.forwarding;
5662 let (bound_client, bound_sink, _bound_rx) = client(200);
5664 let bound = forwarding
5665 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5666 .unwrap();
5667 forwarding
5668 .complete_pending_relay(
5669 fixture.incumbent_connection,
5670 bound.corr,
5671 RouteBindRelayOutcome::Accepted,
5672 )
5673 .unwrap();
5674 let (pending_client, pending_sink, _pending_rx) = client(201);
5675 let mut in_flight = forwarding
5676 .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
5677 .unwrap();
5678 forwarding
5679 .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
5680 .unwrap();
5681
5682 let incumbent = forwarding
5683 .cutover_candidate(MODULE_ID)
5684 .unwrap()
5685 .unwrap()
5686 .incumbent
5687 .unwrap();
5688 let target = forwarding
5689 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
5690 .unwrap()
5691 .expect("the superseded incumbent is still registered");
5692
5693 assert_eq!(target.endpoint, fixture.incumbent);
5694 assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
5695 assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
5696 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5697 assert_eq!(target.abandoned_bindings.len(), 1);
5698 assert_eq!(
5699 target.abandoned_bindings[0].channel,
5700 in_flight.module_channel
5701 );
5702 assert!(matches!(
5703 in_flight.receiver.try_recv(),
5704 Ok(RouteBindRelayOutcome::DaemonRejected(body)) if body.code == "module_reloading"
5705 ));
5706 assert_eq!(
5707 forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
5708 1,
5709 "the incumbent's bound route stays until its drain finishes"
5710 );
5711
5712 let (next_client, next_sink, _next_rx) = client(202);
5713 let next = forwarding
5714 .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
5715 .expect("the promoted candidate keeps accepting routes");
5716 assert_eq!(next.endpoint, fixture.candidate);
5717 }
5718
5719 #[test]
5725 fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
5726 let forwarding = ForwardingTable::default();
5727 let first_connection = ConnectionId::new(70);
5728 let (first_tx, _first_rx) = mpsc::channel(8);
5729 forwarding
5730 .register_module_connection(
5731 first_connection,
5732 MODULE_ID.to_string(),
5733 2,
5734 Concurrency::ModuleManaged,
5735 FrameSink::new(first_tx),
5736 )
5737 .unwrap();
5738 let (client_connection, client_sink, _client_rx) = client(200);
5739 let mut pending = forwarding
5740 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5741 .unwrap();
5742 let (second_tx, _second_rx) = mpsc::channel(8);
5743 forwarding
5744 .register_module_connection(
5745 ConnectionId::new(80),
5746 MODULE_ID.to_string(),
5747 2,
5748 Concurrency::ModuleManaged,
5749 FrameSink::new(second_tx),
5750 )
5751 .unwrap();
5752
5753 assert_eq!(
5754 forwarding
5755 .complete_pending_relay(
5756 first_connection,
5757 pending.corr,
5758 RouteBindRelayOutcome::Accepted
5759 )
5760 .unwrap_err(),
5761 ForwardingError::StaleModuleEndpoint
5762 );
5763 assert!(committed_endpoints(&forwarding).is_empty());
5764 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5765 assert!(matches!(
5766 pending.receiver.try_recv(),
5767 Err(oneshot::error::TryRecvError::Closed)
5768 ));
5769 }
5770
5771 #[test]
5772 fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
5773 let fixture = swap_fixture();
5775 let forwarding = &fixture.forwarding;
5776 assert!(forwarding
5777 .cleanup_connection(fixture.candidate_connection)
5778 .unwrap()
5779 .is_empty());
5780 assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
5781 let (client_connection, client_sink, _client_rx) = client(200);
5782 assert_eq!(
5783 forwarding
5784 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5785 .unwrap()
5786 .endpoint,
5787 fixture.incumbent
5788 );
5789
5790 let fixture = swap_fixture();
5793 let forwarding = &fixture.forwarding;
5794 let (bound_client, bound_sink, _bound_rx) = client(200);
5795 let bound = forwarding
5796 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5797 .unwrap();
5798 forwarding
5799 .complete_pending_relay(
5800 fixture.incumbent_connection,
5801 bound.corr,
5802 RouteBindRelayOutcome::Accepted,
5803 )
5804 .unwrap();
5805 forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5806 let released = forwarding
5807 .cleanup_connection(fixture.incumbent_connection)
5808 .unwrap();
5809 assert_eq!(released.len(), 1);
5810 assert_eq!(released[0].connection_id, bound_client);
5811 assert!(forwarding
5812 .read_inner()
5813 .unwrap()
5814 .superseded_endpoints
5815 .is_empty());
5816 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5817 let (next_client, next_sink, _next_rx) = client(201);
5818 assert_eq!(
5819 forwarding
5820 .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
5821 .unwrap()
5822 .endpoint,
5823 fixture.candidate
5824 );
5825 }
5826}