1use std::{
2 collections::{BTreeMap, HashMap, HashSet},
3 error::Error,
4 fmt,
5 sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
6 time::Duration,
7};
8
9use subc_control::{ClientControlResponse, RouteCloseReason};
10use subc_protocol::{
11 manifest::Concurrency,
12 session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
13 ErrorBody, Flags, FrameType, Principal, Priority,
14};
15use tokio::sync::{oneshot, Semaphore};
16use tokio::time::Instant;
17use tracing::{debug, info, warn};
18
19use crate::{
20 control::{RouteBindBreakers, RouteBindConcurrency},
21 observability::DaemonCounters,
22 registry::ConnectionId,
23 router::FrameSink,
24 scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
25 Frame, ProjectRootId,
26};
27
28const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
30
31const DEFAULT_MODULE_MANAGED_WATCH_WINDOW: usize = 24;
33
34const STATELESS_PARALLEL_WINDOW: usize = 1024;
36
37const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
41
42#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
47pub struct ModuleEndpointId {
48 pub connection_id: ConnectionId,
49 pub generation: u64,
50}
51
52#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
54pub(crate) struct ClientRouteKey {
55 pub connection_id: ConnectionId,
56 pub channel: u16,
57}
58
59#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
61pub(crate) struct ModuleRouteKey {
62 pub endpoint: ModuleEndpointId,
63 pub channel: u16,
64}
65
66#[derive(Debug)]
67pub(crate) struct RouteBinding {
68 pub client_connection_id: ConnectionId,
69 pub client_sink: FrameSink,
70 pub client_negotiated_ver: u8,
71 pub client_channel: u16,
72 pub client_epoch: u32,
73 pub module_id: String,
74 pub module_endpoint: ModuleEndpointId,
75 pub module_sink: FrameSink,
76 pub module_negotiated_ver: u8,
77 pub module_channel: u16,
78 pub module_epoch: u32,
79 pub principal: Principal,
80 #[allow(dead_code)]
82 pub harness: String,
83 #[allow(dead_code)]
85 pub build_class: Option<String>,
86 #[allow(dead_code)]
88 pub operator_app: Option<crate::operator_apps::OperatorAppStamp>,
89 pub project_root: Option<ProjectRootId>,
90 pub bound_at: Instant,
91 pub flow: Arc<ChannelFlow>,
92 pub scope: Option<BoundScope>,
95}
96
97#[derive(Debug, Clone)]
98pub(crate) enum DataRoute {
99 Client(DataRouteState),
100 Module(DataRouteState),
101}
102
103#[derive(Debug, Clone)]
104pub(crate) enum DataRouteState {
105 Bound(Arc<RouteBinding>),
106 Reserved,
107 EpochMismatch,
108 Absent,
109}
110
111#[derive(Debug, Clone, Copy, PartialEq, Eq)]
140pub(crate) enum GoodbyeTargetKind {
141 Client,
142 Module,
143}
144
145#[derive(Debug, Clone)]
146pub(crate) struct GoodbyeTarget {
147 pub connection_id: ConnectionId,
148 pub sink: FrameSink,
149 pub negotiated_ver: u8,
150 pub channel: u16,
151 pub epoch: u32,
152 pub kind: GoodbyeTargetKind,
153 pub module_id: Option<String>,
158}
159
160#[derive(Debug, Clone, Copy)]
163pub(crate) struct UndeliveredFrame<'a> {
164 pub module_id: Option<&'a str>,
166 pub sink: &'a FrameSink,
168}
169
170pub(crate) fn principal_label(principal: &Principal) -> String {
172 match principal {
173 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
174 Principal::Direct => "direct".to_string(),
175 other => format!("{other:?}"),
176 }
177}
178
179fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
182 let labels = inner
183 .client_to_module
184 .iter()
185 .filter(|(key, _)| key.connection_id == connection_id)
186 .map(|(_, route)| principal_label(&route.principal))
187 .collect::<std::collections::BTreeSet<_>>();
188 if labels.is_empty() {
189 "none".to_string()
190 } else {
191 labels.into_iter().collect::<Vec<_>>().join(",")
192 }
193}
194
195impl GoodbyeTarget {
196 pub(crate) fn close_on_delivery_failure(&self) -> bool {
199 matches!(self.kind, GoodbyeTargetKind::Client)
200 }
201}
202
203pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
216
217pub(crate) fn send_module_route_goodbye(
229 counters: &DaemonCounters,
230 sink: &FrameSink,
231 frame: Frame,
232 module_id: Option<&str>,
233 context: &'static str,
234) {
235 let channel = frame.header.channel;
236 let epoch = frame.header.epoch;
237 let Err(err) = sink.try_send(frame.clone()) else {
238 return;
239 };
240 let runtime = match tokio::runtime::Handle::try_current() {
243 Ok(runtime) if !sink.is_closed() => runtime,
244 _ => {
245 counters.increment_goodbye_relay_module_dropped(module_id);
246 warn!(
247 module_id = module_id.unwrap_or("unknown"),
248 route_channel = channel,
249 route_epoch = epoch,
250 error = %err,
251 context,
252 "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
253 );
254 return;
255 }
256 };
257 debug!(
258 module_id = module_id.unwrap_or("unknown"),
259 route_channel = channel,
260 route_epoch = epoch,
261 error = %err,
262 context,
263 "module egress queue refused route GOODBYE; delivering it once the module frees room"
264 );
265 let counters = counters.clone();
266 let sink = sink.clone();
267 let module_id = module_id.map(str::to_string);
268 runtime.spawn(async move {
269 let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
270 let why = match outcome {
271 Ok(Ok(())) => {
272 debug!(
273 module_id = module_id.as_deref().unwrap_or("unknown"),
274 route_channel = channel,
275 route_epoch = epoch,
276 context,
277 "late route GOODBYE delivered to module"
278 );
279 return;
280 }
281 Ok(Err(err)) => err.to_string(),
282 Err(_) => format!(
283 "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
284 ),
285 };
286 counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
287 warn!(
288 module_id = module_id.as_deref().unwrap_or("unknown"),
289 route_channel = channel,
290 route_epoch = epoch,
291 error = %why,
292 context,
293 "route GOODBYE to module dropped under backpressure; not closing shared module connection"
294 );
295 });
296}
297
298#[derive(Debug, Clone)]
304pub(crate) struct EndpointRoute {
305 pub goodbye_target: GoodbyeTarget,
306 pub principal: Principal,
307 pub bound_at: Instant,
308 pub draining: bool,
309 pub drain_reason: Option<RouteCloseReason>,
313}
314
315#[derive(Debug, Clone)]
318pub(crate) struct ScopeDrainedRoute {
319 pub reason: RouteCloseReason,
320 pub scope: BoundScope,
321 pub module_id: String,
322 pub client: GoodbyeTarget,
323 pub module: GoodbyeTarget,
324}
325
326#[derive(Debug)]
327pub(crate) struct PendingRouteBindRelay {
328 pub endpoint: ModuleEndpointId,
329 pub module_sink: FrameSink,
330 pub negotiated_ver: u8,
331 pub client_channel: u16,
332 pub client_epoch: u32,
333 pub module_channel: u16,
334 pub module_epoch: u32,
335 pub corr: u64,
336 pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
337}
338
339#[derive(Debug, Clone)]
340pub(crate) struct ModuleDrainTarget {
341 pub endpoint: ModuleEndpointId,
342 pub sink: FrameSink,
343 pub negotiated_ver: u8,
344 pub abandoned_bindings: Vec<GoodbyeTarget>,
345 pub excluded_subscriptions: u32,
346}
347
348#[cfg(unix)]
351#[derive(Debug, Clone)]
352pub(crate) struct ModuleConnectionTarget {
353 pub module_id: String,
354 pub endpoint: ModuleEndpointId,
355 pub sink: FrameSink,
356 pub negotiated_ver: u8,
357}
358
359#[derive(Debug, Clone)]
360pub(crate) enum RouteBindRelayOutcome {
361 Accepted,
362 Rejected(ErrorBody),
367 DaemonRejected(ErrorBody),
375 ModuleGone(String),
376}
377
378#[derive(Debug, Clone, Copy, PartialEq, Eq)]
380pub(crate) struct ForwardingCutover {
381 pub promoted: ModuleEndpointId,
383 pub incumbent: Option<ModuleEndpointId>,
386}
387
388#[derive(Debug)]
390pub(crate) struct ConnectionCleanup {
391 pub released: Vec<GoodbyeTarget>,
393 pub abandoned_relays: u32,
396}
397
398#[derive(Debug, Clone)]
399pub(crate) struct PendingRelayCompletion {
400 pub settled: bool,
401 pub abandoned: Option<GoodbyeTarget>,
402}
403
404#[derive(Debug)]
405pub(crate) struct PendingModuleControlRpc {
406 pub endpoint: ModuleEndpointId,
407 pub module_sink: FrameSink,
408 pub negotiated_ver: u8,
409 pub corr: u64,
410 pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
411}
412
413#[derive(Debug, Clone)]
414pub(crate) enum ModuleControlRpcOutcome {
415 Response(ModuleControlResponse),
416 Rejected(ErrorBody),
417 ModuleGone(String),
418 MalformedResponse(String),
419 UnexpectedOp { expected: String, actual: String },
420 DeadlineElapsed,
421}
422
423#[derive(Debug, Clone, PartialEq, Eq)]
424pub(crate) enum ModuleControlRpcCompletion {
425 Unknown,
426 Settled,
427 LateHealthAnswer {
428 module_id: String,
429 latency: Duration,
430 },
431}
432
433#[derive(Debug)]
434struct PendingModuleControlRpcEntry {
435 expected_op: String,
436 deadline: Instant,
437 health_probe_started_at: Option<Instant>,
438 sender: oneshot::Sender<ModuleControlRpcOutcome>,
439}
440
441#[derive(Debug)]
442struct HealthProbeTombstone {
443 expected_op: String,
444 module_id: String,
445 probe_started_at: Instant,
446 expires_at: Instant,
447}
448
449#[derive(Debug, Clone)]
450struct RouteReservation {
451 client_key: ClientRouteKey,
452 module_key: ModuleRouteKey,
453 client_epoch: u32,
454 module_epoch: u32,
455 project_root: Option<ProjectRootId>,
456}
457
458#[derive(Debug)]
459struct PendingRouteBindRelayEntry {
460 reservation: RouteReservation,
461 client_sink: FrameSink,
462 client_negotiated_ver: u8,
463 client_permit: crate::router::EgressPermit,
464 route_open_frame: Frame,
465 busy_refusals: bool,
466 request_window: usize,
467 watch_window: Option<usize>,
468 principal: Principal,
469 scope: Option<BoundScope>,
472 harness: String,
473 build_class: Option<String>,
474 operator_app: Option<crate::operator_apps::OperatorAppStamp>,
475 deadline: Instant,
476 relay_enqueued: bool,
487 sender: oneshot::Sender<RouteBindRelayOutcome>,
488}
489
490#[derive(Debug, Clone, Copy, PartialEq, Eq)]
493pub(crate) enum RouteBindRelayMark {
494 Marked,
496 SettledByReply,
500 ReleasedWithoutReply,
504}
505
506#[derive(Debug, Clone)]
507pub(crate) enum RouteRelease {
508 Removed(GoodbyeTarget),
509 Stale,
510 Absent,
511}
512
513#[derive(Debug, Clone)]
514pub(crate) enum RoutePollSnapshot {
515 Bound {
516 module_id: String,
517 status: Option<String>,
518 },
519 Absent,
520}
521
522#[derive(Debug, Clone)]
523struct ModuleConnection {
524 endpoint: ModuleEndpointId,
525 sink: FrameSink,
526 negotiated_ver: u8,
527 concurrency: Concurrency,
528 verified_prefix_owner: Option<String>,
530}
531
532pub(crate) struct ModuleHelloAck {
533 pub(crate) frame: Frame,
534 pub(crate) verified_prefix_owner: Option<String>,
535}
536
537#[derive(Debug, Default)]
538struct ForwardingInner {
539 operator_confirms: Arc<crate::operator_confirm::OperatorConfirms>,
540 daemon_draining: bool,
541 daemon_drain_results: HashMap<ModuleEndpointId, bool>,
546 modules_by_id: HashMap<String, ModuleConnection>,
550 candidates_by_id: HashMap<String, ModuleConnection>,
556 superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
562 endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
563 module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
564 draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
568 closing_connections: HashSet<ConnectionId>,
569 next_generation: u64,
570 reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
571 reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
572 next_client_channel: HashMap<ConnectionId, u16>,
573 next_module_channel: HashMap<ModuleEndpointId, u16>,
574 client_slot_epochs: HashMap<ClientRouteKey, u32>,
575 module_slot_epochs: HashMap<ModuleRouteKey, u32>,
576 last_published_epoch: HashMap<ClientRouteKey, u32>,
577 client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
578 module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
579 status: HashMap<(ClientRouteKey, u32), String>,
580 pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
581 relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
586 scope_tags: HashMap<(String, String), ScopeTag>,
591 next_control_corr: HashMap<ModuleEndpointId, u64>,
592 pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
593 health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
594}
595
596#[derive(Debug, Clone)]
597pub(crate) struct CloseReason {
598 code: &'static str,
599 message: String,
600}
601
602impl CloseReason {
603 pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
604 Self {
605 code,
606 message: message.into(),
607 }
608 }
609}
610
611impl fmt::Display for CloseReason {
612 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
613 write!(f, "{}: {}", self.code, self.message)
614 }
615}
616
617pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
618
619#[derive(Debug, Default)]
621pub struct ForwardingTable {
622 inner: Arc<RwLock<ForwardingInner>>,
623 frame_drop_warnings: Mutex<HashMap<ConnectionId, ChannelDropWarnings>>,
624 #[cfg(feature = "test-support")]
625 route_closing_hold: Mutex<Option<oneshot::Receiver<()>>>,
626 close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
627 counters: DaemonCounters,
628 route_bind_breakers: RouteBindBreakers,
633 route_bind_concurrency: RouteBindConcurrency,
636 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
640}
641
642#[derive(Debug)]
643struct FrameDropWarning {
644 last_warned: tokio::time::Instant,
645 suppressed: u64,
646}
647
648type ChannelDropWarnings = HashMap<(u16, &'static str), FrameDropWarning>;
649
650impl ForwardingTable {
651 pub(crate) fn warn_frame_drop(
654 &self,
655 connection_id: ConnectionId,
656 channel: u16,
657 epoch: u32,
658 direction: &'static str,
659 reason: &'static str,
660 ) {
661 let now = tokio::time::Instant::now();
662 let mut warnings = self
663 .frame_drop_warnings
664 .lock()
665 .unwrap_or_else(std::sync::PoisonError::into_inner);
666 let entry = warnings
667 .entry(connection_id)
668 .or_default()
669 .entry((channel, reason));
670 let suppressed = match entry {
671 std::collections::hash_map::Entry::Occupied(mut entry) => {
672 let state = entry.get_mut();
673 if now.saturating_duration_since(state.last_warned) < Duration::from_secs(60) {
674 state.suppressed = state.suppressed.saturating_add(1);
675 return;
676 }
677 state.last_warned = now;
678 std::mem::take(&mut state.suppressed)
679 }
680 std::collections::hash_map::Entry::Vacant(entry) => {
681 entry.insert(FrameDropWarning {
682 last_warned: now,
683 suppressed: 0,
684 });
685 0
686 }
687 };
688 warn!(
689 connection_id = connection_id.get(),
690 channel, epoch, direction, reason, suppressed, "dropping data-plane frame"
691 );
692 }
693
694 #[cfg(feature = "test-support")]
736 pub fn configure_operator_script(
737 &self,
738 script: &std::path::Path,
739 events: Option<&std::path::Path>,
740 limits: Option<crate::operator_confirm::OperatorLimits>,
741 ) {
742 self.operator_confirms()
743 .configure_script(script, events, limits);
744 }
745
746 #[cfg(feature = "test-support")]
750 pub fn hold_route_closing_for_test(&self) -> oneshot::Sender<()> {
751 let (release, hold) = oneshot::channel();
752 *self.route_closing_hold.lock().unwrap() = Some(hold);
753 release
754 }
755
756 #[cfg(feature = "test-support")]
757 pub(crate) async fn wait_before_route_closing_for_test(&self) {
758 let hold = self.route_closing_hold.lock().unwrap().take();
759 if let Some(hold) = hold {
760 let _ = hold.await;
761 }
762 }
763
764 pub(crate) fn operator_confirms(&self) -> Arc<crate::operator_confirm::OperatorConfirms> {
765 Arc::clone(
766 &self
767 .inner
768 .read()
769 .unwrap_or_else(|p| p.into_inner())
770 .operator_confirms,
771 )
772 }
773
774 #[cfg(test)]
775 pub(crate) fn inject_operator_principal(&self, key: ModuleRouteKey, principal: Principal) {
776 self.alter_operator_binding_for_test(key, |binding| binding.principal = principal);
777 }
778
779 #[cfg(test)]
780 pub(crate) fn alter_operator_binding_for_test(
781 &self,
782 key: ModuleRouteKey,
783 alter: impl FnOnce(&mut RouteBinding),
784 ) {
785 let mut inner = self.write_inner().unwrap();
786 let old = inner.module_to_client.remove(&key).unwrap();
787 let client = ClientRouteKey {
788 connection_id: old.client_connection_id,
789 channel: old.client_channel,
790 };
791 inner.client_to_module.remove(&client);
792 let mut binding = Arc::try_unwrap(old).expect("test binding has no outstanding readers");
793 alter(&mut binding);
794 let binding = Arc::new(binding);
795 inner.client_to_module.insert(client, Arc::clone(&binding));
796 inner.module_to_client.insert(key, binding);
797 }
798
799 #[cfg(test)]
802 pub(crate) fn invalidate_operator_route_for_test(&self, key: ModuleRouteKey, stale: bool) {
803 let mut inner = self.write_inner().unwrap();
804 let old = inner.module_to_client.remove(&key).unwrap();
805 let client = ClientRouteKey {
806 connection_id: old.client_connection_id,
807 channel: old.client_channel,
808 };
809 inner.client_to_module.remove(&client);
810 if stale {
811 let mut binding = Arc::try_unwrap(old).expect("no binding readers at the test pause");
812 binding.module_epoch += 1;
813 let binding = Arc::new(binding);
814 inner.client_to_module.insert(client, binding.clone());
815 inner.module_to_client.insert(key, binding);
816 }
817 }
818
819 pub(crate) fn with_operator_route<T>(
823 &self,
824 connection_id: ConnectionId,
825 channel: u16,
826 epoch: u32,
827 admit: impl FnOnce(Option<&RouteBinding>) -> T,
828 ) -> Result<T, ForwardingError> {
829 self.with_operator_route_epoch(connection_id, channel, |binding| {
830 admit(binding.filter(|binding| binding.module_epoch == epoch))
831 })
832 }
833
834 pub(crate) fn with_operator_route_epoch<T>(
837 &self,
838 connection_id: ConnectionId,
839 channel: u16,
840 decide: impl FnOnce(Option<&RouteBinding>) -> T,
841 ) -> Result<T, ForwardingError> {
842 let inner = self.read_inner()?;
843 let binding = inner
844 .endpoint_by_connection
845 .get(&connection_id)
846 .and_then(|endpoint| {
847 inner.module_to_client.get(&ModuleRouteKey {
848 endpoint: *endpoint,
849 channel,
850 })
851 });
852 Ok(decide(binding.map(Arc::as_ref)))
853 }
854
855 pub(crate) fn counters(&self) -> DaemonCounters {
856 self.counters.clone()
857 }
858
859 pub(crate) fn counters_ref(&self) -> &DaemonCounters {
860 &self.counters
861 }
862
863 pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
864 self.route_bind_breakers.clone()
865 }
866
867 pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
868 self.route_bind_concurrency.clone()
869 }
870
871 pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
872 Arc::clone(&self.route_outages)
873 }
874
875 pub(crate) fn register_connection_close(
876 &self,
877 connection_id: ConnectionId,
878 ) -> ConnectionCloseReceiver {
879 let (sender, receiver) = oneshot::channel();
880 let replaced = self
881 .lock_close_registry()
882 .insert(connection_id, sender)
883 .is_some();
884 if replaced {
885 warn!(
886 connection_id = connection_id.get(),
887 "replaced existing connection close registration"
888 );
889 }
890 receiver
891 }
892
893 pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
894 self.lock_close_registry().remove(&connection_id);
895 }
896
897 #[cfg(unix)]
906 pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
907 let senders: Vec<_> = self.lock_close_registry().drain().collect();
908 let count = senders.len();
909 for (_, sender) in senders {
910 let _ = sender.send(reason.clone());
911 }
912 count
913 }
914
915 #[cfg(unix)]
919 pub(crate) fn module_connections(
920 &self,
921 ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
922 let inner = self.read_inner()?;
923 let mut seen = HashSet::new();
924 Ok(inner
925 .modules_by_id
926 .values()
927 .chain(inner.candidates_by_id.values())
928 .chain(inner.superseded_endpoints.values())
929 .filter(|module| seen.insert(module.endpoint))
930 .map(|module| ModuleConnectionTarget {
931 module_id: inner
932 .module_id_by_endpoint
933 .get(&module.endpoint)
934 .cloned()
935 .unwrap_or_default(),
936 endpoint: module.endpoint,
937 sink: module.sink.clone(),
938 negotiated_ver: module.negotiated_ver,
939 })
940 .collect())
941 }
942
943 pub(crate) fn request_connection_close(
947 &self,
948 connection_id: ConnectionId,
949 reason: CloseReason,
950 ) -> bool {
951 let sender = self.lock_close_registry().remove(&connection_id);
952 if let Some(sender) = sender {
953 debug!(
954 connection_id = connection_id.get(),
955 close_reason = %reason,
956 "requesting connection close"
957 );
958 let _ = sender.send(reason);
959 true
960 } else {
961 debug!(
962 connection_id = connection_id.get(),
963 close_reason = %reason,
964 "connection close request ignored for inactive connection"
965 );
966 false
967 }
968 }
969
970 pub fn register_module_connection(
971 &self,
972 connection_id: ConnectionId,
973 module_id: String,
974 negotiated_ver: u8,
975 concurrency: Concurrency,
976 sink: FrameSink,
977 ) -> Result<ModuleEndpointId, ForwardingError> {
978 self.register_module_connection_inner(
979 connection_id,
980 module_id,
981 negotiated_ver,
982 concurrency,
983 sink,
984 None,
985 )
986 }
987
988 pub(crate) fn register_module_connection_acked(
1001 &self,
1002 connection_id: ConnectionId,
1003 module_id: String,
1004 negotiated_ver: u8,
1005 concurrency: Concurrency,
1006 sink: FrameSink,
1007 hello_ack: ModuleHelloAck,
1008 ) -> Result<ModuleEndpointId, ForwardingError> {
1009 self.register_module_connection_inner(
1010 connection_id,
1011 module_id,
1012 negotiated_ver,
1013 concurrency,
1014 sink,
1015 Some(hello_ack),
1016 )
1017 }
1018
1019 fn register_module_connection_inner(
1020 &self,
1021 connection_id: ConnectionId,
1022 module_id: String,
1023 negotiated_ver: u8,
1024 concurrency: Concurrency,
1025 sink: FrameSink,
1026 hello_ack: Option<ModuleHelloAck>,
1027 ) -> Result<ModuleEndpointId, ForwardingError> {
1028 let mut inner = self.write_inner()?;
1029 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1030 return Err(ForwardingError::ConnectionClosing { connection_id });
1031 }
1032 check_module_connection_role_locked(&inner, connection_id)?;
1033 let verified_prefix_owner = hello_ack
1036 .as_ref()
1037 .and_then(|hello| hello.verified_prefix_owner.clone());
1038 enqueue_hello_ack_locked(&sink, connection_id, hello_ack.map(|hello| hello.frame))?;
1039
1040 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1041 let endpoint = ModuleEndpointId {
1042 connection_id,
1043 generation: inner.next_generation,
1044 };
1045 inner.endpoint_by_connection.insert(connection_id, endpoint);
1046 inner
1047 .module_id_by_endpoint
1048 .insert(endpoint, module_id.clone());
1049 inner.next_module_channel.insert(endpoint, 1);
1050 inner.next_control_corr.insert(endpoint, 1);
1051 inner.modules_by_id.insert(
1052 module_id.clone(),
1053 ModuleConnection {
1054 endpoint,
1055 sink,
1056 negotiated_ver,
1057 concurrency,
1058 verified_prefix_owner,
1059 },
1060 );
1061 drop(inner);
1062
1063 if let Some(discarded) = self
1076 .route_bind_breakers
1077 .reset_for_new_module_connection(&module_id)
1078 {
1079 info!(
1080 module_id = %module_id,
1081 discarded_consecutive_timeouts = discarded,
1082 "route.bind breaker state discarded: a new module connection replaced the process it described"
1083 );
1084 }
1085 Ok(endpoint)
1086 }
1087
1088 #[cfg(test)]
1102 pub(crate) fn register_candidate_module_connection(
1103 &self,
1104 connection_id: ConnectionId,
1105 module_id: String,
1106 negotiated_ver: u8,
1107 concurrency: Concurrency,
1108 sink: FrameSink,
1109 ) -> Result<ModuleEndpointId, ForwardingError> {
1110 self.register_candidate_module_connection_inner(
1111 connection_id,
1112 module_id,
1113 negotiated_ver,
1114 concurrency,
1115 sink,
1116 None,
1117 )
1118 }
1119
1120 pub(crate) fn register_candidate_module_connection_acked(
1127 &self,
1128 connection_id: ConnectionId,
1129 module_id: String,
1130 negotiated_ver: u8,
1131 concurrency: Concurrency,
1132 sink: FrameSink,
1133 hello_ack: Frame,
1134 ) -> Result<ModuleEndpointId, ForwardingError> {
1135 self.register_candidate_module_connection_inner(
1136 connection_id,
1137 module_id,
1138 negotiated_ver,
1139 concurrency,
1140 sink,
1141 Some(hello_ack),
1142 )
1143 }
1144
1145 fn register_candidate_module_connection_inner(
1146 &self,
1147 connection_id: ConnectionId,
1148 module_id: String,
1149 negotiated_ver: u8,
1150 concurrency: Concurrency,
1151 sink: FrameSink,
1152 hello_ack: Option<Frame>,
1153 ) -> Result<ModuleEndpointId, ForwardingError> {
1154 let mut inner = self.write_inner()?;
1155 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1156 return Err(ForwardingError::ConnectionClosing { connection_id });
1157 }
1158 check_module_connection_role_locked(&inner, connection_id)?;
1159 if inner.candidates_by_id.contains_key(&module_id) {
1160 return Err(ForwardingError::CandidateSlotOccupied { module_id });
1161 }
1162 enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1165
1166 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1167 let endpoint = ModuleEndpointId {
1168 connection_id,
1169 generation: inner.next_generation,
1170 };
1171 inner.endpoint_by_connection.insert(connection_id, endpoint);
1172 inner
1173 .module_id_by_endpoint
1174 .insert(endpoint, module_id.clone());
1175 inner.next_module_channel.insert(endpoint, 1);
1176 inner.next_control_corr.insert(endpoint, 1);
1177 inner.candidates_by_id.insert(
1178 module_id,
1179 ModuleConnection {
1180 endpoint,
1181 sink,
1182 negotiated_ver,
1183 concurrency,
1184 verified_prefix_owner: None,
1185 },
1186 );
1187 Ok(endpoint)
1188 }
1189
1190 pub(crate) fn cutover_candidate(
1208 &self,
1209 module_id: &str,
1210 ) -> Result<Option<ForwardingCutover>, ForwardingError> {
1211 let mut inner = self.write_inner()?;
1212 if inner.daemon_draining {
1213 return Err(ForwardingError::ModuleReloading {
1214 module_id: module_id.to_string(),
1215 });
1216 }
1217 let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
1218 return Ok(None);
1219 };
1220 let promoted = candidate.endpoint;
1221 let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
1222 let incumbent = incumbent.map(|incumbent| {
1223 let endpoint = incumbent.endpoint;
1224 inner.superseded_endpoints.insert(endpoint, incumbent);
1225 endpoint
1226 });
1227 drop(inner);
1228
1229 if let Some(discarded) = self
1233 .route_bind_breakers
1234 .reset_for_new_module_connection(module_id)
1235 {
1236 info!(
1237 module_id = %module_id,
1238 discarded_consecutive_timeouts = discarded,
1239 "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
1240 );
1241 }
1242 Ok(Some(ForwardingCutover {
1243 promoted,
1244 incumbent,
1245 }))
1246 }
1247
1248 #[allow(clippy::too_many_arguments)]
1249 pub(crate) async fn begin_route_bind_relay_for(
1250 &self,
1251 client_connection_id: ConnectionId,
1252 client_sink: FrameSink,
1253 client_negotiated_ver: u8,
1254 client_corr: u64,
1255 module_id: &str,
1256 principal: Principal,
1257 scope: Option<BoundScope>,
1258 project_root: Option<ProjectRootId>,
1259 harness: String,
1260 build_class: Option<String>,
1261 operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1262 deadline: Instant,
1263 busy_refusals: bool,
1264 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1265 let client_permit =
1269 client_sink
1270 .reserve_owned()
1271 .await
1272 .map_err(|_| ForwardingError::ClientEgressClosed {
1273 connection_id: client_connection_id,
1274 })?;
1275 self.begin_route_bind_relay_inner(
1276 client_connection_id,
1277 client_sink,
1278 client_negotiated_ver,
1279 client_corr,
1280 module_id,
1281 principal,
1282 scope,
1283 project_root,
1284 harness,
1285 build_class,
1286 operator_app,
1287 deadline,
1288 client_permit,
1289 busy_refusals,
1290 )
1291 }
1292
1293 #[cfg(test)]
1294 pub(crate) fn begin_route_bind_relay_for_test(
1295 &self,
1296 client_connection_id: ConnectionId,
1297 client_sink: FrameSink,
1298 client_corr: u64,
1299 module_id: &str,
1300 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1301 self.begin_route_bind_relay_with_busy_for_test(
1302 client_connection_id,
1303 client_sink,
1304 client_corr,
1305 module_id,
1306 false,
1307 )
1308 }
1309
1310 #[cfg(test)]
1311 pub(crate) fn begin_route_bind_relay_with_busy_for_test(
1312 &self,
1313 client_connection_id: ConnectionId,
1314 client_sink: FrameSink,
1315 client_corr: u64,
1316 module_id: &str,
1317 busy_refusals: bool,
1318 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1319 let permit =
1320 client_sink
1321 .try_reserve_owned()
1322 .map_err(|_| ForwardingError::ClientEgressClosed {
1323 connection_id: client_connection_id,
1324 })?;
1325 self.begin_route_bind_relay_inner(
1326 client_connection_id,
1327 client_sink,
1328 subc_protocol::PROTOCOL_VERSION,
1329 client_corr,
1330 module_id,
1331 Principal::Direct,
1332 None,
1333 None,
1334 String::new(),
1335 None,
1336 None,
1337 Instant::now() + std::time::Duration::from_secs(60),
1338 permit,
1339 busy_refusals,
1340 )
1341 }
1342
1343 pub(crate) fn begin_module_control_rpc_for(
1344 &self,
1345 module_id: &str,
1346 expected_op: &str,
1347 deadline: Instant,
1348 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1349 self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1350 }
1351
1352 pub(crate) fn begin_health_probe_rpc_for(
1353 &self,
1354 module_id: &str,
1355 expected_op: &str,
1356 probe_started_at: Instant,
1357 deadline: Instant,
1358 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1359 self.begin_module_control_rpc_inner(
1360 module_id,
1361 expected_op,
1362 deadline,
1363 Some(probe_started_at),
1364 false,
1365 )
1366 }
1367
1368 pub(crate) fn begin_drain_health_probe_rpc_for(
1369 &self,
1370 module_id: &str,
1371 expected_op: &str,
1372 probe_started_at: Instant,
1373 deadline: Instant,
1374 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1375 self.begin_module_control_rpc_inner(
1376 module_id,
1377 expected_op,
1378 deadline,
1379 Some(probe_started_at),
1380 true,
1381 )
1382 }
1383
1384 pub(crate) fn begin_endpoint_health_probe_rpc_for(
1392 &self,
1393 endpoint: ModuleEndpointId,
1394 expected_op: &str,
1395 probe_started_at: Instant,
1396 deadline: Instant,
1397 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1398 let inner = self.write_inner()?;
1399 let module = module_connection_for_endpoint_locked(&inner, endpoint)
1400 .cloned()
1401 .ok_or(ForwardingError::NoModuleConnection)?;
1402 let module_id = inner
1403 .module_id_by_endpoint
1404 .get(&endpoint)
1405 .cloned()
1406 .unwrap_or_default();
1407 self.begin_control_rpc_locked(
1410 inner,
1411 &module_id,
1412 module,
1413 expected_op,
1414 deadline,
1415 Some(probe_started_at),
1416 true,
1417 )
1418 }
1419
1420 fn begin_module_control_rpc_inner(
1421 &self,
1422 module_id: &str,
1423 expected_op: &str,
1424 deadline: Instant,
1425 health_probe_started_at: Option<Instant>,
1426 allow_draining: bool,
1427 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1428 let inner = self.write_inner()?;
1429 let module = inner
1430 .modules_by_id
1431 .get(module_id)
1432 .cloned()
1433 .ok_or(ForwardingError::NoModuleConnection)?;
1434 self.begin_control_rpc_locked(
1435 inner,
1436 module_id,
1437 module,
1438 expected_op,
1439 deadline,
1440 health_probe_started_at,
1441 allow_draining,
1442 )
1443 }
1444
1445 #[allow(clippy::too_many_arguments)]
1446 fn begin_control_rpc_locked(
1447 &self,
1448 mut inner: RwLockWriteGuard<'_, ForwardingInner>,
1449 module_id: &str,
1450 module: ModuleConnection,
1451 expected_op: &str,
1452 deadline: Instant,
1453 health_probe_started_at: Option<Instant>,
1454 allow_draining: bool,
1455 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1456 if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1457 return Err(ForwardingError::ModuleReloading {
1458 module_id: module_id.to_string(),
1459 });
1460 }
1461 if inner
1462 .closing_connections
1463 .contains(&module.endpoint.connection_id)
1464 {
1465 return Err(ForwardingError::ConnectionClosing {
1466 connection_id: module.endpoint.connection_id,
1467 });
1468 }
1469 if health_probe_started_at.is_some() {
1470 inner
1474 .health_probe_tombstones
1475 .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1476 }
1477 let corr = match inner.allocate_control_corr(module.endpoint) {
1478 Ok(corr) => corr,
1479 Err(err) => {
1480 drop(inner);
1481 self.request_connection_close(
1482 module.endpoint.connection_id,
1483 CloseReason::new(
1484 "control_correlation_exhausted",
1485 "daemon-originated channel-0 correlation space exhausted",
1486 ),
1487 );
1488 return Err(err);
1489 }
1490 };
1491 let (sender, receiver) = oneshot::channel();
1492 inner.pending_control_rpcs.insert(
1493 (module.endpoint, corr),
1494 PendingModuleControlRpcEntry {
1495 expected_op: expected_op.to_string(),
1496 deadline,
1497 health_probe_started_at,
1498 sender,
1499 },
1500 );
1501
1502 Ok(PendingModuleControlRpc {
1503 endpoint: module.endpoint,
1504 module_sink: module.sink,
1505 negotiated_ver: module.negotiated_ver,
1506 corr,
1507 receiver,
1508 })
1509 }
1510
1511 #[allow(clippy::too_many_arguments)]
1512 fn begin_route_bind_relay_inner(
1513 &self,
1514 client_connection_id: ConnectionId,
1515 client_sink: FrameSink,
1516 client_negotiated_ver: u8,
1517 client_corr: u64,
1518 expected_module_id: &str,
1519 principal: Principal,
1520 scope: Option<BoundScope>,
1521 project_root: Option<ProjectRootId>,
1522 harness: String,
1523 build_class: Option<String>,
1524 operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1525 deadline: Instant,
1526 client_permit: crate::router::EgressPermit,
1527 busy_refusals: bool,
1528 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1529 let mut inner = self.write_inner()?;
1530 if inner
1534 .endpoint_by_connection
1535 .contains_key(&client_connection_id)
1536 {
1537 return Err(ForwardingError::ConnectionRoleConflict {
1538 connection_id: client_connection_id,
1539 });
1540 }
1541 if inner.closing_connections.contains(&client_connection_id) {
1542 return Err(ForwardingError::ConnectionClosing {
1543 connection_id: client_connection_id,
1544 });
1545 }
1546 let module = inner
1547 .modules_by_id
1548 .get(expected_module_id)
1549 .cloned()
1550 .ok_or(ForwardingError::NoModuleConnection)?;
1551 if inner.draining_endpoints.contains_key(&module.endpoint) {
1552 return Err(ForwardingError::ModuleReloading {
1553 module_id: expected_module_id.to_string(),
1554 });
1555 }
1556 if inner
1557 .closing_connections
1558 .contains(&module.endpoint.connection_id)
1559 {
1560 return Err(ForwardingError::ConnectionClosing {
1561 connection_id: module.endpoint.connection_id,
1562 });
1563 }
1564
1565 let corr = match inner.allocate_control_corr(module.endpoint) {
1566 Ok(corr) => corr,
1567 Err(err) => {
1568 drop(inner);
1569 self.request_connection_close(
1570 module.endpoint.connection_id,
1571 CloseReason::new(
1572 "control_correlation_exhausted",
1573 "daemon-originated channel-0 correlation space exhausted",
1574 ),
1575 );
1576 return Err(err);
1577 }
1578 };
1579 let (client_channel, client_epoch, module_channel, module_epoch) =
1580 inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1581 let client_key = ClientRouteKey {
1582 connection_id: client_connection_id,
1583 channel: client_channel,
1584 };
1585 let module_key = ModuleRouteKey {
1586 endpoint: module.endpoint,
1587 channel: module_channel,
1588 };
1589 let reservation = RouteReservation {
1590 client_key,
1591 module_key,
1592 client_epoch,
1593 module_epoch,
1594 project_root,
1595 };
1596 let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
1597 request_window: None,
1598 watch_window: None,
1599 route_channel: client_channel,
1600 route_epoch: client_epoch,
1601 })
1602 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1603 let route_open_frame = Frame::build_with_version(
1604 client_negotiated_ver,
1605 FrameType::Response,
1606 Flags::new(false, Priority::Passive, false),
1607 0,
1608 0,
1609 client_corr,
1610 response_body,
1611 )
1612 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1613 let (sender, receiver) = oneshot::channel();
1614 inner.reserved_client.insert(client_key, module_key);
1615 inner.reserved_module.insert(module_key, client_key);
1616 inner.pending_relays.insert(
1617 (module.endpoint, corr),
1618 PendingRouteBindRelayEntry {
1619 reservation,
1620 client_sink,
1621 client_negotiated_ver,
1622 client_permit,
1623 route_open_frame,
1624 busy_refusals,
1625 request_window: window_for(&module.concurrency),
1626 watch_window: (busy_refusals && module.concurrency == Concurrency::ModuleManaged)
1627 .then_some(DEFAULT_MODULE_MANAGED_WATCH_WINDOW),
1628 principal,
1629 scope,
1630 harness,
1631 build_class,
1632 operator_app,
1633 deadline,
1634 relay_enqueued: false,
1635 sender,
1636 },
1637 );
1638
1639 Ok(PendingRouteBindRelay {
1640 endpoint: module.endpoint,
1641 module_sink: module.sink,
1642 negotiated_ver: module.negotiated_ver,
1643 client_channel,
1644 client_epoch,
1645 module_channel,
1646 module_epoch,
1647 corr,
1648 receiver,
1649 })
1650 }
1651
1652 pub(crate) fn mark_route_bind_relay_enqueued(
1662 &self,
1663 endpoint: ModuleEndpointId,
1664 corr: u64,
1665 ) -> Result<RouteBindRelayMark, ForwardingError> {
1666 let mut inner = self.write_inner()?;
1667 if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
1668 pending.relay_enqueued = true;
1669 return Ok(RouteBindRelayMark::Marked);
1670 }
1671 if inner
1672 .relays_released_before_enqueue
1673 .remove(&(endpoint, corr))
1674 {
1675 return Ok(RouteBindRelayMark::ReleasedWithoutReply);
1676 }
1677 Ok(RouteBindRelayMark::SettledByReply)
1678 }
1679
1680 pub(crate) fn release_client_route(
1681 &self,
1682 client_connection_id: ConnectionId,
1683 client_channel: u16,
1684 expected_epoch: u32,
1685 ) -> Result<RouteRelease, ForwardingError> {
1686 let mut inner = self.write_inner()?;
1687 let release = release_client_route_locked(
1688 &mut inner,
1689 ClientRouteKey {
1690 connection_id: client_connection_id,
1691 channel: client_channel,
1692 },
1693 expected_epoch,
1694 );
1695 self.record_route_release(&release);
1696 Ok(release)
1697 }
1698
1699 pub(crate) fn release_module_route(
1700 &self,
1701 module_connection_id: ConnectionId,
1702 module_channel: u16,
1703 expected_epoch: u32,
1704 ) -> Result<RouteRelease, ForwardingError> {
1705 let mut inner = self.write_inner()?;
1706 let Some(endpoint) = inner
1707 .endpoint_by_connection
1708 .get(&module_connection_id)
1709 .copied()
1710 else {
1711 return Ok(RouteRelease::Absent);
1712 };
1713 let release = release_module_route_locked(
1714 &mut inner,
1715 ModuleRouteKey {
1716 endpoint,
1717 channel: module_channel,
1718 },
1719 expected_epoch,
1720 );
1721 self.record_route_release(&release);
1722 Ok(release)
1723 }
1724
1725 pub(crate) fn abort_pending_relay(
1726 &self,
1727 endpoint: ModuleEndpointId,
1728 corr: u64,
1729 outcome: RouteBindRelayOutcome,
1730 ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
1731 let mut inner = self.write_inner()?;
1732 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1733 inner
1738 .relays_released_before_enqueue
1739 .remove(&(endpoint, corr));
1740 return Ok(None);
1741 };
1742 release_reserved_route_locked(
1743 &mut inner,
1744 pending.reservation.client_key,
1745 pending.reservation.module_key,
1746 );
1747 let target = pending
1748 .relay_enqueued
1749 .then(|| abandoned_route_target(&inner, &pending.reservation));
1750 let _ = pending.sender.send(outcome);
1751 Ok(target.flatten())
1752 }
1753
1754 pub(crate) fn cancel_module_control_rpc(
1755 &self,
1756 endpoint: ModuleEndpointId,
1757 corr: u64,
1758 ) -> Result<(), ForwardingError> {
1759 self.write_inner()?
1760 .pending_control_rpcs
1761 .remove(&(endpoint, corr));
1762 Ok(())
1763 }
1764
1765 pub(crate) fn tombstone_health_probe_rpc(
1766 &self,
1767 endpoint: ModuleEndpointId,
1768 corr: u64,
1769 ) -> Result<bool, ForwardingError> {
1770 let key = (endpoint, corr);
1771 let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
1772 {
1773 let mut inner = self.write_inner()?;
1774 let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
1775 return Ok(false);
1776 };
1777 let Some(probe_started_at) = pending.health_probe_started_at else {
1778 inner.pending_control_rpcs.insert(key, pending);
1779 return Ok(false);
1780 };
1781 let module_id = inner
1782 .module_id_by_endpoint
1783 .get(&endpoint)
1784 .cloned()
1785 .unwrap_or_else(|| "unknown".to_string());
1786 inner.health_probe_tombstones.insert(
1787 key,
1788 HealthProbeTombstone {
1789 expected_op: pending.expected_op,
1790 module_id,
1791 probe_started_at,
1792 expires_at,
1793 },
1794 );
1795 }
1796 self.schedule_health_probe_tombstone_expiration(key, expires_at);
1797 Ok(true)
1798 }
1799
1800 fn schedule_health_probe_tombstone_expiration(
1801 &self,
1802 key: (ModuleEndpointId, u64),
1803 expires_at: Instant,
1804 ) {
1805 let inner = Arc::downgrade(&self.inner);
1806 tokio::spawn(async move {
1807 tokio::time::sleep_until(expires_at).await;
1808 let Some(inner) = inner.upgrade() else {
1809 return;
1810 };
1811 let Ok(mut inner) = inner.write() else {
1812 return;
1813 };
1814 let expired = inner
1815 .health_probe_tombstones
1816 .get(&key)
1817 .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
1818 if expired {
1819 inner.health_probe_tombstones.remove(&key);
1820 }
1821 });
1822 }
1823
1824 pub(crate) fn complete_pending_relay(
1825 &self,
1826 connection_id: ConnectionId,
1827 corr: u64,
1828 outcome: RouteBindRelayOutcome,
1829 ) -> Result<PendingRelayCompletion, ForwardingError> {
1830 let mut inner = self.write_inner()?;
1831 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1832 return Ok(PendingRelayCompletion {
1833 settled: false,
1834 abandoned: None,
1835 });
1836 };
1837 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1838 return Ok(PendingRelayCompletion {
1839 settled: false,
1840 abandoned: None,
1841 });
1842 };
1843
1844 if Instant::now() >= pending.deadline {
1845 release_reserved_route_locked(
1846 &mut inner,
1847 pending.reservation.client_key,
1848 pending.reservation.module_key,
1849 );
1850 let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
1851 .then(|| abandoned_route_target(&inner, &pending.reservation))
1852 .flatten();
1853 let _ = pending
1854 .sender
1855 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody {
1856 code: "module_timeout".to_string(),
1857 message: "route.bind response arrived after its daemon deadline".to_string(),
1858 detail: None,
1859 }));
1860 return Ok(PendingRelayCompletion {
1861 settled: true,
1862 abandoned,
1863 });
1864 }
1865
1866 match outcome {
1867 RouteBindRelayOutcome::Accepted
1886 if pending.client_sink.is_closed()
1887 || inner
1888 .closing_connections
1889 .contains(&pending.reservation.client_key.connection_id) =>
1890 {
1891 let reason = if pending.client_sink.is_closed() {
1892 "client egress closed before route publication"
1893 } else {
1894 "client connection is closing before route publication"
1895 };
1896 release_reserved_route_locked(
1897 &mut inner,
1898 pending.reservation.client_key,
1899 pending.reservation.module_key,
1900 );
1901 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1905 let _ = pending
1906 .sender
1907 .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
1908 return Ok(PendingRelayCompletion {
1909 settled: true,
1910 abandoned,
1911 });
1912 }
1913 RouteBindRelayOutcome::Accepted
1936 if inner.superseded_endpoints.contains_key(&endpoint)
1937 || inner.draining_endpoints.contains_key(&endpoint) =>
1938 {
1939 release_reserved_route_locked(
1940 &mut inner,
1941 pending.reservation.client_key,
1942 pending.reservation.module_key,
1943 );
1944 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1947 let module_id = inner
1948 .module_id_by_endpoint
1949 .get(&endpoint)
1950 .cloned()
1951 .unwrap_or_else(|| "unknown".to_string());
1952 let _ = pending
1953 .sender
1954 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
1955 "module_reloading",
1956 format!("module_id '{module_id}' is reloading"),
1957 )));
1958 return Ok(PendingRelayCompletion {
1959 settled: true,
1960 abandoned,
1961 });
1962 }
1963 RouteBindRelayOutcome::Accepted
1972 if pending
1973 .scope
1974 .as_ref()
1975 .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
1976 {
1977 let refusal = pending
1978 .scope
1979 .as_ref()
1980 .and_then(|scope| scope_refusal_locked(&inner, scope))
1981 .expect("guard matched a refusal under the same lock");
1982 release_reserved_route_locked(
1983 &mut inner,
1984 pending.reservation.client_key,
1985 pending.reservation.module_key,
1986 );
1987 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1988 let _ = pending
1989 .sender
1990 .send(RouteBindRelayOutcome::DaemonRejected(refusal));
1991 return Ok(PendingRelayCompletion {
1992 settled: true,
1993 abandoned,
1994 });
1995 }
1996 RouteBindRelayOutcome::Accepted => {
1997 let abandoned = commit_route_locked(&mut inner, pending)?;
1998 return Ok(PendingRelayCompletion {
1999 settled: true,
2000 abandoned,
2001 });
2002 }
2003 terminal => {
2004 release_reserved_route_locked(
2005 &mut inner,
2006 pending.reservation.client_key,
2007 pending.reservation.module_key,
2008 );
2009 let _ = pending.sender.send(terminal);
2010 }
2011 }
2012 Ok(PendingRelayCompletion {
2013 settled: true,
2014 abandoned: None,
2015 })
2016 }
2017
2018 pub(crate) fn pending_module_control_op(
2019 &self,
2020 connection_id: ConnectionId,
2021 corr: u64,
2022 ) -> Result<Option<String>, ForwardingError> {
2023 let inner = self.read_inner()?;
2024 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2025 return Ok(None);
2026 };
2027 let key = (endpoint, corr);
2028 Ok(inner
2029 .pending_control_rpcs
2030 .get(&key)
2031 .map(|pending| pending.expected_op.clone())
2032 .or_else(|| {
2033 inner
2034 .health_probe_tombstones
2035 .get(&key)
2036 .filter(|tombstone| tombstone.expires_at > Instant::now())
2037 .map(|tombstone| tombstone.expected_op.clone())
2038 }))
2039 }
2040
2041 pub(crate) fn complete_module_control_rpc(
2042 &self,
2043 connection_id: ConnectionId,
2044 corr: u64,
2045 actual_op: Option<&str>,
2046 outcome: ModuleControlRpcOutcome,
2047 ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
2048 let now = Instant::now();
2049 let mut inner = self.write_inner()?;
2050 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2051 return Ok(ModuleControlRpcCompletion::Unknown);
2052 };
2053 let key = (endpoint, corr);
2054 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2055 if now >= pending.deadline {
2056 let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
2057 ModuleControlRpcCompletion::LateHealthAnswer {
2058 module_id: inner
2059 .module_id_by_endpoint
2060 .get(&endpoint)
2061 .cloned()
2062 .unwrap_or_else(|| "unknown".to_string()),
2063 latency: now.saturating_duration_since(probe_started_at),
2064 }
2065 });
2066 let _ = pending
2067 .sender
2068 .send(ModuleControlRpcOutcome::DeadlineElapsed);
2069 return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
2070 }
2071 let outcome = match actual_op {
2072 Some(actual) if actual != pending.expected_op => {
2073 ModuleControlRpcOutcome::UnexpectedOp {
2074 expected: pending.expected_op,
2075 actual: actual.to_string(),
2076 }
2077 }
2078 _ => outcome,
2079 };
2080 let _ = pending.sender.send(outcome);
2081 return Ok(ModuleControlRpcCompletion::Settled);
2082 }
2083
2084 let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
2085 return Ok(ModuleControlRpcCompletion::Unknown);
2086 };
2087 if tombstone.expires_at <= now {
2088 return Ok(ModuleControlRpcCompletion::Unknown);
2089 }
2090 Ok(ModuleControlRpcCompletion::LateHealthAnswer {
2091 module_id: tombstone.module_id,
2092 latency: now.saturating_duration_since(tombstone.probe_started_at),
2093 })
2094 }
2095
2096 #[cfg(test)]
2097 pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
2098 Ok(self.read_inner()?.health_probe_tombstones.len())
2099 }
2100
2101 #[cfg(test)]
2102 pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
2103 Ok(self.read_inner()?.closing_connections.len())
2104 }
2105
2106 #[cfg(test)]
2109 pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
2110 let inner = self.read_inner()?;
2111 Ok((inner.reserved_client.len(), inner.reserved_module.len()))
2112 }
2113
2114 pub(crate) fn module_endpoint_for_connection(
2115 &self,
2116 connection_id: ConnectionId,
2117 ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
2118 Ok(self
2119 .read_inner()?
2120 .endpoint_by_connection
2121 .get(&connection_id)
2122 .copied())
2123 }
2124
2125 pub(crate) fn module_id_for_connection(
2128 &self,
2129 connection_id: ConnectionId,
2130 ) -> Result<Option<String>, ForwardingError> {
2131 let inner = self.read_inner()?;
2132 Ok(inner
2133 .endpoint_by_connection
2134 .get(&connection_id)
2135 .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
2136 .cloned())
2137 }
2138
2139 pub(crate) fn module_route_epoch_was_allocated(
2146 &self,
2147 connection_id: ConnectionId,
2148 channel: u16,
2149 epoch: u32,
2150 ) -> Result<bool, ForwardingError> {
2151 let inner = self.read_inner()?;
2152 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2153 return Ok(false);
2154 };
2155 Ok(inner
2156 .module_slot_epochs
2157 .get(&ModuleRouteKey { endpoint, channel })
2158 .is_some_and(|last| epoch != 0 && epoch <= *last))
2159 }
2160
2161 pub(crate) fn has_live_module_connection(
2162 &self,
2163 module_id: &str,
2164 ) -> Result<bool, ForwardingError> {
2165 Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
2166 }
2167
2168 pub(crate) fn try_admit_request(
2171 &self,
2172 route: &Arc<RouteBinding>,
2173 epoch: u32,
2174 corr: u64,
2175 subscription: bool,
2176 ) -> Result<RequestAdmission, ForwardingError> {
2177 let inner = self.read_inner()?;
2178 let key = ClientRouteKey {
2179 connection_id: route.client_connection_id,
2180 channel: route.client_channel,
2181 };
2182 if epoch != route.client_epoch
2183 || !inner
2184 .client_to_module
2185 .get(&key)
2186 .is_some_and(|current| Arc::ptr_eq(current, route))
2187 {
2188 return Ok(RequestAdmission::Stale);
2189 }
2190 let admission = route.flow.try_acquire_tagged(corr, subscription);
2191 Ok(match admission {
2192 RequestAdmission::Stale
2193 if inner
2194 .draining_endpoints
2195 .contains_key(&route.module_endpoint) =>
2196 {
2197 RequestAdmission::ModuleReloading
2198 }
2199 admission => admission,
2200 })
2201 }
2202
2203 pub(crate) fn lookup_data_route(
2204 &self,
2205 connection_id: ConnectionId,
2206 channel: u16,
2207 epoch: u32,
2208 ) -> Result<DataRoute, ForwardingError> {
2209 let inner = self.read_inner()?;
2210 let state = if let Some(endpoint) =
2211 inner.endpoint_by_connection.get(&connection_id).copied()
2212 {
2213 let key = ModuleRouteKey { endpoint, channel };
2214 match inner.module_to_client.get(&key) {
2215 Some(route) if route.module_epoch == epoch => {
2216 DataRouteState::Bound(Arc::clone(route))
2217 }
2218 Some(_) => DataRouteState::EpochMismatch,
2219 None if inner.reserved_module.contains_key(&key)
2220 && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
2221 {
2222 DataRouteState::Reserved
2223 }
2224 None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
2225 None => DataRouteState::Absent,
2226 }
2227 } else {
2228 let key = ClientRouteKey {
2229 connection_id,
2230 channel,
2231 };
2232 match inner.client_to_module.get(&key) {
2233 Some(route) if route.client_epoch == epoch => {
2234 DataRouteState::Bound(Arc::clone(route))
2235 }
2236 Some(_) => DataRouteState::EpochMismatch,
2237 None if inner.reserved_client.contains_key(&key)
2238 && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
2239 {
2240 DataRouteState::Reserved
2241 }
2242 None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
2243 None => DataRouteState::Absent,
2244 }
2245 };
2246 Ok(
2247 if inner.endpoint_by_connection.contains_key(&connection_id) {
2248 DataRoute::Module(state)
2249 } else {
2250 DataRoute::Client(state)
2251 },
2252 )
2253 }
2254
2255 #[cfg(test)]
2256 pub(crate) fn inject_client_slot_epoch(
2257 &self,
2258 connection_id: ConnectionId,
2259 channel: u16,
2260 last_epoch: u32,
2261 ) {
2262 let mut inner = self.write_inner().expect("forwarding lock");
2263 inner.client_slot_epochs.insert(
2264 ClientRouteKey {
2265 connection_id,
2266 channel,
2267 },
2268 last_epoch,
2269 );
2270 inner.next_client_channel.insert(connection_id, channel);
2271 }
2272
2273 #[cfg(test)]
2274 pub(crate) fn inject_module_slot_epoch(
2275 &self,
2276 endpoint: ModuleEndpointId,
2277 channel: u16,
2278 last_epoch: u32,
2279 ) {
2280 let mut inner = self.write_inner().expect("forwarding lock");
2281 inner
2282 .module_slot_epochs
2283 .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
2284 inner.next_module_channel.insert(endpoint, channel);
2285 }
2286
2287 #[cfg(test)]
2288 pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
2289 self.write_inner()
2290 .expect("forwarding lock")
2291 .next_control_corr
2292 .insert(endpoint, next_corr);
2293 }
2294
2295 pub(crate) fn cache_status(
2296 &self,
2297 endpoint: ModuleEndpointId,
2298 module_channel: u16,
2299 module_epoch: u32,
2300 status: String,
2301 ) -> Result<bool, ForwardingError> {
2302 let mut inner = self.write_inner()?;
2303 if !inner.module_id_by_endpoint.contains_key(&endpoint) {
2304 return Err(ForwardingError::StaleModuleEndpoint);
2305 }
2306
2307 let module_key = ModuleRouteKey {
2308 endpoint,
2309 channel: module_channel,
2310 };
2311 let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
2312 (route.module_epoch == module_epoch).then_some((
2313 ClientRouteKey {
2314 connection_id: route.client_connection_id,
2315 channel: route.client_channel,
2316 },
2317 route.client_epoch,
2318 ))
2319 } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
2320 (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
2321 client_key,
2322 inner
2323 .client_slot_epochs
2324 .get(&client_key)
2325 .copied()
2326 .unwrap_or(0),
2327 ))
2328 } else {
2329 None
2330 };
2331
2332 if let Some(handle) = handle {
2333 inner.status.insert(handle, status);
2334 Ok(true)
2335 } else {
2336 debug!(
2337 module_channel,
2338 module_epoch,
2339 generation = endpoint.generation,
2340 connection_id = endpoint.connection_id.get(),
2341 "dropping stale status update for module route handle"
2342 );
2343 Ok(false)
2344 }
2345 }
2346
2347 pub(crate) fn route_poll_snapshot(
2348 &self,
2349 client_connection_id: ConnectionId,
2350 client_channel: u16,
2351 client_epoch: u32,
2352 ) -> Result<RoutePollSnapshot, ForwardingError> {
2353 let inner = self.read_inner()?;
2354 let client_key = ClientRouteKey {
2355 connection_id: client_connection_id,
2356 channel: client_channel,
2357 };
2358 let Some(route) = inner.client_to_module.get(&client_key) else {
2359 return Ok(RoutePollSnapshot::Absent);
2360 };
2361 if route.client_epoch != client_epoch
2362 || !inner
2363 .module_id_by_endpoint
2364 .contains_key(&route.module_endpoint)
2365 {
2366 return Ok(RoutePollSnapshot::Absent);
2367 }
2368 Ok(RoutePollSnapshot::Bound {
2369 module_id: route.module_id.clone(),
2370 status: inner.status.get(&(client_key, client_epoch)).cloned(),
2371 })
2372 }
2373
2374 pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2375 Ok(self.read_inner()?.client_to_module.len())
2376 }
2377
2378 pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2388 let inner = self.read_inner()?;
2389 let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2390 for key in inner.client_to_module.keys() {
2391 *per_connection.entry(key.connection_id).or_insert(0) += 1;
2392 }
2393 let max = per_connection.values().copied().max().unwrap_or(0);
2394 Ok((per_connection.len(), max))
2395 }
2396
2397 pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2398 let inner = self.read_inner()?;
2399 Ok(inner
2400 .client_to_module
2401 .keys()
2402 .any(|key| key.channel == route_channel))
2403 }
2404
2405 pub(crate) fn daemon_drain_result_for_connection(
2409 &self,
2410 connection_id: ConnectionId,
2411 ) -> Result<Option<bool>, ForwardingError> {
2412 let inner = self.read_inner()?;
2413 Ok(inner.daemon_draining.then(|| {
2414 inner
2415 .endpoint_by_connection
2416 .get(&connection_id)
2417 .and_then(|endpoint| inner.daemon_drain_results.get(endpoint))
2418 .copied()
2419 .unwrap_or(false)
2420 }))
2421 }
2422
2423 #[cfg(unix)]
2424 pub(crate) fn record_daemon_drain_result(
2425 &self,
2426 endpoint: ModuleEndpointId,
2427 drained: bool,
2428 ) -> Result<(), ForwardingError> {
2429 let mut inner = self.write_inner()?;
2430 if let Some(result) = inner.daemon_drain_results.get_mut(&endpoint) {
2434 *result = drained;
2435 }
2436 Ok(())
2437 }
2438
2439 #[cfg(unix)]
2442 pub(crate) fn begin_daemon_drain(
2443 &self,
2444 ) -> Result<Vec<(String, ModuleEndpointId)>, ForwardingError> {
2445 let mut inner = self.write_inner()?;
2446 if inner.daemon_draining {
2447 return Ok(Vec::new());
2448 }
2449 inner.daemon_draining = true;
2450 let modules = inner
2451 .modules_by_id
2452 .iter()
2453 .map(|(id, module)| (id.clone(), module.endpoint))
2454 .collect::<Vec<_>>();
2455 for (_, endpoint) in &modules {
2456 inner.daemon_drain_results.insert(*endpoint, false);
2457 inner
2458 .draining_endpoints
2459 .insert(*endpoint, RouteCloseReason::Restart);
2460 }
2461 let off_slot_endpoints = inner
2466 .candidates_by_id
2467 .values()
2468 .map(|module| module.endpoint)
2469 .chain(inner.superseded_endpoints.keys().copied())
2470 .collect::<Vec<_>>();
2471 for endpoint in off_slot_endpoints {
2472 inner
2473 .draining_endpoints
2474 .insert(endpoint, RouteCloseReason::Restart);
2475 }
2476 Ok(modules)
2477 }
2478
2479 pub(crate) fn begin_module_drain(
2486 &self,
2487 module_id: &str,
2488 reason: RouteCloseReason,
2489 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2490 let mut inner = self.write_inner()?;
2491 let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2492 return Ok(None);
2493 };
2494 Ok(Some(begin_drain_locked(
2495 &mut inner, module_id, module, reason,
2496 )))
2497 }
2498
2499 pub(crate) fn begin_endpoint_drain(
2506 &self,
2507 endpoint: ModuleEndpointId,
2508 reason: RouteCloseReason,
2509 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2510 let mut inner = self.write_inner()?;
2511 let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2512 return Ok(None);
2513 };
2514 let module_id = inner
2515 .module_id_by_endpoint
2516 .get(&endpoint)
2517 .cloned()
2518 .expect("an endpoint resolved to a module connection has a module id");
2519 Ok(Some(begin_drain_locked(
2520 &mut inner, &module_id, module, reason,
2521 )))
2522 }
2523}
2524
2525fn begin_drain_locked(
2529 inner: &mut ForwardingInner,
2530 module_id: &str,
2531 module: ModuleConnection,
2532 reason: RouteCloseReason,
2533) -> ModuleDrainTarget {
2534 {
2535 let endpoint = module.endpoint;
2536 inner.draining_endpoints.insert(endpoint, reason);
2537
2538 let flows = inner
2539 .client_to_module
2540 .values()
2541 .filter(|route| route.module_endpoint == endpoint)
2542 .map(|route| Arc::clone(&route.flow))
2543 .collect::<Vec<_>>();
2544 let excluded_subscriptions = flows
2545 .into_iter()
2546 .map(|flow| flow.begin_drain())
2547 .fold(0u32, u32::saturating_add);
2548
2549 let pending_keys = inner
2550 .pending_relays
2551 .keys()
2552 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2553 .copied()
2554 .collect::<Vec<_>>();
2555 let mut abandoned_bindings = Vec::new();
2556 for key in pending_keys {
2557 let Some(pending) = inner.pending_relays.remove(&key) else {
2558 continue;
2559 };
2560 release_reserved_route_locked(
2561 inner,
2562 pending.reservation.client_key,
2563 pending.reservation.module_key,
2564 );
2565 if pending.relay_enqueued {
2566 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2567 abandoned_bindings.push(target);
2568 }
2569 } else {
2570 inner.relays_released_before_enqueue.insert(key);
2574 }
2575 let _ = pending
2576 .sender
2577 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
2578 "module_reloading",
2579 format!("module_id '{module_id}' is reloading"),
2580 )));
2581 }
2582
2583 let pending_control_keys = inner
2584 .pending_control_rpcs
2585 .keys()
2586 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2587 .copied()
2588 .collect::<Vec<_>>();
2589 for key in pending_control_keys {
2590 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2591 let _ = pending
2592 .sender
2593 .send(ModuleControlRpcOutcome::ModuleGone(format!(
2594 "module '{module_id}' began draining during module-control RPC"
2595 )));
2596 }
2597 }
2598
2599 ModuleDrainTarget {
2600 endpoint,
2601 sink: module.sink,
2602 negotiated_ver: module.negotiated_ver,
2603 abandoned_bindings,
2604 excluded_subscriptions,
2605 }
2606 }
2607}
2608
2609#[derive(Debug, Default, PartialEq, Eq)]
2612pub(crate) struct DrainHoldouts {
2613 pub(crate) requests: usize,
2616 pub(crate) routes: usize,
2618 pub(crate) total_routes: usize,
2620 pub(crate) top_connections: Vec<(u64, usize)>,
2623 pub(crate) held: Vec<(u16, u64)>,
2632}
2633
2634pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
2636
2637impl ForwardingTable {
2638 pub(crate) fn endpoint_drain_holdouts(
2640 &self,
2641 endpoint: ModuleEndpointId,
2642 ) -> Result<DrainHoldouts, ForwardingError> {
2643 let inner = self.read_inner()?;
2644 let mut holdouts = DrainHoldouts::default();
2645 let mut by_connection: HashMap<u64, usize> = HashMap::new();
2646 for (key, route) in &inner.client_to_module {
2647 if route.module_endpoint != endpoint {
2648 continue;
2649 }
2650 holdouts.total_routes += 1;
2651 let held = route.flow.drain_in_flight();
2652 if held == 0 {
2653 continue;
2654 }
2655 holdouts.requests += held;
2656 holdouts.routes += 1;
2657 *by_connection.entry(key.connection_id.get()).or_default() += held;
2658 holdouts.held.extend(
2659 route
2660 .flow
2661 .drain_held_corrs()
2662 .into_iter()
2663 .map(|corr| (route.module_channel, corr)),
2664 );
2665 }
2666 holdouts.held.sort_unstable();
2667 holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
2668 let mut connections = by_connection.into_iter().collect::<Vec<_>>();
2669 connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
2670 connections.truncate(3);
2671 holdouts.top_connections = connections;
2672 Ok(holdouts)
2673 }
2674
2675 pub(crate) fn endpoint_in_flight_count(
2676 &self,
2677 endpoint: ModuleEndpointId,
2678 ) -> Result<usize, ForwardingError> {
2679 let inner = self.read_inner()?;
2680 Ok(inner
2681 .client_to_module
2682 .values()
2683 .filter(|route| route.module_endpoint == endpoint)
2684 .map(|route| route.flow.drain_in_flight())
2685 .sum())
2686 }
2687
2688 pub(crate) fn endpoint_is_draining(
2689 &self,
2690 endpoint: ModuleEndpointId,
2691 ) -> Result<bool, ForwardingError> {
2692 Ok(self
2693 .read_inner()?
2694 .draining_endpoints
2695 .contains_key(&endpoint))
2696 }
2697
2698 pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
2699 let inner = self.read_inner()?;
2700 Ok(inner
2701 .modules_by_id
2702 .get(module_id)
2703 .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
2704 }
2705
2706 pub(crate) fn release_module_endpoint_routes(
2707 &self,
2708 endpoint: ModuleEndpointId,
2709 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2710 let mut inner = self.write_inner()?;
2711 let routes = inner
2712 .module_to_client
2713 .iter()
2714 .filter(|(module_key, _)| module_key.endpoint == endpoint)
2715 .map(|(module_key, route)| (*module_key, route.module_epoch))
2716 .collect::<Vec<_>>();
2717 let mut released = Vec::with_capacity(routes.len());
2718 for (module_key, epoch) in routes {
2719 if let RouteRelease::Removed(target) =
2720 release_module_route_locked(&mut inner, module_key, epoch)
2721 {
2722 released.push(target);
2723 }
2724 }
2725 Ok(released)
2726 }
2727
2728 pub(crate) fn endpoint_routes(
2734 &self,
2735 endpoint: ModuleEndpointId,
2736 ) -> Result<Vec<EndpointRoute>, ForwardingError> {
2737 let inner = self.read_inner()?;
2738 Ok(endpoint_routes_locked(&inner, endpoint))
2739 }
2740
2741 pub(crate) fn route_census(
2743 &self,
2744 module_id: Option<&str>,
2745 ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
2746 let inner = self.read_inner()?;
2747 let mut endpoints = inner
2748 .modules_by_id
2749 .iter()
2750 .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
2751 .map(|(id, module)| (id.clone(), module.endpoint))
2752 .collect::<Vec<_>>();
2753 endpoints.sort_by(|left, right| left.0.cmp(&right.0));
2754 Ok(endpoints
2755 .into_iter()
2756 .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
2757 .collect())
2758 }
2759
2760 pub(crate) fn live_roots(
2762 &self,
2763 module_id: &str,
2764 ) -> Result<ModuleControlResponseToModule, ForwardingError> {
2765 let inner = self.read_inner()?;
2766 let endpoint = inner
2767 .modules_by_id
2768 .get(module_id)
2769 .map(|module| module.endpoint);
2770 let mut roots = BTreeMap::new();
2771 let mut unknown_root_bindings = 0;
2772 let mut total_bindings = 0;
2773 if let Some(endpoint) = endpoint {
2774 for binding in inner
2775 .module_to_client
2776 .values()
2777 .filter(|binding| binding.module_endpoint == endpoint)
2778 {
2779 total_bindings += 1;
2780 let workdir = binding
2781 .scope
2782 .as_ref()
2783 .and_then(|scope| scope.workdir.as_deref())
2784 .map(std::path::Path::new);
2785 if let Some(root) = &binding.project_root {
2786 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2787 entry.0 += 1;
2788 } else if workdir.is_none() {
2789 unknown_root_bindings += 1;
2790 }
2791 if let Some(workdir) = workdir.filter(|workdir| {
2792 binding.project_root.as_ref().map(ProjectRootId::as_path) != Some(*workdir)
2793 }) {
2794 roots.entry(workdir.to_path_buf()).or_insert((0, 0)).0 += 1;
2795 }
2796 }
2797 for pending in inner
2798 .pending_relays
2799 .values()
2800 .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
2801 {
2802 total_bindings += 1;
2803 if let Some(root) = &pending.reservation.project_root {
2804 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2805 entry.1 += 1;
2806 } else {
2807 unknown_root_bindings += 1;
2808 }
2809 }
2810 }
2811 Ok(ModuleControlResponseToModule::LiveRoots {
2812 roots: roots
2813 .into_iter()
2814 .map(|(project_root, (bound, pending))| LiveRoot {
2815 project_root,
2816 bound,
2817 pending,
2818 })
2819 .collect(),
2820 unknown_root_bindings,
2821 total_bindings,
2822 })
2823 }
2824
2825 pub(crate) fn connection_has_client_routes(
2831 &self,
2832 connection_id: ConnectionId,
2833 ) -> Result<bool, ForwardingError> {
2834 let inner = self.read_inner()?;
2835 Ok(connection_has_client_routes_locked(&inner, connection_id))
2836 }
2837
2838 pub(crate) fn cleanup_connection(
2839 &self,
2840 connection_id: ConnectionId,
2841 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2842 self.cleanup_connection_counted(connection_id)
2843 .map(|cleanup| cleanup.released)
2844 }
2845
2846 pub(crate) fn cleanup_connection_counted(
2851 &self,
2852 connection_id: ConnectionId,
2853 ) -> Result<ConnectionCleanup, ForwardingError> {
2854 self.frame_drop_warnings
2855 .lock()
2856 .unwrap_or_else(std::sync::PoisonError::into_inner)
2857 .remove(&connection_id);
2858 let mut inner = self.write_inner()?;
2859 inner.closing_connections.insert(connection_id);
2860 let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
2861 remove_module_connection_locked(&mut inner, endpoint)
2862 } else {
2863 ConnectionCleanup {
2864 released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
2865 abandoned_relays: 0,
2866 }
2867 };
2868 inner.closing_connections.remove(&connection_id);
2877 Ok(cleanup)
2878 }
2879
2880 fn cleanup_client_connection_locked(
2881 inner: &mut ForwardingInner,
2882 connection_id: ConnectionId,
2883 ) -> Vec<GoodbyeTarget> {
2884 let routes = inner
2885 .client_to_module
2886 .iter()
2887 .filter(|(key, _)| key.connection_id == connection_id)
2888 .map(|(key, route)| (*key, route.client_epoch))
2889 .collect::<Vec<_>>();
2890 let mut released = Vec::with_capacity(routes.len());
2891 for (client_key, epoch) in routes {
2892 if let RouteRelease::Removed(target) =
2893 release_client_route_locked(inner, client_key, epoch)
2894 {
2895 released.push(target);
2896 }
2897 }
2898
2899 let pending_keys = inner
2900 .pending_relays
2901 .iter()
2902 .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
2903 .map(|(key, _)| *key)
2904 .collect::<Vec<_>>();
2905 for key in pending_keys {
2906 let Some(pending) = inner.pending_relays.remove(&key) else {
2907 continue;
2908 };
2909 release_reserved_route_locked(
2910 inner,
2911 pending.reservation.client_key,
2912 pending.reservation.module_key,
2913 );
2914 if pending.relay_enqueued {
2915 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2916 released.push(target);
2917 }
2918 } else {
2919 inner.relays_released_before_enqueue.insert(key);
2923 }
2924 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
2925 "client connection closed during route.bind relay".to_string(),
2926 ));
2927 }
2928
2929 let orphaned = inner
2930 .reserved_client
2931 .iter()
2932 .filter(|(key, _)| key.connection_id == connection_id)
2933 .map(|(client, module)| (*client, *module))
2934 .collect::<Vec<_>>();
2935 for (client_key, module_key) in orphaned {
2936 release_reserved_route_locked(inner, client_key, module_key);
2937 }
2938 inner.next_client_channel.remove(&connection_id);
2939 inner
2940 .client_slot_epochs
2941 .retain(|key, _| key.connection_id != connection_id);
2942 inner
2943 .last_published_epoch
2944 .retain(|key, _| key.connection_id != connection_id);
2945 inner
2946 .status
2947 .retain(|(key, _), _| key.connection_id != connection_id);
2948
2949 released
2950 }
2951
2952 pub(crate) fn escalate_client_delivery_failure(
2961 &self,
2962 connection_id: ConnectionId,
2963 channel: u16,
2964 expected_epoch: u32,
2965 reason: CloseReason,
2966 undelivered: UndeliveredFrame<'_>,
2967 ) -> Result<bool, ForwardingError> {
2968 self.escalate_client_delivery_failure_diagnosed(
2969 connection_id,
2970 channel,
2971 expected_epoch,
2972 reason,
2973 undelivered,
2974 None,
2975 )
2976 }
2977
2978 pub(crate) fn escalate_client_delivery_failure_diagnosed(
2979 &self,
2980 connection_id: ConnectionId,
2981 channel: u16,
2982 expected_epoch: u32,
2983 reason: CloseReason,
2984 undelivered: UndeliveredFrame<'_>,
2985 diagnosis: Option<&crate::router::ClientDeliveryDiagnosis>,
2986 ) -> Result<bool, ForwardingError> {
2987 let principals = {
2988 let mut inner = self.write_inner()?;
2989 let key = ClientRouteKey {
2990 connection_id,
2991 channel,
2992 };
2993 if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
2994 None
2995 } else {
2996 inner.closing_connections.insert(connection_id);
2997 Some(connection_principals_locked(&inner, connection_id))
2998 }
2999 };
3000 let Some(principals) = principals else {
3001 self.warn_frame_drop(
3005 connection_id,
3006 channel,
3007 expected_epoch,
3008 "module_to_client",
3009 diagnosis
3010 .map(|failure| failure.cause)
3011 .unwrap_or("delivery_failed"),
3012 );
3013 return Ok(false);
3014 };
3015 let backlog = undelivered.sink.backlog();
3016 let close_reason = reason.to_string();
3017 if self.request_connection_close(connection_id, reason) {
3018 warn!(
3019 connection_id = connection_id.get(),
3020 principals = %principals,
3021 module_id = undelivered.module_id.unwrap_or("unknown"),
3022 client_channel = channel,
3023 queued_bytes = backlog.queued_bytes,
3024 queued_frames = backlog.queued_frames,
3025 frame_type = ?diagnosis.map(|failure| failure.frame_type),
3026 frame_flags = ?diagnosis.map(|failure| failure.flags.0),
3027 close_cause = diagnosis.map(|failure| failure.cause).unwrap_or("delivery_failed"),
3028 oldest_queued_ms = backlog
3029 .oldest_age
3030 .map(|age| age.as_millis() as u64)
3031 .unwrap_or(0),
3032 close_reason = %close_reason,
3033 "closing client connection: its egress queue could not take a frame"
3034 );
3035 }
3036 Ok(true)
3037 }
3038
3039 pub(crate) fn publish_scope_changes(
3053 &self,
3054 changes: &[ScopeTagChange],
3055 ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
3056 let mut inner = self.write_inner()?;
3057 let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
3058 for change in changes {
3059 let key = (change.owner.clone(), change.scope_ref.clone());
3060 match change.after {
3061 Some(tag) => {
3062 inner.scope_tags.insert(key, tag);
3063 }
3064 None => {
3065 inner.scope_tags.remove(&key);
3066 }
3067 }
3068 by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
3069 }
3070 let mut selected = Vec::new();
3071 for (client_key, route) in &inner.client_to_module {
3072 let Some(scope) = &route.scope else {
3073 continue;
3074 };
3075 let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
3076 else {
3077 continue;
3078 };
3079 if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
3080 continue;
3081 }
3082 let reason = match &change.drain {
3083 ScopeDrain::Nothing => continue,
3084 ScopeDrain::All(reason) => *reason,
3085 ScopeDrain::Carriers(narrowed) => {
3086 let owner = Principal::Reserved {
3087 module_id: scope.owner.clone(),
3088 };
3089 let hit = route.principal != owner
3090 && narrowed.iter().any(|(principal, allowed)| {
3091 *principal == route.principal
3092 && allowed
3093 .as_ref()
3094 .is_none_or(|targets| !targets.contains(&route.module_id))
3095 });
3096 if !hit {
3097 continue;
3098 }
3099 RouteCloseReason::ScopeCarrierRemoved
3100 }
3101 };
3102 selected.push((*client_key, route.client_epoch, reason));
3103 }
3104 let mut drained = Vec::new();
3105 for (client_key, client_epoch, reason) in selected {
3106 let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
3107 continue;
3108 };
3109 let release = release_client_route_locked(&mut inner, client_key, client_epoch);
3110 self.record_route_release(&release);
3111 if let RouteRelease::Removed(module) = release {
3112 drained.push(ScopeDrainedRoute {
3113 reason,
3114 scope: route
3115 .scope
3116 .clone()
3117 .expect("selected scope drain has a bound scope"),
3118 module_id: route.module_id.clone(),
3119 client: GoodbyeTarget {
3120 connection_id: route.client_connection_id,
3121 sink: route.client_sink.clone(),
3122 negotiated_ver: route.client_negotiated_ver,
3123 channel: route.client_channel,
3124 epoch: route.client_epoch,
3125 kind: GoodbyeTargetKind::Client,
3126 module_id: Some(route.module_id.clone()),
3127 },
3128 module,
3129 });
3130 }
3131 }
3132 Ok(drained)
3133 }
3134
3135 #[cfg(test)]
3137 pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
3138 self.read_inner()
3139 .ok()?
3140 .scope_tags
3141 .get(&(owner.to_string(), scope_ref.to_string()))
3142 .copied()
3143 }
3144
3145 fn record_route_release(&self, release: &RouteRelease) {
3146 match release {
3147 RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
3148 RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
3149 RouteRelease::Absent => {}
3150 }
3151 }
3152
3153 fn read_inner(&self) -> Result<RwLockReadGuard<'_, ForwardingInner>, ForwardingError> {
3154 self.inner.read().map_err(|_| ForwardingError::Poisoned)
3155 }
3156
3157 fn write_inner(&self) -> Result<RwLockWriteGuard<'_, ForwardingInner>, ForwardingError> {
3158 self.inner.write().map_err(|_| ForwardingError::Poisoned)
3159 }
3160
3161 fn lock_close_registry(
3162 &self,
3163 ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
3164 self.close_registry
3165 .lock()
3166 .unwrap_or_else(|poisoned| poisoned.into_inner())
3167 }
3168}
3169
3170impl ForwardingInner {
3171 fn allocate_route_slots(
3172 &mut self,
3173 connection_id: ConnectionId,
3174 endpoint: ModuleEndpointId,
3175 ) -> Result<(u16, u32, u16, u32), ForwardingError> {
3176 let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
3177 let mut client_channel = client_start;
3178 let client_channel = loop {
3179 let key = ClientRouteKey {
3180 connection_id,
3181 channel: client_channel,
3182 };
3183 let eligible = !self.client_to_module.contains_key(&key)
3184 && !self.reserved_client.contains_key(&key)
3185 && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3186 if eligible {
3187 break client_channel;
3188 }
3189 client_channel = next_channel(client_channel);
3190 if client_channel == client_start {
3191 return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
3192 }
3193 };
3194
3195 let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
3196 let mut module_channel = module_start;
3197 let module_channel = loop {
3198 let key = ModuleRouteKey {
3199 endpoint,
3200 channel: module_channel,
3201 };
3202 let eligible = !self.module_to_client.contains_key(&key)
3203 && !self.reserved_module.contains_key(&key)
3204 && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3205 if eligible {
3206 break module_channel;
3207 }
3208 module_channel = next_channel(module_channel);
3209 if module_channel == module_start {
3210 return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
3211 }
3212 };
3213
3214 let client_key = ClientRouteKey {
3215 connection_id,
3216 channel: client_channel,
3217 };
3218 let module_key = ModuleRouteKey {
3219 endpoint,
3220 channel: module_channel,
3221 };
3222 let client_epoch = self
3223 .client_slot_epochs
3224 .get(&client_key)
3225 .copied()
3226 .unwrap_or(0)
3227 + 1;
3228 let module_epoch = self
3229 .module_slot_epochs
3230 .get(&module_key)
3231 .copied()
3232 .unwrap_or(0)
3233 + 1;
3234 self.client_slot_epochs.insert(client_key, client_epoch);
3235 self.module_slot_epochs.insert(module_key, module_epoch);
3236 self.next_client_channel
3237 .insert(connection_id, next_channel(client_channel));
3238 self.next_module_channel
3239 .insert(endpoint, next_channel(module_channel));
3240 Ok((client_channel, client_epoch, module_channel, module_epoch))
3241 }
3242
3243 fn allocate_control_corr(
3244 &mut self,
3245 endpoint: ModuleEndpointId,
3246 ) -> Result<u64, ForwardingError> {
3247 let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
3248 if candidate == 0 {
3249 self.closing_connections.insert(endpoint.connection_id);
3250 return Err(ForwardingError::RelayCorrelationExhausted);
3251 }
3252 self.next_control_corr.insert(
3253 endpoint,
3254 if candidate == u64::MAX {
3255 0
3256 } else {
3257 candidate + 1
3258 },
3259 );
3260 Ok(candidate)
3261 }
3262}
3263
3264fn connection_has_client_routes_locked(
3265 inner: &ForwardingInner,
3266 connection_id: ConnectionId,
3267) -> bool {
3268 inner
3269 .client_to_module
3270 .keys()
3271 .any(|key| key.connection_id == connection_id)
3272 || inner
3273 .reserved_client
3274 .keys()
3275 .any(|key| key.connection_id == connection_id)
3276}
3277
3278fn check_module_connection_role_locked(
3279 inner: &ForwardingInner,
3280 connection_id: ConnectionId,
3281) -> Result<(), ForwardingError> {
3282 if inner.endpoint_by_connection.contains_key(&connection_id)
3283 || connection_has_client_routes_locked(inner, connection_id)
3284 {
3285 return Err(ForwardingError::ConnectionRoleConflict { connection_id });
3286 }
3287 Ok(())
3288}
3289
3290fn next_channel(channel: u16) -> u16 {
3291 let next = channel.wrapping_add(1);
3292 if next == 0 {
3293 1
3294 } else {
3295 next
3296 }
3297}
3298
3299fn endpoint_routes_locked(
3300 inner: &ForwardingInner,
3301 endpoint: ModuleEndpointId,
3302) -> Vec<EndpointRoute> {
3303 let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
3304 let draining = drain_reason.is_some();
3305 let mut routes = inner
3306 .module_to_client
3307 .iter()
3308 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3309 .map(|(_, route)| EndpointRoute {
3310 goodbye_target: GoodbyeTarget {
3311 connection_id: route.client_connection_id,
3312 sink: route.client_sink.clone(),
3313 negotiated_ver: route.client_negotiated_ver,
3314 channel: route.client_channel,
3315 epoch: route.client_epoch,
3316 kind: GoodbyeTargetKind::Client,
3317 module_id: Some(route.module_id.clone()),
3318 },
3319 principal: route.principal.clone(),
3320 bound_at: route.bound_at,
3321 draining,
3322 drain_reason,
3323 })
3324 .collect::<Vec<_>>();
3325 routes.sort_by_key(|route| {
3326 (
3327 route.goodbye_target.connection_id.get(),
3328 route.goodbye_target.channel,
3329 route.goodbye_target.epoch,
3330 )
3331 });
3332 routes
3333}
3334
3335fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
3339 let key = (scope.owner.clone(), scope.scope_ref.clone());
3340 match inner.scope_tags.get(&key) {
3341 Some(current) if *current == scope.tag => None,
3342 Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
3343 subc_protocol::error_codes::SCOPE_CHANGED,
3344 format!(
3345 "scope '{}' of {} changed while the route was being bound; re-open it",
3346 scope.scope_ref, scope.owner
3347 ),
3348 )),
3349 _ => Some(ErrorBody::new(
3350 subc_protocol::error_codes::SCOPE_ENDED,
3351 format!(
3352 "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
3353 scope.scope_ref, scope.owner, scope.tag.scope_epoch
3354 ),
3355 )),
3356 }
3357}
3358
3359fn release_reserved_route_locked(
3360 inner: &mut ForwardingInner,
3361 client_key: ClientRouteKey,
3362 module_key: ModuleRouteKey,
3363) {
3364 if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
3365 inner.reserved_client.remove(&client_key);
3366 }
3367 if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
3368 inner.reserved_module.remove(&module_key);
3369 }
3370 inner.status.retain(|(key, _), _| *key != client_key);
3371}
3372
3373fn release_client_route_locked(
3374 inner: &mut ForwardingInner,
3375 client_key: ClientRouteKey,
3376 expected_epoch: u32,
3377) -> RouteRelease {
3378 let Some(route) = inner.client_to_module.get(&client_key) else {
3379 return RouteRelease::Absent;
3380 };
3381 if route.client_epoch != expected_epoch {
3382 return RouteRelease::Stale;
3383 }
3384 let route = inner
3385 .client_to_module
3386 .remove(&client_key)
3387 .expect("route checked under the same forwarding lock");
3388 route.flow.close();
3389 if let Some(scope) = &route.scope {
3390 scope.release_bind();
3391 }
3392 inner.module_to_client.remove(&ModuleRouteKey {
3393 endpoint: route.module_endpoint,
3394 channel: route.module_channel,
3395 });
3396 inner.operator_confirms.route_closed(ModuleRouteKey {
3397 endpoint: route.module_endpoint,
3398 channel: route.module_channel,
3399 });
3400 inner.status.remove(&(client_key, expected_epoch));
3401 RouteRelease::Removed(GoodbyeTarget {
3402 connection_id: route.module_endpoint.connection_id,
3403 sink: route.module_sink.clone(),
3404 negotiated_ver: route.module_negotiated_ver,
3405 channel: route.module_channel,
3406 epoch: route.module_epoch,
3407 kind: GoodbyeTargetKind::Module,
3408 module_id: Some(route.module_id.clone()),
3409 })
3410}
3411
3412fn release_module_route_locked(
3413 inner: &mut ForwardingInner,
3414 module_key: ModuleRouteKey,
3415 expected_epoch: u32,
3416) -> RouteRelease {
3417 let Some(route) = inner.module_to_client.get(&module_key) else {
3418 return RouteRelease::Absent;
3419 };
3420 if route.module_epoch != expected_epoch {
3421 return RouteRelease::Stale;
3422 }
3423 let route = inner
3424 .module_to_client
3425 .remove(&module_key)
3426 .expect("route checked under the same forwarding lock");
3427 inner.operator_confirms.route_closed(module_key);
3428 route.flow.close();
3429 if let Some(scope) = &route.scope {
3430 scope.release_bind();
3431 }
3432 let client_key = ClientRouteKey {
3433 connection_id: route.client_connection_id,
3434 channel: route.client_channel,
3435 };
3436 inner.client_to_module.remove(&client_key);
3437 inner.status.remove(&(client_key, route.client_epoch));
3438 RouteRelease::Removed(GoodbyeTarget {
3439 connection_id: route.client_connection_id,
3440 sink: route.client_sink.clone(),
3441 negotiated_ver: route.client_negotiated_ver,
3442 channel: route.client_channel,
3443 epoch: route.client_epoch,
3444 kind: GoodbyeTargetKind::Client,
3445 module_id: Some(route.module_id.clone()),
3446 })
3447}
3448
3449fn commit_route_locked(
3450 inner: &mut ForwardingInner,
3451 mut pending: PendingRouteBindRelayEntry,
3452) -> Result<Option<GoodbyeTarget>, ForwardingError> {
3453 let reservation = pending.reservation;
3454 if inner
3455 .closing_connections
3456 .contains(&reservation.client_key.connection_id)
3457 {
3458 return Err(ForwardingError::ConnectionClosing {
3459 connection_id: reservation.client_key.connection_id,
3460 });
3461 }
3462 let module_id = inner
3463 .module_id_by_endpoint
3464 .get(&reservation.module_key.endpoint)
3465 .cloned()
3466 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3467 if inner
3468 .draining_endpoints
3469 .contains_key(&reservation.module_key.endpoint)
3470 {
3471 return Err(ForwardingError::ModuleReloading { module_id });
3472 }
3473 if pending.busy_refusals {
3474 let body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
3475 route_channel: reservation.client_key.channel,
3476 route_epoch: reservation.client_epoch,
3477 request_window: Some(pending.request_window),
3478 watch_window: pending.watch_window,
3479 })
3480 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
3481 pending.route_open_frame = Frame::build_with_version(
3482 pending.client_negotiated_ver,
3483 FrameType::Response,
3484 pending.route_open_frame.header.flags,
3485 0,
3486 0,
3487 pending.route_open_frame.header.corr,
3488 body,
3489 )
3490 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
3491 }
3492 if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
3493 || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
3494 {
3495 return Err(ForwardingError::UnknownReservation {
3496 client_channel: reservation.client_key.channel,
3497 module_channel: reservation.module_key.channel,
3498 });
3499 }
3500 let module = inner
3501 .modules_by_id
3502 .get(&module_id)
3503 .filter(|module| module.endpoint == reservation.module_key.endpoint)
3504 .cloned()
3505 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3506 let binding = Arc::new(RouteBinding {
3507 client_connection_id: reservation.client_key.connection_id,
3508 client_sink: pending.client_sink,
3509 client_negotiated_ver: pending.client_negotiated_ver,
3510 client_channel: reservation.client_key.channel,
3511 client_epoch: reservation.client_epoch,
3512 module_id,
3513 module_endpoint: reservation.module_key.endpoint,
3514 module_sink: module.sink,
3515 module_negotiated_ver: module.negotiated_ver,
3516 module_channel: reservation.module_key.channel,
3517 module_epoch: reservation.module_epoch,
3518 principal: pending.principal,
3519 harness: pending.harness,
3520 build_class: pending.build_class,
3521 operator_app: pending.operator_app,
3522 project_root: reservation.project_root.clone(),
3523 bound_at: Instant::now(),
3524 flow: Arc::new(if pending.busy_refusals {
3525 ChannelFlow::with_admission(pending.request_window, true, pending.watch_window)
3526 } else {
3527 ChannelFlow::new(pending.request_window)
3528 }),
3529 scope: pending.scope,
3530 });
3531 inner
3532 .client_to_module
3533 .insert(reservation.client_key, Arc::clone(&binding));
3534 inner
3535 .module_to_client
3536 .insert(reservation.module_key, Arc::clone(&binding));
3537 let previous_published = inner
3538 .last_published_epoch
3539 .insert(reservation.client_key, reservation.client_epoch);
3540
3541 if let Some(scope) = &binding.scope {
3545 scope.record_served(&binding.module_id, module.verified_prefix_owner.as_deref());
3546 }
3547
3548 let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
3553 if client_writer_closed {
3554 let abandoned = abandoned_route_target(inner, &reservation);
3558 if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
3559 route.flow.close();
3560 }
3561 inner.module_to_client.remove(&reservation.module_key);
3562 inner
3563 .status
3564 .remove(&(reservation.client_key, reservation.client_epoch));
3565 match previous_published {
3566 Some(epoch) => {
3567 inner
3568 .last_published_epoch
3569 .insert(reservation.client_key, epoch);
3570 }
3571 None => {
3572 inner.last_published_epoch.remove(&reservation.client_key);
3573 }
3574 }
3575 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3576 "client egress closed during route publication".to_string(),
3577 ));
3578 return Ok(abandoned);
3579 }
3580
3581 if let Some(scope) = &binding.scope {
3582 scope.commit_presence();
3583 }
3584 let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
3585 Ok(None)
3586}
3587
3588fn module_connection_for_endpoint_locked(
3596 inner: &ForwardingInner,
3597 endpoint: ModuleEndpointId,
3598) -> Option<&ModuleConnection> {
3599 let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
3600 inner
3601 .modules_by_id
3602 .get(module_id)
3603 .filter(|module| module.endpoint == endpoint)
3604 .or_else(|| {
3605 inner
3606 .candidates_by_id
3607 .get(module_id)
3608 .filter(|module| module.endpoint == endpoint)
3609 })
3610 .or_else(|| inner.superseded_endpoints.get(&endpoint))
3611}
3612
3613fn abandoned_route_target(
3614 inner: &ForwardingInner,
3615 reservation: &RouteReservation,
3616) -> Option<GoodbyeTarget> {
3617 let module_id = inner
3618 .module_id_by_endpoint
3619 .get(&reservation.module_key.endpoint)?;
3620 let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
3621 (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
3622 connection_id: module.endpoint.connection_id,
3623 sink: module.sink.clone(),
3624 negotiated_ver: module.negotiated_ver,
3625 channel: reservation.module_key.channel,
3626 epoch: reservation.module_epoch,
3627 kind: GoodbyeTargetKind::Module,
3628 module_id: Some(module_id.clone()),
3629 })
3630}
3631
3632fn enqueue_hello_ack_locked(
3637 sink: &FrameSink,
3638 connection_id: ConnectionId,
3639 hello_ack: Option<Frame>,
3640) -> Result<(), ForwardingError> {
3641 let Some(hello_ack) = hello_ack else {
3642 return Ok(());
3643 };
3644 sink.try_send(hello_ack)
3645 .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
3646}
3647
3648fn remove_module_connection_locked(
3649 inner: &mut ForwardingInner,
3650 endpoint: ModuleEndpointId,
3651) -> ConnectionCleanup {
3652 inner.operator_confirms.module_closed(endpoint);
3655 inner.draining_endpoints.remove(&endpoint);
3656 inner.daemon_drain_results.remove(&endpoint);
3657 let module_id = inner.module_id_by_endpoint.remove(&endpoint);
3658 if let Some(module_id) = module_id.as_ref() {
3659 if inner
3660 .modules_by_id
3661 .get(module_id)
3662 .is_some_and(|module| module.endpoint == endpoint)
3663 {
3664 inner.modules_by_id.remove(module_id);
3665 }
3666 if inner
3667 .candidates_by_id
3668 .get(module_id)
3669 .is_some_and(|module| module.endpoint == endpoint)
3670 {
3671 inner.candidates_by_id.remove(module_id);
3672 }
3673 }
3674 inner.superseded_endpoints.remove(&endpoint);
3675 inner.endpoint_by_connection.remove(&endpoint.connection_id);
3676 inner.next_module_channel.remove(&endpoint);
3677 inner.next_control_corr.remove(&endpoint);
3678 inner
3679 .health_probe_tombstones
3680 .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
3681 inner
3682 .module_slot_epochs
3683 .retain(|key, _| key.endpoint != endpoint);
3684 let reserved_module_keys: Vec<ModuleRouteKey> = inner
3685 .reserved_module
3686 .keys()
3687 .filter(|module_key| module_key.endpoint == endpoint)
3688 .copied()
3689 .collect();
3690 for module_key in reserved_module_keys {
3691 if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
3692 release_reserved_route_locked(inner, client_key, module_key);
3693 }
3694 }
3695
3696 let pending_keys: Vec<_> = inner
3697 .pending_relays
3698 .keys()
3699 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3700 .copied()
3701 .collect();
3702 let pending: Vec<_> = pending_keys
3703 .into_iter()
3704 .filter_map(|key| {
3705 inner
3706 .pending_relays
3707 .remove(&key)
3708 .map(|pending| (key, pending))
3709 })
3710 .collect();
3711 let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
3712 for (key, pending) in pending {
3713 if !pending.relay_enqueued {
3717 inner.relays_released_before_enqueue.insert(key);
3718 }
3719 let module_label = module_id.as_deref().unwrap_or("unknown");
3720 let _ = pending
3721 .sender
3722 .send(RouteBindRelayOutcome::ModuleGone(format!(
3723 "module '{module_label}' connection closed during route.bind relay"
3724 )));
3725 }
3726
3727 let pending_control_keys: Vec<_> = inner
3728 .pending_control_rpcs
3729 .keys()
3730 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3731 .copied()
3732 .collect();
3733 let pending_control: Vec<_> = pending_control_keys
3734 .into_iter()
3735 .filter_map(|key| inner.pending_control_rpcs.remove(&key))
3736 .collect();
3737 for pending in pending_control {
3738 let module_label = module_id.as_deref().unwrap_or("unknown");
3739 let _ = pending
3740 .sender
3741 .send(ModuleControlRpcOutcome::ModuleGone(format!(
3742 "module '{module_label}' connection closed during module-control RPC"
3743 )));
3744 }
3745
3746 let module_routes = inner
3747 .module_to_client
3748 .iter()
3749 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3750 .map(|(module_key, route)| (*module_key, route.module_epoch))
3751 .collect::<Vec<_>>();
3752 let mut released = Vec::with_capacity(module_routes.len());
3753 for (module_key, epoch) in module_routes {
3754 if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
3755 {
3756 released.push(target);
3757 }
3758 }
3759 ConnectionCleanup {
3760 released,
3761 abandoned_relays,
3762 }
3763}
3764
3765#[derive(Debug, Clone, Copy)]
3766struct RequestCredit {
3767 subscription: bool,
3768 watch_credit: bool,
3769 excluded_from_drain: bool,
3770}
3771
3772#[derive(Debug, Default)]
3773struct CreditLedger {
3774 by_corr: HashMap<u64, Vec<RequestCredit>>,
3775}
3776
3777impl CreditLedger {
3778 fn acquire(&mut self, corr: u64, subscription: bool, watch_credit: bool) {
3779 self.by_corr.entry(corr).or_default().push(RequestCredit {
3780 subscription,
3781 watch_credit,
3782 excluded_from_drain: false,
3783 });
3784 }
3785
3786 fn release(&mut self, corr: u64) -> Option<RequestCredit> {
3787 let credits = self.by_corr.get_mut(&corr)?;
3788 let released = credits.pop();
3789 if credits.is_empty() {
3790 self.by_corr.remove(&corr);
3791 }
3792 released
3793 }
3794
3795 fn capture_subscription_exclusions(&mut self) -> u32 {
3796 let mut excluded = 0u32;
3797 for credit in self.by_corr.values_mut().flatten() {
3798 if credit.subscription && !credit.excluded_from_drain {
3799 credit.excluded_from_drain = true;
3800 excluded = excluded.saturating_add(1);
3801 }
3802 }
3803 excluded
3804 }
3805
3806 fn in_flight(&self) -> usize {
3807 self.by_corr.values().map(Vec::len).sum()
3808 }
3809
3810 fn drain_in_flight(&self) -> usize {
3811 self.by_corr
3812 .values()
3813 .flatten()
3814 .filter(|credit| !credit.excluded_from_drain)
3815 .count()
3816 }
3817
3818 fn drain_held_corrs(&self) -> Vec<u64> {
3821 let mut corrs = self
3822 .by_corr
3823 .iter()
3824 .flat_map(|(corr, credits)| {
3825 credits
3826 .iter()
3827 .filter(|credit| !credit.excluded_from_drain)
3828 .map(move |_| *corr)
3829 })
3830 .collect::<Vec<_>>();
3831 corrs.sort_unstable();
3832 corrs
3833 }
3834}
3835
3836#[derive(Debug, Default)]
3837struct ChannelFlowState {
3838 closed: bool,
3839 credits: CreditLedger,
3840 last_busy_warn: Option<tokio::time::Instant>,
3841}
3842
3843#[derive(Debug)]
3845pub(crate) struct ChannelFlow {
3846 sem: Semaphore,
3847 window: usize,
3848 busy_refusals: bool,
3849 watch_sem: Option<Semaphore>,
3850 watch_window: Option<usize>,
3851 state: Mutex<ChannelFlowState>,
3852}
3853
3854impl ChannelFlow {
3855 pub(crate) fn new(window: usize) -> Self {
3856 Self::with_admission(window, false, None)
3857 }
3858
3859 fn with_admission(window: usize, busy_refusals: bool, watch_window: Option<usize>) -> Self {
3860 debug_assert!(window > 0, "flow-control window must be non-zero");
3861 Self {
3862 sem: Semaphore::new(window),
3863 window,
3864 busy_refusals,
3865 watch_sem: watch_window.map(Semaphore::new),
3866 watch_window,
3867 state: Mutex::new(ChannelFlowState::default()),
3868 }
3869 }
3870
3871 pub(crate) fn busy_refusals(&self) -> bool {
3872 self.busy_refusals
3873 }
3874
3875 fn try_acquire_tagged(&self, corr: u64, subscription: bool) -> RequestAdmission {
3876 let mut state = self
3877 .state
3878 .lock()
3879 .unwrap_or_else(|poisoned| poisoned.into_inner());
3880 if state.closed {
3881 return RequestAdmission::Stale;
3882 }
3883 let busy = |state: &ChannelFlowState| {
3884 RequestAdmission::Busy(subc_protocol::RouteBusyDetail {
3885 reason: "no_credit".into(),
3886 request_in_flight: state.credits.in_flight(),
3887 watch_in_flight: state
3888 .credits
3889 .by_corr
3890 .values()
3891 .flatten()
3892 .filter(|credit| credit.subscription)
3893 .count(),
3894 request_window: self.window,
3895 watch_window: self.watch_window,
3896 })
3897 };
3898 let Ok(total) = self.sem.try_acquire() else {
3899 return busy(&state);
3900 };
3901 let watch = if subscription {
3904 if let Some(sem) = &self.watch_sem {
3905 match sem.try_acquire() {
3906 Ok(permit) => Some(permit),
3907 Err(_) => return busy(&state),
3908 }
3909 } else {
3910 None
3911 }
3912 } else {
3913 None
3914 };
3915 state.credits.acquire(corr, subscription, watch.is_some());
3916 total.forget();
3917 if let Some(watch) = watch {
3918 watch.forget();
3919 }
3920 RequestAdmission::Admitted
3921 }
3922
3923 pub(crate) fn should_warn_busy(&self) -> bool {
3924 let mut state = self
3925 .state
3926 .lock()
3927 .unwrap_or_else(|poisoned| poisoned.into_inner());
3928 let now = tokio::time::Instant::now();
3929 if state
3930 .last_busy_warn
3931 .is_some_and(|last| now.duration_since(last) < std::time::Duration::from_secs(60))
3932 {
3933 return false;
3934 }
3935 state.last_busy_warn = Some(now);
3936 true
3937 }
3938
3939 #[cfg(test)]
3940 pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
3941 self.acquire_tagged(0, false).await
3942 }
3943
3944 pub(crate) async fn acquire_tagged(
3945 &self,
3946 corr: u64,
3947 subscription: bool,
3948 ) -> Result<(), ChannelFlowClosed> {
3949 let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
3950 let mut state = self
3951 .state
3952 .lock()
3953 .unwrap_or_else(|poisoned| poisoned.into_inner());
3954 if state.closed {
3955 return Err(ChannelFlowClosed);
3956 }
3957 state.credits.acquire(corr, subscription, false);
3958 permit.forget();
3959 Ok(())
3960 }
3961
3962 #[cfg(test)]
3963 pub(crate) fn release(&self) {
3964 self.release_corr(0);
3965 }
3966
3967 pub(crate) fn release_corr(&self, corr: u64) {
3968 let mut state = self
3969 .state
3970 .lock()
3971 .unwrap_or_else(|poisoned| poisoned.into_inner());
3972 let Some(released) = state.credits.release(corr) else {
3973 warn!(
3977 window = self.window,
3978 available = self.sem.available_permits(),
3979 "flow-control over-release ignored"
3980 );
3981 return;
3982 };
3983 if !self.sem.is_closed() {
3984 self.sem.add_permits(1);
3985 }
3986 if released.watch_credit {
3987 if let Some(watch_sem) = self.watch_sem.as_ref().filter(|sem| !sem.is_closed()) {
3988 watch_sem.add_permits(1);
3989 }
3990 }
3991 }
3992
3993 #[cfg(test)]
3994 pub(crate) fn in_flight(&self) -> usize {
3995 self.state
3996 .lock()
3997 .unwrap_or_else(|poisoned| poisoned.into_inner())
3998 .credits
3999 .in_flight()
4000 }
4001
4002 pub(crate) fn drain_in_flight(&self) -> usize {
4003 self.state
4004 .lock()
4005 .unwrap_or_else(|poisoned| poisoned.into_inner())
4006 .credits
4007 .drain_in_flight()
4008 }
4009
4010 pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
4011 self.state
4012 .lock()
4013 .unwrap_or_else(|poisoned| poisoned.into_inner())
4014 .credits
4015 .drain_held_corrs()
4016 }
4017
4018 #[cfg(test)]
4019 pub(crate) fn available_permits(&self) -> usize {
4020 self.sem.available_permits()
4021 }
4022
4023 pub(crate) fn begin_drain(&self) -> u32 {
4024 let mut state = self
4025 .state
4026 .lock()
4027 .unwrap_or_else(|poisoned| poisoned.into_inner());
4028 state.closed = true;
4029 self.sem.close();
4030 if let Some(sem) = &self.watch_sem {
4031 sem.close();
4032 }
4033 state.credits.capture_subscription_exclusions()
4034 }
4035
4036 pub(crate) fn close(&self) {
4037 let mut state = self
4038 .state
4039 .lock()
4040 .unwrap_or_else(|poisoned| poisoned.into_inner());
4041 state.closed = true;
4042 self.sem.close();
4043 if let Some(sem) = &self.watch_sem {
4044 sem.close();
4045 }
4046 }
4047}
4048
4049#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4050pub(crate) struct ChannelFlowClosed;
4051
4052#[derive(Debug, Clone, PartialEq, Eq)]
4053pub(crate) enum RequestAdmission {
4054 Admitted,
4055 Busy(subc_protocol::RouteBusyDetail),
4056 ModuleReloading,
4057 Stale,
4058}
4059
4060impl fmt::Display for ChannelFlowClosed {
4061 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4062 write!(f, "flow-control window closed")
4063 }
4064}
4065
4066impl Error for ChannelFlowClosed {}
4067
4068#[cfg(test)]
4069#[path = "route_busy_flow_tests.rs"]
4070mod route_busy_flow_tests;
4071
4072fn window_for(concurrency: &Concurrency) -> usize {
4073 match concurrency {
4074 Concurrency::Serial => 1,
4075 Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
4076 Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
4077 }
4078}
4079
4080#[derive(Debug, Clone, PartialEq, Eq)]
4081pub enum ForwardingError {
4082 ConnectionRoleConflict {
4085 connection_id: ConnectionId,
4086 },
4087 NoModuleConnection,
4088 ModuleReloading {
4089 module_id: String,
4090 },
4091 StaleModuleEndpoint,
4092 UnknownReservation {
4093 client_channel: u16,
4094 module_channel: u16,
4095 },
4096 ClientRouteChannelExhausted {
4097 connection_id: ConnectionId,
4098 },
4099 ModuleRouteChannelExhausted {
4100 endpoint: ModuleEndpointId,
4101 },
4102 RelayCorrelationExhausted,
4103 ConnectionClosing {
4104 connection_id: ConnectionId,
4105 },
4106 ClientEgressClosed {
4107 connection_id: ConnectionId,
4108 },
4109 RouteOpenBuild(String),
4110 CandidateSlotOccupied {
4112 module_id: String,
4113 },
4114 ModuleEgressUnavailable {
4117 connection_id: ConnectionId,
4118 },
4119 Poisoned,
4120}
4121
4122impl fmt::Display for ForwardingError {
4123 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4124 match self {
4125 Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
4126 Self::NoModuleConnection => write!(f, "no module connection is registered"),
4127 Self::ModuleReloading { module_id } => {
4128 write!(f, "module_id '{module_id}' is reloading")
4129 }
4130 Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
4131 Self::UnknownReservation {
4132 client_channel,
4133 module_channel,
4134 } => write!(
4135 f,
4136 "route reservation client channel {client_channel} / module channel {module_channel} was not found"
4137 ),
4138 Self::ClientRouteChannelExhausted { connection_id } => write!(
4139 f,
4140 "no client route channels are available for connection {}",
4141 connection_id.get()
4142 ),
4143 Self::ModuleRouteChannelExhausted { endpoint } => write!(
4144 f,
4145 "no module route channels are available for endpoint generation {} on connection {}",
4146 endpoint.generation,
4147 endpoint.connection_id.get()
4148 ),
4149 Self::RelayCorrelationExhausted => {
4150 write!(f, "module control correlation ids are exhausted")
4151 }
4152 Self::ConnectionClosing { connection_id } => write!(
4153 f,
4154 "connection {} is closing and cannot accept route allocation",
4155 connection_id.get()
4156 ),
4157 Self::ClientEgressClosed { connection_id } => write!(
4158 f,
4159 "client connection {} egress is closed",
4160 connection_id.get()
4161 ),
4162 Self::RouteOpenBuild(message) => {
4163 write!(f, "failed to prebuild route.open response: {message}")
4164 }
4165 Self::CandidateSlotOccupied { module_id } => write!(
4166 f,
4167 "module_id '{module_id}' already has a swap candidate registered"
4168 ),
4169 Self::ModuleEgressUnavailable { connection_id } => write!(
4170 f,
4171 "module connection {} egress is unavailable; HELLO_ACK could not be queued",
4172 connection_id.get()
4173 ),
4174 Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
4175 }
4176 }
4177}
4178
4179impl Error for ForwardingError {}
4180
4181#[cfg(test)]
4182mod tests {
4183 use std::time::Duration;
4184
4185 use super::*;
4186 use tokio::sync::mpsc;
4187
4188 #[test]
4189 fn ordinary_long_running_request_is_not_excluded_from_drain() {
4190 let mut ledger = CreditLedger::default();
4191 ledger.acquire(1, false, false);
4192
4193 assert_eq!(ledger.capture_subscription_exclusions(), 0);
4194 assert_eq!(ledger.drain_in_flight(), 1);
4195 }
4196
4197 #[test]
4198 fn bit_set_subscription_is_excluded_and_counted() {
4199 let mut ledger = CreditLedger::default();
4200 ledger.acquire(1, true, false);
4201
4202 assert_eq!(ledger.capture_subscription_exclusions(), 1);
4203 assert_eq!(ledger.drain_in_flight(), 0);
4204 }
4205
4206 #[test]
4207 fn subscription_opened_after_drain_snapshot_is_not_excluded() {
4208 let mut ledger = CreditLedger::default();
4209 ledger.acquire(1, true, false);
4210 assert_eq!(ledger.capture_subscription_exclusions(), 1);
4211
4212 ledger.acquire(2, true, false);
4213
4214 assert_eq!(ledger.drain_in_flight(), 1);
4215 }
4216
4217 #[test]
4218 fn drain_with_no_subscriptions_reports_zero_excluded() {
4219 let mut ledger = CreditLedger::default();
4220 assert_eq!(ledger.capture_subscription_exclusions(), 0);
4221 }
4222
4223 fn test_hello_ack(corr: u64) -> Frame {
4224 Frame::build(
4225 FrameType::HelloAck,
4226 Flags::new(false, Priority::Passive, false),
4227 0,
4228 0,
4229 corr,
4230 Vec::new(),
4231 )
4232 .unwrap()
4233 }
4234
4235 #[test]
4239 fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
4240 let forwarding = ForwardingTable::default();
4241
4242 let (closed_tx, closed_rx) = mpsc::channel(8);
4243 drop(closed_rx);
4244 let closed = ConnectionId::new(1);
4245 assert_eq!(
4246 forwarding.register_module_connection_acked(
4247 closed,
4248 "closed".to_string(),
4249 2,
4250 Concurrency::ModuleManaged,
4251 FrameSink::new(closed_tx),
4252 ModuleHelloAck {
4253 frame: test_hello_ack(1),
4254 verified_prefix_owner: None
4255 },
4256 ),
4257 Err(ForwardingError::ModuleEgressUnavailable {
4258 connection_id: closed
4259 })
4260 );
4261
4262 let (full_tx, _full_rx) = mpsc::channel(1);
4263 let full_sink = FrameSink::new(full_tx);
4264 full_sink.try_send(test_hello_ack(99)).unwrap();
4265 let full = ConnectionId::new(2);
4266 assert_eq!(
4267 forwarding.register_module_connection_acked(
4268 full,
4269 "full".to_string(),
4270 2,
4271 Concurrency::ModuleManaged,
4272 full_sink.clone(),
4273 ModuleHelloAck {
4274 frame: test_hello_ack(2),
4275 verified_prefix_owner: None
4276 },
4277 ),
4278 Err(ForwardingError::ModuleEgressUnavailable {
4279 connection_id: full
4280 })
4281 );
4282 assert_eq!(
4283 forwarding.register_candidate_module_connection_acked(
4284 full,
4285 "full".to_string(),
4286 2,
4287 Concurrency::ModuleManaged,
4288 full_sink,
4289 test_hello_ack(3),
4290 ),
4291 Err(ForwardingError::ModuleEgressUnavailable {
4292 connection_id: full
4293 })
4294 );
4295
4296 for (connection, module_id) in [(closed, "closed"), (full, "full")] {
4297 assert_eq!(
4298 forwarding
4299 .module_endpoint_for_connection(connection)
4300 .unwrap(),
4301 None
4302 );
4303 let (client_tx, _client_rx) = mpsc::channel(8);
4304 assert_eq!(
4305 forwarding
4306 .begin_route_bind_relay_for_test(
4307 ConnectionId::new(50),
4308 FrameSink::new(client_tx),
4309 1,
4310 module_id,
4311 )
4312 .err(),
4313 Some(ForwardingError::NoModuleConnection)
4314 );
4315 }
4316 assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
4317 }
4318
4319 #[test]
4322 fn acked_registration_queues_the_hello_ack_first() {
4323 let forwarding = ForwardingTable::default();
4324 let (active_tx, mut active_rx) = mpsc::channel(8);
4325 forwarding
4326 .register_module_connection_acked(
4327 ConnectionId::new(1),
4328 "acked".to_string(),
4329 2,
4330 Concurrency::ModuleManaged,
4331 FrameSink::new(active_tx),
4332 ModuleHelloAck {
4333 frame: test_hello_ack(11),
4334 verified_prefix_owner: None,
4335 },
4336 )
4337 .unwrap();
4338 let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
4339 forwarding
4340 .register_candidate_module_connection_acked(
4341 ConnectionId::new(2),
4342 "acked".to_string(),
4343 2,
4344 Concurrency::ModuleManaged,
4345 FrameSink::new(candidate_tx),
4346 test_hello_ack(12),
4347 )
4348 .unwrap();
4349
4350 let active_first = active_rx.try_recv().unwrap().frame;
4351 assert_eq!(active_first.header.ty, FrameType::HelloAck);
4352 assert_eq!(active_first.header.corr, 11);
4353 let candidate_first = candidate_rx.try_recv().unwrap().frame;
4354 assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
4355 assert_eq!(candidate_first.header.corr, 12);
4356 }
4357
4358 #[test]
4359 fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
4360 let forwarding = ForwardingTable::default();
4361 let module_connection = ConnectionId::new(10);
4362 let exhausted_client = ConnectionId::new(20);
4363 let second_client = ConnectionId::new(30);
4364 let (module_tx, _module_rx) = mpsc::channel(1);
4365 let endpoint = forwarding
4366 .register_module_connection(
4367 module_connection,
4368 "route-limit-provider".to_string(),
4369 1,
4370 Concurrency::ModuleManaged,
4371 FrameSink::new(module_tx),
4372 )
4373 .unwrap();
4374
4375 {
4376 let mut inner = forwarding.inner.write().unwrap();
4377 for channel in 1..=u16::MAX {
4378 inner.reserved_client.insert(
4379 ClientRouteKey {
4380 connection_id: exhausted_client,
4381 channel,
4382 },
4383 ModuleRouteKey {
4384 endpoint,
4385 channel: 1,
4386 },
4387 );
4388 }
4389 }
4390
4391 let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
4392 let err = forwarding
4393 .begin_route_bind_relay_for_test(
4394 exhausted_client,
4395 FrameSink::new(exhausted_tx),
4396 1,
4397 "route-limit-provider",
4398 )
4399 .unwrap_err();
4400 assert!(matches!(
4401 err,
4402 ForwardingError::ClientRouteChannelExhausted { connection_id }
4403 if connection_id == exhausted_client
4404 ));
4405
4406 let (second_tx, _second_rx) = mpsc::channel(1);
4407 let pending = forwarding
4408 .begin_route_bind_relay_for_test(
4409 second_client,
4410 FrameSink::new(second_tx),
4411 2,
4412 "route-limit-provider",
4413 )
4414 .unwrap();
4415 assert_eq!(pending.client_channel, 1);
4416 }
4417
4418 #[test]
4419 fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
4420 let forwarding = ForwardingTable::default();
4421 let module_connection = ConnectionId::new(40);
4422 let client = ConnectionId::new(50);
4423 let (module_tx, _module_rx) = mpsc::channel(1);
4424 forwarding
4425 .register_module_connection(
4426 module_connection,
4427 "slot-reuse-provider".to_string(),
4428 1,
4429 Concurrency::ModuleManaged,
4430 FrameSink::new(module_tx),
4431 )
4432 .unwrap();
4433
4434 let (client_tx, _client_rx) = mpsc::channel(1);
4435 let client_sink = FrameSink::new(client_tx);
4436 let mut wrapped_channel = None;
4437 for index in 0..=usize::from(u16::MAX) {
4438 let pending = forwarding
4439 .begin_route_bind_relay_for_test(
4440 client,
4441 client_sink.clone(),
4442 index as u64 + 1,
4443 "slot-reuse-provider",
4444 )
4445 .unwrap();
4446 if index == usize::from(u16::MAX) {
4447 wrapped_channel = Some(pending.module_channel);
4448 }
4449 forwarding
4450 .abort_pending_relay(
4451 pending.endpoint,
4452 pending.corr,
4453 RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
4454 )
4455 .unwrap();
4456 }
4457
4458 assert_eq!(wrapped_channel, Some(1));
4459 }
4460
4461 #[test]
4462 fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
4463 let forwarding = ForwardingTable::default();
4464 let client = ConnectionId::new(60);
4465 forwarding
4466 .inner
4467 .write()
4468 .unwrap()
4469 .next_client_channel
4470 .insert(client, 41);
4471
4472 let released = forwarding.cleanup_connection(client).unwrap();
4473
4474 assert!(released.is_empty());
4475 assert!(!forwarding
4476 .inner
4477 .read()
4478 .unwrap()
4479 .next_client_channel
4480 .contains_key(&client));
4481 }
4482
4483 #[test]
4484 fn stale_module_cleanup_preserves_fast_reconnect_successor() {
4485 let forwarding = ForwardingTable::default();
4486 let module_id = "fast-reconnect-provider";
4487 let first_connection = ConnectionId::new(70);
4488 let second_connection = ConnectionId::new(80);
4489 let (first_tx, _first_rx) = mpsc::channel(1);
4490 let first_endpoint = forwarding
4491 .register_module_connection(
4492 first_connection,
4493 module_id.to_string(),
4494 1,
4495 Concurrency::ModuleManaged,
4496 FrameSink::new(first_tx),
4497 )
4498 .unwrap();
4499 let (second_tx, _second_rx) = mpsc::channel(1);
4500 let second_endpoint = forwarding
4501 .register_module_connection(
4502 second_connection,
4503 module_id.to_string(),
4504 1,
4505 Concurrency::ModuleManaged,
4506 FrameSink::new(second_tx),
4507 )
4508 .unwrap();
4509 assert_ne!(first_endpoint, second_endpoint);
4510
4511 let released = forwarding.cleanup_connection(first_connection).unwrap();
4512
4513 assert!(released.is_empty());
4514 assert_eq!(
4515 forwarding
4516 .inner
4517 .read()
4518 .unwrap()
4519 .modules_by_id
4520 .get(module_id)
4521 .map(|module| module.endpoint),
4522 Some(second_endpoint)
4523 );
4524 assert!(forwarding.has_live_module_connection(module_id).unwrap());
4525 let control_rpc = forwarding
4526 .begin_module_control_rpc_for(
4527 module_id,
4528 "health.check",
4529 Instant::now() + Duration::from_secs(1),
4530 )
4531 .unwrap();
4532 assert_eq!(control_rpc.endpoint, second_endpoint);
4533 }
4534
4535 fn route_fixture(
4536 module_id: &str,
4537 ) -> (
4538 ForwardingTable,
4539 ConnectionId,
4540 ModuleEndpointId,
4541 ConnectionId,
4542 FrameSink,
4543 mpsc::Receiver<crate::router::OutboundFrame>,
4544 ) {
4545 let forwarding = ForwardingTable::default();
4546 let module_connection = ConnectionId::new(100);
4547 let client_connection = ConnectionId::new(200);
4548 let (module_tx, _module_rx) = mpsc::channel(8);
4549 let endpoint = forwarding
4550 .register_module_connection(
4551 module_connection,
4552 module_id.to_string(),
4553 2,
4554 Concurrency::ModuleManaged,
4555 FrameSink::new(module_tx),
4556 )
4557 .unwrap();
4558 let (client_tx, client_rx) = mpsc::channel(8);
4559 (
4560 forwarding,
4561 module_connection,
4562 endpoint,
4563 client_connection,
4564 FrameSink::new(client_tx),
4565 client_rx,
4566 )
4567 }
4568
4569 #[test]
4570 #[cfg(unix)]
4571 fn daemon_drain_gates_current_and_racing_provider_registrations() {
4572 let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
4573 assert_eq!(
4574 forwarding.begin_daemon_drain().unwrap(),
4575 [("provider".to_string(), endpoint)]
4576 );
4577 assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
4578 assert!(matches!(
4579 forwarding.register_module_connection(
4580 ConnectionId::new(300),
4581 "late-provider".into(),
4582 2,
4583 Concurrency::ModuleManaged,
4584 sink,
4585 ),
4586 Err(ForwardingError::ConnectionClosing { .. })
4587 ));
4588 }
4589
4590 #[test]
4591 #[cfg(unix)]
4592 fn daemon_drain_results_are_endpoint_fenced_and_removed_on_cleanup() {
4593 let (forwarding, connection, endpoint, _, _, _) = route_fixture("provider");
4594 assert_eq!(
4595 forwarding
4596 .daemon_drain_result_for_connection(connection)
4597 .unwrap(),
4598 None
4599 );
4600 forwarding
4601 .record_daemon_drain_result(endpoint, true)
4602 .unwrap();
4603 forwarding.begin_daemon_drain().unwrap();
4604 assert_eq!(
4605 forwarding
4606 .daemon_drain_result_for_connection(connection)
4607 .unwrap(),
4608 Some(false)
4609 );
4610 let replacement = ModuleEndpointId {
4611 generation: endpoint.generation + 1,
4612 ..endpoint
4613 };
4614 forwarding
4615 .record_daemon_drain_result(replacement, true)
4616 .unwrap();
4617 assert_eq!(
4618 forwarding
4619 .daemon_drain_result_for_connection(connection)
4620 .unwrap(),
4621 Some(false)
4622 );
4623 forwarding
4624 .record_daemon_drain_result(endpoint, true)
4625 .unwrap();
4626 assert_eq!(
4627 forwarding
4628 .daemon_drain_result_for_connection(connection)
4629 .unwrap(),
4630 Some(true)
4631 );
4632 assert_eq!(
4633 forwarding
4634 .daemon_drain_result_for_connection(ConnectionId::new(300))
4635 .unwrap(),
4636 Some(false)
4637 );
4638 forwarding.cleanup_connection(connection).unwrap();
4639 forwarding
4640 .record_daemon_drain_result(endpoint, true)
4641 .unwrap();
4642 assert_eq!(
4643 forwarding
4644 .daemon_drain_result_for_connection(connection)
4645 .unwrap(),
4646 Some(false)
4647 );
4648 assert!(forwarding
4649 .read_inner()
4650 .unwrap()
4651 .daemon_drain_results
4652 .is_empty());
4653 }
4654
4655 #[test]
4656 #[cfg(unix)]
4657 fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
4658 let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
4659 route_fixture("provider");
4660 let mut pending =
4661 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
4662 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
4663 forwarding.begin_daemon_drain().unwrap();
4664 let completion = forwarding
4665 .complete_pending_relay(
4666 module_connection,
4667 pending.corr,
4668 RouteBindRelayOutcome::Accepted,
4669 )
4670 .expect("draining admission is not a fatal module error");
4671 assert!(completion.settled);
4672 let abandoned = completion
4673 .abandoned
4674 .expect("late accepted bind needs a route GOODBYE");
4675 assert_eq!(abandoned.channel, pending.module_channel);
4676 assert!(matches!(pending.receiver.try_recv().unwrap(),
4677 RouteBindRelayOutcome::DaemonRejected(error) if error.code == "module_reloading"));
4678 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4679 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4680 assert!(
4681 rx.try_recv().is_err(),
4682 "no route.open may be published during drain"
4683 );
4684 assert_eq!(
4685 forwarding
4686 .module_endpoint_for_connection(module_connection)
4687 .unwrap(),
4688 Some(endpoint)
4689 );
4690 assert!(
4691 !forwarding
4692 .complete_pending_relay(
4693 module_connection,
4694 pending.corr,
4695 RouteBindRelayOutcome::Accepted,
4696 )
4697 .unwrap()
4698 .settled
4699 );
4700 }
4701
4702 fn test_ping(corr: u64) -> Frame {
4703 Frame::build(
4704 FrameType::Ping,
4705 Flags::new(false, Priority::Passive, false),
4706 0,
4707 0,
4708 corr,
4709 Vec::new(),
4710 )
4711 .unwrap()
4712 }
4713
4714 fn begin_test_route(
4715 forwarding: &ForwardingTable,
4716 client_connection: ConnectionId,
4717 client_sink: FrameSink,
4718 corr: u64,
4719 module_id: &str,
4720 ) -> PendingRouteBindRelay {
4721 forwarding
4722 .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
4723 .unwrap()
4724 }
4725
4726 #[test]
4727 fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
4728 for candidate in [false, true] {
4729 for register_first in [false, true] {
4730 let (forwarding, module_connection, _, client_connection, sink, _rx) =
4731 route_fixture("target");
4732 let register = || {
4733 if candidate {
4734 forwarding.register_candidate_module_connection(
4735 client_connection,
4736 "source".into(),
4737 2,
4738 Concurrency::ModuleManaged,
4739 sink.clone(),
4740 )
4741 } else {
4742 forwarding.register_module_connection(
4743 client_connection,
4744 "source".into(),
4745 2,
4746 Concurrency::ModuleManaged,
4747 sink.clone(),
4748 )
4749 }
4750 };
4751 if register_first {
4752 register().unwrap();
4753 assert!(
4754 forwarding
4755 .begin_route_bind_relay_for_test(
4756 client_connection,
4757 sink.clone(),
4758 1,
4759 "target",
4760 )
4761 .is_err(),
4762 "a deferred route.open cannot reserve after HELLO"
4763 );
4764 assert!(forwarding
4765 .cleanup_connection(client_connection)
4766 .unwrap()
4767 .is_empty());
4768 } else {
4769 let pending =
4770 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
4771 assert!(
4772 register().is_err(),
4773 "HELLO cannot register after a route reservation"
4774 );
4775 forwarding
4776 .complete_pending_relay(
4777 module_connection,
4778 pending.corr,
4779 RouteBindRelayOutcome::Accepted,
4780 )
4781 .unwrap();
4782 assert_eq!(
4783 forwarding
4784 .cleanup_connection(client_connection)
4785 .unwrap()
4786 .len(),
4787 1
4788 );
4789 }
4790 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4791 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4792 }
4793 }
4794 }
4795
4796 #[tokio::test]
4802 async fn pending_route_open_completes_behind_queued_data_frames() {
4803 assert_eq!(
4804 crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
4805 8,
4806 "the per-connection pending route.open limit is its own constant"
4807 );
4808 let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
4809 route_fixture("open-behind-data");
4810 let (sink, mut client_rx) = crate::server::connection_egress();
4811 const DATA_FRAMES: usize = 1_000;
4812 let data = |corr: u64| {
4813 Frame::build(
4814 FrameType::StreamData,
4815 Flags::new(false, Priority::Interactive, false),
4816 9,
4817 1,
4818 corr,
4819 vec![b'x'; 200],
4820 )
4821 .unwrap()
4822 };
4823 for corr in 0..DATA_FRAMES as u64 {
4824 sink.try_send(data(corr)).unwrap();
4825 }
4826 let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
4827 assert_eq!(sink.backlog().queued_bytes, data_bytes);
4828
4829 let pending = tokio::time::timeout(
4830 Duration::from_secs(5),
4831 forwarding.begin_route_bind_relay_for(
4832 client,
4833 sink.clone(),
4834 subc_protocol::PROTOCOL_VERSION,
4835 4_242,
4836 "open-behind-data",
4837 Principal::Direct,
4838 None,
4839 None,
4840 "fixture".into(),
4841 None,
4842 None,
4843 Instant::now() + Duration::from_secs(60),
4844 false,
4845 ),
4846 )
4847 .await
4848 .expect("reserving the route.open slot must not wait behind data frames")
4849 .unwrap();
4850 forwarding
4851 .complete_pending_relay(
4852 module_connection,
4853 pending.corr,
4854 RouteBindRelayOutcome::Accepted,
4855 )
4856 .unwrap();
4857
4858 let backlog = sink.backlog();
4859 assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
4860 assert!(
4861 backlog.queued_bytes > data_bytes,
4862 "the route.open response must be counted in queued bytes: {backlog:?}"
4863 );
4864 for corr in 0..DATA_FRAMES as u64 {
4865 assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
4866 }
4867 let open = client_rx.recv().await.unwrap();
4868 assert_eq!(open.header.corr, 4_242);
4869 assert_eq!(open.header.ty, FrameType::Response);
4870 drop(open);
4871 assert_eq!(sink.backlog().queued_bytes, 0);
4872 assert_eq!(sink.backlog().queued_frames, 0);
4873 }
4874
4875 #[tokio::test]
4880 async fn drain_holdouts_count_held_requests_and_name_the_connection() {
4881 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4882 route_fixture("holdouts");
4883 let mut bound = |corr| {
4884 let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
4885 forwarding
4886 .complete_pending_relay(
4887 module_connection,
4888 route.corr,
4889 RouteBindRelayOutcome::Accepted,
4890 )
4891 .unwrap();
4892 client_rx.try_recv().unwrap();
4893 match forwarding
4894 .lookup_data_route(client, route.client_channel, route.client_epoch)
4895 .unwrap()
4896 {
4897 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4898 other => panic!("expected live route, got {other:?}"),
4899 }
4900 };
4901 let holding = bound(61);
4902 let _idle = bound(62);
4903 holding.flow.acquire_tagged(7, false).await.unwrap();
4904 holding.flow.acquire_tagged(2, false).await.unwrap();
4905 holding.flow.acquire_tagged(3, true).await.unwrap();
4906 forwarding
4907 .begin_module_drain("holdouts", RouteCloseReason::Restart)
4908 .unwrap();
4909
4910 let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
4911 assert_eq!(
4912 holdouts,
4913 DrainHoldouts {
4914 requests: 2,
4915 routes: 1,
4916 total_routes: 2,
4917 top_connections: vec![(client.get(), 2)],
4918 held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
4921 }
4922 );
4923 }
4924
4925 #[test]
4926 fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
4927 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4928 route_fixture("census");
4929 let pending = begin_test_route(&forwarding, client, sink, 1, "census");
4930 forwarding
4931 .complete_pending_relay(
4932 module_connection,
4933 pending.corr,
4934 RouteBindRelayOutcome::Accepted,
4935 )
4936 .unwrap();
4937
4938 let routes = forwarding.endpoint_routes(endpoint).unwrap();
4939 assert_eq!(routes.len(), 1);
4940 assert!(matches!(routes[0].principal, Principal::Direct));
4941 assert_eq!(routes[0].goodbye_target.connection_id, client);
4942 assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
4943 assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
4944 assert!(!routes[0].draining);
4945
4946 forwarding
4947 .begin_module_drain("census", RouteCloseReason::Restart)
4948 .unwrap();
4949 let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
4950 assert_eq!(draining_routes.len(), 1);
4951 assert!(draining_routes[0].draining);
4952 }
4953
4954 #[test]
4955 fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
4956 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
4957 let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
4958 assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
4959 forwarding
4960 .abort_pending_relay(
4961 first.endpoint,
4962 first.corr,
4963 RouteBindRelayOutcome::ModuleGone("abort".into()),
4964 )
4965 .unwrap();
4966 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4967 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4968
4969 let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
4970 assert_eq!(second.client_channel, first.client_channel);
4971 assert_eq!(second.module_channel, first.module_channel);
4972 assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
4973 }
4974
4975 #[test]
4976 fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
4977 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4978 route_fixture("epoch-release");
4979 let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
4980 forwarding
4981 .complete_pending_relay(
4982 module_connection,
4983 first.corr,
4984 RouteBindRelayOutcome::Accepted,
4985 )
4986 .unwrap();
4987 assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
4988 assert!(matches!(
4989 forwarding
4990 .release_client_route(client, first.client_channel, first.client_epoch)
4991 .unwrap(),
4992 RouteRelease::Removed(_)
4993 ));
4994 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4995 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4996
4997 let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
4998 forwarding
4999 .complete_pending_relay(
5000 module_connection,
5001 second.corr,
5002 RouteBindRelayOutcome::Accepted,
5003 )
5004 .unwrap();
5005 assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
5006 assert!(matches!(
5007 forwarding
5008 .release_client_route(client, second.client_channel, first.client_epoch)
5009 .unwrap(),
5010 RouteRelease::Stale
5011 ));
5012 assert!(!forwarding
5013 .cache_status(
5014 endpoint,
5015 second.module_channel,
5016 first.module_epoch,
5017 "stale".into(),
5018 )
5019 .unwrap());
5020 assert!(forwarding
5021 .cache_status(
5022 endpoint,
5023 second.module_channel,
5024 second.module_epoch,
5025 "current".into(),
5026 )
5027 .unwrap());
5028 match forwarding
5029 .route_poll_snapshot(client, second.client_channel, second.client_epoch)
5030 .unwrap()
5031 {
5032 RoutePollSnapshot::Bound { status, .. } => {
5033 assert_eq!(status.as_deref(), Some("current"));
5034 }
5035 RoutePollSnapshot::Absent => panic!("successor binding was removed"),
5036 }
5037 let counters = forwarding.counters().snapshot();
5038 assert_eq!(counters["route_released_epoch_fenced"], 1);
5039 assert_eq!(counters["route_release_stale_skipped"], 1);
5040 }
5041
5042 #[test]
5043 fn max_epoch_reservation_retires_only_that_slot() {
5044 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
5045 forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
5046 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
5047 let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
5048 assert_eq!(
5049 (final_use.client_channel, final_use.client_epoch),
5050 (7, u32::MAX)
5051 );
5052 assert_eq!(
5053 (final_use.module_channel, final_use.module_epoch),
5054 (9, u32::MAX)
5055 );
5056 forwarding
5057 .abort_pending_relay(
5058 endpoint,
5059 final_use.corr,
5060 RouteBindRelayOutcome::ModuleGone("abort".into()),
5061 )
5062 .unwrap();
5063 forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
5064 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
5065 let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
5066 assert_ne!(next.client_channel, 7);
5067 assert_ne!(next.module_channel, 9);
5068 assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
5069 }
5070
5071 #[test]
5072 fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
5073 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
5074 route_fixture("corr-shared");
5075 let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
5076 assert_eq!(bind.corr, 1);
5077 forwarding
5078 .abort_pending_relay(
5079 endpoint,
5080 bind.corr,
5081 RouteBindRelayOutcome::ModuleGone("abort".into()),
5082 )
5083 .unwrap();
5084 let rpc = forwarding
5085 .begin_module_control_rpc_for(
5086 "corr-shared",
5087 "health.check",
5088 Instant::now() - Duration::from_millis(1),
5089 )
5090 .unwrap();
5091 assert_eq!(rpc.corr, 2);
5092 assert_eq!(
5093 forwarding
5094 .complete_module_control_rpc(
5095 module_connection,
5096 rpc.corr,
5097 Some("health.check"),
5098 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
5099 status: subc_protocol::session::HealthStatus::Ok,
5100 detail: None,
5101 metrics: None,
5102 }),
5103 )
5104 .unwrap(),
5105 ModuleControlRpcCompletion::Settled
5106 );
5107 assert!(matches!(
5108 rpc.receiver.blocking_recv().unwrap(),
5109 ModuleControlRpcOutcome::DeadlineElapsed
5110 ));
5111 }
5112
5113 #[tokio::test(start_paused = true)]
5114 async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
5115 let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
5116 let probe_started_at = Instant::now();
5117 let rpc = forwarding
5118 .begin_health_probe_rpc_for(
5119 "tombstone-ttl",
5120 "health.check",
5121 probe_started_at,
5122 probe_started_at + Duration::from_secs(5),
5123 )
5124 .unwrap();
5125 assert!(forwarding
5126 .tombstone_health_probe_rpc(endpoint, rpc.corr)
5127 .unwrap());
5128 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
5129
5130 tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
5131 tokio::task::yield_now().await;
5132
5133 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
5134 }
5135
5136 #[test]
5137 fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
5138 let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
5139 let mut close = forwarding.register_connection_close(endpoint.connection_id);
5140 forwarding.inject_control_corr(endpoint, u64::MAX);
5141 let final_rpc = forwarding
5142 .begin_module_control_rpc_for(
5143 "corr-max",
5144 "health.check",
5145 Instant::now() + Duration::from_secs(1),
5146 )
5147 .unwrap();
5148 assert_eq!(final_rpc.corr, u64::MAX);
5149 forwarding
5150 .cancel_module_control_rpc(endpoint, final_rpc.corr)
5151 .unwrap();
5152 assert!(matches!(
5153 forwarding.begin_module_control_rpc_for(
5154 "corr-max",
5155 "health.check",
5156 Instant::now() + Duration::from_secs(1),
5157 ),
5158 Err(ForwardingError::RelayCorrelationExhausted)
5159 ));
5160 assert!(close.try_recv().is_ok());
5161 }
5162
5163 #[test]
5164 fn publication_epoch_controls_delivery_failure_escalation() {
5165 fn setup_successor(
5166 commit_successor: Option<bool>,
5167 ) -> (ForwardingTable, ConnectionId, u16, u32) {
5168 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5169 route_fixture("escalation");
5170 let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
5171 forwarding
5172 .complete_pending_relay(
5173 module_connection,
5174 first.corr,
5175 RouteBindRelayOutcome::Accepted,
5176 )
5177 .unwrap();
5178 client_rx.try_recv().unwrap();
5179 assert!(matches!(
5180 forwarding
5181 .release_client_route(client, first.client_channel, first.client_epoch)
5182 .unwrap(),
5183 RouteRelease::Removed(_)
5184 ));
5185 if let Some(commit_successor) = commit_successor {
5186 forwarding.inject_client_slot_epoch(
5187 client,
5188 first.client_channel,
5189 first.client_epoch,
5190 );
5191 forwarding.inject_module_slot_epoch(
5192 endpoint,
5193 first.module_channel,
5194 first.module_epoch,
5195 );
5196 let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
5197 if commit_successor {
5198 forwarding
5199 .complete_pending_relay(
5200 module_connection,
5201 successor.corr,
5202 RouteBindRelayOutcome::Accepted,
5203 )
5204 .unwrap();
5205 client_rx.try_recv().unwrap();
5206 } else {
5207 forwarding
5208 .abort_pending_relay(
5209 endpoint,
5210 successor.corr,
5211 RouteBindRelayOutcome::ModuleGone("abort".into()),
5212 )
5213 .unwrap();
5214 }
5215 }
5216 (forwarding, client, first.client_channel, first.client_epoch)
5217 }
5218
5219 let probe_sink = FrameSink::new(mpsc::channel(1).0);
5220 let (no_successor, client, channel, epoch) = setup_successor(None);
5221 let mut close = no_successor.register_connection_close(client);
5222 assert!(no_successor
5223 .escalate_client_delivery_failure(
5224 client,
5225 channel,
5226 epoch,
5227 CloseReason::new("delivery", "failed"),
5228 UndeliveredFrame {
5229 module_id: None,
5230 sink: &probe_sink,
5231 },
5232 )
5233 .unwrap());
5234 assert!(close.try_recv().is_ok());
5235
5236 let (aborted, client, channel, epoch) = setup_successor(Some(false));
5237 let mut close = aborted.register_connection_close(client);
5238 assert!(aborted
5239 .escalate_client_delivery_failure(
5240 client,
5241 channel,
5242 epoch,
5243 CloseReason::new("delivery", "failed"),
5244 UndeliveredFrame {
5245 module_id: None,
5246 sink: &probe_sink,
5247 },
5248 )
5249 .unwrap());
5250 assert!(close.try_recv().is_ok());
5251
5252 let (published, client, channel, epoch) = setup_successor(Some(true));
5253 let mut close = published.register_connection_close(client);
5254 assert!(!published
5255 .escalate_client_delivery_failure(
5256 client,
5257 channel,
5258 epoch,
5259 CloseReason::new("delivery", "stale failure"),
5260 UndeliveredFrame {
5261 module_id: None,
5262 sink: &probe_sink,
5263 },
5264 )
5265 .unwrap());
5266 assert!(close.try_recv().is_err());
5267 }
5268
5269 #[test]
5270 fn route_concentration_separates_client_count_from_routes_per_client() {
5271 let (forwarding, module_connection, _, client, sink, _client_rx) =
5275 route_fixture("concentration");
5276 assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
5277
5278 for corr in [70_u64, 71] {
5279 let pending =
5280 begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
5281 forwarding
5282 .complete_pending_relay(
5283 module_connection,
5284 pending.corr,
5285 RouteBindRelayOutcome::Accepted,
5286 )
5287 .unwrap();
5288 }
5289
5290 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5292 assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
5293 }
5294
5295 #[test]
5296 fn cleanup_and_accepted_resolution_have_one_lock_winner() {
5297 let (forwarding, module_connection, _, client, sink, mut client_rx) =
5298 route_fixture("cleanup-race");
5299 let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
5300 forwarding
5301 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5302 .unwrap();
5303 let released = forwarding.cleanup_connection(client).unwrap();
5304 assert_eq!(released.len(), 1);
5305 let completion = forwarding
5306 .complete_pending_relay(
5307 module_connection,
5308 pending.corr,
5309 RouteBindRelayOutcome::Accepted,
5310 )
5311 .unwrap();
5312 assert!(!completion.settled);
5313 assert!(client_rx.try_recv().is_err());
5314 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5315
5316 let (forwarding, module_connection, _, client, sink, mut client_rx) =
5317 route_fixture("accepted-race");
5318 let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
5319 forwarding
5320 .complete_pending_relay(
5321 module_connection,
5322 pending.corr,
5323 RouteBindRelayOutcome::Accepted,
5324 )
5325 .unwrap();
5326 assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
5327 let released = forwarding.cleanup_connection(client).unwrap();
5328 assert_eq!(released.len(), 1);
5329 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5330 }
5331
5332 #[test]
5333 fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
5334 let (forwarding, module_connection, _, client, sink, mut client_rx) =
5335 route_fixture("drain-gap");
5336 let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
5337 forwarding
5338 .complete_pending_relay(
5339 module_connection,
5340 live.corr,
5341 RouteBindRelayOutcome::Accepted,
5342 )
5343 .unwrap();
5344 client_rx.try_recv().unwrap();
5345 let binding = match forwarding
5346 .lookup_data_route(client, live.client_channel, live.client_epoch)
5347 .unwrap()
5348 {
5349 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
5350 other => panic!("expected live route, got {other:?}"),
5351 };
5352
5353 let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
5354 forwarding
5355 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5356 .unwrap();
5357 let control_rpc = forwarding
5358 .begin_module_control_rpc_for(
5359 "drain-gap",
5360 "health.check",
5361 Instant::now() + Duration::from_secs(1),
5362 )
5363 .unwrap();
5364 let target = forwarding
5365 .begin_module_drain("drain-gap", RouteCloseReason::Reload)
5366 .unwrap()
5367 .unwrap();
5368 assert!(matches!(
5369 control_rpc.receiver.blocking_recv().unwrap(),
5370 ModuleControlRpcOutcome::ModuleGone(_)
5371 ));
5372 assert_eq!(target.abandoned_bindings.len(), 1);
5373 assert!(binding.flow.sem.is_closed());
5374 assert!(
5375 !forwarding
5376 .complete_pending_relay(
5377 module_connection,
5378 pending.corr,
5379 RouteBindRelayOutcome::Accepted,
5380 )
5381 .unwrap()
5382 .settled
5383 );
5384 assert!(matches!(
5385 forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
5386 Err(ForwardingError::ModuleReloading { .. })
5387 ));
5388 let released = forwarding
5389 .release_module_endpoint_routes(target.endpoint)
5390 .unwrap();
5391 assert_eq!(released.len(), 1);
5392 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5393 }
5394
5395 #[test]
5403 fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
5404 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5405 route_fixture("closing-client");
5406
5407 let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
5410 forwarding
5411 .complete_pending_relay(
5412 module_connection,
5413 live.corr,
5414 RouteBindRelayOutcome::Accepted,
5415 )
5416 .unwrap();
5417 client_rx.try_recv().unwrap();
5418
5419 let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
5421 forwarding
5422 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5423 .unwrap();
5424
5425 assert!(forwarding
5427 .escalate_client_delivery_failure(
5428 client,
5429 live.client_channel,
5430 live.client_epoch,
5431 CloseReason::new(
5432 "module_to_client_delivery_failed",
5433 "client egress refused a module frame",
5434 ),
5435 UndeliveredFrame {
5436 module_id: None,
5437 sink: &sink,
5438 },
5439 )
5440 .unwrap());
5441 assert!(!sink.is_closed());
5442
5443 let completion = forwarding
5444 .complete_pending_relay(
5445 module_connection,
5446 pending.corr,
5447 RouteBindRelayOutcome::Accepted,
5448 )
5449 .expect("a closing client must not turn a module's ack into an error");
5450
5451 assert!(completion.settled);
5452 let abandoned = completion
5453 .abandoned
5454 .expect("the module must be told to drop the binding it just created");
5455 assert_eq!(abandoned.connection_id, module_connection);
5456 assert_eq!(abandoned.channel, pending.module_channel);
5457 assert_eq!(abandoned.epoch, pending.module_epoch);
5458 assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
5459 assert!(matches!(
5460 pending.receiver.blocking_recv().unwrap(),
5461 RouteBindRelayOutcome::ModuleGone(_)
5462 ));
5463 assert!(client_rx.try_recv().is_err());
5466 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
5467
5468 assert!(forwarding
5471 .has_live_module_connection("closing-client")
5472 .unwrap());
5473 let cotenant = ConnectionId::new(201);
5474 let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
5475 let cotenant_route = begin_test_route(
5476 &forwarding,
5477 cotenant,
5478 FrameSink::new(cotenant_tx),
5479 62,
5480 "closing-client",
5481 );
5482 assert_eq!(cotenant_route.endpoint, endpoint);
5483 forwarding
5484 .complete_pending_relay(
5485 module_connection,
5486 cotenant_route.corr,
5487 RouteBindRelayOutcome::Accepted,
5488 )
5489 .unwrap();
5490 assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
5491 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5492 }
5493
5494 #[test]
5495 fn pending_route_permit_is_released_on_rejection_and_abort() {
5496 let forwarding = ForwardingTable::default();
5497 let module_connection = ConnectionId::new(300);
5498 let client = ConnectionId::new(301);
5499 let (module_tx, _module_rx) = mpsc::channel(1);
5500 let endpoint = forwarding
5501 .register_module_connection(
5502 module_connection,
5503 "permit".into(),
5504 2,
5505 Concurrency::ModuleManaged,
5506 FrameSink::new(module_tx),
5507 )
5508 .unwrap();
5509 let (client_tx, mut client_rx) = mpsc::channel(1);
5510 let sink = FrameSink::new(client_tx);
5511 let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
5512 assert!(sink.try_send(test_ping(999)).is_err());
5513 forwarding
5514 .complete_pending_relay(
5515 module_connection,
5516 rejected.corr,
5517 RouteBindRelayOutcome::Rejected(ErrorBody {
5518 code: "no".into(),
5519 message: "rejected".into(),
5520 detail: None,
5521 }),
5522 )
5523 .unwrap();
5524 sink.try_send(test_ping(1000)).unwrap();
5525 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
5526
5527 let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
5528 assert!(sink.try_send(test_ping(1001)).is_err());
5529 forwarding
5530 .abort_pending_relay(
5531 endpoint,
5532 aborted.corr,
5533 RouteBindRelayOutcome::ModuleGone("abort".into()),
5534 )
5535 .unwrap();
5536 sink.try_send(test_ping(1002)).unwrap();
5537 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
5538
5539 let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
5540 forwarding
5541 .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
5542 .unwrap();
5543 drop(client_rx);
5544 let completion = forwarding
5545 .complete_pending_relay(
5546 module_connection,
5547 receiver_closed.corr,
5548 RouteBindRelayOutcome::Accepted,
5549 )
5550 .unwrap();
5551 assert!(completion.abandoned.is_some());
5552 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5553 }
5554
5555 #[test]
5561 fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
5562 let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
5563 route_fixture("closing-set-leak");
5564
5565 const CONNECTIONS: u64 = 32;
5566 for index in 0..CONNECTIONS {
5567 let client = ConnectionId::new(1000 + index);
5568 let (client_tx, _client_rx) = mpsc::channel(8);
5569 let route = begin_test_route(
5570 &forwarding,
5571 client,
5572 FrameSink::new(client_tx),
5573 index + 1,
5574 "closing-set-leak",
5575 );
5576 forwarding
5577 .complete_pending_relay(
5578 module_connection,
5579 route.corr,
5580 RouteBindRelayOutcome::Accepted,
5581 )
5582 .unwrap();
5583 forwarding.cleanup_connection(client).unwrap();
5584 }
5585 forwarding.cleanup_connection(module_connection).unwrap();
5586
5587 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5588 }
5589
5590 #[test]
5597 fn closing_connection_is_refused_new_work_until_cleanup_completes() {
5598 let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
5599 route_fixture("closing-gate");
5600
5601 let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
5604 forwarding
5605 .complete_pending_relay(
5606 module_connection,
5607 live.corr,
5608 RouteBindRelayOutcome::Accepted,
5609 )
5610 .unwrap();
5611 client_rx.try_recv().unwrap();
5612
5613 assert!(forwarding
5616 .escalate_client_delivery_failure(
5617 client,
5618 live.client_channel,
5619 live.client_epoch,
5620 CloseReason::new(
5621 "module_to_client_delivery_failed",
5622 "client egress refused a module frame",
5623 ),
5624 UndeliveredFrame {
5625 module_id: None,
5626 sink: &sink,
5627 },
5628 )
5629 .unwrap());
5630 assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
5631
5632 assert!(matches!(
5634 forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
5635 Err(ForwardingError::ConnectionClosing { connection_id })
5636 if connection_id == client
5637 ));
5638 let (late_tx, _late_rx) = mpsc::channel(1);
5640 assert!(matches!(
5641 forwarding.register_module_connection(
5642 client,
5643 "late-module".into(),
5644 2,
5645 Concurrency::ModuleManaged,
5646 FrameSink::new(late_tx),
5647 ),
5648 Err(ForwardingError::ConnectionClosing { connection_id })
5649 if connection_id == client
5650 ));
5651
5652 forwarding.cleanup_connection(client).unwrap();
5656 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5657 }
5658}
5659
5660#[cfg(test)]
5663mod swap_slot_tests {
5664 use std::time::Duration;
5665
5666 use super::*;
5667 use tokio::sync::mpsc;
5668
5669 const MODULE_ID: &str = "swapped";
5670
5671 struct SwapFixture {
5672 forwarding: ForwardingTable,
5673 incumbent_connection: ConnectionId,
5674 incumbent: ModuleEndpointId,
5675 candidate_connection: ConnectionId,
5676 candidate: ModuleEndpointId,
5677 _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
5678 }
5679
5680 fn swap_fixture() -> SwapFixture {
5681 let forwarding = ForwardingTable::default();
5682 let incumbent_connection = ConnectionId::new(100);
5683 let candidate_connection = ConnectionId::new(110);
5684 let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
5685 let incumbent = forwarding
5686 .register_module_connection(
5687 incumbent_connection,
5688 MODULE_ID.to_string(),
5689 2,
5690 Concurrency::ModuleManaged,
5691 FrameSink::new(incumbent_tx),
5692 )
5693 .unwrap();
5694 let (candidate_tx, candidate_rx) = mpsc::channel(8);
5695 let candidate = forwarding
5696 .register_candidate_module_connection(
5697 candidate_connection,
5698 MODULE_ID.to_string(),
5699 2,
5700 Concurrency::ModuleManaged,
5701 FrameSink::new(candidate_tx),
5702 )
5703 .unwrap();
5704 SwapFixture {
5705 forwarding,
5706 incumbent_connection,
5707 incumbent,
5708 candidate_connection,
5709 candidate,
5710 _module_rxs: vec![incumbent_rx, candidate_rx],
5711 }
5712 }
5713
5714 fn client(
5715 raw: u64,
5716 ) -> (
5717 ConnectionId,
5718 FrameSink,
5719 mpsc::Receiver<crate::router::OutboundFrame>,
5720 ) {
5721 let (tx, rx) = mpsc::channel(8);
5722 (ConnectionId::new(raw), FrameSink::new(tx), rx)
5723 }
5724
5725 fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
5726 forwarding
5727 .read_inner()
5728 .unwrap()
5729 .client_to_module
5730 .values()
5731 .map(|route| route.module_endpoint)
5732 .collect()
5733 }
5734
5735 #[test]
5736 fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
5737 let fixture = swap_fixture();
5738 let forwarding = &fixture.forwarding;
5739 assert_ne!(fixture.incumbent, fixture.candidate);
5740
5741 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5743 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5744 let (client_connection, client_sink, _client_rx) = client(200);
5745 let pending = forwarding
5746 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5747 .unwrap();
5748 assert_eq!(pending.endpoint, fixture.incumbent);
5749 let rpc = forwarding
5750 .begin_module_control_rpc_for(
5751 MODULE_ID,
5752 "health.check",
5753 Instant::now() + Duration::from_secs(1),
5754 )
5755 .unwrap();
5756 assert_eq!(rpc.endpoint, fixture.incumbent);
5757 let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
5758 assert_eq!(census.len(), 1, "the census lists one endpoint per id");
5759
5760 assert_eq!(
5762 forwarding
5763 .module_endpoint_for_connection(fixture.candidate_connection)
5764 .unwrap(),
5765 Some(fixture.candidate)
5766 );
5767 assert_eq!(
5768 forwarding
5769 .module_id_for_connection(fixture.candidate_connection)
5770 .unwrap()
5771 .as_deref(),
5772 Some(MODULE_ID)
5773 );
5774
5775 let (other_tx, _other_rx) = mpsc::channel(1);
5777 assert_eq!(
5778 forwarding.register_candidate_module_connection(
5779 ConnectionId::new(120),
5780 MODULE_ID.to_string(),
5781 2,
5782 Concurrency::ModuleManaged,
5783 FrameSink::new(other_tx),
5784 ),
5785 Err(ForwardingError::CandidateSlotOccupied {
5786 module_id: MODULE_ID.to_string()
5787 })
5788 );
5789 }
5790
5791 #[test]
5795 fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
5796 let fixture = swap_fixture();
5797 let forwarding = &fixture.forwarding;
5798 let (early_client, early_sink, _early_rx) = client(200);
5799 let mut early = forwarding
5800 .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
5801 .unwrap();
5802 assert_eq!(early.endpoint, fixture.incumbent);
5803 assert_eq!(
5804 forwarding
5805 .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
5806 .unwrap(),
5807 RouteBindRelayMark::Marked
5808 );
5809
5810 let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5811 assert_eq!(
5812 cutover,
5813 ForwardingCutover {
5814 promoted: fixture.candidate,
5815 incumbent: Some(fixture.incumbent),
5816 }
5817 );
5818
5819 let (late_client, late_sink, _late_rx) = client(201);
5821 let late = forwarding
5822 .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
5823 .unwrap();
5824 assert_eq!(
5825 late.endpoint, fixture.candidate,
5826 "a route.open after cutover was reserved on the incumbent"
5827 );
5828
5829 let completion = forwarding
5831 .complete_pending_relay(
5832 fixture.incumbent_connection,
5833 early.corr,
5834 RouteBindRelayOutcome::Accepted,
5835 )
5836 .expect("a superseded endpoint's ack is not an error on its connection");
5837 assert!(completion.settled);
5838 assert!(
5839 !committed_endpoints(forwarding).contains(&fixture.incumbent),
5840 "a relay reserved before cutover committed a route on the incumbent"
5841 );
5842 let goodbye = completion
5843 .abandoned
5844 .expect("the incumbent is told to drop the binding it just created");
5845 assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
5846 assert_eq!(goodbye.channel, early.module_channel);
5847 assert_eq!(goodbye.epoch, early.module_epoch);
5848 assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
5849 match early.receiver.try_recv() {
5850 Ok(RouteBindRelayOutcome::DaemonRejected(body)) => {
5851 assert_eq!(body.code, "module_reloading")
5852 }
5853 other => panic!("expected a retryable module_reloading answer, got {other:?}"),
5854 }
5855 assert!(matches!(
5856 forwarding
5857 .lookup_data_route(early_client, early.client_channel, early.client_epoch)
5858 .unwrap(),
5859 DataRoute::Client(DataRouteState::Absent)
5860 ));
5861
5862 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
5865 forwarding
5866 .complete_pending_relay(
5867 fixture.candidate_connection,
5868 late.corr,
5869 RouteBindRelayOutcome::Accepted,
5870 )
5871 .unwrap();
5872 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5873 assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
5874 }
5875
5876 #[test]
5877 fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
5878 let fixture = swap_fixture();
5879 let forwarding = &fixture.forwarding;
5880 let (bound_client, bound_sink, _bound_rx) = client(200);
5882 let bound = forwarding
5883 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5884 .unwrap();
5885 forwarding
5886 .complete_pending_relay(
5887 fixture.incumbent_connection,
5888 bound.corr,
5889 RouteBindRelayOutcome::Accepted,
5890 )
5891 .unwrap();
5892 let (pending_client, pending_sink, _pending_rx) = client(201);
5893 let mut in_flight = forwarding
5894 .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
5895 .unwrap();
5896 forwarding
5897 .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
5898 .unwrap();
5899
5900 let incumbent = forwarding
5901 .cutover_candidate(MODULE_ID)
5902 .unwrap()
5903 .unwrap()
5904 .incumbent
5905 .unwrap();
5906 let target = forwarding
5907 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
5908 .unwrap()
5909 .expect("the superseded incumbent is still registered");
5910
5911 assert_eq!(target.endpoint, fixture.incumbent);
5912 assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
5913 assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
5914 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5915 assert_eq!(target.abandoned_bindings.len(), 1);
5916 assert_eq!(
5917 target.abandoned_bindings[0].channel,
5918 in_flight.module_channel
5919 );
5920 assert!(matches!(
5921 in_flight.receiver.try_recv(),
5922 Ok(RouteBindRelayOutcome::DaemonRejected(body)) if body.code == "module_reloading"
5923 ));
5924 assert_eq!(
5925 forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
5926 1,
5927 "the incumbent's bound route stays until its drain finishes"
5928 );
5929
5930 let (next_client, next_sink, _next_rx) = client(202);
5931 let next = forwarding
5932 .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
5933 .expect("the promoted candidate keeps accepting routes");
5934 assert_eq!(next.endpoint, fixture.candidate);
5935 }
5936
5937 #[test]
5943 fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
5944 let forwarding = ForwardingTable::default();
5945 let first_connection = ConnectionId::new(70);
5946 let (first_tx, _first_rx) = mpsc::channel(8);
5947 forwarding
5948 .register_module_connection(
5949 first_connection,
5950 MODULE_ID.to_string(),
5951 2,
5952 Concurrency::ModuleManaged,
5953 FrameSink::new(first_tx),
5954 )
5955 .unwrap();
5956 let (client_connection, client_sink, _client_rx) = client(200);
5957 let mut pending = forwarding
5958 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5959 .unwrap();
5960 let (second_tx, _second_rx) = mpsc::channel(8);
5961 forwarding
5962 .register_module_connection(
5963 ConnectionId::new(80),
5964 MODULE_ID.to_string(),
5965 2,
5966 Concurrency::ModuleManaged,
5967 FrameSink::new(second_tx),
5968 )
5969 .unwrap();
5970
5971 assert_eq!(
5972 forwarding
5973 .complete_pending_relay(
5974 first_connection,
5975 pending.corr,
5976 RouteBindRelayOutcome::Accepted
5977 )
5978 .unwrap_err(),
5979 ForwardingError::StaleModuleEndpoint
5980 );
5981 assert!(committed_endpoints(&forwarding).is_empty());
5982 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5983 assert!(matches!(
5984 pending.receiver.try_recv(),
5985 Err(oneshot::error::TryRecvError::Closed)
5986 ));
5987 }
5988
5989 #[test]
5990 fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
5991 let fixture = swap_fixture();
5993 let forwarding = &fixture.forwarding;
5994 assert!(forwarding
5995 .cleanup_connection(fixture.candidate_connection)
5996 .unwrap()
5997 .is_empty());
5998 assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
5999 let (client_connection, client_sink, _client_rx) = client(200);
6000 assert_eq!(
6001 forwarding
6002 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
6003 .unwrap()
6004 .endpoint,
6005 fixture.incumbent
6006 );
6007
6008 let fixture = swap_fixture();
6011 let forwarding = &fixture.forwarding;
6012 let (bound_client, bound_sink, _bound_rx) = client(200);
6013 let bound = forwarding
6014 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
6015 .unwrap();
6016 forwarding
6017 .complete_pending_relay(
6018 fixture.incumbent_connection,
6019 bound.corr,
6020 RouteBindRelayOutcome::Accepted,
6021 )
6022 .unwrap();
6023 forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
6024 let released = forwarding
6025 .cleanup_connection(fixture.incumbent_connection)
6026 .unwrap();
6027 assert_eq!(released.len(), 1);
6028 assert_eq!(released[0].connection_id, bound_client);
6029 assert!(forwarding
6030 .read_inner()
6031 .unwrap()
6032 .superseded_endpoints
6033 .is_empty());
6034 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
6035 let (next_client, next_sink, _next_rx) = client(201);
6036 assert_eq!(
6037 forwarding
6038 .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
6039 .unwrap()
6040 .endpoint,
6041 fixture.candidate
6042 );
6043 }
6044}