1use std::{
2 collections::{BTreeMap, HashMap, HashSet},
3 error::Error,
4 fmt,
5 sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
6 time::Duration,
7};
8
9use subc_control::{ClientControlResponse, RouteCloseReason};
10use subc_protocol::{
11 manifest::Concurrency,
12 session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
13 ErrorBody, Flags, FrameType, Principal, Priority,
14};
15use tokio::sync::{oneshot, Semaphore};
16use tokio::time::Instant;
17use tracing::{debug, info, warn};
18
19use crate::{
20 control::{RouteBindBreakers, RouteBindConcurrency},
21 observability::DaemonCounters,
22 registry::ConnectionId,
23 router::FrameSink,
24 scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
25 Frame, ProjectRootId,
26};
27
28const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
30
31const DEFAULT_MODULE_MANAGED_WATCH_WINDOW: usize = 24;
33
34const STATELESS_PARALLEL_WINDOW: usize = 1024;
36
37const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
41
42#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
47pub struct ModuleEndpointId {
48 pub connection_id: ConnectionId,
49 pub generation: u64,
50}
51
52#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
54pub(crate) struct ClientRouteKey {
55 pub connection_id: ConnectionId,
56 pub channel: u16,
57}
58
59#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
61pub(crate) struct ModuleRouteKey {
62 pub endpoint: ModuleEndpointId,
63 pub channel: u16,
64}
65
66#[derive(Debug)]
67pub(crate) struct RouteBinding {
68 pub client_connection_id: ConnectionId,
69 pub client_sink: FrameSink,
70 pub client_negotiated_ver: u8,
71 pub client_channel: u16,
72 pub client_epoch: u32,
73 pub module_id: String,
74 pub module_endpoint: ModuleEndpointId,
75 pub module_sink: FrameSink,
76 pub module_negotiated_ver: u8,
77 pub module_channel: u16,
78 pub module_epoch: u32,
79 pub principal: Principal,
80 #[allow(dead_code)]
82 pub harness: String,
83 #[allow(dead_code)]
85 pub build_class: Option<String>,
86 #[allow(dead_code)]
88 pub operator_app: Option<crate::operator_apps::OperatorAppStamp>,
89 pub project_root: Option<ProjectRootId>,
90 pub bound_at: Instant,
91 pub flow: Arc<ChannelFlow>,
92 pub scope: Option<BoundScope>,
95}
96
97#[derive(Debug, Clone)]
98pub(crate) enum DataRoute {
99 Client(DataRouteState),
100 Module(DataRouteState),
101}
102
103#[derive(Debug, Clone)]
104pub(crate) enum DataRouteState {
105 Bound(Arc<RouteBinding>),
106 Reserved,
107 EpochMismatch,
108 Absent,
109}
110
111#[derive(Debug, Clone, Copy, PartialEq, Eq)]
140pub(crate) enum GoodbyeTargetKind {
141 Client,
142 Module,
143}
144
145#[derive(Debug, Clone)]
146pub(crate) struct GoodbyeTarget {
147 pub connection_id: ConnectionId,
148 pub sink: FrameSink,
149 pub negotiated_ver: u8,
150 pub channel: u16,
151 pub epoch: u32,
152 pub kind: GoodbyeTargetKind,
153 pub module_id: Option<String>,
158}
159
160#[derive(Debug, Clone, Copy)]
163pub(crate) struct UndeliveredFrame<'a> {
164 pub module_id: Option<&'a str>,
166 pub sink: &'a FrameSink,
168}
169
170pub(crate) fn principal_label(principal: &Principal) -> String {
172 match principal {
173 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
174 Principal::Direct => "direct".to_string(),
175 other => format!("{other:?}"),
176 }
177}
178
179fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
182 let labels = inner
183 .client_to_module
184 .iter()
185 .filter(|(key, _)| key.connection_id == connection_id)
186 .map(|(_, route)| principal_label(&route.principal))
187 .collect::<std::collections::BTreeSet<_>>();
188 if labels.is_empty() {
189 "none".to_string()
190 } else {
191 labels.into_iter().collect::<Vec<_>>().join(",")
192 }
193}
194
195impl GoodbyeTarget {
196 pub(crate) fn close_on_delivery_failure(&self) -> bool {
199 matches!(self.kind, GoodbyeTargetKind::Client)
200 }
201}
202
203pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
216
217pub(crate) fn send_module_route_goodbye(
229 counters: &DaemonCounters,
230 sink: &FrameSink,
231 frame: Frame,
232 module_id: Option<&str>,
233 context: &'static str,
234) {
235 let channel = frame.header.channel;
236 let epoch = frame.header.epoch;
237 let Err(err) = sink.try_send(frame.clone()) else {
238 return;
239 };
240 let runtime = match tokio::runtime::Handle::try_current() {
243 Ok(runtime) if !sink.is_closed() => runtime,
244 _ => {
245 counters.increment_goodbye_relay_module_dropped(module_id);
246 warn!(
247 module_id = module_id.unwrap_or("unknown"),
248 route_channel = channel,
249 route_epoch = epoch,
250 error = %err,
251 context,
252 "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
253 );
254 return;
255 }
256 };
257 debug!(
258 module_id = module_id.unwrap_or("unknown"),
259 route_channel = channel,
260 route_epoch = epoch,
261 error = %err,
262 context,
263 "module egress queue refused route GOODBYE; delivering it once the module frees room"
264 );
265 let counters = counters.clone();
266 let sink = sink.clone();
267 let module_id = module_id.map(str::to_string);
268 runtime.spawn(async move {
269 let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
270 let why = match outcome {
271 Ok(Ok(())) => {
272 debug!(
273 module_id = module_id.as_deref().unwrap_or("unknown"),
274 route_channel = channel,
275 route_epoch = epoch,
276 context,
277 "late route GOODBYE delivered to module"
278 );
279 return;
280 }
281 Ok(Err(err)) => err.to_string(),
282 Err(_) => format!(
283 "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
284 ),
285 };
286 counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
287 warn!(
288 module_id = module_id.as_deref().unwrap_or("unknown"),
289 route_channel = channel,
290 route_epoch = epoch,
291 error = %why,
292 context,
293 "route GOODBYE to module dropped under backpressure; not closing shared module connection"
294 );
295 });
296}
297
298#[derive(Debug, Clone)]
304pub(crate) struct EndpointRoute {
305 pub goodbye_target: GoodbyeTarget,
306 pub principal: Principal,
307 pub bound_at: Instant,
308 pub draining: bool,
309 pub drain_reason: Option<RouteCloseReason>,
313}
314
315#[derive(Debug, Clone)]
318pub(crate) struct ScopeDrainedRoute {
319 pub reason: RouteCloseReason,
320 pub scope: BoundScope,
321 pub module_id: String,
322 pub client: GoodbyeTarget,
323 pub module: GoodbyeTarget,
324}
325
326#[derive(Debug)]
327pub(crate) struct PendingRouteBindRelay {
328 pub endpoint: ModuleEndpointId,
329 pub module_sink: FrameSink,
330 pub negotiated_ver: u8,
331 pub client_channel: u16,
332 pub client_epoch: u32,
333 pub module_channel: u16,
334 pub module_epoch: u32,
335 pub corr: u64,
336 pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
337}
338
339#[derive(Debug, Clone)]
340pub(crate) struct ModuleDrainTarget {
341 pub endpoint: ModuleEndpointId,
342 pub sink: FrameSink,
343 pub negotiated_ver: u8,
344 pub abandoned_bindings: Vec<GoodbyeTarget>,
345 pub excluded_subscriptions: u32,
346}
347
348#[cfg(unix)]
351#[derive(Debug, Clone)]
352pub(crate) struct ModuleConnectionTarget {
353 pub module_id: String,
354 pub endpoint: ModuleEndpointId,
355 pub sink: FrameSink,
356 pub negotiated_ver: u8,
357}
358
359#[derive(Debug, Clone)]
360pub(crate) enum RouteBindRelayOutcome {
361 Accepted,
362 Rejected(ErrorBody),
367 DaemonRejected(ErrorBody),
375 ModuleGone(String),
376}
377
378#[derive(Debug, Clone, Copy, PartialEq, Eq)]
380pub(crate) struct ForwardingCutover {
381 pub promoted: ModuleEndpointId,
383 pub incumbent: Option<ModuleEndpointId>,
386}
387
388#[derive(Debug)]
390pub(crate) struct ConnectionCleanup {
391 pub released: Vec<GoodbyeTarget>,
393 pub abandoned_relays: u32,
396}
397
398#[derive(Debug, Clone)]
399pub(crate) struct PendingRelayCompletion {
400 pub settled: bool,
401 pub abandoned: Option<GoodbyeTarget>,
402}
403
404#[derive(Debug)]
405pub(crate) struct PendingModuleControlRpc {
406 pub endpoint: ModuleEndpointId,
407 pub module_sink: FrameSink,
408 pub negotiated_ver: u8,
409 pub corr: u64,
410 pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
411}
412
413#[derive(Debug, Clone)]
414pub(crate) enum ModuleControlRpcOutcome {
415 Response(ModuleControlResponse),
416 Rejected(ErrorBody),
417 ModuleGone(String),
418 MalformedResponse(String),
419 UnexpectedOp { expected: String, actual: String },
420 DeadlineElapsed,
421}
422
423#[derive(Debug, Clone, PartialEq, Eq)]
424pub(crate) enum ModuleControlRpcCompletion {
425 Unknown,
426 Settled,
427 LateHealthAnswer {
428 module_id: String,
429 latency: Duration,
430 },
431}
432
433#[derive(Debug)]
434struct PendingModuleControlRpcEntry {
435 expected_op: String,
436 deadline: Instant,
437 health_probe_started_at: Option<Instant>,
438 sender: oneshot::Sender<ModuleControlRpcOutcome>,
439}
440
441#[derive(Debug)]
442struct HealthProbeTombstone {
443 expected_op: String,
444 module_id: String,
445 probe_started_at: Instant,
446 expires_at: Instant,
447}
448
449#[derive(Debug, Clone)]
450struct RouteReservation {
451 client_key: ClientRouteKey,
452 module_key: ModuleRouteKey,
453 client_epoch: u32,
454 module_epoch: u32,
455 project_root: Option<ProjectRootId>,
456}
457
458#[derive(Debug)]
459struct PendingRouteBindRelayEntry {
460 reservation: RouteReservation,
461 client_sink: FrameSink,
462 client_negotiated_ver: u8,
463 client_permit: crate::router::EgressPermit,
464 route_open_frame: Frame,
465 busy_refusals: bool,
466 request_window: usize,
467 watch_window: Option<usize>,
468 principal: Principal,
469 scope: Option<BoundScope>,
472 harness: String,
473 build_class: Option<String>,
474 operator_app: Option<crate::operator_apps::OperatorAppStamp>,
475 deadline: Instant,
476 relay_enqueued: bool,
487 sender: oneshot::Sender<RouteBindRelayOutcome>,
488}
489
490#[derive(Debug, Clone, Copy, PartialEq, Eq)]
493pub(crate) enum RouteBindRelayMark {
494 Marked,
496 SettledByReply,
500 ReleasedWithoutReply,
504}
505
506#[derive(Debug, Clone)]
507pub(crate) enum RouteRelease {
508 Removed(GoodbyeTarget),
509 Stale,
510 Absent,
511}
512
513#[derive(Debug, Clone)]
514pub(crate) enum RoutePollSnapshot {
515 Bound {
516 module_id: String,
517 status: Option<String>,
518 },
519 Absent,
520}
521
522#[derive(Debug, Clone)]
523struct ModuleConnection {
524 endpoint: ModuleEndpointId,
525 sink: FrameSink,
526 negotiated_ver: u8,
527 concurrency: Concurrency,
528}
529
530#[derive(Debug, Default)]
531struct ForwardingInner {
532 operator_confirms: Arc<crate::operator_confirm::OperatorConfirms>,
533 daemon_draining: bool,
534 daemon_drain_results: HashMap<ModuleEndpointId, bool>,
539 modules_by_id: HashMap<String, ModuleConnection>,
543 candidates_by_id: HashMap<String, ModuleConnection>,
549 superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
555 endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
556 module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
557 draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
561 closing_connections: HashSet<ConnectionId>,
562 next_generation: u64,
563 reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
564 reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
565 next_client_channel: HashMap<ConnectionId, u16>,
566 next_module_channel: HashMap<ModuleEndpointId, u16>,
567 client_slot_epochs: HashMap<ClientRouteKey, u32>,
568 module_slot_epochs: HashMap<ModuleRouteKey, u32>,
569 last_published_epoch: HashMap<ClientRouteKey, u32>,
570 client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
571 module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
572 status: HashMap<(ClientRouteKey, u32), String>,
573 pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
574 relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
579 scope_tags: HashMap<(String, String), ScopeTag>,
584 next_control_corr: HashMap<ModuleEndpointId, u64>,
585 pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
586 health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
587}
588
589#[derive(Debug, Clone)]
590pub(crate) struct CloseReason {
591 code: &'static str,
592 message: String,
593}
594
595impl CloseReason {
596 pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
597 Self {
598 code,
599 message: message.into(),
600 }
601 }
602}
603
604impl fmt::Display for CloseReason {
605 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
606 write!(f, "{}: {}", self.code, self.message)
607 }
608}
609
610pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
611
612#[derive(Debug, Default)]
614pub struct ForwardingTable {
615 inner: Arc<RwLock<ForwardingInner>>,
616 frame_drop_warnings: Mutex<HashMap<ConnectionId, ChannelDropWarnings>>,
617 #[cfg(feature = "test-support")]
618 route_closing_hold: Mutex<Option<oneshot::Receiver<()>>>,
619 close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
620 counters: DaemonCounters,
621 route_bind_breakers: RouteBindBreakers,
626 route_bind_concurrency: RouteBindConcurrency,
629 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
633}
634
635#[derive(Debug)]
636struct FrameDropWarning {
637 last_warned: tokio::time::Instant,
638 suppressed: u64,
639}
640
641type ChannelDropWarnings = HashMap<(u16, &'static str), FrameDropWarning>;
642
643impl ForwardingTable {
644 pub(crate) fn warn_frame_drop(
647 &self,
648 connection_id: ConnectionId,
649 channel: u16,
650 epoch: u32,
651 direction: &'static str,
652 reason: &'static str,
653 ) {
654 let now = tokio::time::Instant::now();
655 let mut warnings = self
656 .frame_drop_warnings
657 .lock()
658 .unwrap_or_else(std::sync::PoisonError::into_inner);
659 let entry = warnings
660 .entry(connection_id)
661 .or_default()
662 .entry((channel, reason));
663 let suppressed = match entry {
664 std::collections::hash_map::Entry::Occupied(mut entry) => {
665 let state = entry.get_mut();
666 if now.saturating_duration_since(state.last_warned) < Duration::from_secs(60) {
667 state.suppressed = state.suppressed.saturating_add(1);
668 return;
669 }
670 state.last_warned = now;
671 std::mem::take(&mut state.suppressed)
672 }
673 std::collections::hash_map::Entry::Vacant(entry) => {
674 entry.insert(FrameDropWarning {
675 last_warned: now,
676 suppressed: 0,
677 });
678 0
679 }
680 };
681 warn!(
682 connection_id = connection_id.get(),
683 channel, epoch, direction, reason, suppressed, "dropping data-plane frame"
684 );
685 }
686
687 #[cfg(feature = "test-support")]
729 pub fn configure_operator_script(
730 &self,
731 script: &std::path::Path,
732 events: Option<&std::path::Path>,
733 limits: Option<crate::operator_confirm::OperatorLimits>,
734 ) {
735 self.operator_confirms()
736 .configure_script(script, events, limits);
737 }
738
739 #[cfg(feature = "test-support")]
743 pub fn hold_route_closing_for_test(&self) -> oneshot::Sender<()> {
744 let (release, hold) = oneshot::channel();
745 *self.route_closing_hold.lock().unwrap() = Some(hold);
746 release
747 }
748
749 #[cfg(feature = "test-support")]
750 pub(crate) async fn wait_before_route_closing_for_test(&self) {
751 let hold = self.route_closing_hold.lock().unwrap().take();
752 if let Some(hold) = hold {
753 let _ = hold.await;
754 }
755 }
756
757 pub(crate) fn operator_confirms(&self) -> Arc<crate::operator_confirm::OperatorConfirms> {
758 Arc::clone(
759 &self
760 .inner
761 .read()
762 .unwrap_or_else(|p| p.into_inner())
763 .operator_confirms,
764 )
765 }
766
767 #[cfg(test)]
768 pub(crate) fn inject_operator_principal(&self, key: ModuleRouteKey, principal: Principal) {
769 self.alter_operator_binding_for_test(key, |binding| binding.principal = principal);
770 }
771
772 #[cfg(test)]
773 pub(crate) fn alter_operator_binding_for_test(
774 &self,
775 key: ModuleRouteKey,
776 alter: impl FnOnce(&mut RouteBinding),
777 ) {
778 let mut inner = self.write_inner().unwrap();
779 let old = inner.module_to_client.remove(&key).unwrap();
780 let client = ClientRouteKey {
781 connection_id: old.client_connection_id,
782 channel: old.client_channel,
783 };
784 inner.client_to_module.remove(&client);
785 let mut binding = Arc::try_unwrap(old).expect("test binding has no outstanding readers");
786 alter(&mut binding);
787 let binding = Arc::new(binding);
788 inner.client_to_module.insert(client, Arc::clone(&binding));
789 inner.module_to_client.insert(key, binding);
790 }
791
792 #[cfg(test)]
795 pub(crate) fn invalidate_operator_route_for_test(&self, key: ModuleRouteKey, stale: bool) {
796 let mut inner = self.write_inner().unwrap();
797 let old = inner.module_to_client.remove(&key).unwrap();
798 let client = ClientRouteKey {
799 connection_id: old.client_connection_id,
800 channel: old.client_channel,
801 };
802 inner.client_to_module.remove(&client);
803 if stale {
804 let mut binding = Arc::try_unwrap(old).expect("no binding readers at the test pause");
805 binding.module_epoch += 1;
806 let binding = Arc::new(binding);
807 inner.client_to_module.insert(client, binding.clone());
808 inner.module_to_client.insert(key, binding);
809 }
810 }
811
812 pub(crate) fn with_operator_route<T>(
816 &self,
817 connection_id: ConnectionId,
818 channel: u16,
819 epoch: u32,
820 admit: impl FnOnce(Option<&RouteBinding>) -> T,
821 ) -> Result<T, ForwardingError> {
822 self.with_operator_route_epoch(connection_id, channel, |binding| {
823 admit(binding.filter(|binding| binding.module_epoch == epoch))
824 })
825 }
826
827 pub(crate) fn with_operator_route_epoch<T>(
830 &self,
831 connection_id: ConnectionId,
832 channel: u16,
833 decide: impl FnOnce(Option<&RouteBinding>) -> T,
834 ) -> Result<T, ForwardingError> {
835 let inner = self.read_inner()?;
836 let binding = inner
837 .endpoint_by_connection
838 .get(&connection_id)
839 .and_then(|endpoint| {
840 inner.module_to_client.get(&ModuleRouteKey {
841 endpoint: *endpoint,
842 channel,
843 })
844 });
845 Ok(decide(binding.map(Arc::as_ref)))
846 }
847
848 pub(crate) fn counters(&self) -> DaemonCounters {
849 self.counters.clone()
850 }
851
852 pub(crate) fn counters_ref(&self) -> &DaemonCounters {
853 &self.counters
854 }
855
856 pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
857 self.route_bind_breakers.clone()
858 }
859
860 pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
861 self.route_bind_concurrency.clone()
862 }
863
864 pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
865 Arc::clone(&self.route_outages)
866 }
867
868 pub(crate) fn register_connection_close(
869 &self,
870 connection_id: ConnectionId,
871 ) -> ConnectionCloseReceiver {
872 let (sender, receiver) = oneshot::channel();
873 let replaced = self
874 .lock_close_registry()
875 .insert(connection_id, sender)
876 .is_some();
877 if replaced {
878 warn!(
879 connection_id = connection_id.get(),
880 "replaced existing connection close registration"
881 );
882 }
883 receiver
884 }
885
886 pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
887 self.lock_close_registry().remove(&connection_id);
888 }
889
890 #[cfg(unix)]
899 pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
900 let senders: Vec<_> = self.lock_close_registry().drain().collect();
901 let count = senders.len();
902 for (_, sender) in senders {
903 let _ = sender.send(reason.clone());
904 }
905 count
906 }
907
908 #[cfg(unix)]
912 pub(crate) fn module_connections(
913 &self,
914 ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
915 let inner = self.read_inner()?;
916 let mut seen = HashSet::new();
917 Ok(inner
918 .modules_by_id
919 .values()
920 .chain(inner.candidates_by_id.values())
921 .chain(inner.superseded_endpoints.values())
922 .filter(|module| seen.insert(module.endpoint))
923 .map(|module| ModuleConnectionTarget {
924 module_id: inner
925 .module_id_by_endpoint
926 .get(&module.endpoint)
927 .cloned()
928 .unwrap_or_default(),
929 endpoint: module.endpoint,
930 sink: module.sink.clone(),
931 negotiated_ver: module.negotiated_ver,
932 })
933 .collect())
934 }
935
936 pub(crate) fn request_connection_close(
940 &self,
941 connection_id: ConnectionId,
942 reason: CloseReason,
943 ) -> bool {
944 let sender = self.lock_close_registry().remove(&connection_id);
945 if let Some(sender) = sender {
946 debug!(
947 connection_id = connection_id.get(),
948 close_reason = %reason,
949 "requesting connection close"
950 );
951 let _ = sender.send(reason);
952 true
953 } else {
954 debug!(
955 connection_id = connection_id.get(),
956 close_reason = %reason,
957 "connection close request ignored for inactive connection"
958 );
959 false
960 }
961 }
962
963 pub fn register_module_connection(
964 &self,
965 connection_id: ConnectionId,
966 module_id: String,
967 negotiated_ver: u8,
968 concurrency: Concurrency,
969 sink: FrameSink,
970 ) -> Result<ModuleEndpointId, ForwardingError> {
971 self.register_module_connection_inner(
972 connection_id,
973 module_id,
974 negotiated_ver,
975 concurrency,
976 sink,
977 None,
978 )
979 }
980
981 pub(crate) fn register_module_connection_acked(
994 &self,
995 connection_id: ConnectionId,
996 module_id: String,
997 negotiated_ver: u8,
998 concurrency: Concurrency,
999 sink: FrameSink,
1000 hello_ack: Frame,
1001 ) -> Result<ModuleEndpointId, ForwardingError> {
1002 self.register_module_connection_inner(
1003 connection_id,
1004 module_id,
1005 negotiated_ver,
1006 concurrency,
1007 sink,
1008 Some(hello_ack),
1009 )
1010 }
1011
1012 fn register_module_connection_inner(
1013 &self,
1014 connection_id: ConnectionId,
1015 module_id: String,
1016 negotiated_ver: u8,
1017 concurrency: Concurrency,
1018 sink: FrameSink,
1019 hello_ack: Option<Frame>,
1020 ) -> Result<ModuleEndpointId, ForwardingError> {
1021 let mut inner = self.write_inner()?;
1022 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1023 return Err(ForwardingError::ConnectionClosing { connection_id });
1024 }
1025 check_module_connection_role_locked(&inner, connection_id)?;
1026 enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1029
1030 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1031 let endpoint = ModuleEndpointId {
1032 connection_id,
1033 generation: inner.next_generation,
1034 };
1035 inner.endpoint_by_connection.insert(connection_id, endpoint);
1036 inner
1037 .module_id_by_endpoint
1038 .insert(endpoint, module_id.clone());
1039 inner.next_module_channel.insert(endpoint, 1);
1040 inner.next_control_corr.insert(endpoint, 1);
1041 inner.modules_by_id.insert(
1042 module_id.clone(),
1043 ModuleConnection {
1044 endpoint,
1045 sink,
1046 negotiated_ver,
1047 concurrency,
1048 },
1049 );
1050 drop(inner);
1051
1052 if let Some(discarded) = self
1065 .route_bind_breakers
1066 .reset_for_new_module_connection(&module_id)
1067 {
1068 info!(
1069 module_id = %module_id,
1070 discarded_consecutive_timeouts = discarded,
1071 "route.bind breaker state discarded: a new module connection replaced the process it described"
1072 );
1073 }
1074 Ok(endpoint)
1075 }
1076
1077 #[cfg(test)]
1091 pub(crate) fn register_candidate_module_connection(
1092 &self,
1093 connection_id: ConnectionId,
1094 module_id: String,
1095 negotiated_ver: u8,
1096 concurrency: Concurrency,
1097 sink: FrameSink,
1098 ) -> Result<ModuleEndpointId, ForwardingError> {
1099 self.register_candidate_module_connection_inner(
1100 connection_id,
1101 module_id,
1102 negotiated_ver,
1103 concurrency,
1104 sink,
1105 None,
1106 )
1107 }
1108
1109 pub(crate) fn register_candidate_module_connection_acked(
1116 &self,
1117 connection_id: ConnectionId,
1118 module_id: String,
1119 negotiated_ver: u8,
1120 concurrency: Concurrency,
1121 sink: FrameSink,
1122 hello_ack: Frame,
1123 ) -> Result<ModuleEndpointId, ForwardingError> {
1124 self.register_candidate_module_connection_inner(
1125 connection_id,
1126 module_id,
1127 negotiated_ver,
1128 concurrency,
1129 sink,
1130 Some(hello_ack),
1131 )
1132 }
1133
1134 fn register_candidate_module_connection_inner(
1135 &self,
1136 connection_id: ConnectionId,
1137 module_id: String,
1138 negotiated_ver: u8,
1139 concurrency: Concurrency,
1140 sink: FrameSink,
1141 hello_ack: Option<Frame>,
1142 ) -> Result<ModuleEndpointId, ForwardingError> {
1143 let mut inner = self.write_inner()?;
1144 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1145 return Err(ForwardingError::ConnectionClosing { connection_id });
1146 }
1147 check_module_connection_role_locked(&inner, connection_id)?;
1148 if inner.candidates_by_id.contains_key(&module_id) {
1149 return Err(ForwardingError::CandidateSlotOccupied { module_id });
1150 }
1151 enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1154
1155 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1156 let endpoint = ModuleEndpointId {
1157 connection_id,
1158 generation: inner.next_generation,
1159 };
1160 inner.endpoint_by_connection.insert(connection_id, endpoint);
1161 inner
1162 .module_id_by_endpoint
1163 .insert(endpoint, module_id.clone());
1164 inner.next_module_channel.insert(endpoint, 1);
1165 inner.next_control_corr.insert(endpoint, 1);
1166 inner.candidates_by_id.insert(
1167 module_id,
1168 ModuleConnection {
1169 endpoint,
1170 sink,
1171 negotiated_ver,
1172 concurrency,
1173 },
1174 );
1175 Ok(endpoint)
1176 }
1177
1178 pub(crate) fn cutover_candidate(
1196 &self,
1197 module_id: &str,
1198 ) -> Result<Option<ForwardingCutover>, ForwardingError> {
1199 let mut inner = self.write_inner()?;
1200 if inner.daemon_draining {
1201 return Err(ForwardingError::ModuleReloading {
1202 module_id: module_id.to_string(),
1203 });
1204 }
1205 let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
1206 return Ok(None);
1207 };
1208 let promoted = candidate.endpoint;
1209 let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
1210 let incumbent = incumbent.map(|incumbent| {
1211 let endpoint = incumbent.endpoint;
1212 inner.superseded_endpoints.insert(endpoint, incumbent);
1213 endpoint
1214 });
1215 drop(inner);
1216
1217 if let Some(discarded) = self
1221 .route_bind_breakers
1222 .reset_for_new_module_connection(module_id)
1223 {
1224 info!(
1225 module_id = %module_id,
1226 discarded_consecutive_timeouts = discarded,
1227 "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
1228 );
1229 }
1230 Ok(Some(ForwardingCutover {
1231 promoted,
1232 incumbent,
1233 }))
1234 }
1235
1236 #[allow(clippy::too_many_arguments)]
1237 pub(crate) async fn begin_route_bind_relay_for(
1238 &self,
1239 client_connection_id: ConnectionId,
1240 client_sink: FrameSink,
1241 client_negotiated_ver: u8,
1242 client_corr: u64,
1243 module_id: &str,
1244 principal: Principal,
1245 scope: Option<BoundScope>,
1246 project_root: Option<ProjectRootId>,
1247 harness: String,
1248 build_class: Option<String>,
1249 operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1250 deadline: Instant,
1251 busy_refusals: bool,
1252 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1253 let client_permit =
1257 client_sink
1258 .reserve_owned()
1259 .await
1260 .map_err(|_| ForwardingError::ClientEgressClosed {
1261 connection_id: client_connection_id,
1262 })?;
1263 self.begin_route_bind_relay_inner(
1264 client_connection_id,
1265 client_sink,
1266 client_negotiated_ver,
1267 client_corr,
1268 module_id,
1269 principal,
1270 scope,
1271 project_root,
1272 harness,
1273 build_class,
1274 operator_app,
1275 deadline,
1276 client_permit,
1277 busy_refusals,
1278 )
1279 }
1280
1281 #[cfg(test)]
1282 pub(crate) fn begin_route_bind_relay_for_test(
1283 &self,
1284 client_connection_id: ConnectionId,
1285 client_sink: FrameSink,
1286 client_corr: u64,
1287 module_id: &str,
1288 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1289 self.begin_route_bind_relay_with_busy_for_test(
1290 client_connection_id,
1291 client_sink,
1292 client_corr,
1293 module_id,
1294 false,
1295 )
1296 }
1297
1298 #[cfg(test)]
1299 pub(crate) fn begin_route_bind_relay_with_busy_for_test(
1300 &self,
1301 client_connection_id: ConnectionId,
1302 client_sink: FrameSink,
1303 client_corr: u64,
1304 module_id: &str,
1305 busy_refusals: bool,
1306 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1307 let permit =
1308 client_sink
1309 .try_reserve_owned()
1310 .map_err(|_| ForwardingError::ClientEgressClosed {
1311 connection_id: client_connection_id,
1312 })?;
1313 self.begin_route_bind_relay_inner(
1314 client_connection_id,
1315 client_sink,
1316 subc_protocol::PROTOCOL_VERSION,
1317 client_corr,
1318 module_id,
1319 Principal::Direct,
1320 None,
1321 None,
1322 String::new(),
1323 None,
1324 None,
1325 Instant::now() + std::time::Duration::from_secs(60),
1326 permit,
1327 busy_refusals,
1328 )
1329 }
1330
1331 pub(crate) fn begin_module_control_rpc_for(
1332 &self,
1333 module_id: &str,
1334 expected_op: &str,
1335 deadline: Instant,
1336 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1337 self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1338 }
1339
1340 pub(crate) fn begin_health_probe_rpc_for(
1341 &self,
1342 module_id: &str,
1343 expected_op: &str,
1344 probe_started_at: Instant,
1345 deadline: Instant,
1346 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1347 self.begin_module_control_rpc_inner(
1348 module_id,
1349 expected_op,
1350 deadline,
1351 Some(probe_started_at),
1352 false,
1353 )
1354 }
1355
1356 pub(crate) fn begin_drain_health_probe_rpc_for(
1357 &self,
1358 module_id: &str,
1359 expected_op: &str,
1360 probe_started_at: Instant,
1361 deadline: Instant,
1362 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1363 self.begin_module_control_rpc_inner(
1364 module_id,
1365 expected_op,
1366 deadline,
1367 Some(probe_started_at),
1368 true,
1369 )
1370 }
1371
1372 pub(crate) fn begin_endpoint_health_probe_rpc_for(
1380 &self,
1381 endpoint: ModuleEndpointId,
1382 expected_op: &str,
1383 probe_started_at: Instant,
1384 deadline: Instant,
1385 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1386 let inner = self.write_inner()?;
1387 let module = module_connection_for_endpoint_locked(&inner, endpoint)
1388 .cloned()
1389 .ok_or(ForwardingError::NoModuleConnection)?;
1390 let module_id = inner
1391 .module_id_by_endpoint
1392 .get(&endpoint)
1393 .cloned()
1394 .unwrap_or_default();
1395 self.begin_control_rpc_locked(
1398 inner,
1399 &module_id,
1400 module,
1401 expected_op,
1402 deadline,
1403 Some(probe_started_at),
1404 true,
1405 )
1406 }
1407
1408 fn begin_module_control_rpc_inner(
1409 &self,
1410 module_id: &str,
1411 expected_op: &str,
1412 deadline: Instant,
1413 health_probe_started_at: Option<Instant>,
1414 allow_draining: bool,
1415 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1416 let inner = self.write_inner()?;
1417 let module = inner
1418 .modules_by_id
1419 .get(module_id)
1420 .cloned()
1421 .ok_or(ForwardingError::NoModuleConnection)?;
1422 self.begin_control_rpc_locked(
1423 inner,
1424 module_id,
1425 module,
1426 expected_op,
1427 deadline,
1428 health_probe_started_at,
1429 allow_draining,
1430 )
1431 }
1432
1433 #[allow(clippy::too_many_arguments)]
1434 fn begin_control_rpc_locked(
1435 &self,
1436 mut inner: RwLockWriteGuard<'_, ForwardingInner>,
1437 module_id: &str,
1438 module: ModuleConnection,
1439 expected_op: &str,
1440 deadline: Instant,
1441 health_probe_started_at: Option<Instant>,
1442 allow_draining: bool,
1443 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1444 if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1445 return Err(ForwardingError::ModuleReloading {
1446 module_id: module_id.to_string(),
1447 });
1448 }
1449 if inner
1450 .closing_connections
1451 .contains(&module.endpoint.connection_id)
1452 {
1453 return Err(ForwardingError::ConnectionClosing {
1454 connection_id: module.endpoint.connection_id,
1455 });
1456 }
1457 if health_probe_started_at.is_some() {
1458 inner
1462 .health_probe_tombstones
1463 .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1464 }
1465 let corr = match inner.allocate_control_corr(module.endpoint) {
1466 Ok(corr) => corr,
1467 Err(err) => {
1468 drop(inner);
1469 self.request_connection_close(
1470 module.endpoint.connection_id,
1471 CloseReason::new(
1472 "control_correlation_exhausted",
1473 "daemon-originated channel-0 correlation space exhausted",
1474 ),
1475 );
1476 return Err(err);
1477 }
1478 };
1479 let (sender, receiver) = oneshot::channel();
1480 inner.pending_control_rpcs.insert(
1481 (module.endpoint, corr),
1482 PendingModuleControlRpcEntry {
1483 expected_op: expected_op.to_string(),
1484 deadline,
1485 health_probe_started_at,
1486 sender,
1487 },
1488 );
1489
1490 Ok(PendingModuleControlRpc {
1491 endpoint: module.endpoint,
1492 module_sink: module.sink,
1493 negotiated_ver: module.negotiated_ver,
1494 corr,
1495 receiver,
1496 })
1497 }
1498
1499 #[allow(clippy::too_many_arguments)]
1500 fn begin_route_bind_relay_inner(
1501 &self,
1502 client_connection_id: ConnectionId,
1503 client_sink: FrameSink,
1504 client_negotiated_ver: u8,
1505 client_corr: u64,
1506 expected_module_id: &str,
1507 principal: Principal,
1508 scope: Option<BoundScope>,
1509 project_root: Option<ProjectRootId>,
1510 harness: String,
1511 build_class: Option<String>,
1512 operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1513 deadline: Instant,
1514 client_permit: crate::router::EgressPermit,
1515 busy_refusals: bool,
1516 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1517 let mut inner = self.write_inner()?;
1518 if inner
1522 .endpoint_by_connection
1523 .contains_key(&client_connection_id)
1524 {
1525 return Err(ForwardingError::ConnectionRoleConflict {
1526 connection_id: client_connection_id,
1527 });
1528 }
1529 if inner.closing_connections.contains(&client_connection_id) {
1530 return Err(ForwardingError::ConnectionClosing {
1531 connection_id: client_connection_id,
1532 });
1533 }
1534 let module = inner
1535 .modules_by_id
1536 .get(expected_module_id)
1537 .cloned()
1538 .ok_or(ForwardingError::NoModuleConnection)?;
1539 if inner.draining_endpoints.contains_key(&module.endpoint) {
1540 return Err(ForwardingError::ModuleReloading {
1541 module_id: expected_module_id.to_string(),
1542 });
1543 }
1544 if inner
1545 .closing_connections
1546 .contains(&module.endpoint.connection_id)
1547 {
1548 return Err(ForwardingError::ConnectionClosing {
1549 connection_id: module.endpoint.connection_id,
1550 });
1551 }
1552
1553 let corr = match inner.allocate_control_corr(module.endpoint) {
1554 Ok(corr) => corr,
1555 Err(err) => {
1556 drop(inner);
1557 self.request_connection_close(
1558 module.endpoint.connection_id,
1559 CloseReason::new(
1560 "control_correlation_exhausted",
1561 "daemon-originated channel-0 correlation space exhausted",
1562 ),
1563 );
1564 return Err(err);
1565 }
1566 };
1567 let (client_channel, client_epoch, module_channel, module_epoch) =
1568 inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1569 let client_key = ClientRouteKey {
1570 connection_id: client_connection_id,
1571 channel: client_channel,
1572 };
1573 let module_key = ModuleRouteKey {
1574 endpoint: module.endpoint,
1575 channel: module_channel,
1576 };
1577 let reservation = RouteReservation {
1578 client_key,
1579 module_key,
1580 client_epoch,
1581 module_epoch,
1582 project_root,
1583 };
1584 let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
1585 request_window: None,
1586 watch_window: None,
1587 route_channel: client_channel,
1588 route_epoch: client_epoch,
1589 })
1590 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1591 let route_open_frame = Frame::build_with_version(
1592 client_negotiated_ver,
1593 FrameType::Response,
1594 Flags::new(false, Priority::Passive, false),
1595 0,
1596 0,
1597 client_corr,
1598 response_body,
1599 )
1600 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1601 let (sender, receiver) = oneshot::channel();
1602 inner.reserved_client.insert(client_key, module_key);
1603 inner.reserved_module.insert(module_key, client_key);
1604 inner.pending_relays.insert(
1605 (module.endpoint, corr),
1606 PendingRouteBindRelayEntry {
1607 reservation,
1608 client_sink,
1609 client_negotiated_ver,
1610 client_permit,
1611 route_open_frame,
1612 busy_refusals,
1613 request_window: window_for(&module.concurrency),
1614 watch_window: (busy_refusals && module.concurrency == Concurrency::ModuleManaged)
1615 .then_some(DEFAULT_MODULE_MANAGED_WATCH_WINDOW),
1616 principal,
1617 scope,
1618 harness,
1619 build_class,
1620 operator_app,
1621 deadline,
1622 relay_enqueued: false,
1623 sender,
1624 },
1625 );
1626
1627 Ok(PendingRouteBindRelay {
1628 endpoint: module.endpoint,
1629 module_sink: module.sink,
1630 negotiated_ver: module.negotiated_ver,
1631 client_channel,
1632 client_epoch,
1633 module_channel,
1634 module_epoch,
1635 corr,
1636 receiver,
1637 })
1638 }
1639
1640 pub(crate) fn mark_route_bind_relay_enqueued(
1650 &self,
1651 endpoint: ModuleEndpointId,
1652 corr: u64,
1653 ) -> Result<RouteBindRelayMark, ForwardingError> {
1654 let mut inner = self.write_inner()?;
1655 if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
1656 pending.relay_enqueued = true;
1657 return Ok(RouteBindRelayMark::Marked);
1658 }
1659 if inner
1660 .relays_released_before_enqueue
1661 .remove(&(endpoint, corr))
1662 {
1663 return Ok(RouteBindRelayMark::ReleasedWithoutReply);
1664 }
1665 Ok(RouteBindRelayMark::SettledByReply)
1666 }
1667
1668 pub(crate) fn release_client_route(
1669 &self,
1670 client_connection_id: ConnectionId,
1671 client_channel: u16,
1672 expected_epoch: u32,
1673 ) -> Result<RouteRelease, ForwardingError> {
1674 let mut inner = self.write_inner()?;
1675 let release = release_client_route_locked(
1676 &mut inner,
1677 ClientRouteKey {
1678 connection_id: client_connection_id,
1679 channel: client_channel,
1680 },
1681 expected_epoch,
1682 );
1683 self.record_route_release(&release);
1684 Ok(release)
1685 }
1686
1687 pub(crate) fn release_module_route(
1688 &self,
1689 module_connection_id: ConnectionId,
1690 module_channel: u16,
1691 expected_epoch: u32,
1692 ) -> Result<RouteRelease, ForwardingError> {
1693 let mut inner = self.write_inner()?;
1694 let Some(endpoint) = inner
1695 .endpoint_by_connection
1696 .get(&module_connection_id)
1697 .copied()
1698 else {
1699 return Ok(RouteRelease::Absent);
1700 };
1701 let release = release_module_route_locked(
1702 &mut inner,
1703 ModuleRouteKey {
1704 endpoint,
1705 channel: module_channel,
1706 },
1707 expected_epoch,
1708 );
1709 self.record_route_release(&release);
1710 Ok(release)
1711 }
1712
1713 pub(crate) fn abort_pending_relay(
1714 &self,
1715 endpoint: ModuleEndpointId,
1716 corr: u64,
1717 outcome: RouteBindRelayOutcome,
1718 ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
1719 let mut inner = self.write_inner()?;
1720 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1721 inner
1726 .relays_released_before_enqueue
1727 .remove(&(endpoint, corr));
1728 return Ok(None);
1729 };
1730 release_reserved_route_locked(
1731 &mut inner,
1732 pending.reservation.client_key,
1733 pending.reservation.module_key,
1734 );
1735 let target = pending
1736 .relay_enqueued
1737 .then(|| abandoned_route_target(&inner, &pending.reservation));
1738 let _ = pending.sender.send(outcome);
1739 Ok(target.flatten())
1740 }
1741
1742 pub(crate) fn cancel_module_control_rpc(
1743 &self,
1744 endpoint: ModuleEndpointId,
1745 corr: u64,
1746 ) -> Result<(), ForwardingError> {
1747 self.write_inner()?
1748 .pending_control_rpcs
1749 .remove(&(endpoint, corr));
1750 Ok(())
1751 }
1752
1753 pub(crate) fn tombstone_health_probe_rpc(
1754 &self,
1755 endpoint: ModuleEndpointId,
1756 corr: u64,
1757 ) -> Result<bool, ForwardingError> {
1758 let key = (endpoint, corr);
1759 let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
1760 {
1761 let mut inner = self.write_inner()?;
1762 let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
1763 return Ok(false);
1764 };
1765 let Some(probe_started_at) = pending.health_probe_started_at else {
1766 inner.pending_control_rpcs.insert(key, pending);
1767 return Ok(false);
1768 };
1769 let module_id = inner
1770 .module_id_by_endpoint
1771 .get(&endpoint)
1772 .cloned()
1773 .unwrap_or_else(|| "unknown".to_string());
1774 inner.health_probe_tombstones.insert(
1775 key,
1776 HealthProbeTombstone {
1777 expected_op: pending.expected_op,
1778 module_id,
1779 probe_started_at,
1780 expires_at,
1781 },
1782 );
1783 }
1784 self.schedule_health_probe_tombstone_expiration(key, expires_at);
1785 Ok(true)
1786 }
1787
1788 fn schedule_health_probe_tombstone_expiration(
1789 &self,
1790 key: (ModuleEndpointId, u64),
1791 expires_at: Instant,
1792 ) {
1793 let inner = Arc::downgrade(&self.inner);
1794 tokio::spawn(async move {
1795 tokio::time::sleep_until(expires_at).await;
1796 let Some(inner) = inner.upgrade() else {
1797 return;
1798 };
1799 let Ok(mut inner) = inner.write() else {
1800 return;
1801 };
1802 let expired = inner
1803 .health_probe_tombstones
1804 .get(&key)
1805 .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
1806 if expired {
1807 inner.health_probe_tombstones.remove(&key);
1808 }
1809 });
1810 }
1811
1812 pub(crate) fn complete_pending_relay(
1813 &self,
1814 connection_id: ConnectionId,
1815 corr: u64,
1816 outcome: RouteBindRelayOutcome,
1817 ) -> Result<PendingRelayCompletion, ForwardingError> {
1818 let mut inner = self.write_inner()?;
1819 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1820 return Ok(PendingRelayCompletion {
1821 settled: false,
1822 abandoned: None,
1823 });
1824 };
1825 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1826 return Ok(PendingRelayCompletion {
1827 settled: false,
1828 abandoned: None,
1829 });
1830 };
1831
1832 if Instant::now() >= pending.deadline {
1833 release_reserved_route_locked(
1834 &mut inner,
1835 pending.reservation.client_key,
1836 pending.reservation.module_key,
1837 );
1838 let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
1839 .then(|| abandoned_route_target(&inner, &pending.reservation))
1840 .flatten();
1841 let _ = pending
1842 .sender
1843 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody {
1844 code: "module_timeout".to_string(),
1845 message: "route.bind response arrived after its daemon deadline".to_string(),
1846 detail: None,
1847 }));
1848 return Ok(PendingRelayCompletion {
1849 settled: true,
1850 abandoned,
1851 });
1852 }
1853
1854 match outcome {
1855 RouteBindRelayOutcome::Accepted
1874 if pending.client_sink.is_closed()
1875 || inner
1876 .closing_connections
1877 .contains(&pending.reservation.client_key.connection_id) =>
1878 {
1879 let reason = if pending.client_sink.is_closed() {
1880 "client egress closed before route publication"
1881 } else {
1882 "client connection is closing before route publication"
1883 };
1884 release_reserved_route_locked(
1885 &mut inner,
1886 pending.reservation.client_key,
1887 pending.reservation.module_key,
1888 );
1889 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1893 let _ = pending
1894 .sender
1895 .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
1896 return Ok(PendingRelayCompletion {
1897 settled: true,
1898 abandoned,
1899 });
1900 }
1901 RouteBindRelayOutcome::Accepted
1924 if inner.superseded_endpoints.contains_key(&endpoint)
1925 || inner.draining_endpoints.contains_key(&endpoint) =>
1926 {
1927 release_reserved_route_locked(
1928 &mut inner,
1929 pending.reservation.client_key,
1930 pending.reservation.module_key,
1931 );
1932 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1935 let module_id = inner
1936 .module_id_by_endpoint
1937 .get(&endpoint)
1938 .cloned()
1939 .unwrap_or_else(|| "unknown".to_string());
1940 let _ = pending
1941 .sender
1942 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
1943 "module_reloading",
1944 format!("module_id '{module_id}' is reloading"),
1945 )));
1946 return Ok(PendingRelayCompletion {
1947 settled: true,
1948 abandoned,
1949 });
1950 }
1951 RouteBindRelayOutcome::Accepted
1960 if pending
1961 .scope
1962 .as_ref()
1963 .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
1964 {
1965 let refusal = pending
1966 .scope
1967 .as_ref()
1968 .and_then(|scope| scope_refusal_locked(&inner, scope))
1969 .expect("guard matched a refusal under the same lock");
1970 release_reserved_route_locked(
1971 &mut inner,
1972 pending.reservation.client_key,
1973 pending.reservation.module_key,
1974 );
1975 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1976 let _ = pending
1977 .sender
1978 .send(RouteBindRelayOutcome::DaemonRejected(refusal));
1979 return Ok(PendingRelayCompletion {
1980 settled: true,
1981 abandoned,
1982 });
1983 }
1984 RouteBindRelayOutcome::Accepted => {
1985 let abandoned = commit_route_locked(&mut inner, pending)?;
1986 return Ok(PendingRelayCompletion {
1987 settled: true,
1988 abandoned,
1989 });
1990 }
1991 terminal => {
1992 release_reserved_route_locked(
1993 &mut inner,
1994 pending.reservation.client_key,
1995 pending.reservation.module_key,
1996 );
1997 let _ = pending.sender.send(terminal);
1998 }
1999 }
2000 Ok(PendingRelayCompletion {
2001 settled: true,
2002 abandoned: None,
2003 })
2004 }
2005
2006 pub(crate) fn pending_module_control_op(
2007 &self,
2008 connection_id: ConnectionId,
2009 corr: u64,
2010 ) -> Result<Option<String>, ForwardingError> {
2011 let inner = self.read_inner()?;
2012 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2013 return Ok(None);
2014 };
2015 let key = (endpoint, corr);
2016 Ok(inner
2017 .pending_control_rpcs
2018 .get(&key)
2019 .map(|pending| pending.expected_op.clone())
2020 .or_else(|| {
2021 inner
2022 .health_probe_tombstones
2023 .get(&key)
2024 .filter(|tombstone| tombstone.expires_at > Instant::now())
2025 .map(|tombstone| tombstone.expected_op.clone())
2026 }))
2027 }
2028
2029 pub(crate) fn complete_module_control_rpc(
2030 &self,
2031 connection_id: ConnectionId,
2032 corr: u64,
2033 actual_op: Option<&str>,
2034 outcome: ModuleControlRpcOutcome,
2035 ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
2036 let now = Instant::now();
2037 let mut inner = self.write_inner()?;
2038 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2039 return Ok(ModuleControlRpcCompletion::Unknown);
2040 };
2041 let key = (endpoint, corr);
2042 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2043 if now >= pending.deadline {
2044 let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
2045 ModuleControlRpcCompletion::LateHealthAnswer {
2046 module_id: inner
2047 .module_id_by_endpoint
2048 .get(&endpoint)
2049 .cloned()
2050 .unwrap_or_else(|| "unknown".to_string()),
2051 latency: now.saturating_duration_since(probe_started_at),
2052 }
2053 });
2054 let _ = pending
2055 .sender
2056 .send(ModuleControlRpcOutcome::DeadlineElapsed);
2057 return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
2058 }
2059 let outcome = match actual_op {
2060 Some(actual) if actual != pending.expected_op => {
2061 ModuleControlRpcOutcome::UnexpectedOp {
2062 expected: pending.expected_op,
2063 actual: actual.to_string(),
2064 }
2065 }
2066 _ => outcome,
2067 };
2068 let _ = pending.sender.send(outcome);
2069 return Ok(ModuleControlRpcCompletion::Settled);
2070 }
2071
2072 let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
2073 return Ok(ModuleControlRpcCompletion::Unknown);
2074 };
2075 if tombstone.expires_at <= now {
2076 return Ok(ModuleControlRpcCompletion::Unknown);
2077 }
2078 Ok(ModuleControlRpcCompletion::LateHealthAnswer {
2079 module_id: tombstone.module_id,
2080 latency: now.saturating_duration_since(tombstone.probe_started_at),
2081 })
2082 }
2083
2084 #[cfg(test)]
2085 pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
2086 Ok(self.read_inner()?.health_probe_tombstones.len())
2087 }
2088
2089 #[cfg(test)]
2090 pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
2091 Ok(self.read_inner()?.closing_connections.len())
2092 }
2093
2094 #[cfg(test)]
2097 pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
2098 let inner = self.read_inner()?;
2099 Ok((inner.reserved_client.len(), inner.reserved_module.len()))
2100 }
2101
2102 pub(crate) fn module_endpoint_for_connection(
2103 &self,
2104 connection_id: ConnectionId,
2105 ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
2106 Ok(self
2107 .read_inner()?
2108 .endpoint_by_connection
2109 .get(&connection_id)
2110 .copied())
2111 }
2112
2113 pub(crate) fn module_id_for_connection(
2116 &self,
2117 connection_id: ConnectionId,
2118 ) -> Result<Option<String>, ForwardingError> {
2119 let inner = self.read_inner()?;
2120 Ok(inner
2121 .endpoint_by_connection
2122 .get(&connection_id)
2123 .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
2124 .cloned())
2125 }
2126
2127 pub(crate) fn module_route_epoch_was_allocated(
2134 &self,
2135 connection_id: ConnectionId,
2136 channel: u16,
2137 epoch: u32,
2138 ) -> Result<bool, ForwardingError> {
2139 let inner = self.read_inner()?;
2140 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2141 return Ok(false);
2142 };
2143 Ok(inner
2144 .module_slot_epochs
2145 .get(&ModuleRouteKey { endpoint, channel })
2146 .is_some_and(|last| epoch != 0 && epoch <= *last))
2147 }
2148
2149 pub(crate) fn has_live_module_connection(
2150 &self,
2151 module_id: &str,
2152 ) -> Result<bool, ForwardingError> {
2153 Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
2154 }
2155
2156 pub(crate) fn try_admit_request(
2159 &self,
2160 route: &Arc<RouteBinding>,
2161 epoch: u32,
2162 corr: u64,
2163 subscription: bool,
2164 ) -> Result<RequestAdmission, ForwardingError> {
2165 let inner = self.read_inner()?;
2166 let key = ClientRouteKey {
2167 connection_id: route.client_connection_id,
2168 channel: route.client_channel,
2169 };
2170 if epoch != route.client_epoch
2171 || !inner
2172 .client_to_module
2173 .get(&key)
2174 .is_some_and(|current| Arc::ptr_eq(current, route))
2175 {
2176 return Ok(RequestAdmission::Stale);
2177 }
2178 let admission = route.flow.try_acquire_tagged(corr, subscription);
2179 Ok(match admission {
2180 RequestAdmission::Stale
2181 if inner
2182 .draining_endpoints
2183 .contains_key(&route.module_endpoint) =>
2184 {
2185 RequestAdmission::ModuleReloading
2186 }
2187 admission => admission,
2188 })
2189 }
2190
2191 pub(crate) fn lookup_data_route(
2192 &self,
2193 connection_id: ConnectionId,
2194 channel: u16,
2195 epoch: u32,
2196 ) -> Result<DataRoute, ForwardingError> {
2197 let inner = self.read_inner()?;
2198 let state = if let Some(endpoint) =
2199 inner.endpoint_by_connection.get(&connection_id).copied()
2200 {
2201 let key = ModuleRouteKey { endpoint, channel };
2202 match inner.module_to_client.get(&key) {
2203 Some(route) if route.module_epoch == epoch => {
2204 DataRouteState::Bound(Arc::clone(route))
2205 }
2206 Some(_) => DataRouteState::EpochMismatch,
2207 None if inner.reserved_module.contains_key(&key)
2208 && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
2209 {
2210 DataRouteState::Reserved
2211 }
2212 None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
2213 None => DataRouteState::Absent,
2214 }
2215 } else {
2216 let key = ClientRouteKey {
2217 connection_id,
2218 channel,
2219 };
2220 match inner.client_to_module.get(&key) {
2221 Some(route) if route.client_epoch == epoch => {
2222 DataRouteState::Bound(Arc::clone(route))
2223 }
2224 Some(_) => DataRouteState::EpochMismatch,
2225 None if inner.reserved_client.contains_key(&key)
2226 && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
2227 {
2228 DataRouteState::Reserved
2229 }
2230 None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
2231 None => DataRouteState::Absent,
2232 }
2233 };
2234 Ok(
2235 if inner.endpoint_by_connection.contains_key(&connection_id) {
2236 DataRoute::Module(state)
2237 } else {
2238 DataRoute::Client(state)
2239 },
2240 )
2241 }
2242
2243 #[cfg(test)]
2244 pub(crate) fn inject_client_slot_epoch(
2245 &self,
2246 connection_id: ConnectionId,
2247 channel: u16,
2248 last_epoch: u32,
2249 ) {
2250 let mut inner = self.write_inner().expect("forwarding lock");
2251 inner.client_slot_epochs.insert(
2252 ClientRouteKey {
2253 connection_id,
2254 channel,
2255 },
2256 last_epoch,
2257 );
2258 inner.next_client_channel.insert(connection_id, channel);
2259 }
2260
2261 #[cfg(test)]
2262 pub(crate) fn inject_module_slot_epoch(
2263 &self,
2264 endpoint: ModuleEndpointId,
2265 channel: u16,
2266 last_epoch: u32,
2267 ) {
2268 let mut inner = self.write_inner().expect("forwarding lock");
2269 inner
2270 .module_slot_epochs
2271 .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
2272 inner.next_module_channel.insert(endpoint, channel);
2273 }
2274
2275 #[cfg(test)]
2276 pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
2277 self.write_inner()
2278 .expect("forwarding lock")
2279 .next_control_corr
2280 .insert(endpoint, next_corr);
2281 }
2282
2283 pub(crate) fn cache_status(
2284 &self,
2285 endpoint: ModuleEndpointId,
2286 module_channel: u16,
2287 module_epoch: u32,
2288 status: String,
2289 ) -> Result<bool, ForwardingError> {
2290 let mut inner = self.write_inner()?;
2291 if !inner.module_id_by_endpoint.contains_key(&endpoint) {
2292 return Err(ForwardingError::StaleModuleEndpoint);
2293 }
2294
2295 let module_key = ModuleRouteKey {
2296 endpoint,
2297 channel: module_channel,
2298 };
2299 let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
2300 (route.module_epoch == module_epoch).then_some((
2301 ClientRouteKey {
2302 connection_id: route.client_connection_id,
2303 channel: route.client_channel,
2304 },
2305 route.client_epoch,
2306 ))
2307 } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
2308 (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
2309 client_key,
2310 inner
2311 .client_slot_epochs
2312 .get(&client_key)
2313 .copied()
2314 .unwrap_or(0),
2315 ))
2316 } else {
2317 None
2318 };
2319
2320 if let Some(handle) = handle {
2321 inner.status.insert(handle, status);
2322 Ok(true)
2323 } else {
2324 debug!(
2325 module_channel,
2326 module_epoch,
2327 generation = endpoint.generation,
2328 connection_id = endpoint.connection_id.get(),
2329 "dropping stale status update for module route handle"
2330 );
2331 Ok(false)
2332 }
2333 }
2334
2335 pub(crate) fn route_poll_snapshot(
2336 &self,
2337 client_connection_id: ConnectionId,
2338 client_channel: u16,
2339 client_epoch: u32,
2340 ) -> Result<RoutePollSnapshot, ForwardingError> {
2341 let inner = self.read_inner()?;
2342 let client_key = ClientRouteKey {
2343 connection_id: client_connection_id,
2344 channel: client_channel,
2345 };
2346 let Some(route) = inner.client_to_module.get(&client_key) else {
2347 return Ok(RoutePollSnapshot::Absent);
2348 };
2349 if route.client_epoch != client_epoch
2350 || !inner
2351 .module_id_by_endpoint
2352 .contains_key(&route.module_endpoint)
2353 {
2354 return Ok(RoutePollSnapshot::Absent);
2355 }
2356 Ok(RoutePollSnapshot::Bound {
2357 module_id: route.module_id.clone(),
2358 status: inner.status.get(&(client_key, client_epoch)).cloned(),
2359 })
2360 }
2361
2362 pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2363 Ok(self.read_inner()?.client_to_module.len())
2364 }
2365
2366 pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2376 let inner = self.read_inner()?;
2377 let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2378 for key in inner.client_to_module.keys() {
2379 *per_connection.entry(key.connection_id).or_insert(0) += 1;
2380 }
2381 let max = per_connection.values().copied().max().unwrap_or(0);
2382 Ok((per_connection.len(), max))
2383 }
2384
2385 pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2386 let inner = self.read_inner()?;
2387 Ok(inner
2388 .client_to_module
2389 .keys()
2390 .any(|key| key.channel == route_channel))
2391 }
2392
2393 pub(crate) fn daemon_drain_result_for_connection(
2397 &self,
2398 connection_id: ConnectionId,
2399 ) -> Result<Option<bool>, ForwardingError> {
2400 let inner = self.read_inner()?;
2401 Ok(inner.daemon_draining.then(|| {
2402 inner
2403 .endpoint_by_connection
2404 .get(&connection_id)
2405 .and_then(|endpoint| inner.daemon_drain_results.get(endpoint))
2406 .copied()
2407 .unwrap_or(false)
2408 }))
2409 }
2410
2411 #[cfg(unix)]
2412 pub(crate) fn record_daemon_drain_result(
2413 &self,
2414 endpoint: ModuleEndpointId,
2415 drained: bool,
2416 ) -> Result<(), ForwardingError> {
2417 let mut inner = self.write_inner()?;
2418 if let Some(result) = inner.daemon_drain_results.get_mut(&endpoint) {
2422 *result = drained;
2423 }
2424 Ok(())
2425 }
2426
2427 #[cfg(unix)]
2430 pub(crate) fn begin_daemon_drain(
2431 &self,
2432 ) -> Result<Vec<(String, ModuleEndpointId)>, ForwardingError> {
2433 let mut inner = self.write_inner()?;
2434 if inner.daemon_draining {
2435 return Ok(Vec::new());
2436 }
2437 inner.daemon_draining = true;
2438 let modules = inner
2439 .modules_by_id
2440 .iter()
2441 .map(|(id, module)| (id.clone(), module.endpoint))
2442 .collect::<Vec<_>>();
2443 for (_, endpoint) in &modules {
2444 inner.daemon_drain_results.insert(*endpoint, false);
2445 inner
2446 .draining_endpoints
2447 .insert(*endpoint, RouteCloseReason::Restart);
2448 }
2449 let off_slot_endpoints = inner
2454 .candidates_by_id
2455 .values()
2456 .map(|module| module.endpoint)
2457 .chain(inner.superseded_endpoints.keys().copied())
2458 .collect::<Vec<_>>();
2459 for endpoint in off_slot_endpoints {
2460 inner
2461 .draining_endpoints
2462 .insert(endpoint, RouteCloseReason::Restart);
2463 }
2464 Ok(modules)
2465 }
2466
2467 pub(crate) fn begin_module_drain(
2474 &self,
2475 module_id: &str,
2476 reason: RouteCloseReason,
2477 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2478 let mut inner = self.write_inner()?;
2479 let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2480 return Ok(None);
2481 };
2482 Ok(Some(begin_drain_locked(
2483 &mut inner, module_id, module, reason,
2484 )))
2485 }
2486
2487 pub(crate) fn begin_endpoint_drain(
2494 &self,
2495 endpoint: ModuleEndpointId,
2496 reason: RouteCloseReason,
2497 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2498 let mut inner = self.write_inner()?;
2499 let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2500 return Ok(None);
2501 };
2502 let module_id = inner
2503 .module_id_by_endpoint
2504 .get(&endpoint)
2505 .cloned()
2506 .expect("an endpoint resolved to a module connection has a module id");
2507 Ok(Some(begin_drain_locked(
2508 &mut inner, &module_id, module, reason,
2509 )))
2510 }
2511}
2512
2513fn begin_drain_locked(
2517 inner: &mut ForwardingInner,
2518 module_id: &str,
2519 module: ModuleConnection,
2520 reason: RouteCloseReason,
2521) -> ModuleDrainTarget {
2522 {
2523 let endpoint = module.endpoint;
2524 inner.draining_endpoints.insert(endpoint, reason);
2525
2526 let flows = inner
2527 .client_to_module
2528 .values()
2529 .filter(|route| route.module_endpoint == endpoint)
2530 .map(|route| Arc::clone(&route.flow))
2531 .collect::<Vec<_>>();
2532 let excluded_subscriptions = flows
2533 .into_iter()
2534 .map(|flow| flow.begin_drain())
2535 .fold(0u32, u32::saturating_add);
2536
2537 let pending_keys = inner
2538 .pending_relays
2539 .keys()
2540 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2541 .copied()
2542 .collect::<Vec<_>>();
2543 let mut abandoned_bindings = Vec::new();
2544 for key in pending_keys {
2545 let Some(pending) = inner.pending_relays.remove(&key) else {
2546 continue;
2547 };
2548 release_reserved_route_locked(
2549 inner,
2550 pending.reservation.client_key,
2551 pending.reservation.module_key,
2552 );
2553 if pending.relay_enqueued {
2554 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2555 abandoned_bindings.push(target);
2556 }
2557 } else {
2558 inner.relays_released_before_enqueue.insert(key);
2562 }
2563 let _ = pending
2564 .sender
2565 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
2566 "module_reloading",
2567 format!("module_id '{module_id}' is reloading"),
2568 )));
2569 }
2570
2571 let pending_control_keys = inner
2572 .pending_control_rpcs
2573 .keys()
2574 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2575 .copied()
2576 .collect::<Vec<_>>();
2577 for key in pending_control_keys {
2578 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2579 let _ = pending
2580 .sender
2581 .send(ModuleControlRpcOutcome::ModuleGone(format!(
2582 "module '{module_id}' began draining during module-control RPC"
2583 )));
2584 }
2585 }
2586
2587 ModuleDrainTarget {
2588 endpoint,
2589 sink: module.sink,
2590 negotiated_ver: module.negotiated_ver,
2591 abandoned_bindings,
2592 excluded_subscriptions,
2593 }
2594 }
2595}
2596
2597#[derive(Debug, Default, PartialEq, Eq)]
2600pub(crate) struct DrainHoldouts {
2601 pub(crate) requests: usize,
2604 pub(crate) routes: usize,
2606 pub(crate) total_routes: usize,
2608 pub(crate) top_connections: Vec<(u64, usize)>,
2611 pub(crate) held: Vec<(u16, u64)>,
2620}
2621
2622pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
2624
2625impl ForwardingTable {
2626 pub(crate) fn endpoint_drain_holdouts(
2628 &self,
2629 endpoint: ModuleEndpointId,
2630 ) -> Result<DrainHoldouts, ForwardingError> {
2631 let inner = self.read_inner()?;
2632 let mut holdouts = DrainHoldouts::default();
2633 let mut by_connection: HashMap<u64, usize> = HashMap::new();
2634 for (key, route) in &inner.client_to_module {
2635 if route.module_endpoint != endpoint {
2636 continue;
2637 }
2638 holdouts.total_routes += 1;
2639 let held = route.flow.drain_in_flight();
2640 if held == 0 {
2641 continue;
2642 }
2643 holdouts.requests += held;
2644 holdouts.routes += 1;
2645 *by_connection.entry(key.connection_id.get()).or_default() += held;
2646 holdouts.held.extend(
2647 route
2648 .flow
2649 .drain_held_corrs()
2650 .into_iter()
2651 .map(|corr| (route.module_channel, corr)),
2652 );
2653 }
2654 holdouts.held.sort_unstable();
2655 holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
2656 let mut connections = by_connection.into_iter().collect::<Vec<_>>();
2657 connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
2658 connections.truncate(3);
2659 holdouts.top_connections = connections;
2660 Ok(holdouts)
2661 }
2662
2663 pub(crate) fn endpoint_in_flight_count(
2664 &self,
2665 endpoint: ModuleEndpointId,
2666 ) -> Result<usize, ForwardingError> {
2667 let inner = self.read_inner()?;
2668 Ok(inner
2669 .client_to_module
2670 .values()
2671 .filter(|route| route.module_endpoint == endpoint)
2672 .map(|route| route.flow.drain_in_flight())
2673 .sum())
2674 }
2675
2676 pub(crate) fn endpoint_is_draining(
2677 &self,
2678 endpoint: ModuleEndpointId,
2679 ) -> Result<bool, ForwardingError> {
2680 Ok(self
2681 .read_inner()?
2682 .draining_endpoints
2683 .contains_key(&endpoint))
2684 }
2685
2686 pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
2687 let inner = self.read_inner()?;
2688 Ok(inner
2689 .modules_by_id
2690 .get(module_id)
2691 .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
2692 }
2693
2694 pub(crate) fn release_module_endpoint_routes(
2695 &self,
2696 endpoint: ModuleEndpointId,
2697 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2698 let mut inner = self.write_inner()?;
2699 let routes = inner
2700 .module_to_client
2701 .iter()
2702 .filter(|(module_key, _)| module_key.endpoint == endpoint)
2703 .map(|(module_key, route)| (*module_key, route.module_epoch))
2704 .collect::<Vec<_>>();
2705 let mut released = Vec::with_capacity(routes.len());
2706 for (module_key, epoch) in routes {
2707 if let RouteRelease::Removed(target) =
2708 release_module_route_locked(&mut inner, module_key, epoch)
2709 {
2710 released.push(target);
2711 }
2712 }
2713 Ok(released)
2714 }
2715
2716 pub(crate) fn endpoint_routes(
2722 &self,
2723 endpoint: ModuleEndpointId,
2724 ) -> Result<Vec<EndpointRoute>, ForwardingError> {
2725 let inner = self.read_inner()?;
2726 Ok(endpoint_routes_locked(&inner, endpoint))
2727 }
2728
2729 pub(crate) fn route_census(
2731 &self,
2732 module_id: Option<&str>,
2733 ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
2734 let inner = self.read_inner()?;
2735 let mut endpoints = inner
2736 .modules_by_id
2737 .iter()
2738 .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
2739 .map(|(id, module)| (id.clone(), module.endpoint))
2740 .collect::<Vec<_>>();
2741 endpoints.sort_by(|left, right| left.0.cmp(&right.0));
2742 Ok(endpoints
2743 .into_iter()
2744 .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
2745 .collect())
2746 }
2747
2748 pub(crate) fn live_roots(
2750 &self,
2751 module_id: &str,
2752 ) -> Result<ModuleControlResponseToModule, ForwardingError> {
2753 let inner = self.read_inner()?;
2754 let endpoint = inner
2755 .modules_by_id
2756 .get(module_id)
2757 .map(|module| module.endpoint);
2758 let mut roots = BTreeMap::new();
2759 let mut unknown_root_bindings = 0;
2760 let mut total_bindings = 0;
2761 if let Some(endpoint) = endpoint {
2762 for binding in inner
2763 .module_to_client
2764 .values()
2765 .filter(|binding| binding.module_endpoint == endpoint)
2766 {
2767 total_bindings += 1;
2768 let workdir = binding
2769 .scope
2770 .as_ref()
2771 .and_then(|scope| scope.workdir.as_deref())
2772 .map(std::path::Path::new);
2773 if let Some(root) = &binding.project_root {
2774 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2775 entry.0 += 1;
2776 } else if workdir.is_none() {
2777 unknown_root_bindings += 1;
2778 }
2779 if let Some(workdir) = workdir.filter(|workdir| {
2780 binding.project_root.as_ref().map(ProjectRootId::as_path) != Some(*workdir)
2781 }) {
2782 roots.entry(workdir.to_path_buf()).or_insert((0, 0)).0 += 1;
2783 }
2784 }
2785 for pending in inner
2786 .pending_relays
2787 .values()
2788 .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
2789 {
2790 total_bindings += 1;
2791 if let Some(root) = &pending.reservation.project_root {
2792 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2793 entry.1 += 1;
2794 } else {
2795 unknown_root_bindings += 1;
2796 }
2797 }
2798 }
2799 Ok(ModuleControlResponseToModule::LiveRoots {
2800 roots: roots
2801 .into_iter()
2802 .map(|(project_root, (bound, pending))| LiveRoot {
2803 project_root,
2804 bound,
2805 pending,
2806 })
2807 .collect(),
2808 unknown_root_bindings,
2809 total_bindings,
2810 })
2811 }
2812
2813 pub(crate) fn connection_has_client_routes(
2819 &self,
2820 connection_id: ConnectionId,
2821 ) -> Result<bool, ForwardingError> {
2822 let inner = self.read_inner()?;
2823 Ok(connection_has_client_routes_locked(&inner, connection_id))
2824 }
2825
2826 pub(crate) fn cleanup_connection(
2827 &self,
2828 connection_id: ConnectionId,
2829 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2830 self.cleanup_connection_counted(connection_id)
2831 .map(|cleanup| cleanup.released)
2832 }
2833
2834 pub(crate) fn cleanup_connection_counted(
2839 &self,
2840 connection_id: ConnectionId,
2841 ) -> Result<ConnectionCleanup, ForwardingError> {
2842 self.frame_drop_warnings
2843 .lock()
2844 .unwrap_or_else(std::sync::PoisonError::into_inner)
2845 .remove(&connection_id);
2846 let mut inner = self.write_inner()?;
2847 inner.closing_connections.insert(connection_id);
2848 let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
2849 remove_module_connection_locked(&mut inner, endpoint)
2850 } else {
2851 ConnectionCleanup {
2852 released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
2853 abandoned_relays: 0,
2854 }
2855 };
2856 inner.closing_connections.remove(&connection_id);
2865 Ok(cleanup)
2866 }
2867
2868 fn cleanup_client_connection_locked(
2869 inner: &mut ForwardingInner,
2870 connection_id: ConnectionId,
2871 ) -> Vec<GoodbyeTarget> {
2872 let routes = inner
2873 .client_to_module
2874 .iter()
2875 .filter(|(key, _)| key.connection_id == connection_id)
2876 .map(|(key, route)| (*key, route.client_epoch))
2877 .collect::<Vec<_>>();
2878 let mut released = Vec::with_capacity(routes.len());
2879 for (client_key, epoch) in routes {
2880 if let RouteRelease::Removed(target) =
2881 release_client_route_locked(inner, client_key, epoch)
2882 {
2883 released.push(target);
2884 }
2885 }
2886
2887 let pending_keys = inner
2888 .pending_relays
2889 .iter()
2890 .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
2891 .map(|(key, _)| *key)
2892 .collect::<Vec<_>>();
2893 for key in pending_keys {
2894 let Some(pending) = inner.pending_relays.remove(&key) else {
2895 continue;
2896 };
2897 release_reserved_route_locked(
2898 inner,
2899 pending.reservation.client_key,
2900 pending.reservation.module_key,
2901 );
2902 if pending.relay_enqueued {
2903 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2904 released.push(target);
2905 }
2906 } else {
2907 inner.relays_released_before_enqueue.insert(key);
2911 }
2912 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
2913 "client connection closed during route.bind relay".to_string(),
2914 ));
2915 }
2916
2917 let orphaned = inner
2918 .reserved_client
2919 .iter()
2920 .filter(|(key, _)| key.connection_id == connection_id)
2921 .map(|(client, module)| (*client, *module))
2922 .collect::<Vec<_>>();
2923 for (client_key, module_key) in orphaned {
2924 release_reserved_route_locked(inner, client_key, module_key);
2925 }
2926 inner.next_client_channel.remove(&connection_id);
2927 inner
2928 .client_slot_epochs
2929 .retain(|key, _| key.connection_id != connection_id);
2930 inner
2931 .last_published_epoch
2932 .retain(|key, _| key.connection_id != connection_id);
2933 inner
2934 .status
2935 .retain(|(key, _), _| key.connection_id != connection_id);
2936
2937 released
2938 }
2939
2940 pub(crate) fn escalate_client_delivery_failure(
2949 &self,
2950 connection_id: ConnectionId,
2951 channel: u16,
2952 expected_epoch: u32,
2953 reason: CloseReason,
2954 undelivered: UndeliveredFrame<'_>,
2955 ) -> Result<bool, ForwardingError> {
2956 self.escalate_client_delivery_failure_diagnosed(
2957 connection_id,
2958 channel,
2959 expected_epoch,
2960 reason,
2961 undelivered,
2962 None,
2963 )
2964 }
2965
2966 pub(crate) fn escalate_client_delivery_failure_diagnosed(
2967 &self,
2968 connection_id: ConnectionId,
2969 channel: u16,
2970 expected_epoch: u32,
2971 reason: CloseReason,
2972 undelivered: UndeliveredFrame<'_>,
2973 diagnosis: Option<&crate::router::ClientDeliveryDiagnosis>,
2974 ) -> Result<bool, ForwardingError> {
2975 let principals = {
2976 let mut inner = self.write_inner()?;
2977 let key = ClientRouteKey {
2978 connection_id,
2979 channel,
2980 };
2981 if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
2982 None
2983 } else {
2984 inner.closing_connections.insert(connection_id);
2985 Some(connection_principals_locked(&inner, connection_id))
2986 }
2987 };
2988 let Some(principals) = principals else {
2989 self.warn_frame_drop(
2993 connection_id,
2994 channel,
2995 expected_epoch,
2996 "module_to_client",
2997 diagnosis
2998 .map(|failure| failure.cause)
2999 .unwrap_or("delivery_failed"),
3000 );
3001 return Ok(false);
3002 };
3003 let backlog = undelivered.sink.backlog();
3004 let close_reason = reason.to_string();
3005 if self.request_connection_close(connection_id, reason) {
3006 warn!(
3007 connection_id = connection_id.get(),
3008 principals = %principals,
3009 module_id = undelivered.module_id.unwrap_or("unknown"),
3010 client_channel = channel,
3011 queued_bytes = backlog.queued_bytes,
3012 queued_frames = backlog.queued_frames,
3013 frame_type = ?diagnosis.map(|failure| failure.frame_type),
3014 frame_flags = ?diagnosis.map(|failure| failure.flags.0),
3015 close_cause = diagnosis.map(|failure| failure.cause).unwrap_or("delivery_failed"),
3016 oldest_queued_ms = backlog
3017 .oldest_age
3018 .map(|age| age.as_millis() as u64)
3019 .unwrap_or(0),
3020 close_reason = %close_reason,
3021 "closing client connection: its egress queue could not take a frame"
3022 );
3023 }
3024 Ok(true)
3025 }
3026
3027 pub(crate) fn publish_scope_changes(
3041 &self,
3042 changes: &[ScopeTagChange],
3043 ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
3044 let mut inner = self.write_inner()?;
3045 let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
3046 for change in changes {
3047 let key = (change.owner.clone(), change.scope_ref.clone());
3048 match change.after {
3049 Some(tag) => {
3050 inner.scope_tags.insert(key, tag);
3051 }
3052 None => {
3053 inner.scope_tags.remove(&key);
3054 }
3055 }
3056 by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
3057 }
3058 let mut selected = Vec::new();
3059 for (client_key, route) in &inner.client_to_module {
3060 let Some(scope) = &route.scope else {
3061 continue;
3062 };
3063 let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
3064 else {
3065 continue;
3066 };
3067 if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
3068 continue;
3069 }
3070 let reason = match &change.drain {
3071 ScopeDrain::Nothing => continue,
3072 ScopeDrain::All(reason) => *reason,
3073 ScopeDrain::Carriers(narrowed) => {
3074 let owner = Principal::Reserved {
3075 module_id: scope.owner.clone(),
3076 };
3077 let hit = route.principal != owner
3078 && narrowed.iter().any(|(principal, allowed)| {
3079 *principal == route.principal
3080 && allowed
3081 .as_ref()
3082 .is_none_or(|targets| !targets.contains(&route.module_id))
3083 });
3084 if !hit {
3085 continue;
3086 }
3087 RouteCloseReason::ScopeCarrierRemoved
3088 }
3089 };
3090 selected.push((*client_key, route.client_epoch, reason));
3091 }
3092 let mut drained = Vec::new();
3093 for (client_key, client_epoch, reason) in selected {
3094 let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
3095 continue;
3096 };
3097 let release = release_client_route_locked(&mut inner, client_key, client_epoch);
3098 self.record_route_release(&release);
3099 if let RouteRelease::Removed(module) = release {
3100 drained.push(ScopeDrainedRoute {
3101 reason,
3102 scope: route
3103 .scope
3104 .clone()
3105 .expect("selected scope drain has a bound scope"),
3106 module_id: route.module_id.clone(),
3107 client: GoodbyeTarget {
3108 connection_id: route.client_connection_id,
3109 sink: route.client_sink.clone(),
3110 negotiated_ver: route.client_negotiated_ver,
3111 channel: route.client_channel,
3112 epoch: route.client_epoch,
3113 kind: GoodbyeTargetKind::Client,
3114 module_id: Some(route.module_id.clone()),
3115 },
3116 module,
3117 });
3118 }
3119 }
3120 Ok(drained)
3121 }
3122
3123 #[cfg(test)]
3125 pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
3126 self.read_inner()
3127 .ok()?
3128 .scope_tags
3129 .get(&(owner.to_string(), scope_ref.to_string()))
3130 .copied()
3131 }
3132
3133 fn record_route_release(&self, release: &RouteRelease) {
3134 match release {
3135 RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
3136 RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
3137 RouteRelease::Absent => {}
3138 }
3139 }
3140
3141 fn read_inner(&self) -> Result<RwLockReadGuard<'_, ForwardingInner>, ForwardingError> {
3142 self.inner.read().map_err(|_| ForwardingError::Poisoned)
3143 }
3144
3145 fn write_inner(&self) -> Result<RwLockWriteGuard<'_, ForwardingInner>, ForwardingError> {
3146 self.inner.write().map_err(|_| ForwardingError::Poisoned)
3147 }
3148
3149 fn lock_close_registry(
3150 &self,
3151 ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
3152 self.close_registry
3153 .lock()
3154 .unwrap_or_else(|poisoned| poisoned.into_inner())
3155 }
3156}
3157
3158impl ForwardingInner {
3159 fn allocate_route_slots(
3160 &mut self,
3161 connection_id: ConnectionId,
3162 endpoint: ModuleEndpointId,
3163 ) -> Result<(u16, u32, u16, u32), ForwardingError> {
3164 let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
3165 let mut client_channel = client_start;
3166 let client_channel = loop {
3167 let key = ClientRouteKey {
3168 connection_id,
3169 channel: client_channel,
3170 };
3171 let eligible = !self.client_to_module.contains_key(&key)
3172 && !self.reserved_client.contains_key(&key)
3173 && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3174 if eligible {
3175 break client_channel;
3176 }
3177 client_channel = next_channel(client_channel);
3178 if client_channel == client_start {
3179 return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
3180 }
3181 };
3182
3183 let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
3184 let mut module_channel = module_start;
3185 let module_channel = loop {
3186 let key = ModuleRouteKey {
3187 endpoint,
3188 channel: module_channel,
3189 };
3190 let eligible = !self.module_to_client.contains_key(&key)
3191 && !self.reserved_module.contains_key(&key)
3192 && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3193 if eligible {
3194 break module_channel;
3195 }
3196 module_channel = next_channel(module_channel);
3197 if module_channel == module_start {
3198 return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
3199 }
3200 };
3201
3202 let client_key = ClientRouteKey {
3203 connection_id,
3204 channel: client_channel,
3205 };
3206 let module_key = ModuleRouteKey {
3207 endpoint,
3208 channel: module_channel,
3209 };
3210 let client_epoch = self
3211 .client_slot_epochs
3212 .get(&client_key)
3213 .copied()
3214 .unwrap_or(0)
3215 + 1;
3216 let module_epoch = self
3217 .module_slot_epochs
3218 .get(&module_key)
3219 .copied()
3220 .unwrap_or(0)
3221 + 1;
3222 self.client_slot_epochs.insert(client_key, client_epoch);
3223 self.module_slot_epochs.insert(module_key, module_epoch);
3224 self.next_client_channel
3225 .insert(connection_id, next_channel(client_channel));
3226 self.next_module_channel
3227 .insert(endpoint, next_channel(module_channel));
3228 Ok((client_channel, client_epoch, module_channel, module_epoch))
3229 }
3230
3231 fn allocate_control_corr(
3232 &mut self,
3233 endpoint: ModuleEndpointId,
3234 ) -> Result<u64, ForwardingError> {
3235 let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
3236 if candidate == 0 {
3237 self.closing_connections.insert(endpoint.connection_id);
3238 return Err(ForwardingError::RelayCorrelationExhausted);
3239 }
3240 self.next_control_corr.insert(
3241 endpoint,
3242 if candidate == u64::MAX {
3243 0
3244 } else {
3245 candidate + 1
3246 },
3247 );
3248 Ok(candidate)
3249 }
3250}
3251
3252fn connection_has_client_routes_locked(
3253 inner: &ForwardingInner,
3254 connection_id: ConnectionId,
3255) -> bool {
3256 inner
3257 .client_to_module
3258 .keys()
3259 .any(|key| key.connection_id == connection_id)
3260 || inner
3261 .reserved_client
3262 .keys()
3263 .any(|key| key.connection_id == connection_id)
3264}
3265
3266fn check_module_connection_role_locked(
3267 inner: &ForwardingInner,
3268 connection_id: ConnectionId,
3269) -> Result<(), ForwardingError> {
3270 if inner.endpoint_by_connection.contains_key(&connection_id)
3271 || connection_has_client_routes_locked(inner, connection_id)
3272 {
3273 return Err(ForwardingError::ConnectionRoleConflict { connection_id });
3274 }
3275 Ok(())
3276}
3277
3278fn next_channel(channel: u16) -> u16 {
3279 let next = channel.wrapping_add(1);
3280 if next == 0 {
3281 1
3282 } else {
3283 next
3284 }
3285}
3286
3287fn endpoint_routes_locked(
3288 inner: &ForwardingInner,
3289 endpoint: ModuleEndpointId,
3290) -> Vec<EndpointRoute> {
3291 let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
3292 let draining = drain_reason.is_some();
3293 let mut routes = inner
3294 .module_to_client
3295 .iter()
3296 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3297 .map(|(_, route)| EndpointRoute {
3298 goodbye_target: GoodbyeTarget {
3299 connection_id: route.client_connection_id,
3300 sink: route.client_sink.clone(),
3301 negotiated_ver: route.client_negotiated_ver,
3302 channel: route.client_channel,
3303 epoch: route.client_epoch,
3304 kind: GoodbyeTargetKind::Client,
3305 module_id: Some(route.module_id.clone()),
3306 },
3307 principal: route.principal.clone(),
3308 bound_at: route.bound_at,
3309 draining,
3310 drain_reason,
3311 })
3312 .collect::<Vec<_>>();
3313 routes.sort_by_key(|route| {
3314 (
3315 route.goodbye_target.connection_id.get(),
3316 route.goodbye_target.channel,
3317 route.goodbye_target.epoch,
3318 )
3319 });
3320 routes
3321}
3322
3323fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
3327 let key = (scope.owner.clone(), scope.scope_ref.clone());
3328 match inner.scope_tags.get(&key) {
3329 Some(current) if *current == scope.tag => None,
3330 Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
3331 subc_protocol::error_codes::SCOPE_CHANGED,
3332 format!(
3333 "scope '{}' of {} changed while the route was being bound; re-open it",
3334 scope.scope_ref, scope.owner
3335 ),
3336 )),
3337 _ => Some(ErrorBody::new(
3338 subc_protocol::error_codes::SCOPE_ENDED,
3339 format!(
3340 "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
3341 scope.scope_ref, scope.owner, scope.tag.scope_epoch
3342 ),
3343 )),
3344 }
3345}
3346
3347fn release_reserved_route_locked(
3348 inner: &mut ForwardingInner,
3349 client_key: ClientRouteKey,
3350 module_key: ModuleRouteKey,
3351) {
3352 if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
3353 inner.reserved_client.remove(&client_key);
3354 }
3355 if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
3356 inner.reserved_module.remove(&module_key);
3357 }
3358 inner.status.retain(|(key, _), _| *key != client_key);
3359}
3360
3361fn release_client_route_locked(
3362 inner: &mut ForwardingInner,
3363 client_key: ClientRouteKey,
3364 expected_epoch: u32,
3365) -> RouteRelease {
3366 let Some(route) = inner.client_to_module.get(&client_key) else {
3367 return RouteRelease::Absent;
3368 };
3369 if route.client_epoch != expected_epoch {
3370 return RouteRelease::Stale;
3371 }
3372 let route = inner
3373 .client_to_module
3374 .remove(&client_key)
3375 .expect("route checked under the same forwarding lock");
3376 route.flow.close();
3377 if let Some(scope) = &route.scope {
3378 scope.release_bind();
3379 }
3380 inner.module_to_client.remove(&ModuleRouteKey {
3381 endpoint: route.module_endpoint,
3382 channel: route.module_channel,
3383 });
3384 inner.operator_confirms.route_closed(ModuleRouteKey {
3385 endpoint: route.module_endpoint,
3386 channel: route.module_channel,
3387 });
3388 inner.status.remove(&(client_key, expected_epoch));
3389 RouteRelease::Removed(GoodbyeTarget {
3390 connection_id: route.module_endpoint.connection_id,
3391 sink: route.module_sink.clone(),
3392 negotiated_ver: route.module_negotiated_ver,
3393 channel: route.module_channel,
3394 epoch: route.module_epoch,
3395 kind: GoodbyeTargetKind::Module,
3396 module_id: Some(route.module_id.clone()),
3397 })
3398}
3399
3400fn release_module_route_locked(
3401 inner: &mut ForwardingInner,
3402 module_key: ModuleRouteKey,
3403 expected_epoch: u32,
3404) -> RouteRelease {
3405 let Some(route) = inner.module_to_client.get(&module_key) else {
3406 return RouteRelease::Absent;
3407 };
3408 if route.module_epoch != expected_epoch {
3409 return RouteRelease::Stale;
3410 }
3411 let route = inner
3412 .module_to_client
3413 .remove(&module_key)
3414 .expect("route checked under the same forwarding lock");
3415 inner.operator_confirms.route_closed(module_key);
3416 route.flow.close();
3417 if let Some(scope) = &route.scope {
3418 scope.release_bind();
3419 }
3420 let client_key = ClientRouteKey {
3421 connection_id: route.client_connection_id,
3422 channel: route.client_channel,
3423 };
3424 inner.client_to_module.remove(&client_key);
3425 inner.status.remove(&(client_key, route.client_epoch));
3426 RouteRelease::Removed(GoodbyeTarget {
3427 connection_id: route.client_connection_id,
3428 sink: route.client_sink.clone(),
3429 negotiated_ver: route.client_negotiated_ver,
3430 channel: route.client_channel,
3431 epoch: route.client_epoch,
3432 kind: GoodbyeTargetKind::Client,
3433 module_id: Some(route.module_id.clone()),
3434 })
3435}
3436
3437fn commit_route_locked(
3438 inner: &mut ForwardingInner,
3439 mut pending: PendingRouteBindRelayEntry,
3440) -> Result<Option<GoodbyeTarget>, ForwardingError> {
3441 let reservation = pending.reservation;
3442 if inner
3443 .closing_connections
3444 .contains(&reservation.client_key.connection_id)
3445 {
3446 return Err(ForwardingError::ConnectionClosing {
3447 connection_id: reservation.client_key.connection_id,
3448 });
3449 }
3450 let module_id = inner
3451 .module_id_by_endpoint
3452 .get(&reservation.module_key.endpoint)
3453 .cloned()
3454 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3455 if inner
3456 .draining_endpoints
3457 .contains_key(&reservation.module_key.endpoint)
3458 {
3459 return Err(ForwardingError::ModuleReloading { module_id });
3460 }
3461 if pending.busy_refusals {
3462 let body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
3463 route_channel: reservation.client_key.channel,
3464 route_epoch: reservation.client_epoch,
3465 request_window: Some(pending.request_window),
3466 watch_window: pending.watch_window,
3467 })
3468 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
3469 pending.route_open_frame = Frame::build_with_version(
3470 pending.client_negotiated_ver,
3471 FrameType::Response,
3472 pending.route_open_frame.header.flags,
3473 0,
3474 0,
3475 pending.route_open_frame.header.corr,
3476 body,
3477 )
3478 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
3479 }
3480 if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
3481 || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
3482 {
3483 return Err(ForwardingError::UnknownReservation {
3484 client_channel: reservation.client_key.channel,
3485 module_channel: reservation.module_key.channel,
3486 });
3487 }
3488 let module = inner
3489 .modules_by_id
3490 .get(&module_id)
3491 .filter(|module| module.endpoint == reservation.module_key.endpoint)
3492 .cloned()
3493 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3494 let binding = Arc::new(RouteBinding {
3495 client_connection_id: reservation.client_key.connection_id,
3496 client_sink: pending.client_sink,
3497 client_negotiated_ver: pending.client_negotiated_ver,
3498 client_channel: reservation.client_key.channel,
3499 client_epoch: reservation.client_epoch,
3500 module_id,
3501 module_endpoint: reservation.module_key.endpoint,
3502 module_sink: module.sink,
3503 module_negotiated_ver: module.negotiated_ver,
3504 module_channel: reservation.module_key.channel,
3505 module_epoch: reservation.module_epoch,
3506 principal: pending.principal,
3507 harness: pending.harness,
3508 build_class: pending.build_class,
3509 operator_app: pending.operator_app,
3510 project_root: reservation.project_root.clone(),
3511 bound_at: Instant::now(),
3512 flow: Arc::new(if pending.busy_refusals {
3513 ChannelFlow::with_admission(pending.request_window, true, pending.watch_window)
3514 } else {
3515 ChannelFlow::new(pending.request_window)
3516 }),
3517 scope: pending.scope,
3518 });
3519 inner
3520 .client_to_module
3521 .insert(reservation.client_key, Arc::clone(&binding));
3522 inner
3523 .module_to_client
3524 .insert(reservation.module_key, Arc::clone(&binding));
3525 let previous_published = inner
3526 .last_published_epoch
3527 .insert(reservation.client_key, reservation.client_epoch);
3528
3529 if let Some(scope) = &binding.scope {
3533 scope.record_served(&binding.module_id);
3534 }
3535
3536 let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
3541 if client_writer_closed {
3542 let abandoned = abandoned_route_target(inner, &reservation);
3546 if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
3547 route.flow.close();
3548 }
3549 inner.module_to_client.remove(&reservation.module_key);
3550 inner
3551 .status
3552 .remove(&(reservation.client_key, reservation.client_epoch));
3553 match previous_published {
3554 Some(epoch) => {
3555 inner
3556 .last_published_epoch
3557 .insert(reservation.client_key, epoch);
3558 }
3559 None => {
3560 inner.last_published_epoch.remove(&reservation.client_key);
3561 }
3562 }
3563 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3564 "client egress closed during route publication".to_string(),
3565 ));
3566 return Ok(abandoned);
3567 }
3568
3569 if let Some(scope) = &binding.scope {
3570 scope.commit_presence();
3571 }
3572 let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
3573 Ok(None)
3574}
3575
3576fn module_connection_for_endpoint_locked(
3584 inner: &ForwardingInner,
3585 endpoint: ModuleEndpointId,
3586) -> Option<&ModuleConnection> {
3587 let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
3588 inner
3589 .modules_by_id
3590 .get(module_id)
3591 .filter(|module| module.endpoint == endpoint)
3592 .or_else(|| {
3593 inner
3594 .candidates_by_id
3595 .get(module_id)
3596 .filter(|module| module.endpoint == endpoint)
3597 })
3598 .or_else(|| inner.superseded_endpoints.get(&endpoint))
3599}
3600
3601fn abandoned_route_target(
3602 inner: &ForwardingInner,
3603 reservation: &RouteReservation,
3604) -> Option<GoodbyeTarget> {
3605 let module_id = inner
3606 .module_id_by_endpoint
3607 .get(&reservation.module_key.endpoint)?;
3608 let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
3609 (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
3610 connection_id: module.endpoint.connection_id,
3611 sink: module.sink.clone(),
3612 negotiated_ver: module.negotiated_ver,
3613 channel: reservation.module_key.channel,
3614 epoch: reservation.module_epoch,
3615 kind: GoodbyeTargetKind::Module,
3616 module_id: Some(module_id.clone()),
3617 })
3618}
3619
3620fn enqueue_hello_ack_locked(
3625 sink: &FrameSink,
3626 connection_id: ConnectionId,
3627 hello_ack: Option<Frame>,
3628) -> Result<(), ForwardingError> {
3629 let Some(hello_ack) = hello_ack else {
3630 return Ok(());
3631 };
3632 sink.try_send(hello_ack)
3633 .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
3634}
3635
3636fn remove_module_connection_locked(
3637 inner: &mut ForwardingInner,
3638 endpoint: ModuleEndpointId,
3639) -> ConnectionCleanup {
3640 inner.operator_confirms.module_closed(endpoint);
3643 inner.draining_endpoints.remove(&endpoint);
3644 inner.daemon_drain_results.remove(&endpoint);
3645 let module_id = inner.module_id_by_endpoint.remove(&endpoint);
3646 if let Some(module_id) = module_id.as_ref() {
3647 if inner
3648 .modules_by_id
3649 .get(module_id)
3650 .is_some_and(|module| module.endpoint == endpoint)
3651 {
3652 inner.modules_by_id.remove(module_id);
3653 }
3654 if inner
3655 .candidates_by_id
3656 .get(module_id)
3657 .is_some_and(|module| module.endpoint == endpoint)
3658 {
3659 inner.candidates_by_id.remove(module_id);
3660 }
3661 }
3662 inner.superseded_endpoints.remove(&endpoint);
3663 inner.endpoint_by_connection.remove(&endpoint.connection_id);
3664 inner.next_module_channel.remove(&endpoint);
3665 inner.next_control_corr.remove(&endpoint);
3666 inner
3667 .health_probe_tombstones
3668 .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
3669 inner
3670 .module_slot_epochs
3671 .retain(|key, _| key.endpoint != endpoint);
3672 let reserved_module_keys: Vec<ModuleRouteKey> = inner
3673 .reserved_module
3674 .keys()
3675 .filter(|module_key| module_key.endpoint == endpoint)
3676 .copied()
3677 .collect();
3678 for module_key in reserved_module_keys {
3679 if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
3680 release_reserved_route_locked(inner, client_key, module_key);
3681 }
3682 }
3683
3684 let pending_keys: Vec<_> = inner
3685 .pending_relays
3686 .keys()
3687 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3688 .copied()
3689 .collect();
3690 let pending: Vec<_> = pending_keys
3691 .into_iter()
3692 .filter_map(|key| {
3693 inner
3694 .pending_relays
3695 .remove(&key)
3696 .map(|pending| (key, pending))
3697 })
3698 .collect();
3699 let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
3700 for (key, pending) in pending {
3701 if !pending.relay_enqueued {
3705 inner.relays_released_before_enqueue.insert(key);
3706 }
3707 let module_label = module_id.as_deref().unwrap_or("unknown");
3708 let _ = pending
3709 .sender
3710 .send(RouteBindRelayOutcome::ModuleGone(format!(
3711 "module '{module_label}' connection closed during route.bind relay"
3712 )));
3713 }
3714
3715 let pending_control_keys: Vec<_> = inner
3716 .pending_control_rpcs
3717 .keys()
3718 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3719 .copied()
3720 .collect();
3721 let pending_control: Vec<_> = pending_control_keys
3722 .into_iter()
3723 .filter_map(|key| inner.pending_control_rpcs.remove(&key))
3724 .collect();
3725 for pending in pending_control {
3726 let module_label = module_id.as_deref().unwrap_or("unknown");
3727 let _ = pending
3728 .sender
3729 .send(ModuleControlRpcOutcome::ModuleGone(format!(
3730 "module '{module_label}' connection closed during module-control RPC"
3731 )));
3732 }
3733
3734 let module_routes = inner
3735 .module_to_client
3736 .iter()
3737 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3738 .map(|(module_key, route)| (*module_key, route.module_epoch))
3739 .collect::<Vec<_>>();
3740 let mut released = Vec::with_capacity(module_routes.len());
3741 for (module_key, epoch) in module_routes {
3742 if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
3743 {
3744 released.push(target);
3745 }
3746 }
3747 ConnectionCleanup {
3748 released,
3749 abandoned_relays,
3750 }
3751}
3752
3753#[derive(Debug, Clone, Copy)]
3754struct RequestCredit {
3755 subscription: bool,
3756 watch_credit: bool,
3757 excluded_from_drain: bool,
3758}
3759
3760#[derive(Debug, Default)]
3761struct CreditLedger {
3762 by_corr: HashMap<u64, Vec<RequestCredit>>,
3763}
3764
3765impl CreditLedger {
3766 fn acquire(&mut self, corr: u64, subscription: bool, watch_credit: bool) {
3767 self.by_corr.entry(corr).or_default().push(RequestCredit {
3768 subscription,
3769 watch_credit,
3770 excluded_from_drain: false,
3771 });
3772 }
3773
3774 fn release(&mut self, corr: u64) -> Option<RequestCredit> {
3775 let credits = self.by_corr.get_mut(&corr)?;
3776 let released = credits.pop();
3777 if credits.is_empty() {
3778 self.by_corr.remove(&corr);
3779 }
3780 released
3781 }
3782
3783 fn capture_subscription_exclusions(&mut self) -> u32 {
3784 let mut excluded = 0u32;
3785 for credit in self.by_corr.values_mut().flatten() {
3786 if credit.subscription && !credit.excluded_from_drain {
3787 credit.excluded_from_drain = true;
3788 excluded = excluded.saturating_add(1);
3789 }
3790 }
3791 excluded
3792 }
3793
3794 fn in_flight(&self) -> usize {
3795 self.by_corr.values().map(Vec::len).sum()
3796 }
3797
3798 fn drain_in_flight(&self) -> usize {
3799 self.by_corr
3800 .values()
3801 .flatten()
3802 .filter(|credit| !credit.excluded_from_drain)
3803 .count()
3804 }
3805
3806 fn drain_held_corrs(&self) -> Vec<u64> {
3809 let mut corrs = self
3810 .by_corr
3811 .iter()
3812 .flat_map(|(corr, credits)| {
3813 credits
3814 .iter()
3815 .filter(|credit| !credit.excluded_from_drain)
3816 .map(move |_| *corr)
3817 })
3818 .collect::<Vec<_>>();
3819 corrs.sort_unstable();
3820 corrs
3821 }
3822}
3823
3824#[derive(Debug, Default)]
3825struct ChannelFlowState {
3826 closed: bool,
3827 credits: CreditLedger,
3828 last_busy_warn: Option<tokio::time::Instant>,
3829}
3830
3831#[derive(Debug)]
3833pub(crate) struct ChannelFlow {
3834 sem: Semaphore,
3835 window: usize,
3836 busy_refusals: bool,
3837 watch_sem: Option<Semaphore>,
3838 watch_window: Option<usize>,
3839 state: Mutex<ChannelFlowState>,
3840}
3841
3842impl ChannelFlow {
3843 pub(crate) fn new(window: usize) -> Self {
3844 Self::with_admission(window, false, None)
3845 }
3846
3847 fn with_admission(window: usize, busy_refusals: bool, watch_window: Option<usize>) -> Self {
3848 debug_assert!(window > 0, "flow-control window must be non-zero");
3849 Self {
3850 sem: Semaphore::new(window),
3851 window,
3852 busy_refusals,
3853 watch_sem: watch_window.map(Semaphore::new),
3854 watch_window,
3855 state: Mutex::new(ChannelFlowState::default()),
3856 }
3857 }
3858
3859 pub(crate) fn busy_refusals(&self) -> bool {
3860 self.busy_refusals
3861 }
3862
3863 fn try_acquire_tagged(&self, corr: u64, subscription: bool) -> RequestAdmission {
3864 let mut state = self
3865 .state
3866 .lock()
3867 .unwrap_or_else(|poisoned| poisoned.into_inner());
3868 if state.closed {
3869 return RequestAdmission::Stale;
3870 }
3871 let busy = |state: &ChannelFlowState| {
3872 RequestAdmission::Busy(subc_protocol::RouteBusyDetail {
3873 reason: "no_credit".into(),
3874 request_in_flight: state.credits.in_flight(),
3875 watch_in_flight: state
3876 .credits
3877 .by_corr
3878 .values()
3879 .flatten()
3880 .filter(|credit| credit.subscription)
3881 .count(),
3882 request_window: self.window,
3883 watch_window: self.watch_window,
3884 })
3885 };
3886 let Ok(total) = self.sem.try_acquire() else {
3887 return busy(&state);
3888 };
3889 let watch = if subscription {
3892 if let Some(sem) = &self.watch_sem {
3893 match sem.try_acquire() {
3894 Ok(permit) => Some(permit),
3895 Err(_) => return busy(&state),
3896 }
3897 } else {
3898 None
3899 }
3900 } else {
3901 None
3902 };
3903 state.credits.acquire(corr, subscription, watch.is_some());
3904 total.forget();
3905 if let Some(watch) = watch {
3906 watch.forget();
3907 }
3908 RequestAdmission::Admitted
3909 }
3910
3911 pub(crate) fn should_warn_busy(&self) -> bool {
3912 let mut state = self
3913 .state
3914 .lock()
3915 .unwrap_or_else(|poisoned| poisoned.into_inner());
3916 let now = tokio::time::Instant::now();
3917 if state
3918 .last_busy_warn
3919 .is_some_and(|last| now.duration_since(last) < std::time::Duration::from_secs(60))
3920 {
3921 return false;
3922 }
3923 state.last_busy_warn = Some(now);
3924 true
3925 }
3926
3927 #[cfg(test)]
3928 pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
3929 self.acquire_tagged(0, false).await
3930 }
3931
3932 pub(crate) async fn acquire_tagged(
3933 &self,
3934 corr: u64,
3935 subscription: bool,
3936 ) -> Result<(), ChannelFlowClosed> {
3937 let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
3938 let mut state = self
3939 .state
3940 .lock()
3941 .unwrap_or_else(|poisoned| poisoned.into_inner());
3942 if state.closed {
3943 return Err(ChannelFlowClosed);
3944 }
3945 state.credits.acquire(corr, subscription, false);
3946 permit.forget();
3947 Ok(())
3948 }
3949
3950 #[cfg(test)]
3951 pub(crate) fn release(&self) {
3952 self.release_corr(0);
3953 }
3954
3955 pub(crate) fn release_corr(&self, corr: u64) {
3956 let mut state = self
3957 .state
3958 .lock()
3959 .unwrap_or_else(|poisoned| poisoned.into_inner());
3960 let Some(released) = state.credits.release(corr) else {
3961 warn!(
3965 window = self.window,
3966 available = self.sem.available_permits(),
3967 "flow-control over-release ignored"
3968 );
3969 return;
3970 };
3971 if !self.sem.is_closed() {
3972 self.sem.add_permits(1);
3973 }
3974 if released.watch_credit {
3975 if let Some(watch_sem) = self.watch_sem.as_ref().filter(|sem| !sem.is_closed()) {
3976 watch_sem.add_permits(1);
3977 }
3978 }
3979 }
3980
3981 #[cfg(test)]
3982 pub(crate) fn in_flight(&self) -> usize {
3983 self.state
3984 .lock()
3985 .unwrap_or_else(|poisoned| poisoned.into_inner())
3986 .credits
3987 .in_flight()
3988 }
3989
3990 pub(crate) fn drain_in_flight(&self) -> usize {
3991 self.state
3992 .lock()
3993 .unwrap_or_else(|poisoned| poisoned.into_inner())
3994 .credits
3995 .drain_in_flight()
3996 }
3997
3998 pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
3999 self.state
4000 .lock()
4001 .unwrap_or_else(|poisoned| poisoned.into_inner())
4002 .credits
4003 .drain_held_corrs()
4004 }
4005
4006 #[cfg(test)]
4007 pub(crate) fn available_permits(&self) -> usize {
4008 self.sem.available_permits()
4009 }
4010
4011 pub(crate) fn begin_drain(&self) -> u32 {
4012 let mut state = self
4013 .state
4014 .lock()
4015 .unwrap_or_else(|poisoned| poisoned.into_inner());
4016 state.closed = true;
4017 self.sem.close();
4018 if let Some(sem) = &self.watch_sem {
4019 sem.close();
4020 }
4021 state.credits.capture_subscription_exclusions()
4022 }
4023
4024 pub(crate) fn close(&self) {
4025 let mut state = self
4026 .state
4027 .lock()
4028 .unwrap_or_else(|poisoned| poisoned.into_inner());
4029 state.closed = true;
4030 self.sem.close();
4031 if let Some(sem) = &self.watch_sem {
4032 sem.close();
4033 }
4034 }
4035}
4036
4037#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4038pub(crate) struct ChannelFlowClosed;
4039
4040#[derive(Debug, Clone, PartialEq, Eq)]
4041pub(crate) enum RequestAdmission {
4042 Admitted,
4043 Busy(subc_protocol::RouteBusyDetail),
4044 ModuleReloading,
4045 Stale,
4046}
4047
4048impl fmt::Display for ChannelFlowClosed {
4049 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4050 write!(f, "flow-control window closed")
4051 }
4052}
4053
4054impl Error for ChannelFlowClosed {}
4055
4056#[cfg(test)]
4057#[path = "route_busy_flow_tests.rs"]
4058mod route_busy_flow_tests;
4059
4060fn window_for(concurrency: &Concurrency) -> usize {
4061 match concurrency {
4062 Concurrency::Serial => 1,
4063 Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
4064 Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
4065 }
4066}
4067
4068#[derive(Debug, Clone, PartialEq, Eq)]
4069pub enum ForwardingError {
4070 ConnectionRoleConflict {
4073 connection_id: ConnectionId,
4074 },
4075 NoModuleConnection,
4076 ModuleReloading {
4077 module_id: String,
4078 },
4079 StaleModuleEndpoint,
4080 UnknownReservation {
4081 client_channel: u16,
4082 module_channel: u16,
4083 },
4084 ClientRouteChannelExhausted {
4085 connection_id: ConnectionId,
4086 },
4087 ModuleRouteChannelExhausted {
4088 endpoint: ModuleEndpointId,
4089 },
4090 RelayCorrelationExhausted,
4091 ConnectionClosing {
4092 connection_id: ConnectionId,
4093 },
4094 ClientEgressClosed {
4095 connection_id: ConnectionId,
4096 },
4097 RouteOpenBuild(String),
4098 CandidateSlotOccupied {
4100 module_id: String,
4101 },
4102 ModuleEgressUnavailable {
4105 connection_id: ConnectionId,
4106 },
4107 Poisoned,
4108}
4109
4110impl fmt::Display for ForwardingError {
4111 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4112 match self {
4113 Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
4114 Self::NoModuleConnection => write!(f, "no module connection is registered"),
4115 Self::ModuleReloading { module_id } => {
4116 write!(f, "module_id '{module_id}' is reloading")
4117 }
4118 Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
4119 Self::UnknownReservation {
4120 client_channel,
4121 module_channel,
4122 } => write!(
4123 f,
4124 "route reservation client channel {client_channel} / module channel {module_channel} was not found"
4125 ),
4126 Self::ClientRouteChannelExhausted { connection_id } => write!(
4127 f,
4128 "no client route channels are available for connection {}",
4129 connection_id.get()
4130 ),
4131 Self::ModuleRouteChannelExhausted { endpoint } => write!(
4132 f,
4133 "no module route channels are available for endpoint generation {} on connection {}",
4134 endpoint.generation,
4135 endpoint.connection_id.get()
4136 ),
4137 Self::RelayCorrelationExhausted => {
4138 write!(f, "module control correlation ids are exhausted")
4139 }
4140 Self::ConnectionClosing { connection_id } => write!(
4141 f,
4142 "connection {} is closing and cannot accept route allocation",
4143 connection_id.get()
4144 ),
4145 Self::ClientEgressClosed { connection_id } => write!(
4146 f,
4147 "client connection {} egress is closed",
4148 connection_id.get()
4149 ),
4150 Self::RouteOpenBuild(message) => {
4151 write!(f, "failed to prebuild route.open response: {message}")
4152 }
4153 Self::CandidateSlotOccupied { module_id } => write!(
4154 f,
4155 "module_id '{module_id}' already has a swap candidate registered"
4156 ),
4157 Self::ModuleEgressUnavailable { connection_id } => write!(
4158 f,
4159 "module connection {} egress is unavailable; HELLO_ACK could not be queued",
4160 connection_id.get()
4161 ),
4162 Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
4163 }
4164 }
4165}
4166
4167impl Error for ForwardingError {}
4168
4169#[cfg(test)]
4170mod tests {
4171 use std::time::Duration;
4172
4173 use super::*;
4174 use tokio::sync::mpsc;
4175
4176 #[test]
4177 fn ordinary_long_running_request_is_not_excluded_from_drain() {
4178 let mut ledger = CreditLedger::default();
4179 ledger.acquire(1, false, false);
4180
4181 assert_eq!(ledger.capture_subscription_exclusions(), 0);
4182 assert_eq!(ledger.drain_in_flight(), 1);
4183 }
4184
4185 #[test]
4186 fn bit_set_subscription_is_excluded_and_counted() {
4187 let mut ledger = CreditLedger::default();
4188 ledger.acquire(1, true, false);
4189
4190 assert_eq!(ledger.capture_subscription_exclusions(), 1);
4191 assert_eq!(ledger.drain_in_flight(), 0);
4192 }
4193
4194 #[test]
4195 fn subscription_opened_after_drain_snapshot_is_not_excluded() {
4196 let mut ledger = CreditLedger::default();
4197 ledger.acquire(1, true, false);
4198 assert_eq!(ledger.capture_subscription_exclusions(), 1);
4199
4200 ledger.acquire(2, true, false);
4201
4202 assert_eq!(ledger.drain_in_flight(), 1);
4203 }
4204
4205 #[test]
4206 fn drain_with_no_subscriptions_reports_zero_excluded() {
4207 let mut ledger = CreditLedger::default();
4208 assert_eq!(ledger.capture_subscription_exclusions(), 0);
4209 }
4210
4211 fn test_hello_ack(corr: u64) -> Frame {
4212 Frame::build(
4213 FrameType::HelloAck,
4214 Flags::new(false, Priority::Passive, false),
4215 0,
4216 0,
4217 corr,
4218 Vec::new(),
4219 )
4220 .unwrap()
4221 }
4222
4223 #[test]
4227 fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
4228 let forwarding = ForwardingTable::default();
4229
4230 let (closed_tx, closed_rx) = mpsc::channel(8);
4231 drop(closed_rx);
4232 let closed = ConnectionId::new(1);
4233 assert_eq!(
4234 forwarding.register_module_connection_acked(
4235 closed,
4236 "closed".to_string(),
4237 2,
4238 Concurrency::ModuleManaged,
4239 FrameSink::new(closed_tx),
4240 test_hello_ack(1),
4241 ),
4242 Err(ForwardingError::ModuleEgressUnavailable {
4243 connection_id: closed
4244 })
4245 );
4246
4247 let (full_tx, _full_rx) = mpsc::channel(1);
4248 let full_sink = FrameSink::new(full_tx);
4249 full_sink.try_send(test_hello_ack(99)).unwrap();
4250 let full = ConnectionId::new(2);
4251 assert_eq!(
4252 forwarding.register_module_connection_acked(
4253 full,
4254 "full".to_string(),
4255 2,
4256 Concurrency::ModuleManaged,
4257 full_sink.clone(),
4258 test_hello_ack(2),
4259 ),
4260 Err(ForwardingError::ModuleEgressUnavailable {
4261 connection_id: full
4262 })
4263 );
4264 assert_eq!(
4265 forwarding.register_candidate_module_connection_acked(
4266 full,
4267 "full".to_string(),
4268 2,
4269 Concurrency::ModuleManaged,
4270 full_sink,
4271 test_hello_ack(3),
4272 ),
4273 Err(ForwardingError::ModuleEgressUnavailable {
4274 connection_id: full
4275 })
4276 );
4277
4278 for (connection, module_id) in [(closed, "closed"), (full, "full")] {
4279 assert_eq!(
4280 forwarding
4281 .module_endpoint_for_connection(connection)
4282 .unwrap(),
4283 None
4284 );
4285 let (client_tx, _client_rx) = mpsc::channel(8);
4286 assert_eq!(
4287 forwarding
4288 .begin_route_bind_relay_for_test(
4289 ConnectionId::new(50),
4290 FrameSink::new(client_tx),
4291 1,
4292 module_id,
4293 )
4294 .err(),
4295 Some(ForwardingError::NoModuleConnection)
4296 );
4297 }
4298 assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
4299 }
4300
4301 #[test]
4304 fn acked_registration_queues_the_hello_ack_first() {
4305 let forwarding = ForwardingTable::default();
4306 let (active_tx, mut active_rx) = mpsc::channel(8);
4307 forwarding
4308 .register_module_connection_acked(
4309 ConnectionId::new(1),
4310 "acked".to_string(),
4311 2,
4312 Concurrency::ModuleManaged,
4313 FrameSink::new(active_tx),
4314 test_hello_ack(11),
4315 )
4316 .unwrap();
4317 let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
4318 forwarding
4319 .register_candidate_module_connection_acked(
4320 ConnectionId::new(2),
4321 "acked".to_string(),
4322 2,
4323 Concurrency::ModuleManaged,
4324 FrameSink::new(candidate_tx),
4325 test_hello_ack(12),
4326 )
4327 .unwrap();
4328
4329 let active_first = active_rx.try_recv().unwrap().frame;
4330 assert_eq!(active_first.header.ty, FrameType::HelloAck);
4331 assert_eq!(active_first.header.corr, 11);
4332 let candidate_first = candidate_rx.try_recv().unwrap().frame;
4333 assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
4334 assert_eq!(candidate_first.header.corr, 12);
4335 }
4336
4337 #[test]
4338 fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
4339 let forwarding = ForwardingTable::default();
4340 let module_connection = ConnectionId::new(10);
4341 let exhausted_client = ConnectionId::new(20);
4342 let second_client = ConnectionId::new(30);
4343 let (module_tx, _module_rx) = mpsc::channel(1);
4344 let endpoint = forwarding
4345 .register_module_connection(
4346 module_connection,
4347 "route-limit-provider".to_string(),
4348 1,
4349 Concurrency::ModuleManaged,
4350 FrameSink::new(module_tx),
4351 )
4352 .unwrap();
4353
4354 {
4355 let mut inner = forwarding.inner.write().unwrap();
4356 for channel in 1..=u16::MAX {
4357 inner.reserved_client.insert(
4358 ClientRouteKey {
4359 connection_id: exhausted_client,
4360 channel,
4361 },
4362 ModuleRouteKey {
4363 endpoint,
4364 channel: 1,
4365 },
4366 );
4367 }
4368 }
4369
4370 let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
4371 let err = forwarding
4372 .begin_route_bind_relay_for_test(
4373 exhausted_client,
4374 FrameSink::new(exhausted_tx),
4375 1,
4376 "route-limit-provider",
4377 )
4378 .unwrap_err();
4379 assert!(matches!(
4380 err,
4381 ForwardingError::ClientRouteChannelExhausted { connection_id }
4382 if connection_id == exhausted_client
4383 ));
4384
4385 let (second_tx, _second_rx) = mpsc::channel(1);
4386 let pending = forwarding
4387 .begin_route_bind_relay_for_test(
4388 second_client,
4389 FrameSink::new(second_tx),
4390 2,
4391 "route-limit-provider",
4392 )
4393 .unwrap();
4394 assert_eq!(pending.client_channel, 1);
4395 }
4396
4397 #[test]
4398 fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
4399 let forwarding = ForwardingTable::default();
4400 let module_connection = ConnectionId::new(40);
4401 let client = ConnectionId::new(50);
4402 let (module_tx, _module_rx) = mpsc::channel(1);
4403 forwarding
4404 .register_module_connection(
4405 module_connection,
4406 "slot-reuse-provider".to_string(),
4407 1,
4408 Concurrency::ModuleManaged,
4409 FrameSink::new(module_tx),
4410 )
4411 .unwrap();
4412
4413 let (client_tx, _client_rx) = mpsc::channel(1);
4414 let client_sink = FrameSink::new(client_tx);
4415 let mut wrapped_channel = None;
4416 for index in 0..=usize::from(u16::MAX) {
4417 let pending = forwarding
4418 .begin_route_bind_relay_for_test(
4419 client,
4420 client_sink.clone(),
4421 index as u64 + 1,
4422 "slot-reuse-provider",
4423 )
4424 .unwrap();
4425 if index == usize::from(u16::MAX) {
4426 wrapped_channel = Some(pending.module_channel);
4427 }
4428 forwarding
4429 .abort_pending_relay(
4430 pending.endpoint,
4431 pending.corr,
4432 RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
4433 )
4434 .unwrap();
4435 }
4436
4437 assert_eq!(wrapped_channel, Some(1));
4438 }
4439
4440 #[test]
4441 fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
4442 let forwarding = ForwardingTable::default();
4443 let client = ConnectionId::new(60);
4444 forwarding
4445 .inner
4446 .write()
4447 .unwrap()
4448 .next_client_channel
4449 .insert(client, 41);
4450
4451 let released = forwarding.cleanup_connection(client).unwrap();
4452
4453 assert!(released.is_empty());
4454 assert!(!forwarding
4455 .inner
4456 .read()
4457 .unwrap()
4458 .next_client_channel
4459 .contains_key(&client));
4460 }
4461
4462 #[test]
4463 fn stale_module_cleanup_preserves_fast_reconnect_successor() {
4464 let forwarding = ForwardingTable::default();
4465 let module_id = "fast-reconnect-provider";
4466 let first_connection = ConnectionId::new(70);
4467 let second_connection = ConnectionId::new(80);
4468 let (first_tx, _first_rx) = mpsc::channel(1);
4469 let first_endpoint = forwarding
4470 .register_module_connection(
4471 first_connection,
4472 module_id.to_string(),
4473 1,
4474 Concurrency::ModuleManaged,
4475 FrameSink::new(first_tx),
4476 )
4477 .unwrap();
4478 let (second_tx, _second_rx) = mpsc::channel(1);
4479 let second_endpoint = forwarding
4480 .register_module_connection(
4481 second_connection,
4482 module_id.to_string(),
4483 1,
4484 Concurrency::ModuleManaged,
4485 FrameSink::new(second_tx),
4486 )
4487 .unwrap();
4488 assert_ne!(first_endpoint, second_endpoint);
4489
4490 let released = forwarding.cleanup_connection(first_connection).unwrap();
4491
4492 assert!(released.is_empty());
4493 assert_eq!(
4494 forwarding
4495 .inner
4496 .read()
4497 .unwrap()
4498 .modules_by_id
4499 .get(module_id)
4500 .map(|module| module.endpoint),
4501 Some(second_endpoint)
4502 );
4503 assert!(forwarding.has_live_module_connection(module_id).unwrap());
4504 let control_rpc = forwarding
4505 .begin_module_control_rpc_for(
4506 module_id,
4507 "health.check",
4508 Instant::now() + Duration::from_secs(1),
4509 )
4510 .unwrap();
4511 assert_eq!(control_rpc.endpoint, second_endpoint);
4512 }
4513
4514 fn route_fixture(
4515 module_id: &str,
4516 ) -> (
4517 ForwardingTable,
4518 ConnectionId,
4519 ModuleEndpointId,
4520 ConnectionId,
4521 FrameSink,
4522 mpsc::Receiver<crate::router::OutboundFrame>,
4523 ) {
4524 let forwarding = ForwardingTable::default();
4525 let module_connection = ConnectionId::new(100);
4526 let client_connection = ConnectionId::new(200);
4527 let (module_tx, _module_rx) = mpsc::channel(8);
4528 let endpoint = forwarding
4529 .register_module_connection(
4530 module_connection,
4531 module_id.to_string(),
4532 2,
4533 Concurrency::ModuleManaged,
4534 FrameSink::new(module_tx),
4535 )
4536 .unwrap();
4537 let (client_tx, client_rx) = mpsc::channel(8);
4538 (
4539 forwarding,
4540 module_connection,
4541 endpoint,
4542 client_connection,
4543 FrameSink::new(client_tx),
4544 client_rx,
4545 )
4546 }
4547
4548 #[test]
4549 #[cfg(unix)]
4550 fn daemon_drain_gates_current_and_racing_provider_registrations() {
4551 let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
4552 assert_eq!(
4553 forwarding.begin_daemon_drain().unwrap(),
4554 [("provider".to_string(), endpoint)]
4555 );
4556 assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
4557 assert!(matches!(
4558 forwarding.register_module_connection(
4559 ConnectionId::new(300),
4560 "late-provider".into(),
4561 2,
4562 Concurrency::ModuleManaged,
4563 sink,
4564 ),
4565 Err(ForwardingError::ConnectionClosing { .. })
4566 ));
4567 }
4568
4569 #[test]
4570 #[cfg(unix)]
4571 fn daemon_drain_results_are_endpoint_fenced_and_removed_on_cleanup() {
4572 let (forwarding, connection, endpoint, _, _, _) = route_fixture("provider");
4573 assert_eq!(
4574 forwarding
4575 .daemon_drain_result_for_connection(connection)
4576 .unwrap(),
4577 None
4578 );
4579 forwarding
4580 .record_daemon_drain_result(endpoint, true)
4581 .unwrap();
4582 forwarding.begin_daemon_drain().unwrap();
4583 assert_eq!(
4584 forwarding
4585 .daemon_drain_result_for_connection(connection)
4586 .unwrap(),
4587 Some(false)
4588 );
4589 let replacement = ModuleEndpointId {
4590 generation: endpoint.generation + 1,
4591 ..endpoint
4592 };
4593 forwarding
4594 .record_daemon_drain_result(replacement, true)
4595 .unwrap();
4596 assert_eq!(
4597 forwarding
4598 .daemon_drain_result_for_connection(connection)
4599 .unwrap(),
4600 Some(false)
4601 );
4602 forwarding
4603 .record_daemon_drain_result(endpoint, true)
4604 .unwrap();
4605 assert_eq!(
4606 forwarding
4607 .daemon_drain_result_for_connection(connection)
4608 .unwrap(),
4609 Some(true)
4610 );
4611 assert_eq!(
4612 forwarding
4613 .daemon_drain_result_for_connection(ConnectionId::new(300))
4614 .unwrap(),
4615 Some(false)
4616 );
4617 forwarding.cleanup_connection(connection).unwrap();
4618 forwarding
4619 .record_daemon_drain_result(endpoint, true)
4620 .unwrap();
4621 assert_eq!(
4622 forwarding
4623 .daemon_drain_result_for_connection(connection)
4624 .unwrap(),
4625 Some(false)
4626 );
4627 assert!(forwarding
4628 .read_inner()
4629 .unwrap()
4630 .daemon_drain_results
4631 .is_empty());
4632 }
4633
4634 #[test]
4635 #[cfg(unix)]
4636 fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
4637 let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
4638 route_fixture("provider");
4639 let mut pending =
4640 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
4641 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
4642 forwarding.begin_daemon_drain().unwrap();
4643 let completion = forwarding
4644 .complete_pending_relay(
4645 module_connection,
4646 pending.corr,
4647 RouteBindRelayOutcome::Accepted,
4648 )
4649 .expect("draining admission is not a fatal module error");
4650 assert!(completion.settled);
4651 let abandoned = completion
4652 .abandoned
4653 .expect("late accepted bind needs a route GOODBYE");
4654 assert_eq!(abandoned.channel, pending.module_channel);
4655 assert!(matches!(pending.receiver.try_recv().unwrap(),
4656 RouteBindRelayOutcome::DaemonRejected(error) if error.code == "module_reloading"));
4657 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4658 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4659 assert!(
4660 rx.try_recv().is_err(),
4661 "no route.open may be published during drain"
4662 );
4663 assert_eq!(
4664 forwarding
4665 .module_endpoint_for_connection(module_connection)
4666 .unwrap(),
4667 Some(endpoint)
4668 );
4669 assert!(
4670 !forwarding
4671 .complete_pending_relay(
4672 module_connection,
4673 pending.corr,
4674 RouteBindRelayOutcome::Accepted,
4675 )
4676 .unwrap()
4677 .settled
4678 );
4679 }
4680
4681 fn test_ping(corr: u64) -> Frame {
4682 Frame::build(
4683 FrameType::Ping,
4684 Flags::new(false, Priority::Passive, false),
4685 0,
4686 0,
4687 corr,
4688 Vec::new(),
4689 )
4690 .unwrap()
4691 }
4692
4693 fn begin_test_route(
4694 forwarding: &ForwardingTable,
4695 client_connection: ConnectionId,
4696 client_sink: FrameSink,
4697 corr: u64,
4698 module_id: &str,
4699 ) -> PendingRouteBindRelay {
4700 forwarding
4701 .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
4702 .unwrap()
4703 }
4704
4705 #[test]
4706 fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
4707 for candidate in [false, true] {
4708 for register_first in [false, true] {
4709 let (forwarding, module_connection, _, client_connection, sink, _rx) =
4710 route_fixture("target");
4711 let register = || {
4712 if candidate {
4713 forwarding.register_candidate_module_connection(
4714 client_connection,
4715 "source".into(),
4716 2,
4717 Concurrency::ModuleManaged,
4718 sink.clone(),
4719 )
4720 } else {
4721 forwarding.register_module_connection(
4722 client_connection,
4723 "source".into(),
4724 2,
4725 Concurrency::ModuleManaged,
4726 sink.clone(),
4727 )
4728 }
4729 };
4730 if register_first {
4731 register().unwrap();
4732 assert!(
4733 forwarding
4734 .begin_route_bind_relay_for_test(
4735 client_connection,
4736 sink.clone(),
4737 1,
4738 "target",
4739 )
4740 .is_err(),
4741 "a deferred route.open cannot reserve after HELLO"
4742 );
4743 assert!(forwarding
4744 .cleanup_connection(client_connection)
4745 .unwrap()
4746 .is_empty());
4747 } else {
4748 let pending =
4749 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
4750 assert!(
4751 register().is_err(),
4752 "HELLO cannot register after a route reservation"
4753 );
4754 forwarding
4755 .complete_pending_relay(
4756 module_connection,
4757 pending.corr,
4758 RouteBindRelayOutcome::Accepted,
4759 )
4760 .unwrap();
4761 assert_eq!(
4762 forwarding
4763 .cleanup_connection(client_connection)
4764 .unwrap()
4765 .len(),
4766 1
4767 );
4768 }
4769 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4770 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4771 }
4772 }
4773 }
4774
4775 #[tokio::test]
4781 async fn pending_route_open_completes_behind_queued_data_frames() {
4782 assert_eq!(
4783 crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
4784 8,
4785 "the per-connection pending route.open limit is its own constant"
4786 );
4787 let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
4788 route_fixture("open-behind-data");
4789 let (sink, mut client_rx) = crate::server::connection_egress();
4790 const DATA_FRAMES: usize = 1_000;
4791 let data = |corr: u64| {
4792 Frame::build(
4793 FrameType::StreamData,
4794 Flags::new(false, Priority::Interactive, false),
4795 9,
4796 1,
4797 corr,
4798 vec![b'x'; 200],
4799 )
4800 .unwrap()
4801 };
4802 for corr in 0..DATA_FRAMES as u64 {
4803 sink.try_send(data(corr)).unwrap();
4804 }
4805 let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
4806 assert_eq!(sink.backlog().queued_bytes, data_bytes);
4807
4808 let pending = tokio::time::timeout(
4809 Duration::from_secs(5),
4810 forwarding.begin_route_bind_relay_for(
4811 client,
4812 sink.clone(),
4813 subc_protocol::PROTOCOL_VERSION,
4814 4_242,
4815 "open-behind-data",
4816 Principal::Direct,
4817 None,
4818 None,
4819 "fixture".into(),
4820 None,
4821 None,
4822 Instant::now() + Duration::from_secs(60),
4823 false,
4824 ),
4825 )
4826 .await
4827 .expect("reserving the route.open slot must not wait behind data frames")
4828 .unwrap();
4829 forwarding
4830 .complete_pending_relay(
4831 module_connection,
4832 pending.corr,
4833 RouteBindRelayOutcome::Accepted,
4834 )
4835 .unwrap();
4836
4837 let backlog = sink.backlog();
4838 assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
4839 assert!(
4840 backlog.queued_bytes > data_bytes,
4841 "the route.open response must be counted in queued bytes: {backlog:?}"
4842 );
4843 for corr in 0..DATA_FRAMES as u64 {
4844 assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
4845 }
4846 let open = client_rx.recv().await.unwrap();
4847 assert_eq!(open.header.corr, 4_242);
4848 assert_eq!(open.header.ty, FrameType::Response);
4849 drop(open);
4850 assert_eq!(sink.backlog().queued_bytes, 0);
4851 assert_eq!(sink.backlog().queued_frames, 0);
4852 }
4853
4854 #[tokio::test]
4859 async fn drain_holdouts_count_held_requests_and_name_the_connection() {
4860 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4861 route_fixture("holdouts");
4862 let mut bound = |corr| {
4863 let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
4864 forwarding
4865 .complete_pending_relay(
4866 module_connection,
4867 route.corr,
4868 RouteBindRelayOutcome::Accepted,
4869 )
4870 .unwrap();
4871 client_rx.try_recv().unwrap();
4872 match forwarding
4873 .lookup_data_route(client, route.client_channel, route.client_epoch)
4874 .unwrap()
4875 {
4876 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4877 other => panic!("expected live route, got {other:?}"),
4878 }
4879 };
4880 let holding = bound(61);
4881 let _idle = bound(62);
4882 holding.flow.acquire_tagged(7, false).await.unwrap();
4883 holding.flow.acquire_tagged(2, false).await.unwrap();
4884 holding.flow.acquire_tagged(3, true).await.unwrap();
4885 forwarding
4886 .begin_module_drain("holdouts", RouteCloseReason::Restart)
4887 .unwrap();
4888
4889 let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
4890 assert_eq!(
4891 holdouts,
4892 DrainHoldouts {
4893 requests: 2,
4894 routes: 1,
4895 total_routes: 2,
4896 top_connections: vec![(client.get(), 2)],
4897 held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
4900 }
4901 );
4902 }
4903
4904 #[test]
4905 fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
4906 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4907 route_fixture("census");
4908 let pending = begin_test_route(&forwarding, client, sink, 1, "census");
4909 forwarding
4910 .complete_pending_relay(
4911 module_connection,
4912 pending.corr,
4913 RouteBindRelayOutcome::Accepted,
4914 )
4915 .unwrap();
4916
4917 let routes = forwarding.endpoint_routes(endpoint).unwrap();
4918 assert_eq!(routes.len(), 1);
4919 assert!(matches!(routes[0].principal, Principal::Direct));
4920 assert_eq!(routes[0].goodbye_target.connection_id, client);
4921 assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
4922 assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
4923 assert!(!routes[0].draining);
4924
4925 forwarding
4926 .begin_module_drain("census", RouteCloseReason::Restart)
4927 .unwrap();
4928 let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
4929 assert_eq!(draining_routes.len(), 1);
4930 assert!(draining_routes[0].draining);
4931 }
4932
4933 #[test]
4934 fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
4935 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
4936 let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
4937 assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
4938 forwarding
4939 .abort_pending_relay(
4940 first.endpoint,
4941 first.corr,
4942 RouteBindRelayOutcome::ModuleGone("abort".into()),
4943 )
4944 .unwrap();
4945 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4946 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4947
4948 let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
4949 assert_eq!(second.client_channel, first.client_channel);
4950 assert_eq!(second.module_channel, first.module_channel);
4951 assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
4952 }
4953
4954 #[test]
4955 fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
4956 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4957 route_fixture("epoch-release");
4958 let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
4959 forwarding
4960 .complete_pending_relay(
4961 module_connection,
4962 first.corr,
4963 RouteBindRelayOutcome::Accepted,
4964 )
4965 .unwrap();
4966 assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
4967 assert!(matches!(
4968 forwarding
4969 .release_client_route(client, first.client_channel, first.client_epoch)
4970 .unwrap(),
4971 RouteRelease::Removed(_)
4972 ));
4973 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4974 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4975
4976 let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
4977 forwarding
4978 .complete_pending_relay(
4979 module_connection,
4980 second.corr,
4981 RouteBindRelayOutcome::Accepted,
4982 )
4983 .unwrap();
4984 assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
4985 assert!(matches!(
4986 forwarding
4987 .release_client_route(client, second.client_channel, first.client_epoch)
4988 .unwrap(),
4989 RouteRelease::Stale
4990 ));
4991 assert!(!forwarding
4992 .cache_status(
4993 endpoint,
4994 second.module_channel,
4995 first.module_epoch,
4996 "stale".into(),
4997 )
4998 .unwrap());
4999 assert!(forwarding
5000 .cache_status(
5001 endpoint,
5002 second.module_channel,
5003 second.module_epoch,
5004 "current".into(),
5005 )
5006 .unwrap());
5007 match forwarding
5008 .route_poll_snapshot(client, second.client_channel, second.client_epoch)
5009 .unwrap()
5010 {
5011 RoutePollSnapshot::Bound { status, .. } => {
5012 assert_eq!(status.as_deref(), Some("current"));
5013 }
5014 RoutePollSnapshot::Absent => panic!("successor binding was removed"),
5015 }
5016 let counters = forwarding.counters().snapshot();
5017 assert_eq!(counters["route_released_epoch_fenced"], 1);
5018 assert_eq!(counters["route_release_stale_skipped"], 1);
5019 }
5020
5021 #[test]
5022 fn max_epoch_reservation_retires_only_that_slot() {
5023 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
5024 forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
5025 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
5026 let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
5027 assert_eq!(
5028 (final_use.client_channel, final_use.client_epoch),
5029 (7, u32::MAX)
5030 );
5031 assert_eq!(
5032 (final_use.module_channel, final_use.module_epoch),
5033 (9, u32::MAX)
5034 );
5035 forwarding
5036 .abort_pending_relay(
5037 endpoint,
5038 final_use.corr,
5039 RouteBindRelayOutcome::ModuleGone("abort".into()),
5040 )
5041 .unwrap();
5042 forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
5043 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
5044 let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
5045 assert_ne!(next.client_channel, 7);
5046 assert_ne!(next.module_channel, 9);
5047 assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
5048 }
5049
5050 #[test]
5051 fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
5052 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
5053 route_fixture("corr-shared");
5054 let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
5055 assert_eq!(bind.corr, 1);
5056 forwarding
5057 .abort_pending_relay(
5058 endpoint,
5059 bind.corr,
5060 RouteBindRelayOutcome::ModuleGone("abort".into()),
5061 )
5062 .unwrap();
5063 let rpc = forwarding
5064 .begin_module_control_rpc_for(
5065 "corr-shared",
5066 "health.check",
5067 Instant::now() - Duration::from_millis(1),
5068 )
5069 .unwrap();
5070 assert_eq!(rpc.corr, 2);
5071 assert_eq!(
5072 forwarding
5073 .complete_module_control_rpc(
5074 module_connection,
5075 rpc.corr,
5076 Some("health.check"),
5077 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
5078 status: subc_protocol::session::HealthStatus::Ok,
5079 detail: None,
5080 metrics: None,
5081 }),
5082 )
5083 .unwrap(),
5084 ModuleControlRpcCompletion::Settled
5085 );
5086 assert!(matches!(
5087 rpc.receiver.blocking_recv().unwrap(),
5088 ModuleControlRpcOutcome::DeadlineElapsed
5089 ));
5090 }
5091
5092 #[tokio::test(start_paused = true)]
5093 async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
5094 let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
5095 let probe_started_at = Instant::now();
5096 let rpc = forwarding
5097 .begin_health_probe_rpc_for(
5098 "tombstone-ttl",
5099 "health.check",
5100 probe_started_at,
5101 probe_started_at + Duration::from_secs(5),
5102 )
5103 .unwrap();
5104 assert!(forwarding
5105 .tombstone_health_probe_rpc(endpoint, rpc.corr)
5106 .unwrap());
5107 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
5108
5109 tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
5110 tokio::task::yield_now().await;
5111
5112 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
5113 }
5114
5115 #[test]
5116 fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
5117 let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
5118 let mut close = forwarding.register_connection_close(endpoint.connection_id);
5119 forwarding.inject_control_corr(endpoint, u64::MAX);
5120 let final_rpc = forwarding
5121 .begin_module_control_rpc_for(
5122 "corr-max",
5123 "health.check",
5124 Instant::now() + Duration::from_secs(1),
5125 )
5126 .unwrap();
5127 assert_eq!(final_rpc.corr, u64::MAX);
5128 forwarding
5129 .cancel_module_control_rpc(endpoint, final_rpc.corr)
5130 .unwrap();
5131 assert!(matches!(
5132 forwarding.begin_module_control_rpc_for(
5133 "corr-max",
5134 "health.check",
5135 Instant::now() + Duration::from_secs(1),
5136 ),
5137 Err(ForwardingError::RelayCorrelationExhausted)
5138 ));
5139 assert!(close.try_recv().is_ok());
5140 }
5141
5142 #[test]
5143 fn publication_epoch_controls_delivery_failure_escalation() {
5144 fn setup_successor(
5145 commit_successor: Option<bool>,
5146 ) -> (ForwardingTable, ConnectionId, u16, u32) {
5147 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5148 route_fixture("escalation");
5149 let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
5150 forwarding
5151 .complete_pending_relay(
5152 module_connection,
5153 first.corr,
5154 RouteBindRelayOutcome::Accepted,
5155 )
5156 .unwrap();
5157 client_rx.try_recv().unwrap();
5158 assert!(matches!(
5159 forwarding
5160 .release_client_route(client, first.client_channel, first.client_epoch)
5161 .unwrap(),
5162 RouteRelease::Removed(_)
5163 ));
5164 if let Some(commit_successor) = commit_successor {
5165 forwarding.inject_client_slot_epoch(
5166 client,
5167 first.client_channel,
5168 first.client_epoch,
5169 );
5170 forwarding.inject_module_slot_epoch(
5171 endpoint,
5172 first.module_channel,
5173 first.module_epoch,
5174 );
5175 let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
5176 if commit_successor {
5177 forwarding
5178 .complete_pending_relay(
5179 module_connection,
5180 successor.corr,
5181 RouteBindRelayOutcome::Accepted,
5182 )
5183 .unwrap();
5184 client_rx.try_recv().unwrap();
5185 } else {
5186 forwarding
5187 .abort_pending_relay(
5188 endpoint,
5189 successor.corr,
5190 RouteBindRelayOutcome::ModuleGone("abort".into()),
5191 )
5192 .unwrap();
5193 }
5194 }
5195 (forwarding, client, first.client_channel, first.client_epoch)
5196 }
5197
5198 let probe_sink = FrameSink::new(mpsc::channel(1).0);
5199 let (no_successor, client, channel, epoch) = setup_successor(None);
5200 let mut close = no_successor.register_connection_close(client);
5201 assert!(no_successor
5202 .escalate_client_delivery_failure(
5203 client,
5204 channel,
5205 epoch,
5206 CloseReason::new("delivery", "failed"),
5207 UndeliveredFrame {
5208 module_id: None,
5209 sink: &probe_sink,
5210 },
5211 )
5212 .unwrap());
5213 assert!(close.try_recv().is_ok());
5214
5215 let (aborted, client, channel, epoch) = setup_successor(Some(false));
5216 let mut close = aborted.register_connection_close(client);
5217 assert!(aborted
5218 .escalate_client_delivery_failure(
5219 client,
5220 channel,
5221 epoch,
5222 CloseReason::new("delivery", "failed"),
5223 UndeliveredFrame {
5224 module_id: None,
5225 sink: &probe_sink,
5226 },
5227 )
5228 .unwrap());
5229 assert!(close.try_recv().is_ok());
5230
5231 let (published, client, channel, epoch) = setup_successor(Some(true));
5232 let mut close = published.register_connection_close(client);
5233 assert!(!published
5234 .escalate_client_delivery_failure(
5235 client,
5236 channel,
5237 epoch,
5238 CloseReason::new("delivery", "stale failure"),
5239 UndeliveredFrame {
5240 module_id: None,
5241 sink: &probe_sink,
5242 },
5243 )
5244 .unwrap());
5245 assert!(close.try_recv().is_err());
5246 }
5247
5248 #[test]
5249 fn route_concentration_separates_client_count_from_routes_per_client() {
5250 let (forwarding, module_connection, _, client, sink, _client_rx) =
5254 route_fixture("concentration");
5255 assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
5256
5257 for corr in [70_u64, 71] {
5258 let pending =
5259 begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
5260 forwarding
5261 .complete_pending_relay(
5262 module_connection,
5263 pending.corr,
5264 RouteBindRelayOutcome::Accepted,
5265 )
5266 .unwrap();
5267 }
5268
5269 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5271 assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
5272 }
5273
5274 #[test]
5275 fn cleanup_and_accepted_resolution_have_one_lock_winner() {
5276 let (forwarding, module_connection, _, client, sink, mut client_rx) =
5277 route_fixture("cleanup-race");
5278 let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
5279 forwarding
5280 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5281 .unwrap();
5282 let released = forwarding.cleanup_connection(client).unwrap();
5283 assert_eq!(released.len(), 1);
5284 let completion = forwarding
5285 .complete_pending_relay(
5286 module_connection,
5287 pending.corr,
5288 RouteBindRelayOutcome::Accepted,
5289 )
5290 .unwrap();
5291 assert!(!completion.settled);
5292 assert!(client_rx.try_recv().is_err());
5293 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5294
5295 let (forwarding, module_connection, _, client, sink, mut client_rx) =
5296 route_fixture("accepted-race");
5297 let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
5298 forwarding
5299 .complete_pending_relay(
5300 module_connection,
5301 pending.corr,
5302 RouteBindRelayOutcome::Accepted,
5303 )
5304 .unwrap();
5305 assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
5306 let released = forwarding.cleanup_connection(client).unwrap();
5307 assert_eq!(released.len(), 1);
5308 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5309 }
5310
5311 #[test]
5312 fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
5313 let (forwarding, module_connection, _, client, sink, mut client_rx) =
5314 route_fixture("drain-gap");
5315 let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
5316 forwarding
5317 .complete_pending_relay(
5318 module_connection,
5319 live.corr,
5320 RouteBindRelayOutcome::Accepted,
5321 )
5322 .unwrap();
5323 client_rx.try_recv().unwrap();
5324 let binding = match forwarding
5325 .lookup_data_route(client, live.client_channel, live.client_epoch)
5326 .unwrap()
5327 {
5328 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
5329 other => panic!("expected live route, got {other:?}"),
5330 };
5331
5332 let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
5333 forwarding
5334 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5335 .unwrap();
5336 let control_rpc = forwarding
5337 .begin_module_control_rpc_for(
5338 "drain-gap",
5339 "health.check",
5340 Instant::now() + Duration::from_secs(1),
5341 )
5342 .unwrap();
5343 let target = forwarding
5344 .begin_module_drain("drain-gap", RouteCloseReason::Reload)
5345 .unwrap()
5346 .unwrap();
5347 assert!(matches!(
5348 control_rpc.receiver.blocking_recv().unwrap(),
5349 ModuleControlRpcOutcome::ModuleGone(_)
5350 ));
5351 assert_eq!(target.abandoned_bindings.len(), 1);
5352 assert!(binding.flow.sem.is_closed());
5353 assert!(
5354 !forwarding
5355 .complete_pending_relay(
5356 module_connection,
5357 pending.corr,
5358 RouteBindRelayOutcome::Accepted,
5359 )
5360 .unwrap()
5361 .settled
5362 );
5363 assert!(matches!(
5364 forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
5365 Err(ForwardingError::ModuleReloading { .. })
5366 ));
5367 let released = forwarding
5368 .release_module_endpoint_routes(target.endpoint)
5369 .unwrap();
5370 assert_eq!(released.len(), 1);
5371 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5372 }
5373
5374 #[test]
5382 fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
5383 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5384 route_fixture("closing-client");
5385
5386 let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
5389 forwarding
5390 .complete_pending_relay(
5391 module_connection,
5392 live.corr,
5393 RouteBindRelayOutcome::Accepted,
5394 )
5395 .unwrap();
5396 client_rx.try_recv().unwrap();
5397
5398 let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
5400 forwarding
5401 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5402 .unwrap();
5403
5404 assert!(forwarding
5406 .escalate_client_delivery_failure(
5407 client,
5408 live.client_channel,
5409 live.client_epoch,
5410 CloseReason::new(
5411 "module_to_client_delivery_failed",
5412 "client egress refused a module frame",
5413 ),
5414 UndeliveredFrame {
5415 module_id: None,
5416 sink: &sink,
5417 },
5418 )
5419 .unwrap());
5420 assert!(!sink.is_closed());
5421
5422 let completion = forwarding
5423 .complete_pending_relay(
5424 module_connection,
5425 pending.corr,
5426 RouteBindRelayOutcome::Accepted,
5427 )
5428 .expect("a closing client must not turn a module's ack into an error");
5429
5430 assert!(completion.settled);
5431 let abandoned = completion
5432 .abandoned
5433 .expect("the module must be told to drop the binding it just created");
5434 assert_eq!(abandoned.connection_id, module_connection);
5435 assert_eq!(abandoned.channel, pending.module_channel);
5436 assert_eq!(abandoned.epoch, pending.module_epoch);
5437 assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
5438 assert!(matches!(
5439 pending.receiver.blocking_recv().unwrap(),
5440 RouteBindRelayOutcome::ModuleGone(_)
5441 ));
5442 assert!(client_rx.try_recv().is_err());
5445 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
5446
5447 assert!(forwarding
5450 .has_live_module_connection("closing-client")
5451 .unwrap());
5452 let cotenant = ConnectionId::new(201);
5453 let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
5454 let cotenant_route = begin_test_route(
5455 &forwarding,
5456 cotenant,
5457 FrameSink::new(cotenant_tx),
5458 62,
5459 "closing-client",
5460 );
5461 assert_eq!(cotenant_route.endpoint, endpoint);
5462 forwarding
5463 .complete_pending_relay(
5464 module_connection,
5465 cotenant_route.corr,
5466 RouteBindRelayOutcome::Accepted,
5467 )
5468 .unwrap();
5469 assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
5470 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5471 }
5472
5473 #[test]
5474 fn pending_route_permit_is_released_on_rejection_and_abort() {
5475 let forwarding = ForwardingTable::default();
5476 let module_connection = ConnectionId::new(300);
5477 let client = ConnectionId::new(301);
5478 let (module_tx, _module_rx) = mpsc::channel(1);
5479 let endpoint = forwarding
5480 .register_module_connection(
5481 module_connection,
5482 "permit".into(),
5483 2,
5484 Concurrency::ModuleManaged,
5485 FrameSink::new(module_tx),
5486 )
5487 .unwrap();
5488 let (client_tx, mut client_rx) = mpsc::channel(1);
5489 let sink = FrameSink::new(client_tx);
5490 let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
5491 assert!(sink.try_send(test_ping(999)).is_err());
5492 forwarding
5493 .complete_pending_relay(
5494 module_connection,
5495 rejected.corr,
5496 RouteBindRelayOutcome::Rejected(ErrorBody {
5497 code: "no".into(),
5498 message: "rejected".into(),
5499 detail: None,
5500 }),
5501 )
5502 .unwrap();
5503 sink.try_send(test_ping(1000)).unwrap();
5504 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
5505
5506 let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
5507 assert!(sink.try_send(test_ping(1001)).is_err());
5508 forwarding
5509 .abort_pending_relay(
5510 endpoint,
5511 aborted.corr,
5512 RouteBindRelayOutcome::ModuleGone("abort".into()),
5513 )
5514 .unwrap();
5515 sink.try_send(test_ping(1002)).unwrap();
5516 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
5517
5518 let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
5519 forwarding
5520 .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
5521 .unwrap();
5522 drop(client_rx);
5523 let completion = forwarding
5524 .complete_pending_relay(
5525 module_connection,
5526 receiver_closed.corr,
5527 RouteBindRelayOutcome::Accepted,
5528 )
5529 .unwrap();
5530 assert!(completion.abandoned.is_some());
5531 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5532 }
5533
5534 #[test]
5540 fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
5541 let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
5542 route_fixture("closing-set-leak");
5543
5544 const CONNECTIONS: u64 = 32;
5545 for index in 0..CONNECTIONS {
5546 let client = ConnectionId::new(1000 + index);
5547 let (client_tx, _client_rx) = mpsc::channel(8);
5548 let route = begin_test_route(
5549 &forwarding,
5550 client,
5551 FrameSink::new(client_tx),
5552 index + 1,
5553 "closing-set-leak",
5554 );
5555 forwarding
5556 .complete_pending_relay(
5557 module_connection,
5558 route.corr,
5559 RouteBindRelayOutcome::Accepted,
5560 )
5561 .unwrap();
5562 forwarding.cleanup_connection(client).unwrap();
5563 }
5564 forwarding.cleanup_connection(module_connection).unwrap();
5565
5566 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5567 }
5568
5569 #[test]
5576 fn closing_connection_is_refused_new_work_until_cleanup_completes() {
5577 let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
5578 route_fixture("closing-gate");
5579
5580 let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
5583 forwarding
5584 .complete_pending_relay(
5585 module_connection,
5586 live.corr,
5587 RouteBindRelayOutcome::Accepted,
5588 )
5589 .unwrap();
5590 client_rx.try_recv().unwrap();
5591
5592 assert!(forwarding
5595 .escalate_client_delivery_failure(
5596 client,
5597 live.client_channel,
5598 live.client_epoch,
5599 CloseReason::new(
5600 "module_to_client_delivery_failed",
5601 "client egress refused a module frame",
5602 ),
5603 UndeliveredFrame {
5604 module_id: None,
5605 sink: &sink,
5606 },
5607 )
5608 .unwrap());
5609 assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
5610
5611 assert!(matches!(
5613 forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
5614 Err(ForwardingError::ConnectionClosing { connection_id })
5615 if connection_id == client
5616 ));
5617 let (late_tx, _late_rx) = mpsc::channel(1);
5619 assert!(matches!(
5620 forwarding.register_module_connection(
5621 client,
5622 "late-module".into(),
5623 2,
5624 Concurrency::ModuleManaged,
5625 FrameSink::new(late_tx),
5626 ),
5627 Err(ForwardingError::ConnectionClosing { connection_id })
5628 if connection_id == client
5629 ));
5630
5631 forwarding.cleanup_connection(client).unwrap();
5635 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5636 }
5637}
5638
5639#[cfg(test)]
5642mod swap_slot_tests {
5643 use std::time::Duration;
5644
5645 use super::*;
5646 use tokio::sync::mpsc;
5647
5648 const MODULE_ID: &str = "swapped";
5649
5650 struct SwapFixture {
5651 forwarding: ForwardingTable,
5652 incumbent_connection: ConnectionId,
5653 incumbent: ModuleEndpointId,
5654 candidate_connection: ConnectionId,
5655 candidate: ModuleEndpointId,
5656 _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
5657 }
5658
5659 fn swap_fixture() -> SwapFixture {
5660 let forwarding = ForwardingTable::default();
5661 let incumbent_connection = ConnectionId::new(100);
5662 let candidate_connection = ConnectionId::new(110);
5663 let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
5664 let incumbent = forwarding
5665 .register_module_connection(
5666 incumbent_connection,
5667 MODULE_ID.to_string(),
5668 2,
5669 Concurrency::ModuleManaged,
5670 FrameSink::new(incumbent_tx),
5671 )
5672 .unwrap();
5673 let (candidate_tx, candidate_rx) = mpsc::channel(8);
5674 let candidate = forwarding
5675 .register_candidate_module_connection(
5676 candidate_connection,
5677 MODULE_ID.to_string(),
5678 2,
5679 Concurrency::ModuleManaged,
5680 FrameSink::new(candidate_tx),
5681 )
5682 .unwrap();
5683 SwapFixture {
5684 forwarding,
5685 incumbent_connection,
5686 incumbent,
5687 candidate_connection,
5688 candidate,
5689 _module_rxs: vec![incumbent_rx, candidate_rx],
5690 }
5691 }
5692
5693 fn client(
5694 raw: u64,
5695 ) -> (
5696 ConnectionId,
5697 FrameSink,
5698 mpsc::Receiver<crate::router::OutboundFrame>,
5699 ) {
5700 let (tx, rx) = mpsc::channel(8);
5701 (ConnectionId::new(raw), FrameSink::new(tx), rx)
5702 }
5703
5704 fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
5705 forwarding
5706 .read_inner()
5707 .unwrap()
5708 .client_to_module
5709 .values()
5710 .map(|route| route.module_endpoint)
5711 .collect()
5712 }
5713
5714 #[test]
5715 fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
5716 let fixture = swap_fixture();
5717 let forwarding = &fixture.forwarding;
5718 assert_ne!(fixture.incumbent, fixture.candidate);
5719
5720 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5722 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5723 let (client_connection, client_sink, _client_rx) = client(200);
5724 let pending = forwarding
5725 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5726 .unwrap();
5727 assert_eq!(pending.endpoint, fixture.incumbent);
5728 let rpc = forwarding
5729 .begin_module_control_rpc_for(
5730 MODULE_ID,
5731 "health.check",
5732 Instant::now() + Duration::from_secs(1),
5733 )
5734 .unwrap();
5735 assert_eq!(rpc.endpoint, fixture.incumbent);
5736 let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
5737 assert_eq!(census.len(), 1, "the census lists one endpoint per id");
5738
5739 assert_eq!(
5741 forwarding
5742 .module_endpoint_for_connection(fixture.candidate_connection)
5743 .unwrap(),
5744 Some(fixture.candidate)
5745 );
5746 assert_eq!(
5747 forwarding
5748 .module_id_for_connection(fixture.candidate_connection)
5749 .unwrap()
5750 .as_deref(),
5751 Some(MODULE_ID)
5752 );
5753
5754 let (other_tx, _other_rx) = mpsc::channel(1);
5756 assert_eq!(
5757 forwarding.register_candidate_module_connection(
5758 ConnectionId::new(120),
5759 MODULE_ID.to_string(),
5760 2,
5761 Concurrency::ModuleManaged,
5762 FrameSink::new(other_tx),
5763 ),
5764 Err(ForwardingError::CandidateSlotOccupied {
5765 module_id: MODULE_ID.to_string()
5766 })
5767 );
5768 }
5769
5770 #[test]
5774 fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
5775 let fixture = swap_fixture();
5776 let forwarding = &fixture.forwarding;
5777 let (early_client, early_sink, _early_rx) = client(200);
5778 let mut early = forwarding
5779 .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
5780 .unwrap();
5781 assert_eq!(early.endpoint, fixture.incumbent);
5782 assert_eq!(
5783 forwarding
5784 .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
5785 .unwrap(),
5786 RouteBindRelayMark::Marked
5787 );
5788
5789 let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5790 assert_eq!(
5791 cutover,
5792 ForwardingCutover {
5793 promoted: fixture.candidate,
5794 incumbent: Some(fixture.incumbent),
5795 }
5796 );
5797
5798 let (late_client, late_sink, _late_rx) = client(201);
5800 let late = forwarding
5801 .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
5802 .unwrap();
5803 assert_eq!(
5804 late.endpoint, fixture.candidate,
5805 "a route.open after cutover was reserved on the incumbent"
5806 );
5807
5808 let completion = forwarding
5810 .complete_pending_relay(
5811 fixture.incumbent_connection,
5812 early.corr,
5813 RouteBindRelayOutcome::Accepted,
5814 )
5815 .expect("a superseded endpoint's ack is not an error on its connection");
5816 assert!(completion.settled);
5817 assert!(
5818 !committed_endpoints(forwarding).contains(&fixture.incumbent),
5819 "a relay reserved before cutover committed a route on the incumbent"
5820 );
5821 let goodbye = completion
5822 .abandoned
5823 .expect("the incumbent is told to drop the binding it just created");
5824 assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
5825 assert_eq!(goodbye.channel, early.module_channel);
5826 assert_eq!(goodbye.epoch, early.module_epoch);
5827 assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
5828 match early.receiver.try_recv() {
5829 Ok(RouteBindRelayOutcome::DaemonRejected(body)) => {
5830 assert_eq!(body.code, "module_reloading")
5831 }
5832 other => panic!("expected a retryable module_reloading answer, got {other:?}"),
5833 }
5834 assert!(matches!(
5835 forwarding
5836 .lookup_data_route(early_client, early.client_channel, early.client_epoch)
5837 .unwrap(),
5838 DataRoute::Client(DataRouteState::Absent)
5839 ));
5840
5841 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
5844 forwarding
5845 .complete_pending_relay(
5846 fixture.candidate_connection,
5847 late.corr,
5848 RouteBindRelayOutcome::Accepted,
5849 )
5850 .unwrap();
5851 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5852 assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
5853 }
5854
5855 #[test]
5856 fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
5857 let fixture = swap_fixture();
5858 let forwarding = &fixture.forwarding;
5859 let (bound_client, bound_sink, _bound_rx) = client(200);
5861 let bound = forwarding
5862 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5863 .unwrap();
5864 forwarding
5865 .complete_pending_relay(
5866 fixture.incumbent_connection,
5867 bound.corr,
5868 RouteBindRelayOutcome::Accepted,
5869 )
5870 .unwrap();
5871 let (pending_client, pending_sink, _pending_rx) = client(201);
5872 let mut in_flight = forwarding
5873 .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
5874 .unwrap();
5875 forwarding
5876 .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
5877 .unwrap();
5878
5879 let incumbent = forwarding
5880 .cutover_candidate(MODULE_ID)
5881 .unwrap()
5882 .unwrap()
5883 .incumbent
5884 .unwrap();
5885 let target = forwarding
5886 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
5887 .unwrap()
5888 .expect("the superseded incumbent is still registered");
5889
5890 assert_eq!(target.endpoint, fixture.incumbent);
5891 assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
5892 assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
5893 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5894 assert_eq!(target.abandoned_bindings.len(), 1);
5895 assert_eq!(
5896 target.abandoned_bindings[0].channel,
5897 in_flight.module_channel
5898 );
5899 assert!(matches!(
5900 in_flight.receiver.try_recv(),
5901 Ok(RouteBindRelayOutcome::DaemonRejected(body)) if body.code == "module_reloading"
5902 ));
5903 assert_eq!(
5904 forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
5905 1,
5906 "the incumbent's bound route stays until its drain finishes"
5907 );
5908
5909 let (next_client, next_sink, _next_rx) = client(202);
5910 let next = forwarding
5911 .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
5912 .expect("the promoted candidate keeps accepting routes");
5913 assert_eq!(next.endpoint, fixture.candidate);
5914 }
5915
5916 #[test]
5922 fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
5923 let forwarding = ForwardingTable::default();
5924 let first_connection = ConnectionId::new(70);
5925 let (first_tx, _first_rx) = mpsc::channel(8);
5926 forwarding
5927 .register_module_connection(
5928 first_connection,
5929 MODULE_ID.to_string(),
5930 2,
5931 Concurrency::ModuleManaged,
5932 FrameSink::new(first_tx),
5933 )
5934 .unwrap();
5935 let (client_connection, client_sink, _client_rx) = client(200);
5936 let mut pending = forwarding
5937 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5938 .unwrap();
5939 let (second_tx, _second_rx) = mpsc::channel(8);
5940 forwarding
5941 .register_module_connection(
5942 ConnectionId::new(80),
5943 MODULE_ID.to_string(),
5944 2,
5945 Concurrency::ModuleManaged,
5946 FrameSink::new(second_tx),
5947 )
5948 .unwrap();
5949
5950 assert_eq!(
5951 forwarding
5952 .complete_pending_relay(
5953 first_connection,
5954 pending.corr,
5955 RouteBindRelayOutcome::Accepted
5956 )
5957 .unwrap_err(),
5958 ForwardingError::StaleModuleEndpoint
5959 );
5960 assert!(committed_endpoints(&forwarding).is_empty());
5961 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5962 assert!(matches!(
5963 pending.receiver.try_recv(),
5964 Err(oneshot::error::TryRecvError::Closed)
5965 ));
5966 }
5967
5968 #[test]
5969 fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
5970 let fixture = swap_fixture();
5972 let forwarding = &fixture.forwarding;
5973 assert!(forwarding
5974 .cleanup_connection(fixture.candidate_connection)
5975 .unwrap()
5976 .is_empty());
5977 assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
5978 let (client_connection, client_sink, _client_rx) = client(200);
5979 assert_eq!(
5980 forwarding
5981 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5982 .unwrap()
5983 .endpoint,
5984 fixture.incumbent
5985 );
5986
5987 let fixture = swap_fixture();
5990 let forwarding = &fixture.forwarding;
5991 let (bound_client, bound_sink, _bound_rx) = client(200);
5992 let bound = forwarding
5993 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5994 .unwrap();
5995 forwarding
5996 .complete_pending_relay(
5997 fixture.incumbent_connection,
5998 bound.corr,
5999 RouteBindRelayOutcome::Accepted,
6000 )
6001 .unwrap();
6002 forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
6003 let released = forwarding
6004 .cleanup_connection(fixture.incumbent_connection)
6005 .unwrap();
6006 assert_eq!(released.len(), 1);
6007 assert_eq!(released[0].connection_id, bound_client);
6008 assert!(forwarding
6009 .read_inner()
6010 .unwrap()
6011 .superseded_endpoints
6012 .is_empty());
6013 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
6014 let (next_client, next_sink, _next_rx) = client(201);
6015 assert_eq!(
6016 forwarding
6017 .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
6018 .unwrap()
6019 .endpoint,
6020 fixture.candidate
6021 );
6022 }
6023}