1use std::{
10 collections::{BTreeMap, HashMap, HashSet},
11 error::Error,
12 fmt,
13 sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
14 time::Duration,
15};
16
17use subc_control::{ClientControlResponse, RouteCloseReason};
18use subc_protocol::{
19 manifest::Concurrency,
20 session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
21 ErrorBody, Flags, FrameType, Principal, Priority,
22};
23use tokio::sync::{oneshot, Semaphore};
24use tokio::time::Instant;
25use tracing::{debug, info, warn};
26
27use crate::{
28 control::{RouteBindBreakers, RouteBindConcurrency},
29 observability::DaemonCounters,
30 registry::ConnectionId,
31 router::FrameSink,
32 scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
33 Frame, ProjectRootId,
34};
35
36const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
38
39const DEFAULT_MODULE_MANAGED_WATCH_WINDOW: usize = 24;
41
42const STATELESS_PARALLEL_WINDOW: usize = 1024;
44
45const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
49
50#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
55pub struct ModuleEndpointId {
56 pub connection_id: ConnectionId,
57 pub generation: u64,
58}
59
60#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
62pub(crate) struct ClientRouteKey {
63 pub connection_id: ConnectionId,
64 pub channel: u16,
65}
66
67#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
69pub(crate) struct ModuleRouteKey {
70 pub endpoint: ModuleEndpointId,
71 pub channel: u16,
72}
73
74#[derive(Debug)]
75pub(crate) struct RouteBinding {
76 pub client_connection_id: ConnectionId,
77 pub client_sink: FrameSink,
78 pub client_negotiated_ver: u8,
79 pub client_channel: u16,
80 pub client_epoch: u32,
81 pub module_id: String,
82 pub module_endpoint: ModuleEndpointId,
83 pub module_sink: FrameSink,
84 pub module_negotiated_ver: u8,
85 pub module_channel: u16,
86 pub module_epoch: u32,
87 pub principal: Principal,
88 #[allow(dead_code)]
90 pub harness: String,
91 #[allow(dead_code)]
93 pub build_class: Option<String>,
94 #[allow(dead_code)]
96 pub operator_app: Option<crate::operator_apps::OperatorAppStamp>,
97 pub project_root: Option<ProjectRootId>,
98 pub bound_at: Instant,
99 pub flow: Arc<ChannelFlow>,
100 pub scope: Option<BoundScope>,
103}
104
105#[derive(Debug, Clone)]
106pub(crate) enum DataRoute {
107 Client(DataRouteState),
108 Module(DataRouteState),
109}
110
111#[derive(Debug, Clone)]
112pub(crate) enum DataRouteState {
113 Bound(Arc<RouteBinding>),
114 Reserved,
115 EpochMismatch,
116 Absent,
117}
118
119#[derive(Debug, Clone, Copy, PartialEq, Eq)]
148pub(crate) enum GoodbyeTargetKind {
149 Client,
150 Module,
151}
152
153#[derive(Debug, Clone)]
154pub(crate) struct GoodbyeTarget {
155 pub connection_id: ConnectionId,
156 pub sink: FrameSink,
157 pub negotiated_ver: u8,
158 pub channel: u16,
159 pub epoch: u32,
160 pub kind: GoodbyeTargetKind,
161 pub module_id: Option<String>,
166}
167
168#[derive(Debug, Clone, Copy)]
171pub(crate) struct UndeliveredFrame<'a> {
172 pub module_id: Option<&'a str>,
174 pub sink: &'a FrameSink,
176}
177
178pub(crate) fn principal_label(principal: &Principal) -> String {
180 match principal {
181 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
182 Principal::Direct => "direct".to_string(),
183 other => format!("{other:?}"),
184 }
185}
186
187fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
190 let labels = inner
191 .client_to_module
192 .iter()
193 .filter(|(key, _)| key.connection_id == connection_id)
194 .map(|(_, route)| principal_label(&route.principal))
195 .collect::<std::collections::BTreeSet<_>>();
196 if labels.is_empty() {
197 "none".to_string()
198 } else {
199 labels.into_iter().collect::<Vec<_>>().join(",")
200 }
201}
202
203impl GoodbyeTarget {
204 pub(crate) fn close_on_delivery_failure(&self) -> bool {
207 matches!(self.kind, GoodbyeTargetKind::Client)
208 }
209}
210
211pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
224
225pub(crate) fn send_module_route_goodbye(
237 counters: &DaemonCounters,
238 sink: &FrameSink,
239 frame: Frame,
240 module_id: Option<&str>,
241 context: &'static str,
242) {
243 let channel = frame.header.channel;
244 let epoch = frame.header.epoch;
245 let Err(err) = sink.try_send(frame.clone()) else {
246 return;
247 };
248 let runtime = match tokio::runtime::Handle::try_current() {
251 Ok(runtime) if !sink.is_closed() => runtime,
252 _ => {
253 counters.increment_goodbye_relay_module_dropped(module_id);
254 warn!(
255 module_id = module_id.unwrap_or("unknown"),
256 route_channel = channel,
257 route_epoch = epoch,
258 error = %err,
259 context,
260 "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
261 );
262 return;
263 }
264 };
265 debug!(
266 module_id = module_id.unwrap_or("unknown"),
267 route_channel = channel,
268 route_epoch = epoch,
269 error = %err,
270 context,
271 "module egress queue refused route GOODBYE; delivering it once the module frees room"
272 );
273 let counters = counters.clone();
274 let sink = sink.clone();
275 let module_id = module_id.map(str::to_string);
276 runtime.spawn(async move {
277 let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
278 let why = match outcome {
279 Ok(Ok(())) => {
280 debug!(
281 module_id = module_id.as_deref().unwrap_or("unknown"),
282 route_channel = channel,
283 route_epoch = epoch,
284 context,
285 "late route GOODBYE delivered to module"
286 );
287 return;
288 }
289 Ok(Err(err)) => err.to_string(),
290 Err(_) => format!(
291 "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
292 ),
293 };
294 counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
295 warn!(
296 module_id = module_id.as_deref().unwrap_or("unknown"),
297 route_channel = channel,
298 route_epoch = epoch,
299 error = %why,
300 context,
301 "route GOODBYE to module dropped under backpressure; not closing shared module connection"
302 );
303 });
304}
305
306#[derive(Debug, Clone)]
312pub(crate) struct EndpointRoute {
313 pub goodbye_target: GoodbyeTarget,
314 pub principal: Principal,
315 pub bound_at: Instant,
316 pub draining: bool,
317 pub drain_reason: Option<RouteCloseReason>,
321}
322
323#[derive(Debug, Clone)]
326pub(crate) struct ScopeDrainedRoute {
327 pub reason: RouteCloseReason,
328 pub scope: BoundScope,
329 pub module_id: String,
330 pub client: GoodbyeTarget,
331 pub module: GoodbyeTarget,
332}
333
334#[derive(Debug)]
335pub(crate) struct PendingRouteBindRelay {
336 pub endpoint: ModuleEndpointId,
337 pub module_sink: FrameSink,
338 pub negotiated_ver: u8,
339 pub client_channel: u16,
340 pub client_epoch: u32,
341 pub module_channel: u16,
342 pub module_epoch: u32,
343 pub corr: u64,
344 pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
345}
346
347#[derive(Debug, Clone)]
348pub(crate) struct ModuleDrainTarget {
349 pub endpoint: ModuleEndpointId,
350 pub sink: FrameSink,
351 pub negotiated_ver: u8,
352 pub abandoned_bindings: Vec<GoodbyeTarget>,
353 pub excluded_subscriptions: u32,
354}
355
356#[cfg(unix)]
358#[derive(Debug, Clone)]
359pub(crate) struct ModuleConnectionTarget {
360 pub module_id: String,
361 pub endpoint: ModuleEndpointId,
362 pub sink: FrameSink,
363 pub negotiated_ver: u8,
364}
365
366#[derive(Debug, Clone)]
367pub(crate) enum RouteBindRelayOutcome {
368 Accepted,
369 Rejected(ErrorBody),
374 DaemonRejected(ErrorBody),
382 ModuleGone(String),
383}
384
385#[derive(Debug, Clone, Copy, PartialEq, Eq)]
387pub(crate) struct ForwardingCutover {
388 pub promoted: ModuleEndpointId,
390 pub incumbent: Option<ModuleEndpointId>,
393}
394
395#[derive(Debug)]
397pub(crate) struct ConnectionCleanup {
398 pub released: Vec<GoodbyeTarget>,
400 pub abandoned_relays: u32,
403}
404
405#[derive(Debug, Clone)]
406pub(crate) struct PendingRelayCompletion {
407 pub settled: bool,
408 pub abandoned: Option<GoodbyeTarget>,
409}
410
411#[derive(Debug)]
412pub(crate) struct PendingModuleControlRpc {
413 pub endpoint: ModuleEndpointId,
414 pub module_sink: FrameSink,
415 pub negotiated_ver: u8,
416 pub corr: u64,
417 pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
418}
419
420#[derive(Debug, Clone)]
421pub(crate) enum ModuleControlRpcOutcome {
422 RawResponse(Vec<u8>),
423 Response(ModuleControlResponse),
424 Rejected(ErrorBody),
425 ModuleGone(String),
426 MalformedResponse(String),
427 UnexpectedOp { expected: String, actual: String },
428 DeadlineElapsed,
429}
430
431#[derive(Debug, Clone, PartialEq, Eq)]
432pub(crate) enum ModuleControlRpcCompletion {
433 Unknown,
434 Settled,
435 LateHealthAnswer {
436 module_id: String,
437 latency: Duration,
438 },
439}
440
441#[derive(Debug)]
442struct PendingModuleControlRpcEntry {
443 expected_op: String,
444 deadline: Instant,
445 health_probe_started_at: Option<Instant>,
446 sender: oneshot::Sender<ModuleControlRpcOutcome>,
447}
448
449#[derive(Debug)]
450struct HealthProbeTombstone {
451 expected_op: String,
452 module_id: String,
453 probe_started_at: Instant,
454 expires_at: Instant,
455}
456
457#[derive(Debug, Clone)]
458struct RouteReservation {
459 client_key: ClientRouteKey,
460 module_key: ModuleRouteKey,
461 client_epoch: u32,
462 module_epoch: u32,
463 project_root: Option<ProjectRootId>,
464}
465
466#[derive(Debug)]
467struct PendingRouteBindRelayEntry {
468 reservation: RouteReservation,
469 client_sink: FrameSink,
470 client_negotiated_ver: u8,
471 client_permit: crate::router::EgressPermit,
472 route_open_frame: Frame,
473 busy_refusals: bool,
474 request_window: usize,
475 watch_window: Option<usize>,
476 principal: Principal,
477 scope: Option<BoundScope>,
480 harness: String,
481 build_class: Option<String>,
482 operator_app: Option<crate::operator_apps::OperatorAppStamp>,
483 deadline: Instant,
484 relay_enqueued: bool,
495 sender: oneshot::Sender<RouteBindRelayOutcome>,
496}
497
498#[derive(Debug, Clone, Copy, PartialEq, Eq)]
501pub(crate) enum RouteBindRelayMark {
502 Marked,
504 SettledByReply,
508 ReleasedWithoutReply,
512}
513
514#[derive(Debug, Clone)]
515pub(crate) enum RouteRelease {
516 Removed(GoodbyeTarget),
517 Stale,
518 Absent,
519}
520
521#[derive(Debug, Clone)]
522pub(crate) enum RoutePollSnapshot {
523 Bound {
524 module_id: String,
525 status: Option<String>,
526 },
527 Absent,
528}
529
530#[derive(Debug, Clone)]
531struct ModuleConnection {
532 endpoint: ModuleEndpointId,
533 sink: FrameSink,
534 negotiated_ver: u8,
535 concurrency: Concurrency,
536 verified_prefix_owner: Option<String>,
538}
539
540pub(crate) struct ModuleHelloAck {
541 pub(crate) frame: Frame,
542 pub(crate) verified_prefix_owner: Option<String>,
543}
544
545#[derive(Debug, Default)]
546struct ForwardingInner {
547 operator_confirms: Arc<crate::operator_confirm::OperatorConfirms>,
548 daemon_draining: bool,
549 daemon_drain_results: HashMap<ModuleEndpointId, bool>,
554 modules_by_id: HashMap<String, ModuleConnection>,
558 candidates_by_id: HashMap<String, ModuleConnection>,
564 superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
570 endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
571 module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
572 draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
576 closing_connections: HashSet<ConnectionId>,
577 spawn_connections: HashMap<ConnectionId, std::sync::Weak<crate::supervise::RegistrationSpawn>>,
579 next_generation: u64,
580 reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
581 reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
582 next_client_channel: HashMap<ConnectionId, u16>,
583 next_module_channel: HashMap<ModuleEndpointId, u16>,
584 client_slot_epochs: HashMap<ClientRouteKey, u32>,
585 module_slot_epochs: HashMap<ModuleRouteKey, u32>,
586 last_published_epoch: HashMap<ClientRouteKey, u32>,
587 client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
588 module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
589 status: HashMap<(ClientRouteKey, u32), String>,
590 pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
591 relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
596 scope_tags: HashMap<(String, String), ScopeTag>,
601 next_control_corr: HashMap<ModuleEndpointId, u64>,
602 pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
603 health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
604}
605
606#[derive(Debug, Clone)]
607pub(crate) struct CloseReason {
608 code: &'static str,
609 message: String,
610}
611
612impl CloseReason {
613 pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
614 Self {
615 code,
616 message: message.into(),
617 }
618 }
619}
620
621impl fmt::Display for CloseReason {
622 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
623 write!(f, "{}: {}", self.code, self.message)
624 }
625}
626
627pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
628
629#[derive(Debug, Default)]
631pub struct ForwardingTable {
632 #[cfg(all(test, target_os = "linux"))]
633 route_commit_pause: Mutex<Option<RouteCommitPause>>,
634 #[cfg(all(test, target_os = "linux"))]
635 cleanup_tie_observer: Mutex<Option<oneshot::Sender<(bool, usize)>>>,
636 #[cfg(all(test, target_os = "linux"))]
637 cleanup_pause: Mutex<Option<RouteCommitPause>>,
638 #[cfg(test)]
639 policy_rebind_before_write: Mutex<Option<Arc<std::sync::Barrier>>>,
640 inner: Arc<RwLock<ForwardingInner>>,
641 frame_drop_warnings: Mutex<HashMap<ConnectionId, ChannelDropWarnings>>,
642 #[cfg(feature = "test-support")]
643 route_closing_hold: Mutex<Option<oneshot::Receiver<()>>>,
644 close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
645 counters: DaemonCounters,
646 route_bind_breakers: RouteBindBreakers,
651 route_bind_concurrency: RouteBindConcurrency,
654 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
658}
659
660#[cfg(all(test, target_os = "linux"))]
661#[derive(Debug)]
662struct RouteCommitPause {
663 reached: oneshot::Sender<()>,
664 release: std::sync::mpsc::Receiver<()>,
665}
666
667pub(crate) type PolicyRevocation = (Vec<EndpointRoute>, Vec<GoodbyeTarget>);
668
669#[derive(Debug)]
670struct FrameDropWarning {
671 last_warned: tokio::time::Instant,
672 suppressed: u64,
673}
674
675type ChannelDropWarnings = HashMap<(u16, &'static str), FrameDropWarning>;
676
677impl ForwardingTable {
678 #[cfg(all(test, target_os = "linux"))]
679 pub(crate) fn pause_cleanup_for_test(
680 &self,
681 ) -> (oneshot::Receiver<()>, std::sync::mpsc::Sender<()>) {
682 let (reached, observed) = oneshot::channel();
683 let (release, paused) = std::sync::mpsc::channel();
684 *self.cleanup_pause.lock().unwrap() = Some(RouteCommitPause {
685 reached,
686 release: paused,
687 });
688 (observed, release)
689 }
690 #[cfg(all(test, target_os = "linux"))]
691 pub(crate) fn observe_cleanup_tie_for_test(&self) -> oneshot::Receiver<(bool, usize)> {
692 let (send, receive) = oneshot::channel();
693 *self.cleanup_tie_observer.lock().unwrap() = Some(send);
694 receive
695 }
696 #[cfg(all(test, target_os = "linux"))]
697 pub(crate) fn spawn_connection_recorded_for_test(&self, connection: ConnectionId) -> bool {
698 self.read_inner()
699 .unwrap()
700 .spawn_connections
701 .contains_key(&connection)
702 }
703 #[cfg(all(test, target_os = "linux"))]
704 pub(crate) fn pause_route_commit_for_test(
705 &self,
706 ) -> (oneshot::Receiver<()>, std::sync::mpsc::Sender<()>) {
707 let (reached, observed) = oneshot::channel();
708 let (release, paused) = std::sync::mpsc::channel();
709 *self.route_commit_pause.lock().unwrap() = Some(RouteCommitPause {
710 reached,
711 release: paused,
712 });
713 (observed, release)
714 }
715 pub(crate) fn bind_spawn(
716 &self,
717 connection: ConnectionId,
718 spawn: &Arc<crate::supervise::RegistrationSpawn>,
719 ) -> Result<(), ForwardingError> {
720 self.write_inner()?
721 .spawn_connections
722 .insert(connection, Arc::downgrade(spawn));
723 Ok(())
724 }
725 pub(crate) fn active_module_endpoint(
726 &self,
727 module_id: &str,
728 ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
729 Ok(self
730 .read_inner()?
731 .modules_by_id
732 .get(module_id)
733 .map(|module| module.endpoint))
734 }
735
736 pub(crate) fn config_delivery_sink(
737 &self,
738 module: &str,
739 connection: ConnectionId,
740 ) -> Result<Option<FrameSink>, ForwardingError> {
741 let inner = self.read_inner()?;
742 Ok(inner
743 .modules_by_id
744 .get(module)
745 .filter(|target| {
746 target.endpoint.connection_id == connection
747 && !inner.closing_connections.contains(&connection)
748 })
749 .map(|target| target.sink.clone()))
750 }
751 pub(crate) fn warn_frame_drop(
754 &self,
755 connection_id: ConnectionId,
756 channel: u16,
757 epoch: u32,
758 direction: &'static str,
759 reason: &'static str,
760 ) {
761 let now = tokio::time::Instant::now();
762 let mut warnings = self
763 .frame_drop_warnings
764 .lock()
765 .unwrap_or_else(std::sync::PoisonError::into_inner);
766 let entry = warnings
767 .entry(connection_id)
768 .or_default()
769 .entry((channel, reason));
770 let suppressed = match entry {
771 std::collections::hash_map::Entry::Occupied(mut entry) => {
772 let state = entry.get_mut();
773 if now.saturating_duration_since(state.last_warned) < Duration::from_secs(60) {
774 state.suppressed = state.suppressed.saturating_add(1);
775 return;
776 }
777 state.last_warned = now;
778 std::mem::take(&mut state.suppressed)
779 }
780 std::collections::hash_map::Entry::Vacant(entry) => {
781 entry.insert(FrameDropWarning {
782 last_warned: now,
783 suppressed: 0,
784 });
785 0
786 }
787 };
788 warn!(
789 connection_id = connection_id.get(),
790 channel, epoch, direction, reason, suppressed, "dropping data-plane frame"
791 );
792 }
793
794 #[cfg(feature = "test-support")]
836 pub fn configure_operator_script(
837 &self,
838 script: &std::path::Path,
839 events: Option<&std::path::Path>,
840 limits: Option<crate::operator_confirm::OperatorLimits>,
841 ) {
842 self.operator_confirms()
843 .configure_script(script, events, limits);
844 }
845
846 #[cfg(feature = "test-support")]
850 pub fn hold_route_closing_for_test(&self) -> oneshot::Sender<()> {
851 let (release, hold) = oneshot::channel();
852 *self.route_closing_hold.lock().unwrap() = Some(hold);
853 release
854 }
855
856 #[cfg(feature = "test-support")]
857 pub(crate) async fn wait_before_route_closing_for_test(&self) {
858 let hold = self.route_closing_hold.lock().unwrap().take();
859 if let Some(hold) = hold {
860 let _ = hold.await;
861 }
862 }
863
864 pub(crate) fn operator_confirms(&self) -> Arc<crate::operator_confirm::OperatorConfirms> {
865 Arc::clone(
866 &self
867 .inner
868 .read()
869 .unwrap_or_else(|p| p.into_inner())
870 .operator_confirms,
871 )
872 }
873
874 #[cfg(test)]
875 pub(crate) fn inject_operator_principal(&self, key: ModuleRouteKey, principal: Principal) {
876 self.alter_operator_binding_for_test(key, |binding| binding.principal = principal);
877 }
878
879 #[cfg(test)]
880 pub(crate) fn alter_operator_binding_for_test(
881 &self,
882 key: ModuleRouteKey,
883 alter: impl FnOnce(&mut RouteBinding),
884 ) {
885 let mut inner = self.write_inner().unwrap();
886 let old = inner.module_to_client.remove(&key).unwrap();
887 let client = ClientRouteKey {
888 connection_id: old.client_connection_id,
889 channel: old.client_channel,
890 };
891 inner.client_to_module.remove(&client);
892 let mut binding = Arc::try_unwrap(old).expect("test binding has no outstanding readers");
893 alter(&mut binding);
894 let binding = Arc::new(binding);
895 inner.client_to_module.insert(client, Arc::clone(&binding));
896 inner.module_to_client.insert(key, binding);
897 }
898
899 #[cfg(test)]
902 pub(crate) fn invalidate_operator_route_for_test(&self, key: ModuleRouteKey, stale: bool) {
903 let mut inner = self.write_inner().unwrap();
904 let old = inner.module_to_client.remove(&key).unwrap();
905 let client = ClientRouteKey {
906 connection_id: old.client_connection_id,
907 channel: old.client_channel,
908 };
909 inner.client_to_module.remove(&client);
910 if stale {
911 let mut binding = Arc::try_unwrap(old).expect("no binding readers at the test pause");
912 binding.module_epoch += 1;
913 let binding = Arc::new(binding);
914 inner.client_to_module.insert(client, binding.clone());
915 inner.module_to_client.insert(key, binding);
916 }
917 }
918
919 pub(crate) fn with_operator_route<T>(
923 &self,
924 connection_id: ConnectionId,
925 channel: u16,
926 epoch: u32,
927 admit: impl FnOnce(Option<&RouteBinding>) -> T,
928 ) -> Result<T, ForwardingError> {
929 self.with_operator_route_epoch(connection_id, channel, |binding| {
930 admit(binding.filter(|binding| binding.module_epoch == epoch))
931 })
932 }
933
934 pub(crate) fn with_operator_route_epoch<T>(
937 &self,
938 connection_id: ConnectionId,
939 channel: u16,
940 decide: impl FnOnce(Option<&RouteBinding>) -> T,
941 ) -> Result<T, ForwardingError> {
942 let inner = self.read_inner()?;
943 let binding = inner
944 .endpoint_by_connection
945 .get(&connection_id)
946 .and_then(|endpoint| {
947 inner.module_to_client.get(&ModuleRouteKey {
948 endpoint: *endpoint,
949 channel,
950 })
951 });
952 Ok(decide(binding.map(Arc::as_ref)))
953 }
954
955 pub(crate) fn counters(&self) -> DaemonCounters {
956 self.counters.clone()
957 }
958
959 pub(crate) fn counters_ref(&self) -> &DaemonCounters {
960 &self.counters
961 }
962
963 pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
964 self.route_bind_breakers.clone()
965 }
966
967 pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
968 self.route_bind_concurrency.clone()
969 }
970
971 pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
972 Arc::clone(&self.route_outages)
973 }
974
975 pub(crate) fn register_connection_close(
976 &self,
977 connection_id: ConnectionId,
978 ) -> ConnectionCloseReceiver {
979 let (sender, receiver) = oneshot::channel();
980 let replaced = self
981 .lock_close_registry()
982 .insert(connection_id, sender)
983 .is_some();
984 if replaced {
985 warn!(
986 connection_id = connection_id.get(),
987 "replaced existing connection close registration"
988 );
989 }
990 receiver
991 }
992
993 pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
994 self.lock_close_registry().remove(&connection_id);
995 if let Ok(mut inner) = self.write_inner() {
1000 inner.spawn_connections.remove(&connection_id);
1001 }
1002 }
1003
1004 #[cfg(unix)]
1013 pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
1014 let senders: Vec<_> = self.lock_close_registry().drain().collect();
1015 let count = senders.len();
1016 for (_, sender) in senders {
1017 let _ = sender.send(reason.clone());
1018 }
1019 count
1020 }
1021
1022 #[cfg(unix)]
1026 pub(crate) fn module_connections(
1027 &self,
1028 ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
1029 let inner = self.read_inner()?;
1030 let mut seen = HashSet::new();
1031 Ok(inner
1032 .modules_by_id
1033 .values()
1034 .chain(inner.candidates_by_id.values())
1035 .chain(inner.superseded_endpoints.values())
1036 .filter(|module| seen.insert(module.endpoint))
1037 .map(|module| ModuleConnectionTarget {
1038 module_id: inner
1039 .module_id_by_endpoint
1040 .get(&module.endpoint)
1041 .cloned()
1042 .unwrap_or_default(),
1043 endpoint: module.endpoint,
1044 sink: module.sink.clone(),
1045 negotiated_ver: module.negotiated_ver,
1046 })
1047 .collect())
1048 }
1049
1050 pub(crate) fn request_connection_close(
1054 &self,
1055 connection_id: ConnectionId,
1056 reason: CloseReason,
1057 ) -> bool {
1058 let sender = self.lock_close_registry().remove(&connection_id);
1059 if let Some(sender) = sender {
1060 debug!(
1061 connection_id = connection_id.get(),
1062 close_reason = %reason,
1063 "requesting connection close"
1064 );
1065 let _ = sender.send(reason);
1066 true
1067 } else {
1068 debug!(
1069 connection_id = connection_id.get(),
1070 close_reason = %reason,
1071 "connection close request ignored for inactive connection"
1072 );
1073 false
1074 }
1075 }
1076
1077 pub fn register_module_connection(
1078 &self,
1079 connection_id: ConnectionId,
1080 module_id: String,
1081 negotiated_ver: u8,
1082 concurrency: Concurrency,
1083 sink: FrameSink,
1084 ) -> Result<ModuleEndpointId, ForwardingError> {
1085 self.register_module_connection_inner(
1086 connection_id,
1087 module_id,
1088 negotiated_ver,
1089 concurrency,
1090 sink,
1091 None,
1092 )
1093 }
1094
1095 pub(crate) fn register_module_connection_acked(
1108 &self,
1109 connection_id: ConnectionId,
1110 module_id: String,
1111 negotiated_ver: u8,
1112 concurrency: Concurrency,
1113 sink: FrameSink,
1114 hello_ack: ModuleHelloAck,
1115 ) -> Result<ModuleEndpointId, ForwardingError> {
1116 self.register_module_connection_inner(
1117 connection_id,
1118 module_id,
1119 negotiated_ver,
1120 concurrency,
1121 sink,
1122 Some(hello_ack),
1123 )
1124 }
1125
1126 fn register_module_connection_inner(
1127 &self,
1128 connection_id: ConnectionId,
1129 module_id: String,
1130 negotiated_ver: u8,
1131 concurrency: Concurrency,
1132 sink: FrameSink,
1133 hello_ack: Option<ModuleHelloAck>,
1134 ) -> Result<ModuleEndpointId, ForwardingError> {
1135 let mut inner = self.write_inner()?;
1136 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1137 return Err(ForwardingError::ConnectionClosing { connection_id });
1138 }
1139 check_module_connection_role_locked(&inner, connection_id)?;
1140 let verified_prefix_owner = hello_ack
1143 .as_ref()
1144 .and_then(|hello| hello.verified_prefix_owner.clone());
1145 enqueue_hello_ack_locked(&sink, connection_id, hello_ack.map(|hello| hello.frame))?;
1146
1147 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1148 let endpoint = ModuleEndpointId {
1149 connection_id,
1150 generation: inner.next_generation,
1151 };
1152 inner.endpoint_by_connection.insert(connection_id, endpoint);
1153 inner
1154 .module_id_by_endpoint
1155 .insert(endpoint, module_id.clone());
1156 inner.next_module_channel.insert(endpoint, 1);
1157 inner.next_control_corr.insert(endpoint, 1);
1158 inner.modules_by_id.insert(
1159 module_id.clone(),
1160 ModuleConnection {
1161 endpoint,
1162 sink,
1163 negotiated_ver,
1164 concurrency,
1165 verified_prefix_owner,
1166 },
1167 );
1168 drop(inner);
1169
1170 if let Some(discarded) = self
1183 .route_bind_breakers
1184 .reset_for_new_module_connection(&module_id)
1185 {
1186 info!(
1187 module_id = %module_id,
1188 discarded_consecutive_timeouts = discarded,
1189 "route.bind breaker state discarded: a new module connection replaced the process it described"
1190 );
1191 }
1192 Ok(endpoint)
1193 }
1194
1195 #[cfg(test)]
1209 pub(crate) fn register_candidate_module_connection(
1210 &self,
1211 connection_id: ConnectionId,
1212 module_id: String,
1213 negotiated_ver: u8,
1214 concurrency: Concurrency,
1215 sink: FrameSink,
1216 ) -> Result<ModuleEndpointId, ForwardingError> {
1217 self.register_candidate_module_connection_inner(
1218 connection_id,
1219 module_id,
1220 negotiated_ver,
1221 concurrency,
1222 sink,
1223 None,
1224 )
1225 }
1226
1227 pub(crate) fn register_candidate_module_connection_acked(
1234 &self,
1235 connection_id: ConnectionId,
1236 module_id: String,
1237 negotiated_ver: u8,
1238 concurrency: Concurrency,
1239 sink: FrameSink,
1240 hello_ack: Frame,
1241 ) -> Result<ModuleEndpointId, ForwardingError> {
1242 self.register_candidate_module_connection_inner(
1243 connection_id,
1244 module_id,
1245 negotiated_ver,
1246 concurrency,
1247 sink,
1248 Some(hello_ack),
1249 )
1250 }
1251
1252 fn register_candidate_module_connection_inner(
1253 &self,
1254 connection_id: ConnectionId,
1255 module_id: String,
1256 negotiated_ver: u8,
1257 concurrency: Concurrency,
1258 sink: FrameSink,
1259 hello_ack: Option<Frame>,
1260 ) -> Result<ModuleEndpointId, ForwardingError> {
1261 let mut inner = self.write_inner()?;
1262 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1263 return Err(ForwardingError::ConnectionClosing { connection_id });
1264 }
1265 check_module_connection_role_locked(&inner, connection_id)?;
1266 if inner.candidates_by_id.contains_key(&module_id) {
1267 return Err(ForwardingError::CandidateSlotOccupied { module_id });
1268 }
1269 enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1272
1273 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1274 let endpoint = ModuleEndpointId {
1275 connection_id,
1276 generation: inner.next_generation,
1277 };
1278 inner.endpoint_by_connection.insert(connection_id, endpoint);
1279 inner
1280 .module_id_by_endpoint
1281 .insert(endpoint, module_id.clone());
1282 inner.next_module_channel.insert(endpoint, 1);
1283 inner.next_control_corr.insert(endpoint, 1);
1284 inner.candidates_by_id.insert(
1285 module_id,
1286 ModuleConnection {
1287 endpoint,
1288 sink,
1289 negotiated_ver,
1290 concurrency,
1291 verified_prefix_owner: None,
1292 },
1293 );
1294 Ok(endpoint)
1295 }
1296
1297 pub(crate) fn cutover_candidate(
1315 &self,
1316 module_id: &str,
1317 ) -> Result<Option<ForwardingCutover>, ForwardingError> {
1318 let mut inner = self.write_inner()?;
1319 if inner.daemon_draining {
1320 return Err(ForwardingError::ModuleReloading {
1321 module_id: module_id.to_string(),
1322 });
1323 }
1324 let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
1325 return Ok(None);
1326 };
1327 let promoted = candidate.endpoint;
1328 let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
1329 let incumbent = incumbent.map(|incumbent| {
1330 let endpoint = incumbent.endpoint;
1331 inner.superseded_endpoints.insert(endpoint, incumbent);
1332 endpoint
1333 });
1334 drop(inner);
1335
1336 if let Some(discarded) = self
1340 .route_bind_breakers
1341 .reset_for_new_module_connection(module_id)
1342 {
1343 info!(
1344 module_id = %module_id,
1345 discarded_consecutive_timeouts = discarded,
1346 "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
1347 );
1348 }
1349 Ok(Some(ForwardingCutover {
1350 promoted,
1351 incumbent,
1352 }))
1353 }
1354
1355 #[allow(clippy::too_many_arguments)]
1356 pub(crate) async fn begin_route_bind_relay_for(
1357 &self,
1358 client_connection_id: ConnectionId,
1359 client_sink: FrameSink,
1360 client_negotiated_ver: u8,
1361 client_corr: u64,
1362 module_id: &str,
1363 principal: Principal,
1364 scope: Option<BoundScope>,
1365 project_root: Option<ProjectRootId>,
1366 harness: String,
1367 build_class: Option<String>,
1368 operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1369 deadline: Instant,
1370 busy_refusals: bool,
1371 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1372 let client_permit =
1376 client_sink
1377 .reserve_owned()
1378 .await
1379 .map_err(|_| ForwardingError::ClientEgressClosed {
1380 connection_id: client_connection_id,
1381 })?;
1382 self.begin_route_bind_relay_inner(
1383 client_connection_id,
1384 client_sink,
1385 client_negotiated_ver,
1386 client_corr,
1387 module_id,
1388 principal,
1389 scope,
1390 project_root,
1391 harness,
1392 build_class,
1393 operator_app,
1394 deadline,
1395 client_permit,
1396 busy_refusals,
1397 )
1398 }
1399
1400 #[cfg(test)]
1401 pub(crate) fn begin_route_bind_relay_for_test(
1402 &self,
1403 client_connection_id: ConnectionId,
1404 client_sink: FrameSink,
1405 client_corr: u64,
1406 module_id: &str,
1407 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1408 self.begin_route_bind_relay_with_busy_for_test(
1409 client_connection_id,
1410 client_sink,
1411 client_corr,
1412 module_id,
1413 false,
1414 )
1415 }
1416
1417 #[cfg(test)]
1418 pub(crate) fn begin_route_bind_relay_with_busy_for_test(
1419 &self,
1420 client_connection_id: ConnectionId,
1421 client_sink: FrameSink,
1422 client_corr: u64,
1423 module_id: &str,
1424 busy_refusals: bool,
1425 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1426 let permit =
1427 client_sink
1428 .try_reserve_owned()
1429 .map_err(|_| ForwardingError::ClientEgressClosed {
1430 connection_id: client_connection_id,
1431 })?;
1432 self.begin_route_bind_relay_inner(
1433 client_connection_id,
1434 client_sink,
1435 subc_protocol::PROTOCOL_VERSION,
1436 client_corr,
1437 module_id,
1438 Principal::Direct,
1439 None,
1440 None,
1441 String::new(),
1442 None,
1443 None,
1444 Instant::now() + std::time::Duration::from_secs(60),
1445 permit,
1446 busy_refusals,
1447 )
1448 }
1449
1450 pub(crate) fn begin_module_control_rpc_for(
1451 &self,
1452 module_id: &str,
1453 expected_op: &str,
1454 deadline: Instant,
1455 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1456 self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1457 }
1458
1459 pub(crate) fn begin_health_probe_rpc_for(
1460 &self,
1461 module_id: &str,
1462 expected_op: &str,
1463 probe_started_at: Instant,
1464 deadline: Instant,
1465 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1466 self.begin_module_control_rpc_inner(
1467 module_id,
1468 expected_op,
1469 deadline,
1470 Some(probe_started_at),
1471 false,
1472 )
1473 }
1474
1475 pub(crate) fn begin_drain_health_probe_rpc_for(
1476 &self,
1477 module_id: &str,
1478 expected_op: &str,
1479 probe_started_at: Instant,
1480 deadline: Instant,
1481 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1482 self.begin_module_control_rpc_inner(
1483 module_id,
1484 expected_op,
1485 deadline,
1486 Some(probe_started_at),
1487 true,
1488 )
1489 }
1490
1491 pub(crate) fn begin_endpoint_health_probe_rpc_for(
1499 &self,
1500 endpoint: ModuleEndpointId,
1501 expected_op: &str,
1502 probe_started_at: Instant,
1503 deadline: Instant,
1504 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1505 let inner = self.write_inner()?;
1506 let module = module_connection_for_endpoint_locked(&inner, endpoint)
1507 .cloned()
1508 .ok_or(ForwardingError::NoModuleConnection)?;
1509 let module_id = inner
1510 .module_id_by_endpoint
1511 .get(&endpoint)
1512 .cloned()
1513 .unwrap_or_default();
1514 self.begin_control_rpc_locked(
1517 inner,
1518 &module_id,
1519 module,
1520 expected_op,
1521 deadline,
1522 Some(probe_started_at),
1523 true,
1524 )
1525 }
1526
1527 fn begin_module_control_rpc_inner(
1528 &self,
1529 module_id: &str,
1530 expected_op: &str,
1531 deadline: Instant,
1532 health_probe_started_at: Option<Instant>,
1533 allow_draining: bool,
1534 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1535 let inner = self.write_inner()?;
1536 let module = inner
1537 .modules_by_id
1538 .get(module_id)
1539 .cloned()
1540 .ok_or(ForwardingError::NoModuleConnection)?;
1541 self.begin_control_rpc_locked(
1542 inner,
1543 module_id,
1544 module,
1545 expected_op,
1546 deadline,
1547 health_probe_started_at,
1548 allow_draining,
1549 )
1550 }
1551
1552 #[allow(clippy::too_many_arguments)]
1553 fn begin_control_rpc_locked(
1554 &self,
1555 mut inner: RwLockWriteGuard<'_, ForwardingInner>,
1556 module_id: &str,
1557 module: ModuleConnection,
1558 expected_op: &str,
1559 deadline: Instant,
1560 health_probe_started_at: Option<Instant>,
1561 allow_draining: bool,
1562 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1563 if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1564 return Err(ForwardingError::ModuleReloading {
1565 module_id: module_id.to_string(),
1566 });
1567 }
1568 if inner
1569 .closing_connections
1570 .contains(&module.endpoint.connection_id)
1571 {
1572 return Err(ForwardingError::ConnectionClosing {
1573 connection_id: module.endpoint.connection_id,
1574 });
1575 }
1576 if health_probe_started_at.is_some() {
1577 inner
1581 .health_probe_tombstones
1582 .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1583 }
1584 let corr = match inner.allocate_control_corr(module.endpoint) {
1585 Ok(corr) => corr,
1586 Err(err) => {
1587 drop(inner);
1588 self.request_connection_close(
1589 module.endpoint.connection_id,
1590 CloseReason::new(
1591 "control_correlation_exhausted",
1592 "daemon-originated channel-0 correlation space exhausted",
1593 ),
1594 );
1595 return Err(err);
1596 }
1597 };
1598 let (sender, receiver) = oneshot::channel();
1599 inner.pending_control_rpcs.insert(
1600 (module.endpoint, corr),
1601 PendingModuleControlRpcEntry {
1602 expected_op: expected_op.to_string(),
1603 deadline,
1604 health_probe_started_at,
1605 sender,
1606 },
1607 );
1608
1609 Ok(PendingModuleControlRpc {
1610 endpoint: module.endpoint,
1611 module_sink: module.sink,
1612 negotiated_ver: module.negotiated_ver,
1613 corr,
1614 receiver,
1615 })
1616 }
1617
1618 #[allow(clippy::too_many_arguments)]
1619 fn begin_route_bind_relay_inner(
1620 &self,
1621 client_connection_id: ConnectionId,
1622 client_sink: FrameSink,
1623 client_negotiated_ver: u8,
1624 client_corr: u64,
1625 expected_module_id: &str,
1626 principal: Principal,
1627 scope: Option<BoundScope>,
1628 project_root: Option<ProjectRootId>,
1629 harness: String,
1630 build_class: Option<String>,
1631 operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1632 deadline: Instant,
1633 client_permit: crate::router::EgressPermit,
1634 busy_refusals: bool,
1635 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1636 let mut inner = self.write_inner()?;
1637 if inner
1641 .endpoint_by_connection
1642 .contains_key(&client_connection_id)
1643 {
1644 return Err(ForwardingError::ConnectionRoleConflict {
1645 connection_id: client_connection_id,
1646 });
1647 }
1648 if inner.closing_connections.contains(&client_connection_id) {
1649 return Err(ForwardingError::ConnectionClosing {
1650 connection_id: client_connection_id,
1651 });
1652 }
1653 let module = inner
1654 .modules_by_id
1655 .get(expected_module_id)
1656 .cloned()
1657 .ok_or(ForwardingError::NoModuleConnection)?;
1658 if inner.draining_endpoints.contains_key(&module.endpoint) {
1659 return Err(ForwardingError::ModuleReloading {
1660 module_id: expected_module_id.to_string(),
1661 });
1662 }
1663 if inner
1664 .closing_connections
1665 .contains(&module.endpoint.connection_id)
1666 {
1667 return Err(ForwardingError::ConnectionClosing {
1668 connection_id: module.endpoint.connection_id,
1669 });
1670 }
1671
1672 let corr = match inner.allocate_control_corr(module.endpoint) {
1673 Ok(corr) => corr,
1674 Err(err) => {
1675 drop(inner);
1676 self.request_connection_close(
1677 module.endpoint.connection_id,
1678 CloseReason::new(
1679 "control_correlation_exhausted",
1680 "daemon-originated channel-0 correlation space exhausted",
1681 ),
1682 );
1683 return Err(err);
1684 }
1685 };
1686 let (client_channel, client_epoch, module_channel, module_epoch) =
1687 inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1688 let client_key = ClientRouteKey {
1689 connection_id: client_connection_id,
1690 channel: client_channel,
1691 };
1692 let module_key = ModuleRouteKey {
1693 endpoint: module.endpoint,
1694 channel: module_channel,
1695 };
1696 let reservation = RouteReservation {
1697 client_key,
1698 module_key,
1699 client_epoch,
1700 module_epoch,
1701 project_root,
1702 };
1703 let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
1704 request_window: None,
1705 watch_window: None,
1706 route_channel: client_channel,
1707 route_epoch: client_epoch,
1708 })
1709 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1710 let route_open_frame = Frame::build_with_version(
1711 client_negotiated_ver,
1712 FrameType::Response,
1713 Flags::new(false, Priority::Passive, false),
1714 0,
1715 0,
1716 client_corr,
1717 response_body,
1718 )
1719 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1720 let (sender, receiver) = oneshot::channel();
1721 inner.reserved_client.insert(client_key, module_key);
1722 inner.reserved_module.insert(module_key, client_key);
1723 inner.pending_relays.insert(
1724 (module.endpoint, corr),
1725 PendingRouteBindRelayEntry {
1726 reservation,
1727 client_sink,
1728 client_negotiated_ver,
1729 client_permit,
1730 route_open_frame,
1731 busy_refusals,
1732 request_window: window_for(&module.concurrency),
1733 watch_window: (busy_refusals && module.concurrency == Concurrency::ModuleManaged)
1734 .then_some(DEFAULT_MODULE_MANAGED_WATCH_WINDOW),
1735 principal,
1736 scope,
1737 harness,
1738 build_class,
1739 operator_app,
1740 deadline,
1741 relay_enqueued: false,
1742 sender,
1743 },
1744 );
1745
1746 Ok(PendingRouteBindRelay {
1747 endpoint: module.endpoint,
1748 module_sink: module.sink,
1749 negotiated_ver: module.negotiated_ver,
1750 client_channel,
1751 client_epoch,
1752 module_channel,
1753 module_epoch,
1754 corr,
1755 receiver,
1756 })
1757 }
1758
1759 pub(crate) fn mark_route_bind_relay_enqueued(
1769 &self,
1770 endpoint: ModuleEndpointId,
1771 corr: u64,
1772 ) -> Result<RouteBindRelayMark, ForwardingError> {
1773 let mut inner = self.write_inner()?;
1774 if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
1775 pending.relay_enqueued = true;
1776 return Ok(RouteBindRelayMark::Marked);
1777 }
1778 if inner
1779 .relays_released_before_enqueue
1780 .remove(&(endpoint, corr))
1781 {
1782 return Ok(RouteBindRelayMark::ReleasedWithoutReply);
1783 }
1784 Ok(RouteBindRelayMark::SettledByReply)
1785 }
1786
1787 pub(crate) fn release_client_route(
1788 &self,
1789 client_connection_id: ConnectionId,
1790 client_channel: u16,
1791 expected_epoch: u32,
1792 ) -> Result<RouteRelease, ForwardingError> {
1793 let mut inner = self.write_inner()?;
1794 let release = release_client_route_locked(
1795 &mut inner,
1796 ClientRouteKey {
1797 connection_id: client_connection_id,
1798 channel: client_channel,
1799 },
1800 expected_epoch,
1801 );
1802 self.record_route_release(&release);
1803 Ok(release)
1804 }
1805
1806 pub(crate) fn release_module_route(
1807 &self,
1808 module_connection_id: ConnectionId,
1809 module_channel: u16,
1810 expected_epoch: u32,
1811 ) -> Result<RouteRelease, ForwardingError> {
1812 let mut inner = self.write_inner()?;
1813 let Some(endpoint) = inner
1814 .endpoint_by_connection
1815 .get(&module_connection_id)
1816 .copied()
1817 else {
1818 return Ok(RouteRelease::Absent);
1819 };
1820 let release = release_module_route_locked(
1821 &mut inner,
1822 ModuleRouteKey {
1823 endpoint,
1824 channel: module_channel,
1825 },
1826 expected_epoch,
1827 );
1828 self.record_route_release(&release);
1829 Ok(release)
1830 }
1831
1832 pub(crate) fn abort_pending_relay(
1833 &self,
1834 endpoint: ModuleEndpointId,
1835 corr: u64,
1836 outcome: RouteBindRelayOutcome,
1837 ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
1838 let mut inner = self.write_inner()?;
1839 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1840 inner
1845 .relays_released_before_enqueue
1846 .remove(&(endpoint, corr));
1847 return Ok(None);
1848 };
1849 release_reserved_route_locked(
1850 &mut inner,
1851 pending.reservation.client_key,
1852 pending.reservation.module_key,
1853 );
1854 let target = pending
1855 .relay_enqueued
1856 .then(|| abandoned_route_target(&inner, &pending.reservation));
1857 let _ = pending.sender.send(outcome);
1858 Ok(target.flatten())
1859 }
1860
1861 pub(crate) fn cancel_module_control_rpc(
1862 &self,
1863 endpoint: ModuleEndpointId,
1864 corr: u64,
1865 ) -> Result<(), ForwardingError> {
1866 self.write_inner()?
1867 .pending_control_rpcs
1868 .remove(&(endpoint, corr));
1869 Ok(())
1870 }
1871
1872 pub(crate) fn tombstone_health_probe_rpc(
1873 &self,
1874 endpoint: ModuleEndpointId,
1875 corr: u64,
1876 ) -> Result<bool, ForwardingError> {
1877 let key = (endpoint, corr);
1878 let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
1879 {
1880 let mut inner = self.write_inner()?;
1881 let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
1882 return Ok(false);
1883 };
1884 let Some(probe_started_at) = pending.health_probe_started_at else {
1885 inner.pending_control_rpcs.insert(key, pending);
1886 return Ok(false);
1887 };
1888 let module_id = inner
1889 .module_id_by_endpoint
1890 .get(&endpoint)
1891 .cloned()
1892 .unwrap_or_else(|| "unknown".to_string());
1893 inner.health_probe_tombstones.insert(
1894 key,
1895 HealthProbeTombstone {
1896 expected_op: pending.expected_op,
1897 module_id,
1898 probe_started_at,
1899 expires_at,
1900 },
1901 );
1902 }
1903 self.schedule_health_probe_tombstone_expiration(key, expires_at);
1904 Ok(true)
1905 }
1906
1907 fn schedule_health_probe_tombstone_expiration(
1908 &self,
1909 key: (ModuleEndpointId, u64),
1910 expires_at: Instant,
1911 ) {
1912 let inner = Arc::downgrade(&self.inner);
1913 tokio::spawn(async move {
1914 tokio::time::sleep_until(expires_at).await;
1915 let Some(inner) = inner.upgrade() else {
1916 return;
1917 };
1918 let Ok(mut inner) = inner.write() else {
1919 return;
1920 };
1921 let expired = inner
1922 .health_probe_tombstones
1923 .get(&key)
1924 .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
1925 if expired {
1926 inner.health_probe_tombstones.remove(&key);
1927 }
1928 });
1929 }
1930
1931 pub(crate) fn complete_pending_relay(
1932 &self,
1933 connection_id: ConnectionId,
1934 corr: u64,
1935 outcome: RouteBindRelayOutcome,
1936 ) -> Result<PendingRelayCompletion, ForwardingError> {
1937 let mut inner = self.write_inner()?;
1938 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1939 return Ok(PendingRelayCompletion {
1940 settled: false,
1941 abandoned: None,
1942 });
1943 };
1944 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1945 return Ok(PendingRelayCompletion {
1946 settled: false,
1947 abandoned: None,
1948 });
1949 };
1950
1951 if Instant::now() >= pending.deadline {
1952 release_reserved_route_locked(
1953 &mut inner,
1954 pending.reservation.client_key,
1955 pending.reservation.module_key,
1956 );
1957 let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
1958 .then(|| abandoned_route_target(&inner, &pending.reservation))
1959 .flatten();
1960 let _ = pending
1961 .sender
1962 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody {
1963 code: "module_timeout".to_string(),
1964 message: "route.bind response arrived after its daemon deadline".to_string(),
1965 detail: None,
1966 }));
1967 return Ok(PendingRelayCompletion {
1968 settled: true,
1969 abandoned,
1970 });
1971 }
1972
1973 match outcome {
1982 RouteBindRelayOutcome::Accepted
2002 if pending.client_sink.is_closed()
2003 || spawn_connection_dead(
2004 &inner,
2005 pending.reservation.client_key.connection_id,
2006 )
2007 || inner
2008 .closing_connections
2009 .contains(&pending.reservation.client_key.connection_id) =>
2010 {
2011 let reason = if pending.client_sink.is_closed() {
2012 "client egress closed before route publication"
2013 } else {
2014 "client connection is closing before route publication"
2015 };
2016 release_reserved_route_locked(
2017 &mut inner,
2018 pending.reservation.client_key,
2019 pending.reservation.module_key,
2020 );
2021 let abandoned = abandoned_route_target(&inner, &pending.reservation);
2025 let _ = pending
2026 .sender
2027 .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
2028 return Ok(PendingRelayCompletion {
2029 settled: true,
2030 abandoned,
2031 });
2032 }
2033 RouteBindRelayOutcome::Accepted
2056 if inner.superseded_endpoints.contains_key(&endpoint)
2057 || spawn_connection_dead(&inner, connection_id)
2058 || inner.draining_endpoints.contains_key(&endpoint) =>
2059 {
2060 release_reserved_route_locked(
2061 &mut inner,
2062 pending.reservation.client_key,
2063 pending.reservation.module_key,
2064 );
2065 let abandoned = abandoned_route_target(&inner, &pending.reservation);
2068 let module_id = inner
2069 .module_id_by_endpoint
2070 .get(&endpoint)
2071 .cloned()
2072 .unwrap_or_else(|| "unknown".to_string());
2073 let _ = pending
2074 .sender
2075 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
2076 "module_reloading",
2077 format!("module_id '{module_id}' is reloading"),
2078 )));
2079 return Ok(PendingRelayCompletion {
2080 settled: true,
2081 abandoned,
2082 });
2083 }
2084 RouteBindRelayOutcome::Accepted
2093 if pending
2094 .scope
2095 .as_ref()
2096 .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
2097 {
2098 let refusal = pending
2099 .scope
2100 .as_ref()
2101 .and_then(|scope| scope_refusal_locked(&inner, scope))
2102 .expect("guard matched a refusal under the same lock");
2103 release_reserved_route_locked(
2104 &mut inner,
2105 pending.reservation.client_key,
2106 pending.reservation.module_key,
2107 );
2108 let abandoned = abandoned_route_target(&inner, &pending.reservation);
2109 let _ = pending
2110 .sender
2111 .send(RouteBindRelayOutcome::DaemonRejected(refusal));
2112 return Ok(PendingRelayCompletion {
2113 settled: true,
2114 abandoned,
2115 });
2116 }
2117 RouteBindRelayOutcome::Accepted => {
2118 #[cfg(all(test, target_os = "linux"))]
2119 if let Some(pause) = self.route_commit_pause.lock().unwrap().take() {
2120 let _ = pause.reached.send(());
2121 pause
2122 .release
2123 .recv_timeout(Duration::from_secs(5))
2124 .expect("test must release bind publication");
2125 }
2126 let abandoned = commit_route_locked(&mut inner, pending)?;
2127 return Ok(PendingRelayCompletion {
2128 settled: true,
2129 abandoned,
2130 });
2131 }
2132 terminal => {
2133 release_reserved_route_locked(
2134 &mut inner,
2135 pending.reservation.client_key,
2136 pending.reservation.module_key,
2137 );
2138 let _ = pending.sender.send(terminal);
2139 }
2140 }
2141 Ok(PendingRelayCompletion {
2142 settled: true,
2143 abandoned: None,
2144 })
2145 }
2146
2147 pub(crate) fn pending_module_control_op(
2148 &self,
2149 connection_id: ConnectionId,
2150 corr: u64,
2151 ) -> Result<Option<String>, ForwardingError> {
2152 let inner = self.read_inner()?;
2153 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2154 return Ok(None);
2155 };
2156 let key = (endpoint, corr);
2157 Ok(inner
2158 .pending_control_rpcs
2159 .get(&key)
2160 .map(|pending| pending.expected_op.clone())
2161 .or_else(|| {
2162 inner
2163 .health_probe_tombstones
2164 .get(&key)
2165 .filter(|tombstone| tombstone.expires_at > Instant::now())
2166 .map(|tombstone| tombstone.expected_op.clone())
2167 }))
2168 }
2169
2170 pub(crate) fn complete_module_control_rpc(
2171 &self,
2172 connection_id: ConnectionId,
2173 corr: u64,
2174 actual_op: Option<&str>,
2175 outcome: ModuleControlRpcOutcome,
2176 ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
2177 let now = Instant::now();
2178 let mut inner = self.write_inner()?;
2179 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2180 return Ok(ModuleControlRpcCompletion::Unknown);
2181 };
2182 let key = (endpoint, corr);
2183 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2184 if now >= pending.deadline {
2185 let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
2186 ModuleControlRpcCompletion::LateHealthAnswer {
2187 module_id: inner
2188 .module_id_by_endpoint
2189 .get(&endpoint)
2190 .cloned()
2191 .unwrap_or_else(|| "unknown".to_string()),
2192 latency: now.saturating_duration_since(probe_started_at),
2193 }
2194 });
2195 let _ = pending
2196 .sender
2197 .send(ModuleControlRpcOutcome::DeadlineElapsed);
2198 return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
2199 }
2200 let outcome = match actual_op {
2201 Some(actual) if actual != pending.expected_op => {
2202 ModuleControlRpcOutcome::UnexpectedOp {
2203 expected: pending.expected_op,
2204 actual: actual.to_string(),
2205 }
2206 }
2207 _ => outcome,
2208 };
2209 let _ = pending.sender.send(outcome);
2210 return Ok(ModuleControlRpcCompletion::Settled);
2211 }
2212
2213 let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
2214 return Ok(ModuleControlRpcCompletion::Unknown);
2215 };
2216 if tombstone.expires_at <= now {
2217 return Ok(ModuleControlRpcCompletion::Unknown);
2218 }
2219 Ok(ModuleControlRpcCompletion::LateHealthAnswer {
2220 module_id: tombstone.module_id,
2221 latency: now.saturating_duration_since(tombstone.probe_started_at),
2222 })
2223 }
2224
2225 #[cfg(test)]
2226 pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
2227 Ok(self.read_inner()?.health_probe_tombstones.len())
2228 }
2229
2230 #[cfg(test)]
2231 pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
2232 Ok(self.read_inner()?.closing_connections.len())
2233 }
2234
2235 #[cfg(test)]
2238 pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
2239 let inner = self.read_inner()?;
2240 Ok((inner.reserved_client.len(), inner.reserved_module.len()))
2241 }
2242
2243 pub(crate) fn module_endpoint_for_connection(
2244 &self,
2245 connection_id: ConnectionId,
2246 ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
2247 Ok(self
2248 .read_inner()?
2249 .endpoint_by_connection
2250 .get(&connection_id)
2251 .copied())
2252 }
2253
2254 pub(crate) fn module_id_for_connection(
2257 &self,
2258 connection_id: ConnectionId,
2259 ) -> Result<Option<String>, ForwardingError> {
2260 let inner = self.read_inner()?;
2261 Ok(inner
2262 .endpoint_by_connection
2263 .get(&connection_id)
2264 .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
2265 .cloned())
2266 }
2267
2268 pub(crate) fn module_route_epoch_was_allocated(
2275 &self,
2276 connection_id: ConnectionId,
2277 channel: u16,
2278 epoch: u32,
2279 ) -> Result<bool, ForwardingError> {
2280 let inner = self.read_inner()?;
2281 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2282 return Ok(false);
2283 };
2284 Ok(inner
2285 .module_slot_epochs
2286 .get(&ModuleRouteKey { endpoint, channel })
2287 .is_some_and(|last| epoch != 0 && epoch <= *last))
2288 }
2289
2290 pub(crate) fn has_live_module_connection(
2291 &self,
2292 module_id: &str,
2293 ) -> Result<bool, ForwardingError> {
2294 Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
2295 }
2296
2297 pub(crate) fn try_admit_request(
2300 &self,
2301 route: &Arc<RouteBinding>,
2302 epoch: u32,
2303 corr: u64,
2304 subscription: bool,
2305 ) -> Result<RequestAdmission, ForwardingError> {
2306 let inner = self.read_inner()?;
2307 let key = ClientRouteKey {
2308 connection_id: route.client_connection_id,
2309 channel: route.client_channel,
2310 };
2311 if epoch != route.client_epoch
2312 || !inner
2313 .client_to_module
2314 .get(&key)
2315 .is_some_and(|current| Arc::ptr_eq(current, route))
2316 {
2317 return Ok(RequestAdmission::Stale);
2318 }
2319 let admission = route.flow.try_acquire_tagged(corr, subscription);
2320 Ok(match admission {
2321 RequestAdmission::Stale
2322 if inner
2323 .draining_endpoints
2324 .contains_key(&route.module_endpoint) =>
2325 {
2326 RequestAdmission::ModuleReloading
2327 }
2328 admission => admission,
2329 })
2330 }
2331
2332 pub(crate) fn lookup_data_route(
2333 &self,
2334 connection_id: ConnectionId,
2335 channel: u16,
2336 epoch: u32,
2337 ) -> Result<DataRoute, ForwardingError> {
2338 let inner = self.read_inner()?;
2339 let state = if let Some(endpoint) =
2340 inner.endpoint_by_connection.get(&connection_id).copied()
2341 {
2342 let key = ModuleRouteKey { endpoint, channel };
2343 match inner.module_to_client.get(&key) {
2344 Some(route) if route.module_epoch == epoch => {
2345 DataRouteState::Bound(Arc::clone(route))
2346 }
2347 Some(_) => DataRouteState::EpochMismatch,
2348 None if inner.reserved_module.contains_key(&key)
2349 && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
2350 {
2351 DataRouteState::Reserved
2352 }
2353 None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
2354 None => DataRouteState::Absent,
2355 }
2356 } else {
2357 let key = ClientRouteKey {
2358 connection_id,
2359 channel,
2360 };
2361 match inner.client_to_module.get(&key) {
2362 Some(route) if route.client_epoch == epoch => {
2363 DataRouteState::Bound(Arc::clone(route))
2364 }
2365 Some(_) => DataRouteState::EpochMismatch,
2366 None if inner.reserved_client.contains_key(&key)
2367 && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
2368 {
2369 DataRouteState::Reserved
2370 }
2371 None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
2372 None => DataRouteState::Absent,
2373 }
2374 };
2375 Ok(
2376 if inner.endpoint_by_connection.contains_key(&connection_id) {
2377 DataRoute::Module(state)
2378 } else {
2379 DataRoute::Client(state)
2380 },
2381 )
2382 }
2383
2384 #[cfg(test)]
2385 pub(crate) fn inject_client_slot_epoch(
2386 &self,
2387 connection_id: ConnectionId,
2388 channel: u16,
2389 last_epoch: u32,
2390 ) {
2391 let mut inner = self.write_inner().expect("forwarding lock");
2392 inner.client_slot_epochs.insert(
2393 ClientRouteKey {
2394 connection_id,
2395 channel,
2396 },
2397 last_epoch,
2398 );
2399 inner.next_client_channel.insert(connection_id, channel);
2400 }
2401
2402 #[cfg(test)]
2403 pub(crate) fn inject_module_slot_epoch(
2404 &self,
2405 endpoint: ModuleEndpointId,
2406 channel: u16,
2407 last_epoch: u32,
2408 ) {
2409 let mut inner = self.write_inner().expect("forwarding lock");
2410 inner
2411 .module_slot_epochs
2412 .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
2413 inner.next_module_channel.insert(endpoint, channel);
2414 }
2415
2416 #[cfg(test)]
2417 pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
2418 self.write_inner()
2419 .expect("forwarding lock")
2420 .next_control_corr
2421 .insert(endpoint, next_corr);
2422 }
2423
2424 pub(crate) fn cache_status(
2425 &self,
2426 endpoint: ModuleEndpointId,
2427 module_channel: u16,
2428 module_epoch: u32,
2429 status: String,
2430 ) -> Result<bool, ForwardingError> {
2431 let mut inner = self.write_inner()?;
2432 if !inner.module_id_by_endpoint.contains_key(&endpoint) {
2433 return Err(ForwardingError::StaleModuleEndpoint);
2434 }
2435
2436 let module_key = ModuleRouteKey {
2437 endpoint,
2438 channel: module_channel,
2439 };
2440 let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
2441 (route.module_epoch == module_epoch).then_some((
2442 ClientRouteKey {
2443 connection_id: route.client_connection_id,
2444 channel: route.client_channel,
2445 },
2446 route.client_epoch,
2447 ))
2448 } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
2449 (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
2450 client_key,
2451 inner
2452 .client_slot_epochs
2453 .get(&client_key)
2454 .copied()
2455 .unwrap_or(0),
2456 ))
2457 } else {
2458 None
2459 };
2460
2461 if let Some(handle) = handle {
2462 inner.status.insert(handle, status);
2463 Ok(true)
2464 } else {
2465 debug!(
2466 module_channel,
2467 module_epoch,
2468 generation = endpoint.generation,
2469 connection_id = endpoint.connection_id.get(),
2470 "dropping stale status update for module route handle"
2471 );
2472 Ok(false)
2473 }
2474 }
2475
2476 pub(crate) fn route_poll_snapshot(
2477 &self,
2478 client_connection_id: ConnectionId,
2479 client_channel: u16,
2480 client_epoch: u32,
2481 ) -> Result<RoutePollSnapshot, ForwardingError> {
2482 let inner = self.read_inner()?;
2483 let client_key = ClientRouteKey {
2484 connection_id: client_connection_id,
2485 channel: client_channel,
2486 };
2487 let Some(route) = inner.client_to_module.get(&client_key) else {
2488 return Ok(RoutePollSnapshot::Absent);
2489 };
2490 if route.client_epoch != client_epoch
2491 || !inner
2492 .module_id_by_endpoint
2493 .contains_key(&route.module_endpoint)
2494 {
2495 return Ok(RoutePollSnapshot::Absent);
2496 }
2497 Ok(RoutePollSnapshot::Bound {
2498 module_id: route.module_id.clone(),
2499 status: inner.status.get(&(client_key, client_epoch)).cloned(),
2500 })
2501 }
2502
2503 pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2504 Ok(self.read_inner()?.client_to_module.len())
2505 }
2506
2507 pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2517 let inner = self.read_inner()?;
2518 let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2519 for key in inner.client_to_module.keys() {
2520 *per_connection.entry(key.connection_id).or_insert(0) += 1;
2521 }
2522 let max = per_connection.values().copied().max().unwrap_or(0);
2523 Ok((per_connection.len(), max))
2524 }
2525
2526 pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2527 let inner = self.read_inner()?;
2528 Ok(inner
2529 .client_to_module
2530 .keys()
2531 .any(|key| key.channel == route_channel))
2532 }
2533
2534 pub(crate) fn daemon_drain_result_for_connection(
2538 &self,
2539 connection_id: ConnectionId,
2540 ) -> Result<Option<bool>, ForwardingError> {
2541 let inner = self.read_inner()?;
2542 Ok(inner.daemon_draining.then(|| {
2543 inner
2544 .endpoint_by_connection
2545 .get(&connection_id)
2546 .and_then(|endpoint| inner.daemon_drain_results.get(endpoint))
2547 .copied()
2548 .unwrap_or(false)
2549 }))
2550 }
2551
2552 #[cfg(unix)]
2553 pub(crate) fn record_daemon_drain_result(
2554 &self,
2555 endpoint: ModuleEndpointId,
2556 drained: bool,
2557 ) -> Result<(), ForwardingError> {
2558 let mut inner = self.write_inner()?;
2559 if let Some(result) = inner.daemon_drain_results.get_mut(&endpoint) {
2563 *result = drained;
2564 }
2565 Ok(())
2566 }
2567
2568 #[cfg(unix)]
2571 pub(crate) fn begin_daemon_drain(
2572 &self,
2573 ) -> Result<Vec<(String, ModuleEndpointId)>, ForwardingError> {
2574 let mut inner = self.write_inner()?;
2575 if inner.daemon_draining {
2576 return Ok(Vec::new());
2577 }
2578 inner.daemon_draining = true;
2579 let modules = inner
2580 .modules_by_id
2581 .iter()
2582 .map(|(id, module)| (id.clone(), module.endpoint))
2583 .collect::<Vec<_>>();
2584 for (_, endpoint) in &modules {
2585 inner.daemon_drain_results.insert(*endpoint, false);
2586 inner
2587 .draining_endpoints
2588 .insert(*endpoint, RouteCloseReason::Restart);
2589 }
2590 let off_slot_endpoints = inner
2595 .candidates_by_id
2596 .values()
2597 .map(|module| module.endpoint)
2598 .chain(inner.superseded_endpoints.keys().copied())
2599 .collect::<Vec<_>>();
2600 for endpoint in off_slot_endpoints {
2601 inner
2602 .draining_endpoints
2603 .insert(endpoint, RouteCloseReason::Restart);
2604 }
2605 Ok(modules)
2606 }
2607
2608 pub(crate) fn begin_module_drain(
2615 &self,
2616 module_id: &str,
2617 reason: RouteCloseReason,
2618 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2619 let mut inner = self.write_inner()?;
2620 let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2621 return Ok(None);
2622 };
2623 Ok(Some(begin_drain_locked(
2624 &mut inner, module_id, module, reason,
2625 )))
2626 }
2627
2628 pub(crate) fn begin_endpoint_drain(
2635 &self,
2636 endpoint: ModuleEndpointId,
2637 reason: RouteCloseReason,
2638 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2639 let mut inner = self.write_inner()?;
2640 let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2641 return Ok(None);
2642 };
2643 let module_id = inner
2644 .module_id_by_endpoint
2645 .get(&endpoint)
2646 .cloned()
2647 .expect("an endpoint resolved to a module connection has a module id");
2648 Ok(Some(begin_drain_locked(
2649 &mut inner, &module_id, module, reason,
2650 )))
2651 }
2652}
2653
2654fn begin_drain_locked(
2658 inner: &mut ForwardingInner,
2659 module_id: &str,
2660 module: ModuleConnection,
2661 reason: RouteCloseReason,
2662) -> ModuleDrainTarget {
2663 {
2664 let endpoint = module.endpoint;
2665 inner.draining_endpoints.insert(endpoint, reason);
2666
2667 let flows = inner
2668 .client_to_module
2669 .values()
2670 .filter(|route| route.module_endpoint == endpoint)
2671 .map(|route| Arc::clone(&route.flow))
2672 .collect::<Vec<_>>();
2673 let excluded_subscriptions = flows
2674 .into_iter()
2675 .map(|flow| flow.begin_drain())
2676 .fold(0u32, u32::saturating_add);
2677
2678 let pending_keys = inner
2679 .pending_relays
2680 .keys()
2681 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2682 .copied()
2683 .collect::<Vec<_>>();
2684 let mut abandoned_bindings = Vec::new();
2685 for key in pending_keys {
2686 let Some(pending) = inner.pending_relays.remove(&key) else {
2687 continue;
2688 };
2689 release_reserved_route_locked(
2690 inner,
2691 pending.reservation.client_key,
2692 pending.reservation.module_key,
2693 );
2694 if pending.relay_enqueued {
2695 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2696 abandoned_bindings.push(target);
2697 }
2698 } else {
2699 inner.relays_released_before_enqueue.insert(key);
2703 }
2704 let _ = pending
2705 .sender
2706 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
2707 "module_reloading",
2708 format!("module_id '{module_id}' is reloading"),
2709 )));
2710 }
2711
2712 let pending_control_keys = inner
2713 .pending_control_rpcs
2714 .keys()
2715 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2716 .copied()
2717 .collect::<Vec<_>>();
2718 for key in pending_control_keys {
2719 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2720 let _ = pending
2721 .sender
2722 .send(ModuleControlRpcOutcome::ModuleGone(format!(
2723 "module '{module_id}' began draining during module-control RPC"
2724 )));
2725 }
2726 }
2727
2728 ModuleDrainTarget {
2729 endpoint,
2730 sink: module.sink,
2731 negotiated_ver: module.negotiated_ver,
2732 abandoned_bindings,
2733 excluded_subscriptions,
2734 }
2735 }
2736}
2737
2738#[derive(Debug, Default, PartialEq, Eq)]
2741pub(crate) struct DrainHoldouts {
2742 pub(crate) requests: usize,
2745 pub(crate) routes: usize,
2747 pub(crate) total_routes: usize,
2749 pub(crate) top_connections: Vec<(u64, usize)>,
2752 pub(crate) held: Vec<(u16, u64)>,
2761}
2762
2763pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
2765
2766impl ForwardingTable {
2767 pub(crate) fn endpoint_drain_holdouts(
2769 &self,
2770 endpoint: ModuleEndpointId,
2771 ) -> Result<DrainHoldouts, ForwardingError> {
2772 let inner = self.read_inner()?;
2773 let mut holdouts = DrainHoldouts::default();
2774 let mut by_connection: HashMap<u64, usize> = HashMap::new();
2775 for (key, route) in &inner.client_to_module {
2776 if route.module_endpoint != endpoint {
2777 continue;
2778 }
2779 holdouts.total_routes += 1;
2780 let held = route.flow.drain_in_flight();
2781 if held == 0 {
2782 continue;
2783 }
2784 holdouts.requests += held;
2785 holdouts.routes += 1;
2786 *by_connection.entry(key.connection_id.get()).or_default() += held;
2787 holdouts.held.extend(
2788 route
2789 .flow
2790 .drain_held_corrs()
2791 .into_iter()
2792 .map(|corr| (route.module_channel, corr)),
2793 );
2794 }
2795 holdouts.held.sort_unstable();
2796 holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
2797 let mut connections = by_connection.into_iter().collect::<Vec<_>>();
2798 connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
2799 connections.truncate(3);
2800 holdouts.top_connections = connections;
2801 Ok(holdouts)
2802 }
2803
2804 pub(crate) fn endpoint_in_flight_count(
2805 &self,
2806 endpoint: ModuleEndpointId,
2807 ) -> Result<usize, ForwardingError> {
2808 let inner = self.read_inner()?;
2809 Ok(inner
2810 .client_to_module
2811 .values()
2812 .filter(|route| route.module_endpoint == endpoint)
2813 .map(|route| route.flow.drain_in_flight())
2814 .sum())
2815 }
2816
2817 pub(crate) fn endpoint_is_draining(
2818 &self,
2819 endpoint: ModuleEndpointId,
2820 ) -> Result<bool, ForwardingError> {
2821 Ok(self
2822 .read_inner()?
2823 .draining_endpoints
2824 .contains_key(&endpoint))
2825 }
2826
2827 pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
2828 let inner = self.read_inner()?;
2829 Ok(inner
2830 .modules_by_id
2831 .get(module_id)
2832 .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
2833 }
2834
2835 pub(crate) fn release_module_endpoint_routes(
2836 &self,
2837 endpoint: ModuleEndpointId,
2838 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2839 let mut inner = self.write_inner()?;
2840 let routes = inner
2841 .module_to_client
2842 .iter()
2843 .filter(|(module_key, _)| module_key.endpoint == endpoint)
2844 .map(|(module_key, route)| (*module_key, route.module_epoch))
2845 .collect::<Vec<_>>();
2846 let mut released = Vec::with_capacity(routes.len());
2847 for (module_key, epoch) in routes {
2848 if let RouteRelease::Removed(target) =
2849 release_module_route_locked(&mut inner, module_key, epoch)
2850 {
2851 released.push(target);
2852 }
2853 }
2854 Ok(released)
2855 }
2856
2857 pub(crate) fn rebind_module_policy(
2860 &self,
2861 module_id: &str,
2862 expected_connection: Option<ConnectionId>,
2863 authorized: impl FnOnce() -> bool,
2864 ) -> Result<Option<PolicyRevocation>, ForwardingError> {
2865 #[cfg(test)]
2866 if let Some(hold) = self.policy_rebind_before_write.lock().unwrap().take() {
2867 hold.wait();
2868 hold.wait();
2869 }
2870 let mut inner = self.write_inner()?;
2871 let module = inner
2872 .modules_by_id
2873 .get(module_id)
2874 .cloned()
2875 .ok_or(ForwardingError::NoModuleConnection)?;
2876 if expected_connection.is_some_and(|connection| module.endpoint.connection_id != connection)
2877 || !authorized()
2878 {
2879 return Ok(None);
2880 }
2881 let endpoint = module.endpoint;
2882 if inner.draining_endpoints.contains_key(&endpoint) {
2883 return Err(ForwardingError::ModuleReloading {
2884 module_id: module_id.to_owned(),
2885 });
2886 }
2887 let target = begin_drain_locked(&mut inner, module_id, module, RouteCloseReason::Reload);
2888 let routes = endpoint_routes_locked(&inner, endpoint);
2889 let mut goodbyes = target.abandoned_bindings;
2890 for route in &routes {
2891 let target = &route.goodbye_target;
2892 if let RouteRelease::Removed(module) = release_client_route_locked(
2893 &mut inner,
2894 ClientRouteKey {
2895 connection_id: target.connection_id,
2896 channel: target.channel,
2897 },
2898 target.epoch,
2899 ) {
2900 goodbyes.push(module);
2901 goodbyes.push(target.clone());
2902 }
2903 }
2904 inner.draining_endpoints.remove(&endpoint);
2905 Ok(Some((routes, goodbyes)))
2906 }
2907
2908 #[cfg(test)]
2909 pub(crate) fn pause_policy_rebind_before_write(&self, hold: Arc<std::sync::Barrier>) {
2910 *self.policy_rebind_before_write.lock().unwrap() = Some(hold);
2911 }
2912
2913 #[cfg(test)]
2914 pub(crate) fn with_forwarding_write_lock_for_test(&self, work: impl FnOnce()) {
2915 let _guard = self.write_inner().unwrap();
2916 work();
2917 }
2918
2919 pub(crate) fn endpoint_routes(
2925 &self,
2926 endpoint: ModuleEndpointId,
2927 ) -> Result<Vec<EndpointRoute>, ForwardingError> {
2928 let inner = self.read_inner()?;
2929 Ok(endpoint_routes_locked(&inner, endpoint))
2930 }
2931
2932 pub(crate) fn route_census(
2934 &self,
2935 module_id: Option<&str>,
2936 ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
2937 let inner = self.read_inner()?;
2938 let mut endpoints = inner
2939 .modules_by_id
2940 .iter()
2941 .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
2942 .map(|(id, module)| (id.clone(), module.endpoint))
2943 .collect::<Vec<_>>();
2944 endpoints.sort_by(|left, right| left.0.cmp(&right.0));
2945 Ok(endpoints
2946 .into_iter()
2947 .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
2948 .collect())
2949 }
2950
2951 pub(crate) fn live_roots(
2953 &self,
2954 module_id: &str,
2955 ) -> Result<ModuleControlResponseToModule, ForwardingError> {
2956 let inner = self.read_inner()?;
2957 let endpoint = inner
2958 .modules_by_id
2959 .get(module_id)
2960 .map(|module| module.endpoint);
2961 let mut roots = BTreeMap::new();
2962 let mut unknown_root_bindings = 0;
2963 let mut total_bindings = 0;
2964 if let Some(endpoint) = endpoint {
2965 for binding in inner
2966 .module_to_client
2967 .values()
2968 .filter(|binding| binding.module_endpoint == endpoint)
2969 {
2970 total_bindings += 1;
2971 let workdir = binding
2972 .scope
2973 .as_ref()
2974 .and_then(|scope| scope.workdir.as_deref())
2975 .map(std::path::Path::new);
2976 if let Some(root) = &binding.project_root {
2977 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2978 entry.0 += 1;
2979 } else if workdir.is_none() {
2980 unknown_root_bindings += 1;
2981 }
2982 if let Some(workdir) = workdir.filter(|workdir| {
2983 binding.project_root.as_ref().map(ProjectRootId::as_path) != Some(*workdir)
2984 }) {
2985 roots.entry(workdir.to_path_buf()).or_insert((0, 0)).0 += 1;
2986 }
2987 }
2988 for pending in inner
2989 .pending_relays
2990 .values()
2991 .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
2992 {
2993 total_bindings += 1;
2994 if let Some(root) = &pending.reservation.project_root {
2995 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2996 entry.1 += 1;
2997 } else {
2998 unknown_root_bindings += 1;
2999 }
3000 }
3001 }
3002 Ok(ModuleControlResponseToModule::LiveRoots {
3003 roots: roots
3004 .into_iter()
3005 .map(|(project_root, (bound, pending))| LiveRoot {
3006 project_root,
3007 bound,
3008 pending,
3009 })
3010 .collect(),
3011 unknown_root_bindings,
3012 total_bindings,
3013 })
3014 }
3015
3016 pub(crate) fn connection_has_client_routes(
3022 &self,
3023 connection_id: ConnectionId,
3024 ) -> Result<bool, ForwardingError> {
3025 let inner = self.read_inner()?;
3026 Ok(connection_has_client_routes_locked(&inner, connection_id))
3027 }
3028
3029 pub(crate) fn cleanup_connection(
3030 &self,
3031 connection_id: ConnectionId,
3032 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
3033 self.cleanup_connection_counted(connection_id)
3034 .map(|cleanup| cleanup.released)
3035 }
3036
3037 pub(crate) fn cleanup_connection_counted(
3042 &self,
3043 connection_id: ConnectionId,
3044 ) -> Result<ConnectionCleanup, ForwardingError> {
3045 self.frame_drop_warnings
3046 .lock()
3047 .unwrap_or_else(std::sync::PoisonError::into_inner)
3048 .remove(&connection_id);
3049 let mut inner = self.write_inner()?;
3050 inner.closing_connections.insert(connection_id);
3051 #[cfg(all(test, target_os = "linux"))]
3052 if let Some(pause) = self.cleanup_pause.lock().unwrap().take() {
3053 let _ = pause.reached.send(());
3054 pause
3055 .release
3056 .recv_timeout(Duration::from_secs(5))
3057 .expect("test must release connection cleanup");
3058 }
3059 #[cfg(all(test, target_os = "linux"))]
3060 if let Some(observe) = self.cleanup_tie_observer.lock().unwrap().take() {
3061 let pending = inner
3062 .pending_relays
3063 .values()
3064 .filter(|relay| relay.reservation.client_key.connection_id == connection_id)
3065 .count();
3066 let _ = observe.send((
3067 inner.spawn_connections.contains_key(&connection_id),
3068 pending,
3069 ));
3070 }
3071 let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
3072 remove_module_connection_locked(&mut inner, endpoint)
3073 } else {
3074 ConnectionCleanup {
3075 released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
3076 abandoned_relays: 0,
3077 }
3078 };
3079 inner.closing_connections.remove(&connection_id);
3088 Ok(cleanup)
3089 }
3090
3091 fn cleanup_client_connection_locked(
3092 inner: &mut ForwardingInner,
3093 connection_id: ConnectionId,
3094 ) -> Vec<GoodbyeTarget> {
3095 let routes = inner
3096 .client_to_module
3097 .iter()
3098 .filter(|(key, _)| key.connection_id == connection_id)
3099 .map(|(key, route)| (*key, route.client_epoch))
3100 .collect::<Vec<_>>();
3101 let mut released = Vec::with_capacity(routes.len());
3102 for (client_key, epoch) in routes {
3103 if let RouteRelease::Removed(target) =
3104 release_client_route_locked(inner, client_key, epoch)
3105 {
3106 released.push(target);
3107 }
3108 }
3109
3110 let pending_keys = inner
3111 .pending_relays
3112 .iter()
3113 .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
3114 .map(|(key, _)| *key)
3115 .collect::<Vec<_>>();
3116 for key in pending_keys {
3117 let Some(pending) = inner.pending_relays.remove(&key) else {
3118 continue;
3119 };
3120 release_reserved_route_locked(
3121 inner,
3122 pending.reservation.client_key,
3123 pending.reservation.module_key,
3124 );
3125 if pending.relay_enqueued {
3126 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
3127 released.push(target);
3128 }
3129 } else {
3130 inner.relays_released_before_enqueue.insert(key);
3134 }
3135 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3136 "client connection closed during route.bind relay".to_string(),
3137 ));
3138 }
3139
3140 let orphaned = inner
3141 .reserved_client
3142 .iter()
3143 .filter(|(key, _)| key.connection_id == connection_id)
3144 .map(|(client, module)| (*client, *module))
3145 .collect::<Vec<_>>();
3146 for (client_key, module_key) in orphaned {
3147 release_reserved_route_locked(inner, client_key, module_key);
3148 }
3149 inner.next_client_channel.remove(&connection_id);
3150 inner
3151 .client_slot_epochs
3152 .retain(|key, _| key.connection_id != connection_id);
3153 inner
3154 .last_published_epoch
3155 .retain(|key, _| key.connection_id != connection_id);
3156 inner
3157 .status
3158 .retain(|(key, _), _| key.connection_id != connection_id);
3159
3160 released
3161 }
3162
3163 pub(crate) fn escalate_client_delivery_failure(
3172 &self,
3173 connection_id: ConnectionId,
3174 channel: u16,
3175 expected_epoch: u32,
3176 reason: CloseReason,
3177 undelivered: UndeliveredFrame<'_>,
3178 ) -> Result<bool, ForwardingError> {
3179 self.escalate_client_delivery_failure_diagnosed(
3180 connection_id,
3181 channel,
3182 expected_epoch,
3183 reason,
3184 undelivered,
3185 None,
3186 )
3187 }
3188
3189 pub(crate) fn escalate_client_delivery_failure_diagnosed(
3190 &self,
3191 connection_id: ConnectionId,
3192 channel: u16,
3193 expected_epoch: u32,
3194 reason: CloseReason,
3195 undelivered: UndeliveredFrame<'_>,
3196 diagnosis: Option<&crate::router::ClientDeliveryDiagnosis>,
3197 ) -> Result<bool, ForwardingError> {
3198 let principals = {
3199 let mut inner = self.write_inner()?;
3200 let key = ClientRouteKey {
3201 connection_id,
3202 channel,
3203 };
3204 if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
3205 None
3206 } else {
3207 inner.closing_connections.insert(connection_id);
3208 Some(connection_principals_locked(&inner, connection_id))
3209 }
3210 };
3211 let Some(principals) = principals else {
3212 self.warn_frame_drop(
3216 connection_id,
3217 channel,
3218 expected_epoch,
3219 "module_to_client",
3220 diagnosis
3221 .map(|failure| failure.cause)
3222 .unwrap_or("delivery_failed"),
3223 );
3224 return Ok(false);
3225 };
3226 let backlog = undelivered.sink.backlog();
3227 let close_reason = reason.to_string();
3228 if self.request_connection_close(connection_id, reason) {
3229 warn!(
3230 connection_id = connection_id.get(),
3231 principals = %principals,
3232 module_id = undelivered.module_id.unwrap_or("unknown"),
3233 client_channel = channel,
3234 queued_bytes = backlog.queued_bytes,
3235 queued_frames = backlog.queued_frames,
3236 frame_type = ?diagnosis.map(|failure| failure.frame_type),
3237 frame_flags = ?diagnosis.map(|failure| failure.flags.0),
3238 close_cause = diagnosis.map(|failure| failure.cause).unwrap_or("delivery_failed"),
3239 oldest_queued_ms = backlog
3240 .oldest_age
3241 .map(|age| age.as_millis() as u64)
3242 .unwrap_or(0),
3243 close_reason = %close_reason,
3244 "closing client connection: its egress queue could not take a frame"
3245 );
3246 }
3247 Ok(true)
3248 }
3249
3250 pub(crate) fn publish_scope_changes(
3264 &self,
3265 changes: &[ScopeTagChange],
3266 ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
3267 let mut inner = self.write_inner()?;
3268 let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
3269 for change in changes {
3270 let key = (change.owner.clone(), change.scope_ref.clone());
3271 match change.after {
3272 Some(tag) => {
3273 inner.scope_tags.insert(key, tag);
3274 }
3275 None => {
3276 inner.scope_tags.remove(&key);
3277 }
3278 }
3279 by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
3280 }
3281 let mut selected = Vec::new();
3282 for (client_key, route) in &inner.client_to_module {
3283 let Some(scope) = &route.scope else {
3284 continue;
3285 };
3286 let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
3287 else {
3288 continue;
3289 };
3290 if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
3291 continue;
3292 }
3293 let reason = match &change.drain {
3294 ScopeDrain::Nothing => continue,
3295 ScopeDrain::All(reason) => *reason,
3296 ScopeDrain::Carriers(narrowed) => {
3297 let owner = Principal::Reserved {
3298 module_id: scope.owner.clone(),
3299 };
3300 let hit = route.principal != owner
3301 && narrowed.iter().any(|(principal, allowed)| {
3302 *principal == route.principal
3303 && allowed
3304 .as_ref()
3305 .is_none_or(|targets| !targets.contains(&route.module_id))
3306 });
3307 if !hit {
3308 continue;
3309 }
3310 RouteCloseReason::ScopeCarrierRemoved
3311 }
3312 };
3313 selected.push((*client_key, route.client_epoch, reason));
3314 }
3315 let mut drained = Vec::new();
3316 for (client_key, client_epoch, reason) in selected {
3317 let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
3318 continue;
3319 };
3320 let release = release_client_route_locked(&mut inner, client_key, client_epoch);
3321 self.record_route_release(&release);
3322 if let RouteRelease::Removed(module) = release {
3323 drained.push(ScopeDrainedRoute {
3324 reason,
3325 scope: route
3326 .scope
3327 .clone()
3328 .expect("selected scope drain has a bound scope"),
3329 module_id: route.module_id.clone(),
3330 client: GoodbyeTarget {
3331 connection_id: route.client_connection_id,
3332 sink: route.client_sink.clone(),
3333 negotiated_ver: route.client_negotiated_ver,
3334 channel: route.client_channel,
3335 epoch: route.client_epoch,
3336 kind: GoodbyeTargetKind::Client,
3337 module_id: Some(route.module_id.clone()),
3338 },
3339 module,
3340 });
3341 }
3342 }
3343 Ok(drained)
3344 }
3345
3346 #[cfg(test)]
3348 pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
3349 self.read_inner()
3350 .ok()?
3351 .scope_tags
3352 .get(&(owner.to_string(), scope_ref.to_string()))
3353 .copied()
3354 }
3355
3356 fn record_route_release(&self, release: &RouteRelease) {
3357 match release {
3358 RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
3359 RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
3360 RouteRelease::Absent => {}
3361 }
3362 }
3363
3364 fn read_inner(&self) -> Result<RwLockReadGuard<'_, ForwardingInner>, ForwardingError> {
3365 self.inner.read().map_err(|_| ForwardingError::Poisoned)
3366 }
3367
3368 fn write_inner(&self) -> Result<RwLockWriteGuard<'_, ForwardingInner>, ForwardingError> {
3369 self.inner.write().map_err(|_| ForwardingError::Poisoned)
3370 }
3371
3372 fn lock_close_registry(
3373 &self,
3374 ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
3375 self.close_registry
3376 .lock()
3377 .unwrap_or_else(|poisoned| poisoned.into_inner())
3378 }
3379}
3380
3381impl ForwardingInner {
3382 fn allocate_route_slots(
3383 &mut self,
3384 connection_id: ConnectionId,
3385 endpoint: ModuleEndpointId,
3386 ) -> Result<(u16, u32, u16, u32), ForwardingError> {
3387 let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
3388 let mut client_channel = client_start;
3389 let client_channel = loop {
3390 let key = ClientRouteKey {
3391 connection_id,
3392 channel: client_channel,
3393 };
3394 let eligible = !self.client_to_module.contains_key(&key)
3395 && !self.reserved_client.contains_key(&key)
3396 && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3397 if eligible {
3398 break client_channel;
3399 }
3400 client_channel = next_channel(client_channel);
3401 if client_channel == client_start {
3402 return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
3403 }
3404 };
3405
3406 let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
3407 let mut module_channel = module_start;
3408 let module_channel = loop {
3409 let key = ModuleRouteKey {
3410 endpoint,
3411 channel: module_channel,
3412 };
3413 let eligible = !self.module_to_client.contains_key(&key)
3414 && !self.reserved_module.contains_key(&key)
3415 && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3416 if eligible {
3417 break module_channel;
3418 }
3419 module_channel = next_channel(module_channel);
3420 if module_channel == module_start {
3421 return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
3422 }
3423 };
3424
3425 let client_key = ClientRouteKey {
3426 connection_id,
3427 channel: client_channel,
3428 };
3429 let module_key = ModuleRouteKey {
3430 endpoint,
3431 channel: module_channel,
3432 };
3433 let client_epoch = self
3434 .client_slot_epochs
3435 .get(&client_key)
3436 .copied()
3437 .unwrap_or(0)
3438 + 1;
3439 let module_epoch = self
3440 .module_slot_epochs
3441 .get(&module_key)
3442 .copied()
3443 .unwrap_or(0)
3444 + 1;
3445 self.client_slot_epochs.insert(client_key, client_epoch);
3446 self.module_slot_epochs.insert(module_key, module_epoch);
3447 self.next_client_channel
3448 .insert(connection_id, next_channel(client_channel));
3449 self.next_module_channel
3450 .insert(endpoint, next_channel(module_channel));
3451 Ok((client_channel, client_epoch, module_channel, module_epoch))
3452 }
3453
3454 fn allocate_control_corr(
3455 &mut self,
3456 endpoint: ModuleEndpointId,
3457 ) -> Result<u64, ForwardingError> {
3458 let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
3459 if candidate == 0 {
3460 self.closing_connections.insert(endpoint.connection_id);
3461 return Err(ForwardingError::RelayCorrelationExhausted);
3462 }
3463 self.next_control_corr.insert(
3464 endpoint,
3465 if candidate == u64::MAX {
3466 0
3467 } else {
3468 candidate + 1
3469 },
3470 );
3471 Ok(candidate)
3472 }
3473}
3474
3475fn connection_has_client_routes_locked(
3476 inner: &ForwardingInner,
3477 connection_id: ConnectionId,
3478) -> bool {
3479 inner
3480 .client_to_module
3481 .keys()
3482 .any(|key| key.connection_id == connection_id)
3483 || inner
3484 .reserved_client
3485 .keys()
3486 .any(|key| key.connection_id == connection_id)
3487}
3488
3489fn check_module_connection_role_locked(
3490 inner: &ForwardingInner,
3491 connection_id: ConnectionId,
3492) -> Result<(), ForwardingError> {
3493 if inner.endpoint_by_connection.contains_key(&connection_id)
3494 || connection_has_client_routes_locked(inner, connection_id)
3495 {
3496 return Err(ForwardingError::ConnectionRoleConflict { connection_id });
3497 }
3498 Ok(())
3499}
3500
3501fn next_channel(channel: u16) -> u16 {
3502 let next = channel.wrapping_add(1);
3503 if next == 0 {
3504 1
3505 } else {
3506 next
3507 }
3508}
3509
3510fn endpoint_routes_locked(
3511 inner: &ForwardingInner,
3512 endpoint: ModuleEndpointId,
3513) -> Vec<EndpointRoute> {
3514 let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
3515 let draining = drain_reason.is_some();
3516 let mut routes = inner
3517 .module_to_client
3518 .iter()
3519 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3520 .map(|(_, route)| EndpointRoute {
3521 goodbye_target: GoodbyeTarget {
3522 connection_id: route.client_connection_id,
3523 sink: route.client_sink.clone(),
3524 negotiated_ver: route.client_negotiated_ver,
3525 channel: route.client_channel,
3526 epoch: route.client_epoch,
3527 kind: GoodbyeTargetKind::Client,
3528 module_id: Some(route.module_id.clone()),
3529 },
3530 principal: route.principal.clone(),
3531 bound_at: route.bound_at,
3532 draining,
3533 drain_reason,
3534 })
3535 .collect::<Vec<_>>();
3536 routes.sort_by_key(|route| {
3537 (
3538 route.goodbye_target.connection_id.get(),
3539 route.goodbye_target.channel,
3540 route.goodbye_target.epoch,
3541 )
3542 });
3543 routes
3544}
3545
3546fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
3550 let key = (scope.owner.clone(), scope.scope_ref.clone());
3551 match inner.scope_tags.get(&key) {
3552 Some(current) if *current == scope.tag => None,
3553 Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
3554 subc_protocol::error_codes::SCOPE_CHANGED,
3555 format!(
3556 "scope '{}' of {} changed while the route was being bound; re-open it",
3557 scope.scope_ref, scope.owner
3558 ),
3559 )),
3560 _ => Some(ErrorBody::new(
3561 subc_protocol::error_codes::SCOPE_ENDED,
3562 format!(
3563 "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
3564 scope.scope_ref, scope.owner, scope.tag.scope_epoch
3565 ),
3566 )),
3567 }
3568}
3569
3570fn release_reserved_route_locked(
3571 inner: &mut ForwardingInner,
3572 client_key: ClientRouteKey,
3573 module_key: ModuleRouteKey,
3574) {
3575 if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
3576 inner.reserved_client.remove(&client_key);
3577 }
3578 if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
3579 inner.reserved_module.remove(&module_key);
3580 }
3581 inner.status.retain(|(key, _), _| *key != client_key);
3582}
3583
3584fn release_client_route_locked(
3585 inner: &mut ForwardingInner,
3586 client_key: ClientRouteKey,
3587 expected_epoch: u32,
3588) -> RouteRelease {
3589 let Some(route) = inner.client_to_module.get(&client_key) else {
3590 return RouteRelease::Absent;
3591 };
3592 if route.client_epoch != expected_epoch {
3593 return RouteRelease::Stale;
3594 }
3595 let route = inner
3596 .client_to_module
3597 .remove(&client_key)
3598 .expect("route checked under the same forwarding lock");
3599 route.flow.close();
3600 if let Some(scope) = &route.scope {
3601 scope.release_bind();
3602 }
3603 inner.module_to_client.remove(&ModuleRouteKey {
3604 endpoint: route.module_endpoint,
3605 channel: route.module_channel,
3606 });
3607 inner.operator_confirms.route_closed(ModuleRouteKey {
3608 endpoint: route.module_endpoint,
3609 channel: route.module_channel,
3610 });
3611 inner.status.remove(&(client_key, expected_epoch));
3612 RouteRelease::Removed(GoodbyeTarget {
3613 connection_id: route.module_endpoint.connection_id,
3614 sink: route.module_sink.clone(),
3615 negotiated_ver: route.module_negotiated_ver,
3616 channel: route.module_channel,
3617 epoch: route.module_epoch,
3618 kind: GoodbyeTargetKind::Module,
3619 module_id: Some(route.module_id.clone()),
3620 })
3621}
3622
3623fn release_module_route_locked(
3624 inner: &mut ForwardingInner,
3625 module_key: ModuleRouteKey,
3626 expected_epoch: u32,
3627) -> RouteRelease {
3628 let Some(route) = inner.module_to_client.get(&module_key) else {
3629 return RouteRelease::Absent;
3630 };
3631 if route.module_epoch != expected_epoch {
3632 return RouteRelease::Stale;
3633 }
3634 let route = inner
3635 .module_to_client
3636 .remove(&module_key)
3637 .expect("route checked under the same forwarding lock");
3638 inner.operator_confirms.route_closed(module_key);
3639 route.flow.close();
3640 if let Some(scope) = &route.scope {
3641 scope.release_bind();
3642 }
3643 let client_key = ClientRouteKey {
3644 connection_id: route.client_connection_id,
3645 channel: route.client_channel,
3646 };
3647 inner.client_to_module.remove(&client_key);
3648 inner.status.remove(&(client_key, route.client_epoch));
3649 RouteRelease::Removed(GoodbyeTarget {
3650 connection_id: route.client_connection_id,
3651 sink: route.client_sink.clone(),
3652 negotiated_ver: route.client_negotiated_ver,
3653 channel: route.client_channel,
3654 epoch: route.client_epoch,
3655 kind: GoodbyeTargetKind::Client,
3656 module_id: Some(route.module_id.clone()),
3657 })
3658}
3659
3660fn commit_route_locked(
3661 inner: &mut ForwardingInner,
3662 mut pending: PendingRouteBindRelayEntry,
3663) -> Result<Option<GoodbyeTarget>, ForwardingError> {
3664 let reservation = pending.reservation;
3665 if inner
3666 .closing_connections
3667 .contains(&reservation.client_key.connection_id)
3668 {
3669 return Err(ForwardingError::ConnectionClosing {
3670 connection_id: reservation.client_key.connection_id,
3671 });
3672 }
3673 let module_id = inner
3674 .module_id_by_endpoint
3675 .get(&reservation.module_key.endpoint)
3676 .cloned()
3677 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3678 if inner
3679 .draining_endpoints
3680 .contains_key(&reservation.module_key.endpoint)
3681 {
3682 return Err(ForwardingError::ModuleReloading { module_id });
3683 }
3684 if pending.busy_refusals {
3685 let body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
3686 route_channel: reservation.client_key.channel,
3687 route_epoch: reservation.client_epoch,
3688 request_window: Some(pending.request_window),
3689 watch_window: pending.watch_window,
3690 })
3691 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
3692 pending.route_open_frame = Frame::build_with_version(
3693 pending.client_negotiated_ver,
3694 FrameType::Response,
3695 pending.route_open_frame.header.flags,
3696 0,
3697 0,
3698 pending.route_open_frame.header.corr,
3699 body,
3700 )
3701 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
3702 }
3703 if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
3704 || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
3705 {
3706 return Err(ForwardingError::UnknownReservation {
3707 client_channel: reservation.client_key.channel,
3708 module_channel: reservation.module_key.channel,
3709 });
3710 }
3711 let module = inner
3712 .modules_by_id
3713 .get(&module_id)
3714 .filter(|module| module.endpoint == reservation.module_key.endpoint)
3715 .cloned()
3716 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3717 let binding = Arc::new(RouteBinding {
3718 client_connection_id: reservation.client_key.connection_id,
3719 client_sink: pending.client_sink,
3720 client_negotiated_ver: pending.client_negotiated_ver,
3721 client_channel: reservation.client_key.channel,
3722 client_epoch: reservation.client_epoch,
3723 module_id,
3724 module_endpoint: reservation.module_key.endpoint,
3725 module_sink: module.sink,
3726 module_negotiated_ver: module.negotiated_ver,
3727 module_channel: reservation.module_key.channel,
3728 module_epoch: reservation.module_epoch,
3729 principal: pending.principal,
3730 harness: pending.harness,
3731 build_class: pending.build_class,
3732 operator_app: pending.operator_app,
3733 project_root: reservation.project_root.clone(),
3734 bound_at: Instant::now(),
3735 flow: Arc::new(if pending.busy_refusals {
3736 ChannelFlow::with_admission(pending.request_window, true, pending.watch_window)
3737 } else {
3738 ChannelFlow::new(pending.request_window)
3739 }),
3740 scope: pending.scope,
3741 });
3742 inner
3743 .client_to_module
3744 .insert(reservation.client_key, Arc::clone(&binding));
3745 inner
3746 .module_to_client
3747 .insert(reservation.module_key, Arc::clone(&binding));
3748 let previous_published = inner
3749 .last_published_epoch
3750 .insert(reservation.client_key, reservation.client_epoch);
3751
3752 if let Some(scope) = &binding.scope {
3756 scope.record_served(&binding.module_id, module.verified_prefix_owner.as_deref());
3757 }
3758
3759 let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
3764 if client_writer_closed {
3765 let abandoned = abandoned_route_target(inner, &reservation);
3769 if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
3770 route.flow.close();
3771 }
3772 inner.module_to_client.remove(&reservation.module_key);
3773 inner
3774 .status
3775 .remove(&(reservation.client_key, reservation.client_epoch));
3776 match previous_published {
3777 Some(epoch) => {
3778 inner
3779 .last_published_epoch
3780 .insert(reservation.client_key, epoch);
3781 }
3782 None => {
3783 inner.last_published_epoch.remove(&reservation.client_key);
3784 }
3785 }
3786 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3787 "client egress closed during route publication".to_string(),
3788 ));
3789 return Ok(abandoned);
3790 }
3791
3792 if let Some(scope) = &binding.scope {
3793 scope.commit_presence();
3794 }
3795 let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
3796 Ok(None)
3797}
3798
3799fn spawn_connection_dead(inner: &ForwardingInner, connection: ConnectionId) -> bool {
3800 inner
3801 .spawn_connections
3802 .get(&connection)
3803 .is_some_and(|spawn| spawn.upgrade().is_none_or(|spawn| !spawn.launch_ready()))
3804}
3805
3806fn module_connection_for_endpoint_locked(
3814 inner: &ForwardingInner,
3815 endpoint: ModuleEndpointId,
3816) -> Option<&ModuleConnection> {
3817 let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
3818 inner
3819 .modules_by_id
3820 .get(module_id)
3821 .filter(|module| module.endpoint == endpoint)
3822 .or_else(|| {
3823 inner
3824 .candidates_by_id
3825 .get(module_id)
3826 .filter(|module| module.endpoint == endpoint)
3827 })
3828 .or_else(|| inner.superseded_endpoints.get(&endpoint))
3829}
3830
3831fn abandoned_route_target(
3832 inner: &ForwardingInner,
3833 reservation: &RouteReservation,
3834) -> Option<GoodbyeTarget> {
3835 let module_id = inner
3836 .module_id_by_endpoint
3837 .get(&reservation.module_key.endpoint)?;
3838 let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
3839 (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
3840 connection_id: module.endpoint.connection_id,
3841 sink: module.sink.clone(),
3842 negotiated_ver: module.negotiated_ver,
3843 channel: reservation.module_key.channel,
3844 epoch: reservation.module_epoch,
3845 kind: GoodbyeTargetKind::Module,
3846 module_id: Some(module_id.clone()),
3847 })
3848}
3849
3850fn enqueue_hello_ack_locked(
3855 sink: &FrameSink,
3856 connection_id: ConnectionId,
3857 hello_ack: Option<Frame>,
3858) -> Result<(), ForwardingError> {
3859 let Some(hello_ack) = hello_ack else {
3860 return Ok(());
3861 };
3862 sink.try_send(hello_ack)
3863 .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
3864}
3865
3866fn remove_module_connection_locked(
3867 inner: &mut ForwardingInner,
3868 endpoint: ModuleEndpointId,
3869) -> ConnectionCleanup {
3870 inner.operator_confirms.module_closed(endpoint);
3873 inner.draining_endpoints.remove(&endpoint);
3874 inner.daemon_drain_results.remove(&endpoint);
3875 let module_id = inner.module_id_by_endpoint.remove(&endpoint);
3876 if let Some(module_id) = module_id.as_ref() {
3877 if inner
3878 .modules_by_id
3879 .get(module_id)
3880 .is_some_and(|module| module.endpoint == endpoint)
3881 {
3882 inner.modules_by_id.remove(module_id);
3883 }
3884 if inner
3885 .candidates_by_id
3886 .get(module_id)
3887 .is_some_and(|module| module.endpoint == endpoint)
3888 {
3889 inner.candidates_by_id.remove(module_id);
3890 }
3891 }
3892 inner.superseded_endpoints.remove(&endpoint);
3893 inner.endpoint_by_connection.remove(&endpoint.connection_id);
3894 inner.next_module_channel.remove(&endpoint);
3895 inner.next_control_corr.remove(&endpoint);
3896 inner
3897 .health_probe_tombstones
3898 .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
3899 inner
3900 .module_slot_epochs
3901 .retain(|key, _| key.endpoint != endpoint);
3902 let reserved_module_keys: Vec<ModuleRouteKey> = inner
3903 .reserved_module
3904 .keys()
3905 .filter(|module_key| module_key.endpoint == endpoint)
3906 .copied()
3907 .collect();
3908 for module_key in reserved_module_keys {
3909 if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
3910 release_reserved_route_locked(inner, client_key, module_key);
3911 }
3912 }
3913
3914 let pending_keys: Vec<_> = inner
3915 .pending_relays
3916 .keys()
3917 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3918 .copied()
3919 .collect();
3920 let pending: Vec<_> = pending_keys
3921 .into_iter()
3922 .filter_map(|key| {
3923 inner
3924 .pending_relays
3925 .remove(&key)
3926 .map(|pending| (key, pending))
3927 })
3928 .collect();
3929 let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
3930 for (key, pending) in pending {
3931 if !pending.relay_enqueued {
3935 inner.relays_released_before_enqueue.insert(key);
3936 }
3937 let module_label = module_id.as_deref().unwrap_or("unknown");
3938 let _ = pending
3939 .sender
3940 .send(RouteBindRelayOutcome::ModuleGone(format!(
3941 "module '{module_label}' connection closed during route.bind relay"
3942 )));
3943 }
3944
3945 let pending_control_keys: Vec<_> = inner
3946 .pending_control_rpcs
3947 .keys()
3948 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3949 .copied()
3950 .collect();
3951 let pending_control: Vec<_> = pending_control_keys
3952 .into_iter()
3953 .filter_map(|key| inner.pending_control_rpcs.remove(&key))
3954 .collect();
3955 for pending in pending_control {
3956 let module_label = module_id.as_deref().unwrap_or("unknown");
3957 let _ = pending
3958 .sender
3959 .send(ModuleControlRpcOutcome::ModuleGone(format!(
3960 "module '{module_label}' connection closed during module-control RPC"
3961 )));
3962 }
3963
3964 let module_routes = inner
3965 .module_to_client
3966 .iter()
3967 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3968 .map(|(module_key, route)| (*module_key, route.module_epoch))
3969 .collect::<Vec<_>>();
3970 let mut released = Vec::with_capacity(module_routes.len());
3971 for (module_key, epoch) in module_routes {
3972 if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
3973 {
3974 released.push(target);
3975 }
3976 }
3977 ConnectionCleanup {
3978 released,
3979 abandoned_relays,
3980 }
3981}
3982
3983#[derive(Debug, Clone, Copy)]
3984struct RequestCredit {
3985 subscription: bool,
3986 watch_credit: bool,
3987 excluded_from_drain: bool,
3988}
3989
3990#[derive(Debug, Default)]
3991struct CreditLedger {
3992 by_corr: HashMap<u64, Vec<RequestCredit>>,
3993}
3994
3995impl CreditLedger {
3996 fn acquire(&mut self, corr: u64, subscription: bool, watch_credit: bool) {
3997 self.by_corr.entry(corr).or_default().push(RequestCredit {
3998 subscription,
3999 watch_credit,
4000 excluded_from_drain: false,
4001 });
4002 }
4003
4004 fn release(&mut self, corr: u64) -> Option<RequestCredit> {
4005 let credits = self.by_corr.get_mut(&corr)?;
4006 let released = credits.pop();
4007 if credits.is_empty() {
4008 self.by_corr.remove(&corr);
4009 }
4010 released
4011 }
4012
4013 fn capture_subscription_exclusions(&mut self) -> u32 {
4014 let mut excluded = 0u32;
4015 for credit in self.by_corr.values_mut().flatten() {
4016 if credit.subscription && !credit.excluded_from_drain {
4017 credit.excluded_from_drain = true;
4018 excluded = excluded.saturating_add(1);
4019 }
4020 }
4021 excluded
4022 }
4023
4024 fn in_flight(&self) -> usize {
4025 self.by_corr.values().map(Vec::len).sum()
4026 }
4027
4028 fn drain_in_flight(&self) -> usize {
4029 self.by_corr
4030 .values()
4031 .flatten()
4032 .filter(|credit| !credit.excluded_from_drain)
4033 .count()
4034 }
4035
4036 fn drain_held_corrs(&self) -> Vec<u64> {
4039 let mut corrs = self
4040 .by_corr
4041 .iter()
4042 .flat_map(|(corr, credits)| {
4043 credits
4044 .iter()
4045 .filter(|credit| !credit.excluded_from_drain)
4046 .map(move |_| *corr)
4047 })
4048 .collect::<Vec<_>>();
4049 corrs.sort_unstable();
4050 corrs
4051 }
4052}
4053
4054#[derive(Debug, Default)]
4055struct ChannelFlowState {
4056 closed: bool,
4057 credits: CreditLedger,
4058 last_busy_warn: Option<tokio::time::Instant>,
4059}
4060
4061#[derive(Debug)]
4063pub(crate) struct ChannelFlow {
4064 sem: Semaphore,
4065 window: usize,
4066 busy_refusals: bool,
4067 watch_sem: Option<Semaphore>,
4068 watch_window: Option<usize>,
4069 state: Mutex<ChannelFlowState>,
4070}
4071
4072impl ChannelFlow {
4073 pub(crate) fn new(window: usize) -> Self {
4074 Self::with_admission(window, false, None)
4075 }
4076
4077 fn with_admission(window: usize, busy_refusals: bool, watch_window: Option<usize>) -> Self {
4078 debug_assert!(window > 0, "flow-control window must be non-zero");
4079 Self {
4080 sem: Semaphore::new(window),
4081 window,
4082 busy_refusals,
4083 watch_sem: watch_window.map(Semaphore::new),
4084 watch_window,
4085 state: Mutex::new(ChannelFlowState::default()),
4086 }
4087 }
4088
4089 pub(crate) fn busy_refusals(&self) -> bool {
4090 self.busy_refusals
4091 }
4092
4093 fn try_acquire_tagged(&self, corr: u64, subscription: bool) -> RequestAdmission {
4094 let mut state = self
4095 .state
4096 .lock()
4097 .unwrap_or_else(|poisoned| poisoned.into_inner());
4098 if state.closed {
4099 return RequestAdmission::Stale;
4100 }
4101 let busy = |state: &ChannelFlowState| {
4102 RequestAdmission::Busy(subc_protocol::RouteBusyDetail {
4103 reason: "no_credit".into(),
4104 request_in_flight: state.credits.in_flight(),
4105 watch_in_flight: state
4106 .credits
4107 .by_corr
4108 .values()
4109 .flatten()
4110 .filter(|credit| credit.subscription)
4111 .count(),
4112 request_window: self.window,
4113 watch_window: self.watch_window,
4114 })
4115 };
4116 let Ok(total) = self.sem.try_acquire() else {
4117 return busy(&state);
4118 };
4119 let watch = if subscription {
4122 if let Some(sem) = &self.watch_sem {
4123 match sem.try_acquire() {
4124 Ok(permit) => Some(permit),
4125 Err(_) => return busy(&state),
4126 }
4127 } else {
4128 None
4129 }
4130 } else {
4131 None
4132 };
4133 state.credits.acquire(corr, subscription, watch.is_some());
4134 total.forget();
4135 if let Some(watch) = watch {
4136 watch.forget();
4137 }
4138 RequestAdmission::Admitted
4139 }
4140
4141 pub(crate) fn should_warn_busy(&self) -> bool {
4142 let mut state = self
4143 .state
4144 .lock()
4145 .unwrap_or_else(|poisoned| poisoned.into_inner());
4146 let now = tokio::time::Instant::now();
4147 if state
4148 .last_busy_warn
4149 .is_some_and(|last| now.duration_since(last) < std::time::Duration::from_secs(60))
4150 {
4151 return false;
4152 }
4153 state.last_busy_warn = Some(now);
4154 true
4155 }
4156
4157 #[cfg(test)]
4158 pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
4159 self.acquire_tagged(0, false).await
4160 }
4161
4162 pub(crate) async fn acquire_tagged(
4163 &self,
4164 corr: u64,
4165 subscription: bool,
4166 ) -> Result<(), ChannelFlowClosed> {
4167 let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
4168 let mut state = self
4169 .state
4170 .lock()
4171 .unwrap_or_else(|poisoned| poisoned.into_inner());
4172 if state.closed {
4173 return Err(ChannelFlowClosed);
4174 }
4175 state.credits.acquire(corr, subscription, false);
4176 permit.forget();
4177 Ok(())
4178 }
4179
4180 #[cfg(test)]
4181 pub(crate) fn release(&self) {
4182 self.release_corr(0);
4183 }
4184
4185 pub(crate) fn release_corr(&self, corr: u64) {
4186 let mut state = self
4187 .state
4188 .lock()
4189 .unwrap_or_else(|poisoned| poisoned.into_inner());
4190 let Some(released) = state.credits.release(corr) else {
4191 warn!(
4195 window = self.window,
4196 available = self.sem.available_permits(),
4197 "flow-control over-release ignored"
4198 );
4199 return;
4200 };
4201 if !self.sem.is_closed() {
4202 self.sem.add_permits(1);
4203 }
4204 if released.watch_credit {
4205 if let Some(watch_sem) = self.watch_sem.as_ref().filter(|sem| !sem.is_closed()) {
4206 watch_sem.add_permits(1);
4207 }
4208 }
4209 }
4210
4211 #[cfg(test)]
4212 pub(crate) fn in_flight(&self) -> usize {
4213 self.state
4214 .lock()
4215 .unwrap_or_else(|poisoned| poisoned.into_inner())
4216 .credits
4217 .in_flight()
4218 }
4219
4220 pub(crate) fn drain_in_flight(&self) -> usize {
4221 self.state
4222 .lock()
4223 .unwrap_or_else(|poisoned| poisoned.into_inner())
4224 .credits
4225 .drain_in_flight()
4226 }
4227
4228 pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
4229 self.state
4230 .lock()
4231 .unwrap_or_else(|poisoned| poisoned.into_inner())
4232 .credits
4233 .drain_held_corrs()
4234 }
4235
4236 #[cfg(test)]
4237 pub(crate) fn available_permits(&self) -> usize {
4238 self.sem.available_permits()
4239 }
4240
4241 pub(crate) fn begin_drain(&self) -> u32 {
4242 let mut state = self
4243 .state
4244 .lock()
4245 .unwrap_or_else(|poisoned| poisoned.into_inner());
4246 state.closed = true;
4247 self.sem.close();
4248 if let Some(sem) = &self.watch_sem {
4249 sem.close();
4250 }
4251 state.credits.capture_subscription_exclusions()
4252 }
4253
4254 pub(crate) fn close(&self) {
4255 let mut state = self
4256 .state
4257 .lock()
4258 .unwrap_or_else(|poisoned| poisoned.into_inner());
4259 state.closed = true;
4260 self.sem.close();
4261 if let Some(sem) = &self.watch_sem {
4262 sem.close();
4263 }
4264 }
4265}
4266
4267#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4268pub(crate) struct ChannelFlowClosed;
4269
4270#[derive(Debug, Clone, PartialEq, Eq)]
4271pub(crate) enum RequestAdmission {
4272 Admitted,
4273 Busy(subc_protocol::RouteBusyDetail),
4274 ModuleReloading,
4275 Stale,
4276}
4277
4278impl fmt::Display for ChannelFlowClosed {
4279 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4280 write!(f, "flow-control window closed")
4281 }
4282}
4283
4284impl Error for ChannelFlowClosed {}
4285
4286#[cfg(test)]
4287#[path = "route_busy_flow_tests.rs"]
4288mod route_busy_flow_tests;
4289
4290fn window_for(concurrency: &Concurrency) -> usize {
4291 match concurrency {
4292 Concurrency::Serial => 1,
4293 Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
4294 Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
4295 }
4296}
4297
4298#[derive(Debug, Clone, PartialEq, Eq)]
4299pub enum ForwardingError {
4300 ConnectionRoleConflict {
4303 connection_id: ConnectionId,
4304 },
4305 NoModuleConnection,
4306 ModuleReloading {
4307 module_id: String,
4308 },
4309 StaleModuleEndpoint,
4310 UnknownReservation {
4311 client_channel: u16,
4312 module_channel: u16,
4313 },
4314 ClientRouteChannelExhausted {
4315 connection_id: ConnectionId,
4316 },
4317 ModuleRouteChannelExhausted {
4318 endpoint: ModuleEndpointId,
4319 },
4320 RelayCorrelationExhausted,
4321 ConnectionClosing {
4322 connection_id: ConnectionId,
4323 },
4324 ClientEgressClosed {
4325 connection_id: ConnectionId,
4326 },
4327 RouteOpenBuild(String),
4328 CandidateSlotOccupied {
4330 module_id: String,
4331 },
4332 ModuleEgressUnavailable {
4335 connection_id: ConnectionId,
4336 },
4337 Poisoned,
4338}
4339
4340impl fmt::Display for ForwardingError {
4341 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4342 match self {
4343 Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
4344 Self::NoModuleConnection => write!(f, "no module connection is registered"),
4345 Self::ModuleReloading { module_id } => {
4346 write!(f, "module_id '{module_id}' is reloading")
4347 }
4348 Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
4349 Self::UnknownReservation {
4350 client_channel,
4351 module_channel,
4352 } => write!(
4353 f,
4354 "route reservation client channel {client_channel} / module channel {module_channel} was not found"
4355 ),
4356 Self::ClientRouteChannelExhausted { connection_id } => write!(
4357 f,
4358 "no client route channels are available for connection {}",
4359 connection_id.get()
4360 ),
4361 Self::ModuleRouteChannelExhausted { endpoint } => write!(
4362 f,
4363 "no module route channels are available for endpoint generation {} on connection {}",
4364 endpoint.generation,
4365 endpoint.connection_id.get()
4366 ),
4367 Self::RelayCorrelationExhausted => {
4368 write!(f, "module control correlation ids are exhausted")
4369 }
4370 Self::ConnectionClosing { connection_id } => write!(
4371 f,
4372 "connection {} is closing and cannot accept route allocation",
4373 connection_id.get()
4374 ),
4375 Self::ClientEgressClosed { connection_id } => write!(
4376 f,
4377 "client connection {} egress is closed",
4378 connection_id.get()
4379 ),
4380 Self::RouteOpenBuild(message) => {
4381 write!(f, "failed to prebuild route.open response: {message}")
4382 }
4383 Self::CandidateSlotOccupied { module_id } => write!(
4384 f,
4385 "module_id '{module_id}' already has a swap candidate registered"
4386 ),
4387 Self::ModuleEgressUnavailable { connection_id } => write!(
4388 f,
4389 "module connection {} egress is unavailable; HELLO_ACK could not be queued",
4390 connection_id.get()
4391 ),
4392 Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
4393 }
4394 }
4395}
4396
4397impl Error for ForwardingError {}
4398
4399#[cfg(test)]
4400mod tests {
4401 use std::time::Duration;
4402
4403 use super::*;
4404 use tokio::sync::mpsc;
4405
4406 #[test]
4407 fn ordinary_long_running_request_is_not_excluded_from_drain() {
4408 let mut ledger = CreditLedger::default();
4409 ledger.acquire(1, false, false);
4410
4411 assert_eq!(ledger.capture_subscription_exclusions(), 0);
4412 assert_eq!(ledger.drain_in_flight(), 1);
4413 }
4414
4415 #[test]
4416 fn bit_set_subscription_is_excluded_and_counted() {
4417 let mut ledger = CreditLedger::default();
4418 ledger.acquire(1, true, false);
4419
4420 assert_eq!(ledger.capture_subscription_exclusions(), 1);
4421 assert_eq!(ledger.drain_in_flight(), 0);
4422 }
4423
4424 #[test]
4425 fn subscription_opened_after_drain_snapshot_is_not_excluded() {
4426 let mut ledger = CreditLedger::default();
4427 ledger.acquire(1, true, false);
4428 assert_eq!(ledger.capture_subscription_exclusions(), 1);
4429
4430 ledger.acquire(2, true, false);
4431
4432 assert_eq!(ledger.drain_in_flight(), 1);
4433 }
4434
4435 #[test]
4436 fn drain_with_no_subscriptions_reports_zero_excluded() {
4437 let mut ledger = CreditLedger::default();
4438 assert_eq!(ledger.capture_subscription_exclusions(), 0);
4439 }
4440
4441 fn test_hello_ack(corr: u64) -> Frame {
4442 Frame::build(
4443 FrameType::HelloAck,
4444 Flags::new(false, Priority::Passive, false),
4445 0,
4446 0,
4447 corr,
4448 Vec::new(),
4449 )
4450 .unwrap()
4451 }
4452
4453 #[test]
4457 fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
4458 let forwarding = ForwardingTable::default();
4459
4460 let (closed_tx, closed_rx) = mpsc::channel(8);
4461 drop(closed_rx);
4462 let closed = ConnectionId::new(1);
4463 assert_eq!(
4464 forwarding.register_module_connection_acked(
4465 closed,
4466 "closed".to_string(),
4467 2,
4468 Concurrency::ModuleManaged,
4469 FrameSink::new(closed_tx),
4470 ModuleHelloAck {
4471 frame: test_hello_ack(1),
4472 verified_prefix_owner: None
4473 },
4474 ),
4475 Err(ForwardingError::ModuleEgressUnavailable {
4476 connection_id: closed
4477 })
4478 );
4479
4480 let (full_tx, _full_rx) = mpsc::channel(1);
4481 let full_sink = FrameSink::new(full_tx);
4482 full_sink.try_send(test_hello_ack(99)).unwrap();
4483 let full = ConnectionId::new(2);
4484 assert_eq!(
4485 forwarding.register_module_connection_acked(
4486 full,
4487 "full".to_string(),
4488 2,
4489 Concurrency::ModuleManaged,
4490 full_sink.clone(),
4491 ModuleHelloAck {
4492 frame: test_hello_ack(2),
4493 verified_prefix_owner: None
4494 },
4495 ),
4496 Err(ForwardingError::ModuleEgressUnavailable {
4497 connection_id: full
4498 })
4499 );
4500 assert_eq!(
4501 forwarding.register_candidate_module_connection_acked(
4502 full,
4503 "full".to_string(),
4504 2,
4505 Concurrency::ModuleManaged,
4506 full_sink,
4507 test_hello_ack(3),
4508 ),
4509 Err(ForwardingError::ModuleEgressUnavailable {
4510 connection_id: full
4511 })
4512 );
4513
4514 for (connection, module_id) in [(closed, "closed"), (full, "full")] {
4515 assert_eq!(
4516 forwarding
4517 .module_endpoint_for_connection(connection)
4518 .unwrap(),
4519 None
4520 );
4521 let (client_tx, _client_rx) = mpsc::channel(8);
4522 assert_eq!(
4523 forwarding
4524 .begin_route_bind_relay_for_test(
4525 ConnectionId::new(50),
4526 FrameSink::new(client_tx),
4527 1,
4528 module_id,
4529 )
4530 .err(),
4531 Some(ForwardingError::NoModuleConnection)
4532 );
4533 }
4534 assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
4535 }
4536
4537 #[test]
4540 fn acked_registration_queues_the_hello_ack_first() {
4541 let forwarding = ForwardingTable::default();
4542 let (active_tx, mut active_rx) = mpsc::channel(8);
4543 forwarding
4544 .register_module_connection_acked(
4545 ConnectionId::new(1),
4546 "acked".to_string(),
4547 2,
4548 Concurrency::ModuleManaged,
4549 FrameSink::new(active_tx),
4550 ModuleHelloAck {
4551 frame: test_hello_ack(11),
4552 verified_prefix_owner: None,
4553 },
4554 )
4555 .unwrap();
4556 let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
4557 forwarding
4558 .register_candidate_module_connection_acked(
4559 ConnectionId::new(2),
4560 "acked".to_string(),
4561 2,
4562 Concurrency::ModuleManaged,
4563 FrameSink::new(candidate_tx),
4564 test_hello_ack(12),
4565 )
4566 .unwrap();
4567
4568 let active_first = active_rx.try_recv().unwrap().frame;
4569 assert_eq!(active_first.header.ty, FrameType::HelloAck);
4570 assert_eq!(active_first.header.corr, 11);
4571 let candidate_first = candidate_rx.try_recv().unwrap().frame;
4572 assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
4573 assert_eq!(candidate_first.header.corr, 12);
4574 }
4575
4576 #[test]
4577 fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
4578 let forwarding = ForwardingTable::default();
4579 let module_connection = ConnectionId::new(10);
4580 let exhausted_client = ConnectionId::new(20);
4581 let second_client = ConnectionId::new(30);
4582 let (module_tx, _module_rx) = mpsc::channel(1);
4583 let endpoint = forwarding
4584 .register_module_connection(
4585 module_connection,
4586 "route-limit-provider".to_string(),
4587 1,
4588 Concurrency::ModuleManaged,
4589 FrameSink::new(module_tx),
4590 )
4591 .unwrap();
4592
4593 {
4594 let mut inner = forwarding.inner.write().unwrap();
4595 for channel in 1..=u16::MAX {
4596 inner.reserved_client.insert(
4597 ClientRouteKey {
4598 connection_id: exhausted_client,
4599 channel,
4600 },
4601 ModuleRouteKey {
4602 endpoint,
4603 channel: 1,
4604 },
4605 );
4606 }
4607 }
4608
4609 let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
4610 let err = forwarding
4611 .begin_route_bind_relay_for_test(
4612 exhausted_client,
4613 FrameSink::new(exhausted_tx),
4614 1,
4615 "route-limit-provider",
4616 )
4617 .unwrap_err();
4618 assert!(matches!(
4619 err,
4620 ForwardingError::ClientRouteChannelExhausted { connection_id }
4621 if connection_id == exhausted_client
4622 ));
4623
4624 let (second_tx, _second_rx) = mpsc::channel(1);
4625 let pending = forwarding
4626 .begin_route_bind_relay_for_test(
4627 second_client,
4628 FrameSink::new(second_tx),
4629 2,
4630 "route-limit-provider",
4631 )
4632 .unwrap();
4633 assert_eq!(pending.client_channel, 1);
4634 }
4635
4636 #[test]
4637 fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
4638 let forwarding = ForwardingTable::default();
4639 let module_connection = ConnectionId::new(40);
4640 let client = ConnectionId::new(50);
4641 let (module_tx, _module_rx) = mpsc::channel(1);
4642 forwarding
4643 .register_module_connection(
4644 module_connection,
4645 "slot-reuse-provider".to_string(),
4646 1,
4647 Concurrency::ModuleManaged,
4648 FrameSink::new(module_tx),
4649 )
4650 .unwrap();
4651
4652 let (client_tx, _client_rx) = mpsc::channel(1);
4653 let client_sink = FrameSink::new(client_tx);
4654 let mut wrapped_channel = None;
4655 for index in 0..=usize::from(u16::MAX) {
4656 let pending = forwarding
4657 .begin_route_bind_relay_for_test(
4658 client,
4659 client_sink.clone(),
4660 index as u64 + 1,
4661 "slot-reuse-provider",
4662 )
4663 .unwrap();
4664 if index == usize::from(u16::MAX) {
4665 wrapped_channel = Some(pending.module_channel);
4666 }
4667 forwarding
4668 .abort_pending_relay(
4669 pending.endpoint,
4670 pending.corr,
4671 RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
4672 )
4673 .unwrap();
4674 }
4675
4676 assert_eq!(wrapped_channel, Some(1));
4677 }
4678
4679 #[test]
4680 fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
4681 let forwarding = ForwardingTable::default();
4682 let client = ConnectionId::new(60);
4683 forwarding
4684 .inner
4685 .write()
4686 .unwrap()
4687 .next_client_channel
4688 .insert(client, 41);
4689
4690 let released = forwarding.cleanup_connection(client).unwrap();
4691
4692 assert!(released.is_empty());
4693 assert!(!forwarding
4694 .inner
4695 .read()
4696 .unwrap()
4697 .next_client_channel
4698 .contains_key(&client));
4699 }
4700
4701 #[test]
4702 fn stale_module_cleanup_preserves_fast_reconnect_successor() {
4703 let forwarding = ForwardingTable::default();
4704 let module_id = "fast-reconnect-provider";
4705 let first_connection = ConnectionId::new(70);
4706 let second_connection = ConnectionId::new(80);
4707 let (first_tx, _first_rx) = mpsc::channel(1);
4708 let first_endpoint = forwarding
4709 .register_module_connection(
4710 first_connection,
4711 module_id.to_string(),
4712 1,
4713 Concurrency::ModuleManaged,
4714 FrameSink::new(first_tx),
4715 )
4716 .unwrap();
4717 let (second_tx, _second_rx) = mpsc::channel(1);
4718 let second_endpoint = forwarding
4719 .register_module_connection(
4720 second_connection,
4721 module_id.to_string(),
4722 1,
4723 Concurrency::ModuleManaged,
4724 FrameSink::new(second_tx),
4725 )
4726 .unwrap();
4727 assert_ne!(first_endpoint, second_endpoint);
4728
4729 let released = forwarding.cleanup_connection(first_connection).unwrap();
4730
4731 assert!(released.is_empty());
4732 assert_eq!(
4733 forwarding
4734 .inner
4735 .read()
4736 .unwrap()
4737 .modules_by_id
4738 .get(module_id)
4739 .map(|module| module.endpoint),
4740 Some(second_endpoint)
4741 );
4742 assert!(forwarding.has_live_module_connection(module_id).unwrap());
4743 let control_rpc = forwarding
4744 .begin_module_control_rpc_for(
4745 module_id,
4746 "health.check",
4747 Instant::now() + Duration::from_secs(1),
4748 )
4749 .unwrap();
4750 assert_eq!(control_rpc.endpoint, second_endpoint);
4751 }
4752
4753 fn route_fixture(
4754 module_id: &str,
4755 ) -> (
4756 ForwardingTable,
4757 ConnectionId,
4758 ModuleEndpointId,
4759 ConnectionId,
4760 FrameSink,
4761 mpsc::Receiver<crate::router::OutboundFrame>,
4762 ) {
4763 let forwarding = ForwardingTable::default();
4764 let module_connection = ConnectionId::new(100);
4765 let client_connection = ConnectionId::new(200);
4766 let (module_tx, _module_rx) = mpsc::channel(8);
4767 let endpoint = forwarding
4768 .register_module_connection(
4769 module_connection,
4770 module_id.to_string(),
4771 2,
4772 Concurrency::ModuleManaged,
4773 FrameSink::new(module_tx),
4774 )
4775 .unwrap();
4776 let (client_tx, client_rx) = mpsc::channel(8);
4777 (
4778 forwarding,
4779 module_connection,
4780 endpoint,
4781 client_connection,
4782 FrameSink::new(client_tx),
4783 client_rx,
4784 )
4785 }
4786
4787 #[test]
4788 #[cfg(unix)]
4789 fn daemon_drain_gates_current_and_racing_provider_registrations() {
4790 let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
4791 assert_eq!(
4792 forwarding.begin_daemon_drain().unwrap(),
4793 [("provider".to_string(), endpoint)]
4794 );
4795 assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
4796 assert!(matches!(
4797 forwarding.register_module_connection(
4798 ConnectionId::new(300),
4799 "late-provider".into(),
4800 2,
4801 Concurrency::ModuleManaged,
4802 sink,
4803 ),
4804 Err(ForwardingError::ConnectionClosing { .. })
4805 ));
4806 }
4807
4808 #[test]
4809 #[cfg(unix)]
4810 fn daemon_drain_results_are_endpoint_fenced_and_removed_on_cleanup() {
4811 let (forwarding, connection, endpoint, _, _, _) = route_fixture("provider");
4812 assert_eq!(
4813 forwarding
4814 .daemon_drain_result_for_connection(connection)
4815 .unwrap(),
4816 None
4817 );
4818 forwarding
4819 .record_daemon_drain_result(endpoint, true)
4820 .unwrap();
4821 forwarding.begin_daemon_drain().unwrap();
4822 assert_eq!(
4823 forwarding
4824 .daemon_drain_result_for_connection(connection)
4825 .unwrap(),
4826 Some(false)
4827 );
4828 let replacement = ModuleEndpointId {
4829 generation: endpoint.generation + 1,
4830 ..endpoint
4831 };
4832 forwarding
4833 .record_daemon_drain_result(replacement, true)
4834 .unwrap();
4835 assert_eq!(
4836 forwarding
4837 .daemon_drain_result_for_connection(connection)
4838 .unwrap(),
4839 Some(false)
4840 );
4841 forwarding
4842 .record_daemon_drain_result(endpoint, true)
4843 .unwrap();
4844 assert_eq!(
4845 forwarding
4846 .daemon_drain_result_for_connection(connection)
4847 .unwrap(),
4848 Some(true)
4849 );
4850 assert_eq!(
4851 forwarding
4852 .daemon_drain_result_for_connection(ConnectionId::new(300))
4853 .unwrap(),
4854 Some(false)
4855 );
4856 forwarding.cleanup_connection(connection).unwrap();
4857 forwarding
4858 .record_daemon_drain_result(endpoint, true)
4859 .unwrap();
4860 assert_eq!(
4861 forwarding
4862 .daemon_drain_result_for_connection(connection)
4863 .unwrap(),
4864 Some(false)
4865 );
4866 assert!(forwarding
4867 .read_inner()
4868 .unwrap()
4869 .daemon_drain_results
4870 .is_empty());
4871 }
4872
4873 #[test]
4874 #[cfg(unix)]
4875 fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
4876 let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
4877 route_fixture("provider");
4878 let mut pending =
4879 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
4880 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
4881 forwarding.begin_daemon_drain().unwrap();
4882 let completion = forwarding
4883 .complete_pending_relay(
4884 module_connection,
4885 pending.corr,
4886 RouteBindRelayOutcome::Accepted,
4887 )
4888 .expect("draining admission is not a fatal module error");
4889 assert!(completion.settled);
4890 let abandoned = completion
4891 .abandoned
4892 .expect("late accepted bind needs a route GOODBYE");
4893 assert_eq!(abandoned.channel, pending.module_channel);
4894 assert!(matches!(pending.receiver.try_recv().unwrap(),
4895 RouteBindRelayOutcome::DaemonRejected(error) if error.code == "module_reloading"));
4896 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4897 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4898 assert!(
4899 rx.try_recv().is_err(),
4900 "no route.open may be published during drain"
4901 );
4902 assert_eq!(
4903 forwarding
4904 .module_endpoint_for_connection(module_connection)
4905 .unwrap(),
4906 Some(endpoint)
4907 );
4908 assert!(
4909 !forwarding
4910 .complete_pending_relay(
4911 module_connection,
4912 pending.corr,
4913 RouteBindRelayOutcome::Accepted,
4914 )
4915 .unwrap()
4916 .settled
4917 );
4918 }
4919
4920 fn test_ping(corr: u64) -> Frame {
4921 Frame::build(
4922 FrameType::Ping,
4923 Flags::new(false, Priority::Passive, false),
4924 0,
4925 0,
4926 corr,
4927 Vec::new(),
4928 )
4929 .unwrap()
4930 }
4931
4932 fn begin_test_route(
4933 forwarding: &ForwardingTable,
4934 client_connection: ConnectionId,
4935 client_sink: FrameSink,
4936 corr: u64,
4937 module_id: &str,
4938 ) -> PendingRouteBindRelay {
4939 forwarding
4940 .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
4941 .unwrap()
4942 }
4943
4944 #[test]
4945 fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
4946 for candidate in [false, true] {
4947 for register_first in [false, true] {
4948 let (forwarding, module_connection, _, client_connection, sink, _rx) =
4949 route_fixture("target");
4950 let register = || {
4951 if candidate {
4952 forwarding.register_candidate_module_connection(
4953 client_connection,
4954 "source".into(),
4955 2,
4956 Concurrency::ModuleManaged,
4957 sink.clone(),
4958 )
4959 } else {
4960 forwarding.register_module_connection(
4961 client_connection,
4962 "source".into(),
4963 2,
4964 Concurrency::ModuleManaged,
4965 sink.clone(),
4966 )
4967 }
4968 };
4969 if register_first {
4970 register().unwrap();
4971 assert!(
4972 forwarding
4973 .begin_route_bind_relay_for_test(
4974 client_connection,
4975 sink.clone(),
4976 1,
4977 "target",
4978 )
4979 .is_err(),
4980 "a deferred route.open cannot reserve after HELLO"
4981 );
4982 assert!(forwarding
4983 .cleanup_connection(client_connection)
4984 .unwrap()
4985 .is_empty());
4986 } else {
4987 let pending =
4988 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
4989 assert!(
4990 register().is_err(),
4991 "HELLO cannot register after a route reservation"
4992 );
4993 forwarding
4994 .complete_pending_relay(
4995 module_connection,
4996 pending.corr,
4997 RouteBindRelayOutcome::Accepted,
4998 )
4999 .unwrap();
5000 assert_eq!(
5001 forwarding
5002 .cleanup_connection(client_connection)
5003 .unwrap()
5004 .len(),
5005 1
5006 );
5007 }
5008 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5009 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5010 }
5011 }
5012 }
5013
5014 #[tokio::test]
5020 async fn pending_route_open_completes_behind_queued_data_frames() {
5021 assert_eq!(
5022 crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
5023 8,
5024 "the per-connection pending route.open limit is its own constant"
5025 );
5026 let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
5027 route_fixture("open-behind-data");
5028 let (sink, mut client_rx) = crate::server::connection_egress();
5029 const DATA_FRAMES: usize = 1_000;
5030 let data = |corr: u64| {
5031 Frame::build(
5032 FrameType::StreamData,
5033 Flags::new(false, Priority::Interactive, false),
5034 9,
5035 1,
5036 corr,
5037 vec![b'x'; 200],
5038 )
5039 .unwrap()
5040 };
5041 for corr in 0..DATA_FRAMES as u64 {
5042 sink.try_send(data(corr)).unwrap();
5043 }
5044 let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
5045 assert_eq!(sink.backlog().queued_bytes, data_bytes);
5046
5047 let pending = tokio::time::timeout(
5048 Duration::from_secs(5),
5049 forwarding.begin_route_bind_relay_for(
5050 client,
5051 sink.clone(),
5052 subc_protocol::PROTOCOL_VERSION,
5053 4_242,
5054 "open-behind-data",
5055 Principal::Direct,
5056 None,
5057 None,
5058 "fixture".into(),
5059 None,
5060 None,
5061 Instant::now() + Duration::from_secs(60),
5062 false,
5063 ),
5064 )
5065 .await
5066 .expect("reserving the route.open slot must not wait behind data frames")
5067 .unwrap();
5068 forwarding
5069 .complete_pending_relay(
5070 module_connection,
5071 pending.corr,
5072 RouteBindRelayOutcome::Accepted,
5073 )
5074 .unwrap();
5075
5076 let backlog = sink.backlog();
5077 assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
5078 assert!(
5079 backlog.queued_bytes > data_bytes,
5080 "the route.open response must be counted in queued bytes: {backlog:?}"
5081 );
5082 for corr in 0..DATA_FRAMES as u64 {
5083 assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
5084 }
5085 let open = client_rx.recv().await.unwrap();
5086 assert_eq!(open.header.corr, 4_242);
5087 assert_eq!(open.header.ty, FrameType::Response);
5088 drop(open);
5089 assert_eq!(sink.backlog().queued_bytes, 0);
5090 assert_eq!(sink.backlog().queued_frames, 0);
5091 }
5092
5093 #[tokio::test]
5098 async fn drain_holdouts_count_held_requests_and_name_the_connection() {
5099 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5100 route_fixture("holdouts");
5101 let mut bound = |corr| {
5102 let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
5103 forwarding
5104 .complete_pending_relay(
5105 module_connection,
5106 route.corr,
5107 RouteBindRelayOutcome::Accepted,
5108 )
5109 .unwrap();
5110 client_rx.try_recv().unwrap();
5111 match forwarding
5112 .lookup_data_route(client, route.client_channel, route.client_epoch)
5113 .unwrap()
5114 {
5115 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
5116 other => panic!("expected live route, got {other:?}"),
5117 }
5118 };
5119 let holding = bound(61);
5120 let _idle = bound(62);
5121 holding.flow.acquire_tagged(7, false).await.unwrap();
5122 holding.flow.acquire_tagged(2, false).await.unwrap();
5123 holding.flow.acquire_tagged(3, true).await.unwrap();
5124 forwarding
5125 .begin_module_drain("holdouts", RouteCloseReason::Restart)
5126 .unwrap();
5127
5128 let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
5129 assert_eq!(
5130 holdouts,
5131 DrainHoldouts {
5132 requests: 2,
5133 routes: 1,
5134 total_routes: 2,
5135 top_connections: vec![(client.get(), 2)],
5136 held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
5139 }
5140 );
5141 }
5142
5143 #[test]
5144 fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
5145 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
5146 route_fixture("census");
5147 let pending = begin_test_route(&forwarding, client, sink, 1, "census");
5148 forwarding
5149 .complete_pending_relay(
5150 module_connection,
5151 pending.corr,
5152 RouteBindRelayOutcome::Accepted,
5153 )
5154 .unwrap();
5155
5156 let routes = forwarding.endpoint_routes(endpoint).unwrap();
5157 assert_eq!(routes.len(), 1);
5158 assert!(matches!(routes[0].principal, Principal::Direct));
5159 assert_eq!(routes[0].goodbye_target.connection_id, client);
5160 assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
5161 assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
5162 assert!(!routes[0].draining);
5163
5164 forwarding
5165 .begin_module_drain("census", RouteCloseReason::Restart)
5166 .unwrap();
5167 let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
5168 assert_eq!(draining_routes.len(), 1);
5169 assert!(draining_routes[0].draining);
5170 }
5171
5172 #[test]
5173 fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
5174 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
5175 let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
5176 assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
5177 forwarding
5178 .abort_pending_relay(
5179 first.endpoint,
5180 first.corr,
5181 RouteBindRelayOutcome::ModuleGone("abort".into()),
5182 )
5183 .unwrap();
5184 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
5185 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
5186
5187 let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
5188 assert_eq!(second.client_channel, first.client_channel);
5189 assert_eq!(second.module_channel, first.module_channel);
5190 assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
5191 }
5192
5193 #[test]
5194 fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
5195 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5196 route_fixture("epoch-release");
5197 let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
5198 forwarding
5199 .complete_pending_relay(
5200 module_connection,
5201 first.corr,
5202 RouteBindRelayOutcome::Accepted,
5203 )
5204 .unwrap();
5205 assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
5206 assert!(matches!(
5207 forwarding
5208 .release_client_route(client, first.client_channel, first.client_epoch)
5209 .unwrap(),
5210 RouteRelease::Removed(_)
5211 ));
5212 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
5213 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
5214
5215 let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
5216 forwarding
5217 .complete_pending_relay(
5218 module_connection,
5219 second.corr,
5220 RouteBindRelayOutcome::Accepted,
5221 )
5222 .unwrap();
5223 assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
5224 assert!(matches!(
5225 forwarding
5226 .release_client_route(client, second.client_channel, first.client_epoch)
5227 .unwrap(),
5228 RouteRelease::Stale
5229 ));
5230 assert!(!forwarding
5231 .cache_status(
5232 endpoint,
5233 second.module_channel,
5234 first.module_epoch,
5235 "stale".into(),
5236 )
5237 .unwrap());
5238 assert!(forwarding
5239 .cache_status(
5240 endpoint,
5241 second.module_channel,
5242 second.module_epoch,
5243 "current".into(),
5244 )
5245 .unwrap());
5246 match forwarding
5247 .route_poll_snapshot(client, second.client_channel, second.client_epoch)
5248 .unwrap()
5249 {
5250 RoutePollSnapshot::Bound { status, .. } => {
5251 assert_eq!(status.as_deref(), Some("current"));
5252 }
5253 RoutePollSnapshot::Absent => panic!("successor binding was removed"),
5254 }
5255 let counters = forwarding.counters().snapshot();
5256 assert_eq!(counters["route_released_epoch_fenced"], 1);
5257 assert_eq!(counters["route_release_stale_skipped"], 1);
5258 }
5259
5260 #[test]
5261 fn max_epoch_reservation_retires_only_that_slot() {
5262 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
5263 forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
5264 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
5265 let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
5266 assert_eq!(
5267 (final_use.client_channel, final_use.client_epoch),
5268 (7, u32::MAX)
5269 );
5270 assert_eq!(
5271 (final_use.module_channel, final_use.module_epoch),
5272 (9, u32::MAX)
5273 );
5274 forwarding
5275 .abort_pending_relay(
5276 endpoint,
5277 final_use.corr,
5278 RouteBindRelayOutcome::ModuleGone("abort".into()),
5279 )
5280 .unwrap();
5281 forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
5282 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
5283 let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
5284 assert_ne!(next.client_channel, 7);
5285 assert_ne!(next.module_channel, 9);
5286 assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
5287 }
5288
5289 #[test]
5290 fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
5291 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
5292 route_fixture("corr-shared");
5293 let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
5294 assert_eq!(bind.corr, 1);
5295 forwarding
5296 .abort_pending_relay(
5297 endpoint,
5298 bind.corr,
5299 RouteBindRelayOutcome::ModuleGone("abort".into()),
5300 )
5301 .unwrap();
5302 let rpc = forwarding
5303 .begin_module_control_rpc_for(
5304 "corr-shared",
5305 "health.check",
5306 Instant::now() - Duration::from_millis(1),
5307 )
5308 .unwrap();
5309 assert_eq!(rpc.corr, 2);
5310 assert_eq!(
5311 forwarding
5312 .complete_module_control_rpc(
5313 module_connection,
5314 rpc.corr,
5315 Some("health.check"),
5316 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
5317 status: subc_protocol::session::HealthStatus::Ok,
5318 detail: None,
5319 metrics: None,
5320 }),
5321 )
5322 .unwrap(),
5323 ModuleControlRpcCompletion::Settled
5324 );
5325 assert!(matches!(
5326 rpc.receiver.blocking_recv().unwrap(),
5327 ModuleControlRpcOutcome::DeadlineElapsed
5328 ));
5329 }
5330
5331 #[tokio::test(start_paused = true)]
5332 async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
5333 let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
5334 let probe_started_at = Instant::now();
5335 let rpc = forwarding
5336 .begin_health_probe_rpc_for(
5337 "tombstone-ttl",
5338 "health.check",
5339 probe_started_at,
5340 probe_started_at + Duration::from_secs(5),
5341 )
5342 .unwrap();
5343 assert!(forwarding
5344 .tombstone_health_probe_rpc(endpoint, rpc.corr)
5345 .unwrap());
5346 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
5347
5348 tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
5349 tokio::task::yield_now().await;
5350
5351 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
5352 }
5353
5354 #[test]
5355 fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
5356 let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
5357 let mut close = forwarding.register_connection_close(endpoint.connection_id);
5358 forwarding.inject_control_corr(endpoint, u64::MAX);
5359 let final_rpc = forwarding
5360 .begin_module_control_rpc_for(
5361 "corr-max",
5362 "health.check",
5363 Instant::now() + Duration::from_secs(1),
5364 )
5365 .unwrap();
5366 assert_eq!(final_rpc.corr, u64::MAX);
5367 forwarding
5368 .cancel_module_control_rpc(endpoint, final_rpc.corr)
5369 .unwrap();
5370 assert!(matches!(
5371 forwarding.begin_module_control_rpc_for(
5372 "corr-max",
5373 "health.check",
5374 Instant::now() + Duration::from_secs(1),
5375 ),
5376 Err(ForwardingError::RelayCorrelationExhausted)
5377 ));
5378 assert!(close.try_recv().is_ok());
5379 }
5380
5381 #[test]
5382 fn publication_epoch_controls_delivery_failure_escalation() {
5383 fn setup_successor(
5384 commit_successor: Option<bool>,
5385 ) -> (ForwardingTable, ConnectionId, u16, u32) {
5386 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5387 route_fixture("escalation");
5388 let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
5389 forwarding
5390 .complete_pending_relay(
5391 module_connection,
5392 first.corr,
5393 RouteBindRelayOutcome::Accepted,
5394 )
5395 .unwrap();
5396 client_rx.try_recv().unwrap();
5397 assert!(matches!(
5398 forwarding
5399 .release_client_route(client, first.client_channel, first.client_epoch)
5400 .unwrap(),
5401 RouteRelease::Removed(_)
5402 ));
5403 if let Some(commit_successor) = commit_successor {
5404 forwarding.inject_client_slot_epoch(
5405 client,
5406 first.client_channel,
5407 first.client_epoch,
5408 );
5409 forwarding.inject_module_slot_epoch(
5410 endpoint,
5411 first.module_channel,
5412 first.module_epoch,
5413 );
5414 let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
5415 if commit_successor {
5416 forwarding
5417 .complete_pending_relay(
5418 module_connection,
5419 successor.corr,
5420 RouteBindRelayOutcome::Accepted,
5421 )
5422 .unwrap();
5423 client_rx.try_recv().unwrap();
5424 } else {
5425 forwarding
5426 .abort_pending_relay(
5427 endpoint,
5428 successor.corr,
5429 RouteBindRelayOutcome::ModuleGone("abort".into()),
5430 )
5431 .unwrap();
5432 }
5433 }
5434 (forwarding, client, first.client_channel, first.client_epoch)
5435 }
5436
5437 let probe_sink = FrameSink::new(mpsc::channel(1).0);
5438 let (no_successor, client, channel, epoch) = setup_successor(None);
5439 let mut close = no_successor.register_connection_close(client);
5440 assert!(no_successor
5441 .escalate_client_delivery_failure(
5442 client,
5443 channel,
5444 epoch,
5445 CloseReason::new("delivery", "failed"),
5446 UndeliveredFrame {
5447 module_id: None,
5448 sink: &probe_sink,
5449 },
5450 )
5451 .unwrap());
5452 assert!(close.try_recv().is_ok());
5453
5454 let (aborted, client, channel, epoch) = setup_successor(Some(false));
5455 let mut close = aborted.register_connection_close(client);
5456 assert!(aborted
5457 .escalate_client_delivery_failure(
5458 client,
5459 channel,
5460 epoch,
5461 CloseReason::new("delivery", "failed"),
5462 UndeliveredFrame {
5463 module_id: None,
5464 sink: &probe_sink,
5465 },
5466 )
5467 .unwrap());
5468 assert!(close.try_recv().is_ok());
5469
5470 let (published, client, channel, epoch) = setup_successor(Some(true));
5471 let mut close = published.register_connection_close(client);
5472 assert!(!published
5473 .escalate_client_delivery_failure(
5474 client,
5475 channel,
5476 epoch,
5477 CloseReason::new("delivery", "stale failure"),
5478 UndeliveredFrame {
5479 module_id: None,
5480 sink: &probe_sink,
5481 },
5482 )
5483 .unwrap());
5484 assert!(close.try_recv().is_err());
5485 }
5486
5487 #[test]
5488 fn route_concentration_separates_client_count_from_routes_per_client() {
5489 let (forwarding, module_connection, _, client, sink, _client_rx) =
5493 route_fixture("concentration");
5494 assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
5495
5496 for corr in [70_u64, 71] {
5497 let pending =
5498 begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
5499 forwarding
5500 .complete_pending_relay(
5501 module_connection,
5502 pending.corr,
5503 RouteBindRelayOutcome::Accepted,
5504 )
5505 .unwrap();
5506 }
5507
5508 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5510 assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
5511 }
5512
5513 #[test]
5514 fn cleanup_and_accepted_resolution_have_one_lock_winner() {
5515 let (forwarding, module_connection, _, client, sink, mut client_rx) =
5516 route_fixture("cleanup-race");
5517 let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
5518 forwarding
5519 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5520 .unwrap();
5521 let released = forwarding.cleanup_connection(client).unwrap();
5522 assert_eq!(released.len(), 1);
5523 let completion = forwarding
5524 .complete_pending_relay(
5525 module_connection,
5526 pending.corr,
5527 RouteBindRelayOutcome::Accepted,
5528 )
5529 .unwrap();
5530 assert!(!completion.settled);
5531 assert!(client_rx.try_recv().is_err());
5532 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5533
5534 let (forwarding, module_connection, _, client, sink, mut client_rx) =
5535 route_fixture("accepted-race");
5536 let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
5537 forwarding
5538 .complete_pending_relay(
5539 module_connection,
5540 pending.corr,
5541 RouteBindRelayOutcome::Accepted,
5542 )
5543 .unwrap();
5544 assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
5545 let released = forwarding.cleanup_connection(client).unwrap();
5546 assert_eq!(released.len(), 1);
5547 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5548 }
5549
5550 #[test]
5551 fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
5552 let (forwarding, module_connection, _, client, sink, mut client_rx) =
5553 route_fixture("drain-gap");
5554 let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
5555 forwarding
5556 .complete_pending_relay(
5557 module_connection,
5558 live.corr,
5559 RouteBindRelayOutcome::Accepted,
5560 )
5561 .unwrap();
5562 client_rx.try_recv().unwrap();
5563 let binding = match forwarding
5564 .lookup_data_route(client, live.client_channel, live.client_epoch)
5565 .unwrap()
5566 {
5567 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
5568 other => panic!("expected live route, got {other:?}"),
5569 };
5570
5571 let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
5572 forwarding
5573 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5574 .unwrap();
5575 let control_rpc = forwarding
5576 .begin_module_control_rpc_for(
5577 "drain-gap",
5578 "health.check",
5579 Instant::now() + Duration::from_secs(1),
5580 )
5581 .unwrap();
5582 let target = forwarding
5583 .begin_module_drain("drain-gap", RouteCloseReason::Reload)
5584 .unwrap()
5585 .unwrap();
5586 assert!(matches!(
5587 control_rpc.receiver.blocking_recv().unwrap(),
5588 ModuleControlRpcOutcome::ModuleGone(_)
5589 ));
5590 assert_eq!(target.abandoned_bindings.len(), 1);
5591 assert!(binding.flow.sem.is_closed());
5592 assert!(
5593 !forwarding
5594 .complete_pending_relay(
5595 module_connection,
5596 pending.corr,
5597 RouteBindRelayOutcome::Accepted,
5598 )
5599 .unwrap()
5600 .settled
5601 );
5602 assert!(matches!(
5603 forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
5604 Err(ForwardingError::ModuleReloading { .. })
5605 ));
5606 let released = forwarding
5607 .release_module_endpoint_routes(target.endpoint)
5608 .unwrap();
5609 assert_eq!(released.len(), 1);
5610 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5611 }
5612
5613 #[test]
5621 fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
5622 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5623 route_fixture("closing-client");
5624
5625 let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
5628 forwarding
5629 .complete_pending_relay(
5630 module_connection,
5631 live.corr,
5632 RouteBindRelayOutcome::Accepted,
5633 )
5634 .unwrap();
5635 client_rx.try_recv().unwrap();
5636
5637 let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
5639 forwarding
5640 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5641 .unwrap();
5642
5643 assert!(forwarding
5645 .escalate_client_delivery_failure(
5646 client,
5647 live.client_channel,
5648 live.client_epoch,
5649 CloseReason::new(
5650 "module_to_client_delivery_failed",
5651 "client egress refused a module frame",
5652 ),
5653 UndeliveredFrame {
5654 module_id: None,
5655 sink: &sink,
5656 },
5657 )
5658 .unwrap());
5659 assert!(!sink.is_closed());
5660
5661 let completion = forwarding
5662 .complete_pending_relay(
5663 module_connection,
5664 pending.corr,
5665 RouteBindRelayOutcome::Accepted,
5666 )
5667 .expect("a closing client must not turn a module's ack into an error");
5668
5669 assert!(completion.settled);
5670 let abandoned = completion
5671 .abandoned
5672 .expect("the module must be told to drop the binding it just created");
5673 assert_eq!(abandoned.connection_id, module_connection);
5674 assert_eq!(abandoned.channel, pending.module_channel);
5675 assert_eq!(abandoned.epoch, pending.module_epoch);
5676 assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
5677 assert!(matches!(
5678 pending.receiver.blocking_recv().unwrap(),
5679 RouteBindRelayOutcome::ModuleGone(_)
5680 ));
5681 assert!(client_rx.try_recv().is_err());
5684 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
5685
5686 assert!(forwarding
5689 .has_live_module_connection("closing-client")
5690 .unwrap());
5691 let cotenant = ConnectionId::new(201);
5692 let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
5693 let cotenant_route = begin_test_route(
5694 &forwarding,
5695 cotenant,
5696 FrameSink::new(cotenant_tx),
5697 62,
5698 "closing-client",
5699 );
5700 assert_eq!(cotenant_route.endpoint, endpoint);
5701 forwarding
5702 .complete_pending_relay(
5703 module_connection,
5704 cotenant_route.corr,
5705 RouteBindRelayOutcome::Accepted,
5706 )
5707 .unwrap();
5708 assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
5709 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5710 }
5711
5712 #[test]
5713 fn pending_route_permit_is_released_on_rejection_and_abort() {
5714 let forwarding = ForwardingTable::default();
5715 let module_connection = ConnectionId::new(300);
5716 let client = ConnectionId::new(301);
5717 let (module_tx, _module_rx) = mpsc::channel(1);
5718 let endpoint = forwarding
5719 .register_module_connection(
5720 module_connection,
5721 "permit".into(),
5722 2,
5723 Concurrency::ModuleManaged,
5724 FrameSink::new(module_tx),
5725 )
5726 .unwrap();
5727 let (client_tx, mut client_rx) = mpsc::channel(1);
5728 let sink = FrameSink::new(client_tx);
5729 let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
5730 assert!(sink.try_send(test_ping(999)).is_err());
5731 forwarding
5732 .complete_pending_relay(
5733 module_connection,
5734 rejected.corr,
5735 RouteBindRelayOutcome::Rejected(ErrorBody {
5736 code: "no".into(),
5737 message: "rejected".into(),
5738 detail: None,
5739 }),
5740 )
5741 .unwrap();
5742 sink.try_send(test_ping(1000)).unwrap();
5743 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
5744
5745 let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
5746 assert!(sink.try_send(test_ping(1001)).is_err());
5747 forwarding
5748 .abort_pending_relay(
5749 endpoint,
5750 aborted.corr,
5751 RouteBindRelayOutcome::ModuleGone("abort".into()),
5752 )
5753 .unwrap();
5754 sink.try_send(test_ping(1002)).unwrap();
5755 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
5756
5757 let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
5758 forwarding
5759 .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
5760 .unwrap();
5761 drop(client_rx);
5762 let completion = forwarding
5763 .complete_pending_relay(
5764 module_connection,
5765 receiver_closed.corr,
5766 RouteBindRelayOutcome::Accepted,
5767 )
5768 .unwrap();
5769 assert!(completion.abandoned.is_some());
5770 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5771 }
5772
5773 #[test]
5779 fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
5780 let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
5781 route_fixture("closing-set-leak");
5782
5783 const CONNECTIONS: u64 = 32;
5784 for index in 0..CONNECTIONS {
5785 let client = ConnectionId::new(1000 + index);
5786 let (client_tx, _client_rx) = mpsc::channel(8);
5787 let route = begin_test_route(
5788 &forwarding,
5789 client,
5790 FrameSink::new(client_tx),
5791 index + 1,
5792 "closing-set-leak",
5793 );
5794 forwarding
5795 .complete_pending_relay(
5796 module_connection,
5797 route.corr,
5798 RouteBindRelayOutcome::Accepted,
5799 )
5800 .unwrap();
5801 forwarding.cleanup_connection(client).unwrap();
5802 }
5803 forwarding.cleanup_connection(module_connection).unwrap();
5804
5805 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5806 }
5807
5808 #[test]
5815 fn closing_connection_is_refused_new_work_until_cleanup_completes() {
5816 let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
5817 route_fixture("closing-gate");
5818
5819 let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
5822 forwarding
5823 .complete_pending_relay(
5824 module_connection,
5825 live.corr,
5826 RouteBindRelayOutcome::Accepted,
5827 )
5828 .unwrap();
5829 client_rx.try_recv().unwrap();
5830
5831 assert!(forwarding
5834 .escalate_client_delivery_failure(
5835 client,
5836 live.client_channel,
5837 live.client_epoch,
5838 CloseReason::new(
5839 "module_to_client_delivery_failed",
5840 "client egress refused a module frame",
5841 ),
5842 UndeliveredFrame {
5843 module_id: None,
5844 sink: &sink,
5845 },
5846 )
5847 .unwrap());
5848 assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
5849
5850 assert!(matches!(
5852 forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
5853 Err(ForwardingError::ConnectionClosing { connection_id })
5854 if connection_id == client
5855 ));
5856 let (late_tx, _late_rx) = mpsc::channel(1);
5858 assert!(matches!(
5859 forwarding.register_module_connection(
5860 client,
5861 "late-module".into(),
5862 2,
5863 Concurrency::ModuleManaged,
5864 FrameSink::new(late_tx),
5865 ),
5866 Err(ForwardingError::ConnectionClosing { connection_id })
5867 if connection_id == client
5868 ));
5869
5870 forwarding.cleanup_connection(client).unwrap();
5874 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5875 }
5876}
5877
5878#[cfg(test)]
5881mod swap_slot_tests {
5882 use std::time::Duration;
5883
5884 use super::*;
5885 use tokio::sync::mpsc;
5886
5887 const MODULE_ID: &str = "swapped";
5888
5889 struct SwapFixture {
5890 forwarding: ForwardingTable,
5891 incumbent_connection: ConnectionId,
5892 incumbent: ModuleEndpointId,
5893 candidate_connection: ConnectionId,
5894 candidate: ModuleEndpointId,
5895 _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
5896 }
5897
5898 fn swap_fixture() -> SwapFixture {
5899 let forwarding = ForwardingTable::default();
5900 let incumbent_connection = ConnectionId::new(100);
5901 let candidate_connection = ConnectionId::new(110);
5902 let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
5903 let incumbent = forwarding
5904 .register_module_connection(
5905 incumbent_connection,
5906 MODULE_ID.to_string(),
5907 2,
5908 Concurrency::ModuleManaged,
5909 FrameSink::new(incumbent_tx),
5910 )
5911 .unwrap();
5912 let (candidate_tx, candidate_rx) = mpsc::channel(8);
5913 let candidate = forwarding
5914 .register_candidate_module_connection(
5915 candidate_connection,
5916 MODULE_ID.to_string(),
5917 2,
5918 Concurrency::ModuleManaged,
5919 FrameSink::new(candidate_tx),
5920 )
5921 .unwrap();
5922 SwapFixture {
5923 forwarding,
5924 incumbent_connection,
5925 incumbent,
5926 candidate_connection,
5927 candidate,
5928 _module_rxs: vec![incumbent_rx, candidate_rx],
5929 }
5930 }
5931
5932 fn client(
5933 raw: u64,
5934 ) -> (
5935 ConnectionId,
5936 FrameSink,
5937 mpsc::Receiver<crate::router::OutboundFrame>,
5938 ) {
5939 let (tx, rx) = mpsc::channel(8);
5940 (ConnectionId::new(raw), FrameSink::new(tx), rx)
5941 }
5942
5943 fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
5944 forwarding
5945 .read_inner()
5946 .unwrap()
5947 .client_to_module
5948 .values()
5949 .map(|route| route.module_endpoint)
5950 .collect()
5951 }
5952
5953 #[test]
5954 fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
5955 let fixture = swap_fixture();
5956 let forwarding = &fixture.forwarding;
5957 assert_ne!(fixture.incumbent, fixture.candidate);
5958
5959 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5961 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5962 let (client_connection, client_sink, _client_rx) = client(200);
5963 let pending = forwarding
5964 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5965 .unwrap();
5966 assert_eq!(pending.endpoint, fixture.incumbent);
5967 let rpc = forwarding
5968 .begin_module_control_rpc_for(
5969 MODULE_ID,
5970 "health.check",
5971 Instant::now() + Duration::from_secs(1),
5972 )
5973 .unwrap();
5974 assert_eq!(rpc.endpoint, fixture.incumbent);
5975 let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
5976 assert_eq!(census.len(), 1, "the census lists one endpoint per id");
5977
5978 assert_eq!(
5980 forwarding
5981 .module_endpoint_for_connection(fixture.candidate_connection)
5982 .unwrap(),
5983 Some(fixture.candidate)
5984 );
5985 assert_eq!(
5986 forwarding
5987 .module_id_for_connection(fixture.candidate_connection)
5988 .unwrap()
5989 .as_deref(),
5990 Some(MODULE_ID)
5991 );
5992
5993 let (other_tx, _other_rx) = mpsc::channel(1);
5995 assert_eq!(
5996 forwarding.register_candidate_module_connection(
5997 ConnectionId::new(120),
5998 MODULE_ID.to_string(),
5999 2,
6000 Concurrency::ModuleManaged,
6001 FrameSink::new(other_tx),
6002 ),
6003 Err(ForwardingError::CandidateSlotOccupied {
6004 module_id: MODULE_ID.to_string()
6005 })
6006 );
6007 }
6008
6009 #[test]
6013 fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
6014 let fixture = swap_fixture();
6015 let forwarding = &fixture.forwarding;
6016 let (early_client, early_sink, _early_rx) = client(200);
6017 let mut early = forwarding
6018 .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
6019 .unwrap();
6020 assert_eq!(early.endpoint, fixture.incumbent);
6021 assert_eq!(
6022 forwarding
6023 .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
6024 .unwrap(),
6025 RouteBindRelayMark::Marked
6026 );
6027
6028 let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
6029 assert_eq!(
6030 cutover,
6031 ForwardingCutover {
6032 promoted: fixture.candidate,
6033 incumbent: Some(fixture.incumbent),
6034 }
6035 );
6036
6037 let (late_client, late_sink, _late_rx) = client(201);
6039 let late = forwarding
6040 .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
6041 .unwrap();
6042 assert_eq!(
6043 late.endpoint, fixture.candidate,
6044 "a route.open after cutover was reserved on the incumbent"
6045 );
6046
6047 let completion = forwarding
6049 .complete_pending_relay(
6050 fixture.incumbent_connection,
6051 early.corr,
6052 RouteBindRelayOutcome::Accepted,
6053 )
6054 .expect("a superseded endpoint's ack is not an error on its connection");
6055 assert!(completion.settled);
6056 assert!(
6057 !committed_endpoints(forwarding).contains(&fixture.incumbent),
6058 "a relay reserved before cutover committed a route on the incumbent"
6059 );
6060 let goodbye = completion
6061 .abandoned
6062 .expect("the incumbent is told to drop the binding it just created");
6063 assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
6064 assert_eq!(goodbye.channel, early.module_channel);
6065 assert_eq!(goodbye.epoch, early.module_epoch);
6066 assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
6067 match early.receiver.try_recv() {
6068 Ok(RouteBindRelayOutcome::DaemonRejected(body)) => {
6069 assert_eq!(body.code, "module_reloading")
6070 }
6071 other => panic!("expected a retryable module_reloading answer, got {other:?}"),
6072 }
6073 assert!(matches!(
6074 forwarding
6075 .lookup_data_route(early_client, early.client_channel, early.client_epoch)
6076 .unwrap(),
6077 DataRoute::Client(DataRouteState::Absent)
6078 ));
6079
6080 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
6083 forwarding
6084 .complete_pending_relay(
6085 fixture.candidate_connection,
6086 late.corr,
6087 RouteBindRelayOutcome::Accepted,
6088 )
6089 .unwrap();
6090 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
6091 assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
6092 }
6093
6094 #[test]
6095 fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
6096 let fixture = swap_fixture();
6097 let forwarding = &fixture.forwarding;
6098 let (bound_client, bound_sink, _bound_rx) = client(200);
6100 let bound = forwarding
6101 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
6102 .unwrap();
6103 forwarding
6104 .complete_pending_relay(
6105 fixture.incumbent_connection,
6106 bound.corr,
6107 RouteBindRelayOutcome::Accepted,
6108 )
6109 .unwrap();
6110 let (pending_client, pending_sink, _pending_rx) = client(201);
6111 let mut in_flight = forwarding
6112 .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
6113 .unwrap();
6114 forwarding
6115 .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
6116 .unwrap();
6117
6118 let incumbent = forwarding
6119 .cutover_candidate(MODULE_ID)
6120 .unwrap()
6121 .unwrap()
6122 .incumbent
6123 .unwrap();
6124 let target = forwarding
6125 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
6126 .unwrap()
6127 .expect("the superseded incumbent is still registered");
6128
6129 assert_eq!(target.endpoint, fixture.incumbent);
6130 assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
6131 assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
6132 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
6133 assert_eq!(target.abandoned_bindings.len(), 1);
6134 assert_eq!(
6135 target.abandoned_bindings[0].channel,
6136 in_flight.module_channel
6137 );
6138 assert!(matches!(
6139 in_flight.receiver.try_recv(),
6140 Ok(RouteBindRelayOutcome::DaemonRejected(body)) if body.code == "module_reloading"
6141 ));
6142 assert_eq!(
6143 forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
6144 1,
6145 "the incumbent's bound route stays until its drain finishes"
6146 );
6147
6148 let (next_client, next_sink, _next_rx) = client(202);
6149 let next = forwarding
6150 .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
6151 .expect("the promoted candidate keeps accepting routes");
6152 assert_eq!(next.endpoint, fixture.candidate);
6153 }
6154
6155 #[test]
6161 fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
6162 let forwarding = ForwardingTable::default();
6163 let first_connection = ConnectionId::new(70);
6164 let (first_tx, _first_rx) = mpsc::channel(8);
6165 forwarding
6166 .register_module_connection(
6167 first_connection,
6168 MODULE_ID.to_string(),
6169 2,
6170 Concurrency::ModuleManaged,
6171 FrameSink::new(first_tx),
6172 )
6173 .unwrap();
6174 let (client_connection, client_sink, _client_rx) = client(200);
6175 let mut pending = forwarding
6176 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
6177 .unwrap();
6178 let (second_tx, _second_rx) = mpsc::channel(8);
6179 forwarding
6180 .register_module_connection(
6181 ConnectionId::new(80),
6182 MODULE_ID.to_string(),
6183 2,
6184 Concurrency::ModuleManaged,
6185 FrameSink::new(second_tx),
6186 )
6187 .unwrap();
6188
6189 assert_eq!(
6190 forwarding
6191 .complete_pending_relay(
6192 first_connection,
6193 pending.corr,
6194 RouteBindRelayOutcome::Accepted
6195 )
6196 .unwrap_err(),
6197 ForwardingError::StaleModuleEndpoint
6198 );
6199 assert!(committed_endpoints(&forwarding).is_empty());
6200 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
6201 assert!(matches!(
6202 pending.receiver.try_recv(),
6203 Err(oneshot::error::TryRecvError::Closed)
6204 ));
6205 }
6206
6207 #[test]
6208 fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
6209 let fixture = swap_fixture();
6211 let forwarding = &fixture.forwarding;
6212 assert!(forwarding
6213 .cleanup_connection(fixture.candidate_connection)
6214 .unwrap()
6215 .is_empty());
6216 assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
6217 let (client_connection, client_sink, _client_rx) = client(200);
6218 assert_eq!(
6219 forwarding
6220 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
6221 .unwrap()
6222 .endpoint,
6223 fixture.incumbent
6224 );
6225
6226 let fixture = swap_fixture();
6229 let forwarding = &fixture.forwarding;
6230 let (bound_client, bound_sink, _bound_rx) = client(200);
6231 let bound = forwarding
6232 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
6233 .unwrap();
6234 forwarding
6235 .complete_pending_relay(
6236 fixture.incumbent_connection,
6237 bound.corr,
6238 RouteBindRelayOutcome::Accepted,
6239 )
6240 .unwrap();
6241 forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
6242 let released = forwarding
6243 .cleanup_connection(fixture.incumbent_connection)
6244 .unwrap();
6245 assert_eq!(released.len(), 1);
6246 assert_eq!(released[0].connection_id, bound_client);
6247 assert!(forwarding
6248 .read_inner()
6249 .unwrap()
6250 .superseded_endpoints
6251 .is_empty());
6252 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
6253 let (next_client, next_sink, _next_rx) = client(201);
6254 assert_eq!(
6255 forwarding
6256 .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
6257 .unwrap()
6258 .endpoint,
6259 fixture.candidate
6260 );
6261 }
6262}