1use std::{
2 collections::{BTreeMap, HashMap, HashSet},
3 error::Error,
4 fmt,
5 sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
6 time::Duration,
7};
8
9use subc_control::{ClientControlResponse, RouteCloseReason};
10use subc_protocol::{
11 manifest::Concurrency,
12 session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
13 ErrorBody, Flags, FrameType, Principal, Priority,
14};
15use tokio::sync::{oneshot, Semaphore};
16use tokio::time::Instant;
17use tracing::{debug, info, warn};
18
19use crate::{
20 control::{RouteBindBreakers, RouteBindConcurrency},
21 observability::DaemonCounters,
22 registry::ConnectionId,
23 router::FrameSink,
24 scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
25 Frame, ProjectRootId,
26};
27
28const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
30
31const STATELESS_PARALLEL_WINDOW: usize = 1024;
33
34const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
38
39#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
44pub struct ModuleEndpointId {
45 pub connection_id: ConnectionId,
46 pub generation: u64,
47}
48
49#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
51pub(crate) struct ClientRouteKey {
52 pub connection_id: ConnectionId,
53 pub channel: u16,
54}
55
56#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
58pub(crate) struct ModuleRouteKey {
59 pub endpoint: ModuleEndpointId,
60 pub channel: u16,
61}
62
63#[derive(Debug)]
64pub(crate) struct RouteBinding {
65 pub client_connection_id: ConnectionId,
66 pub client_sink: FrameSink,
67 pub client_negotiated_ver: u8,
68 pub client_channel: u16,
69 pub client_epoch: u32,
70 pub module_id: String,
71 pub module_endpoint: ModuleEndpointId,
72 pub module_sink: FrameSink,
73 pub module_negotiated_ver: u8,
74 pub module_channel: u16,
75 pub module_epoch: u32,
76 pub principal: Principal,
77 pub project_root: Option<ProjectRootId>,
78 pub bound_at: Instant,
79 pub flow: Arc<ChannelFlow>,
80 pub scope: Option<BoundScope>,
83}
84
85#[derive(Debug, Clone)]
86pub(crate) enum DataRoute {
87 Client(DataRouteState),
88 Module(DataRouteState),
89}
90
91#[derive(Debug, Clone)]
92pub(crate) enum DataRouteState {
93 Bound(Arc<RouteBinding>),
94 Reserved,
95 EpochMismatch,
96 Absent,
97}
98
99#[derive(Debug, Clone, Copy, PartialEq, Eq)]
128pub(crate) enum GoodbyeTargetKind {
129 Client,
130 Module,
131}
132
133#[derive(Debug, Clone)]
134pub(crate) struct GoodbyeTarget {
135 pub connection_id: ConnectionId,
136 pub sink: FrameSink,
137 pub negotiated_ver: u8,
138 pub channel: u16,
139 pub epoch: u32,
140 pub kind: GoodbyeTargetKind,
141 pub module_id: Option<String>,
146}
147
148#[derive(Debug, Clone, Copy)]
151pub(crate) struct UndeliveredFrame<'a> {
152 pub module_id: Option<&'a str>,
154 pub sink: &'a FrameSink,
156}
157
158fn principal_label(principal: &Principal) -> String {
160 match principal {
161 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
162 Principal::Direct => "direct".to_string(),
163 other => format!("{other:?}"),
164 }
165}
166
167fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
170 let labels = inner
171 .client_to_module
172 .iter()
173 .filter(|(key, _)| key.connection_id == connection_id)
174 .map(|(_, route)| principal_label(&route.principal))
175 .collect::<std::collections::BTreeSet<_>>();
176 if labels.is_empty() {
177 "none".to_string()
178 } else {
179 labels.into_iter().collect::<Vec<_>>().join(",")
180 }
181}
182
183impl GoodbyeTarget {
184 pub(crate) fn close_on_delivery_failure(&self) -> bool {
187 matches!(self.kind, GoodbyeTargetKind::Client)
188 }
189}
190
191pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
204
205pub(crate) fn send_module_route_goodbye(
217 counters: &DaemonCounters,
218 sink: &FrameSink,
219 frame: Frame,
220 module_id: Option<&str>,
221 context: &'static str,
222) {
223 let channel = frame.header.channel;
224 let epoch = frame.header.epoch;
225 let Err(err) = sink.try_send(frame.clone()) else {
226 return;
227 };
228 let runtime = match tokio::runtime::Handle::try_current() {
231 Ok(runtime) if !sink.is_closed() => runtime,
232 _ => {
233 counters.increment_goodbye_relay_module_dropped(module_id);
234 warn!(
235 module_id = module_id.unwrap_or("unknown"),
236 route_channel = channel,
237 route_epoch = epoch,
238 error = %err,
239 context,
240 "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
241 );
242 return;
243 }
244 };
245 debug!(
246 module_id = module_id.unwrap_or("unknown"),
247 route_channel = channel,
248 route_epoch = epoch,
249 error = %err,
250 context,
251 "module egress queue refused route GOODBYE; delivering it once the module frees room"
252 );
253 let counters = counters.clone();
254 let sink = sink.clone();
255 let module_id = module_id.map(str::to_string);
256 runtime.spawn(async move {
257 let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
258 let why = match outcome {
259 Ok(Ok(())) => {
260 debug!(
261 module_id = module_id.as_deref().unwrap_or("unknown"),
262 route_channel = channel,
263 route_epoch = epoch,
264 context,
265 "late route GOODBYE delivered to module"
266 );
267 return;
268 }
269 Ok(Err(err)) => err.to_string(),
270 Err(_) => format!(
271 "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
272 ),
273 };
274 counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
275 warn!(
276 module_id = module_id.as_deref().unwrap_or("unknown"),
277 route_channel = channel,
278 route_epoch = epoch,
279 error = %why,
280 context,
281 "route GOODBYE to module dropped under backpressure; not closing shared module connection"
282 );
283 });
284}
285
286#[derive(Debug, Clone)]
292pub(crate) struct EndpointRoute {
293 pub goodbye_target: GoodbyeTarget,
294 pub principal: Principal,
295 pub bound_at: Instant,
296 pub draining: bool,
297 pub drain_reason: Option<RouteCloseReason>,
301}
302
303#[derive(Debug, Clone)]
306pub(crate) struct ScopeDrainedRoute {
307 pub reason: RouteCloseReason,
308 pub module_id: String,
309 pub client: GoodbyeTarget,
310 pub module: GoodbyeTarget,
311}
312
313#[derive(Debug)]
314pub(crate) struct PendingRouteBindRelay {
315 pub endpoint: ModuleEndpointId,
316 pub module_sink: FrameSink,
317 pub negotiated_ver: u8,
318 pub client_channel: u16,
319 pub client_epoch: u32,
320 pub module_channel: u16,
321 pub module_epoch: u32,
322 pub corr: u64,
323 pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
324}
325
326#[derive(Debug, Clone)]
327pub(crate) struct ModuleDrainTarget {
328 pub endpoint: ModuleEndpointId,
329 pub sink: FrameSink,
330 pub negotiated_ver: u8,
331 pub abandoned_bindings: Vec<GoodbyeTarget>,
332 pub excluded_subscriptions: u32,
333}
334
335#[cfg(unix)]
338#[derive(Debug, Clone)]
339pub(crate) struct ModuleConnectionTarget {
340 pub module_id: String,
341 pub endpoint: ModuleEndpointId,
342 pub sink: FrameSink,
343 pub negotiated_ver: u8,
344}
345
346#[derive(Debug, Clone)]
347pub(crate) enum RouteBindRelayOutcome {
348 Accepted,
349 Rejected(ErrorBody),
350 ModuleGone(String),
351}
352
353#[derive(Debug, Clone, Copy, PartialEq, Eq)]
355pub(crate) struct ForwardingCutover {
356 pub promoted: ModuleEndpointId,
358 pub incumbent: Option<ModuleEndpointId>,
361}
362
363#[derive(Debug)]
365pub(crate) struct ConnectionCleanup {
366 pub released: Vec<GoodbyeTarget>,
368 pub abandoned_relays: u32,
371}
372
373#[derive(Debug, Clone)]
374pub(crate) struct PendingRelayCompletion {
375 pub settled: bool,
376 pub abandoned: Option<GoodbyeTarget>,
377}
378
379#[derive(Debug)]
380pub(crate) struct PendingModuleControlRpc {
381 pub endpoint: ModuleEndpointId,
382 pub module_sink: FrameSink,
383 pub negotiated_ver: u8,
384 pub corr: u64,
385 pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
386}
387
388#[derive(Debug, Clone)]
389pub(crate) enum ModuleControlRpcOutcome {
390 Response(ModuleControlResponse),
391 Rejected(ErrorBody),
392 ModuleGone(String),
393 MalformedResponse(String),
394 UnexpectedOp { expected: String, actual: String },
395 DeadlineElapsed,
396}
397
398#[derive(Debug, Clone, PartialEq, Eq)]
399pub(crate) enum ModuleControlRpcCompletion {
400 Unknown,
401 Settled,
402 LateHealthAnswer {
403 module_id: String,
404 latency: Duration,
405 },
406}
407
408#[derive(Debug)]
409struct PendingModuleControlRpcEntry {
410 expected_op: String,
411 deadline: Instant,
412 health_probe_started_at: Option<Instant>,
413 sender: oneshot::Sender<ModuleControlRpcOutcome>,
414}
415
416#[derive(Debug)]
417struct HealthProbeTombstone {
418 expected_op: String,
419 module_id: String,
420 probe_started_at: Instant,
421 expires_at: Instant,
422}
423
424#[derive(Debug, Clone)]
425struct RouteReservation {
426 client_key: ClientRouteKey,
427 module_key: ModuleRouteKey,
428 client_epoch: u32,
429 module_epoch: u32,
430 project_root: Option<ProjectRootId>,
431}
432
433#[derive(Debug)]
434struct PendingRouteBindRelayEntry {
435 reservation: RouteReservation,
436 client_sink: FrameSink,
437 client_negotiated_ver: u8,
438 client_permit: crate::router::EgressPermit,
439 route_open_frame: Frame,
440 principal: Principal,
441 scope: Option<BoundScope>,
444 deadline: Instant,
445 relay_enqueued: bool,
456 sender: oneshot::Sender<RouteBindRelayOutcome>,
457}
458
459#[derive(Debug, Clone, Copy, PartialEq, Eq)]
462pub(crate) enum RouteBindRelayMark {
463 Marked,
465 SettledByReply,
469 ReleasedWithoutReply,
473}
474
475#[derive(Debug, Clone)]
476pub(crate) enum RouteRelease {
477 Removed(GoodbyeTarget),
478 Stale,
479 Absent,
480}
481
482#[derive(Debug, Clone)]
483pub(crate) enum RoutePollSnapshot {
484 Bound {
485 module_id: String,
486 status: Option<String>,
487 },
488 Absent,
489}
490
491#[derive(Debug, Clone)]
492struct ModuleConnection {
493 endpoint: ModuleEndpointId,
494 sink: FrameSink,
495 negotiated_ver: u8,
496 concurrency: Concurrency,
497}
498
499#[derive(Debug, Default)]
500struct ForwardingInner {
501 daemon_draining: bool,
502 modules_by_id: HashMap<String, ModuleConnection>,
506 candidates_by_id: HashMap<String, ModuleConnection>,
512 superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
518 endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
519 module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
520 draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
524 closing_connections: HashSet<ConnectionId>,
525 next_generation: u64,
526 reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
527 reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
528 next_client_channel: HashMap<ConnectionId, u16>,
529 next_module_channel: HashMap<ModuleEndpointId, u16>,
530 client_slot_epochs: HashMap<ClientRouteKey, u32>,
531 module_slot_epochs: HashMap<ModuleRouteKey, u32>,
532 last_published_epoch: HashMap<ClientRouteKey, u32>,
533 client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
534 module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
535 status: HashMap<(ClientRouteKey, u32), String>,
536 pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
537 relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
542 scope_tags: HashMap<(String, String), ScopeTag>,
547 next_control_corr: HashMap<ModuleEndpointId, u64>,
548 pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
549 health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
550}
551
552#[derive(Debug, Clone)]
553pub(crate) struct CloseReason {
554 code: &'static str,
555 message: String,
556}
557
558impl CloseReason {
559 pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
560 Self {
561 code,
562 message: message.into(),
563 }
564 }
565}
566
567impl fmt::Display for CloseReason {
568 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
569 write!(f, "{}: {}", self.code, self.message)
570 }
571}
572
573pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
574
575#[derive(Debug, Default)]
577pub struct ForwardingTable {
578 inner: Arc<RwLock<ForwardingInner>>,
579 close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
580 counters: DaemonCounters,
581 route_bind_breakers: RouteBindBreakers,
586 route_bind_concurrency: RouteBindConcurrency,
589 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
593}
594
595impl ForwardingTable {
596 pub(crate) fn counters(&self) -> DaemonCounters {
597 self.counters.clone()
598 }
599
600 pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
601 self.route_bind_breakers.clone()
602 }
603
604 pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
605 self.route_bind_concurrency.clone()
606 }
607
608 pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
609 Arc::clone(&self.route_outages)
610 }
611
612 pub(crate) fn register_connection_close(
613 &self,
614 connection_id: ConnectionId,
615 ) -> ConnectionCloseReceiver {
616 let (sender, receiver) = oneshot::channel();
617 let replaced = self
618 .lock_close_registry()
619 .insert(connection_id, sender)
620 .is_some();
621 if replaced {
622 warn!(
623 connection_id = connection_id.get(),
624 "replaced existing connection close registration"
625 );
626 }
627 receiver
628 }
629
630 pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
631 self.lock_close_registry().remove(&connection_id);
632 }
633
634 #[cfg(unix)]
643 pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
644 let senders: Vec<_> = self.lock_close_registry().drain().collect();
645 let count = senders.len();
646 for (_, sender) in senders {
647 let _ = sender.send(reason.clone());
648 }
649 count
650 }
651
652 #[cfg(unix)]
656 pub(crate) fn module_connections(
657 &self,
658 ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
659 let inner = self.read_inner()?;
660 let mut seen = HashSet::new();
661 Ok(inner
662 .modules_by_id
663 .values()
664 .chain(inner.candidates_by_id.values())
665 .chain(inner.superseded_endpoints.values())
666 .filter(|module| seen.insert(module.endpoint))
667 .map(|module| ModuleConnectionTarget {
668 module_id: inner
669 .module_id_by_endpoint
670 .get(&module.endpoint)
671 .cloned()
672 .unwrap_or_default(),
673 endpoint: module.endpoint,
674 sink: module.sink.clone(),
675 negotiated_ver: module.negotiated_ver,
676 })
677 .collect())
678 }
679
680 pub(crate) fn request_connection_close(
684 &self,
685 connection_id: ConnectionId,
686 reason: CloseReason,
687 ) -> bool {
688 let sender = self.lock_close_registry().remove(&connection_id);
689 if let Some(sender) = sender {
690 debug!(
691 connection_id = connection_id.get(),
692 close_reason = %reason,
693 "requesting connection close"
694 );
695 let _ = sender.send(reason);
696 true
697 } else {
698 debug!(
699 connection_id = connection_id.get(),
700 close_reason = %reason,
701 "connection close request ignored for inactive connection"
702 );
703 false
704 }
705 }
706
707 pub fn register_module_connection(
708 &self,
709 connection_id: ConnectionId,
710 module_id: String,
711 negotiated_ver: u8,
712 concurrency: Concurrency,
713 sink: FrameSink,
714 ) -> Result<ModuleEndpointId, ForwardingError> {
715 self.register_module_connection_inner(
716 connection_id,
717 module_id,
718 negotiated_ver,
719 concurrency,
720 sink,
721 None,
722 )
723 }
724
725 pub(crate) fn register_module_connection_acked(
738 &self,
739 connection_id: ConnectionId,
740 module_id: String,
741 negotiated_ver: u8,
742 concurrency: Concurrency,
743 sink: FrameSink,
744 hello_ack: Frame,
745 ) -> Result<ModuleEndpointId, ForwardingError> {
746 self.register_module_connection_inner(
747 connection_id,
748 module_id,
749 negotiated_ver,
750 concurrency,
751 sink,
752 Some(hello_ack),
753 )
754 }
755
756 fn register_module_connection_inner(
757 &self,
758 connection_id: ConnectionId,
759 module_id: String,
760 negotiated_ver: u8,
761 concurrency: Concurrency,
762 sink: FrameSink,
763 hello_ack: Option<Frame>,
764 ) -> Result<ModuleEndpointId, ForwardingError> {
765 let mut inner = self.write_inner()?;
766 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
767 return Err(ForwardingError::ConnectionClosing { connection_id });
768 }
769 check_module_connection_role_locked(&inner, connection_id)?;
770 enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
773
774 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
775 let endpoint = ModuleEndpointId {
776 connection_id,
777 generation: inner.next_generation,
778 };
779 inner.endpoint_by_connection.insert(connection_id, endpoint);
780 inner
781 .module_id_by_endpoint
782 .insert(endpoint, module_id.clone());
783 inner.next_module_channel.insert(endpoint, 1);
784 inner.next_control_corr.insert(endpoint, 1);
785 inner.modules_by_id.insert(
786 module_id.clone(),
787 ModuleConnection {
788 endpoint,
789 sink,
790 negotiated_ver,
791 concurrency,
792 },
793 );
794 drop(inner);
795
796 if let Some(discarded) = self
809 .route_bind_breakers
810 .reset_for_new_module_connection(&module_id)
811 {
812 info!(
813 module_id = %module_id,
814 discarded_consecutive_timeouts = discarded,
815 "route.bind breaker state discarded: a new module connection replaced the process it described"
816 );
817 }
818 Ok(endpoint)
819 }
820
821 #[cfg(test)]
835 pub(crate) fn register_candidate_module_connection(
836 &self,
837 connection_id: ConnectionId,
838 module_id: String,
839 negotiated_ver: u8,
840 concurrency: Concurrency,
841 sink: FrameSink,
842 ) -> Result<ModuleEndpointId, ForwardingError> {
843 self.register_candidate_module_connection_inner(
844 connection_id,
845 module_id,
846 negotiated_ver,
847 concurrency,
848 sink,
849 None,
850 )
851 }
852
853 pub(crate) fn register_candidate_module_connection_acked(
860 &self,
861 connection_id: ConnectionId,
862 module_id: String,
863 negotiated_ver: u8,
864 concurrency: Concurrency,
865 sink: FrameSink,
866 hello_ack: Frame,
867 ) -> Result<ModuleEndpointId, ForwardingError> {
868 self.register_candidate_module_connection_inner(
869 connection_id,
870 module_id,
871 negotiated_ver,
872 concurrency,
873 sink,
874 Some(hello_ack),
875 )
876 }
877
878 fn register_candidate_module_connection_inner(
879 &self,
880 connection_id: ConnectionId,
881 module_id: String,
882 negotiated_ver: u8,
883 concurrency: Concurrency,
884 sink: FrameSink,
885 hello_ack: Option<Frame>,
886 ) -> Result<ModuleEndpointId, ForwardingError> {
887 let mut inner = self.write_inner()?;
888 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
889 return Err(ForwardingError::ConnectionClosing { connection_id });
890 }
891 check_module_connection_role_locked(&inner, connection_id)?;
892 if inner.candidates_by_id.contains_key(&module_id) {
893 return Err(ForwardingError::CandidateSlotOccupied { module_id });
894 }
895 enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
898
899 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
900 let endpoint = ModuleEndpointId {
901 connection_id,
902 generation: inner.next_generation,
903 };
904 inner.endpoint_by_connection.insert(connection_id, endpoint);
905 inner
906 .module_id_by_endpoint
907 .insert(endpoint, module_id.clone());
908 inner.next_module_channel.insert(endpoint, 1);
909 inner.next_control_corr.insert(endpoint, 1);
910 inner.candidates_by_id.insert(
911 module_id,
912 ModuleConnection {
913 endpoint,
914 sink,
915 negotiated_ver,
916 concurrency,
917 },
918 );
919 Ok(endpoint)
920 }
921
922 pub(crate) fn cutover_candidate(
940 &self,
941 module_id: &str,
942 ) -> Result<Option<ForwardingCutover>, ForwardingError> {
943 let mut inner = self.write_inner()?;
944 if inner.daemon_draining {
945 return Err(ForwardingError::ModuleReloading {
946 module_id: module_id.to_string(),
947 });
948 }
949 let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
950 return Ok(None);
951 };
952 let promoted = candidate.endpoint;
953 let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
954 let incumbent = incumbent.map(|incumbent| {
955 let endpoint = incumbent.endpoint;
956 inner.superseded_endpoints.insert(endpoint, incumbent);
957 endpoint
958 });
959 drop(inner);
960
961 if let Some(discarded) = self
965 .route_bind_breakers
966 .reset_for_new_module_connection(module_id)
967 {
968 info!(
969 module_id = %module_id,
970 discarded_consecutive_timeouts = discarded,
971 "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
972 );
973 }
974 Ok(Some(ForwardingCutover {
975 promoted,
976 incumbent,
977 }))
978 }
979
980 #[allow(clippy::too_many_arguments)]
981 pub(crate) async fn begin_route_bind_relay_for(
982 &self,
983 client_connection_id: ConnectionId,
984 client_sink: FrameSink,
985 client_negotiated_ver: u8,
986 client_corr: u64,
987 module_id: &str,
988 principal: Principal,
989 scope: Option<BoundScope>,
990 project_root: Option<ProjectRootId>,
991 deadline: Instant,
992 ) -> Result<PendingRouteBindRelay, ForwardingError> {
993 let client_permit =
997 client_sink
998 .reserve_owned()
999 .await
1000 .map_err(|_| ForwardingError::ClientEgressClosed {
1001 connection_id: client_connection_id,
1002 })?;
1003 self.begin_route_bind_relay_inner(
1004 client_connection_id,
1005 client_sink,
1006 client_negotiated_ver,
1007 client_corr,
1008 module_id,
1009 principal,
1010 scope,
1011 project_root,
1012 deadline,
1013 client_permit,
1014 )
1015 }
1016
1017 #[cfg(test)]
1018 pub(crate) fn begin_route_bind_relay_for_test(
1019 &self,
1020 client_connection_id: ConnectionId,
1021 client_sink: FrameSink,
1022 client_corr: u64,
1023 module_id: &str,
1024 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1025 let permit =
1026 client_sink
1027 .try_reserve_owned()
1028 .map_err(|_| ForwardingError::ClientEgressClosed {
1029 connection_id: client_connection_id,
1030 })?;
1031 self.begin_route_bind_relay_inner(
1032 client_connection_id,
1033 client_sink,
1034 subc_protocol::PROTOCOL_VERSION,
1035 client_corr,
1036 module_id,
1037 Principal::Direct,
1038 None,
1039 None,
1040 Instant::now() + std::time::Duration::from_secs(60),
1041 permit,
1042 )
1043 }
1044
1045 pub(crate) fn begin_module_control_rpc_for(
1046 &self,
1047 module_id: &str,
1048 expected_op: &str,
1049 deadline: Instant,
1050 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1051 self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1052 }
1053
1054 pub(crate) fn begin_health_probe_rpc_for(
1055 &self,
1056 module_id: &str,
1057 expected_op: &str,
1058 probe_started_at: Instant,
1059 deadline: Instant,
1060 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1061 self.begin_module_control_rpc_inner(
1062 module_id,
1063 expected_op,
1064 deadline,
1065 Some(probe_started_at),
1066 false,
1067 )
1068 }
1069
1070 pub(crate) fn begin_drain_health_probe_rpc_for(
1071 &self,
1072 module_id: &str,
1073 expected_op: &str,
1074 probe_started_at: Instant,
1075 deadline: Instant,
1076 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1077 self.begin_module_control_rpc_inner(
1078 module_id,
1079 expected_op,
1080 deadline,
1081 Some(probe_started_at),
1082 true,
1083 )
1084 }
1085
1086 pub(crate) fn begin_endpoint_health_probe_rpc_for(
1094 &self,
1095 endpoint: ModuleEndpointId,
1096 expected_op: &str,
1097 probe_started_at: Instant,
1098 deadline: Instant,
1099 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1100 let inner = self.write_inner()?;
1101 let module = module_connection_for_endpoint_locked(&inner, endpoint)
1102 .cloned()
1103 .ok_or(ForwardingError::NoModuleConnection)?;
1104 let module_id = inner
1105 .module_id_by_endpoint
1106 .get(&endpoint)
1107 .cloned()
1108 .unwrap_or_default();
1109 self.begin_control_rpc_locked(
1112 inner,
1113 &module_id,
1114 module,
1115 expected_op,
1116 deadline,
1117 Some(probe_started_at),
1118 true,
1119 )
1120 }
1121
1122 fn begin_module_control_rpc_inner(
1123 &self,
1124 module_id: &str,
1125 expected_op: &str,
1126 deadline: Instant,
1127 health_probe_started_at: Option<Instant>,
1128 allow_draining: bool,
1129 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1130 let inner = self.write_inner()?;
1131 let module = inner
1132 .modules_by_id
1133 .get(module_id)
1134 .cloned()
1135 .ok_or(ForwardingError::NoModuleConnection)?;
1136 self.begin_control_rpc_locked(
1137 inner,
1138 module_id,
1139 module,
1140 expected_op,
1141 deadline,
1142 health_probe_started_at,
1143 allow_draining,
1144 )
1145 }
1146
1147 #[allow(clippy::too_many_arguments)]
1148 fn begin_control_rpc_locked(
1149 &self,
1150 mut inner: RwLockWriteGuard<'_, ForwardingInner>,
1151 module_id: &str,
1152 module: ModuleConnection,
1153 expected_op: &str,
1154 deadline: Instant,
1155 health_probe_started_at: Option<Instant>,
1156 allow_draining: bool,
1157 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1158 if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1159 return Err(ForwardingError::ModuleReloading {
1160 module_id: module_id.to_string(),
1161 });
1162 }
1163 if inner
1164 .closing_connections
1165 .contains(&module.endpoint.connection_id)
1166 {
1167 return Err(ForwardingError::ConnectionClosing {
1168 connection_id: module.endpoint.connection_id,
1169 });
1170 }
1171 if health_probe_started_at.is_some() {
1172 inner
1176 .health_probe_tombstones
1177 .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1178 }
1179 let corr = match inner.allocate_control_corr(module.endpoint) {
1180 Ok(corr) => corr,
1181 Err(err) => {
1182 drop(inner);
1183 self.request_connection_close(
1184 module.endpoint.connection_id,
1185 CloseReason::new(
1186 "control_correlation_exhausted",
1187 "daemon-originated channel-0 correlation space exhausted",
1188 ),
1189 );
1190 return Err(err);
1191 }
1192 };
1193 let (sender, receiver) = oneshot::channel();
1194 inner.pending_control_rpcs.insert(
1195 (module.endpoint, corr),
1196 PendingModuleControlRpcEntry {
1197 expected_op: expected_op.to_string(),
1198 deadline,
1199 health_probe_started_at,
1200 sender,
1201 },
1202 );
1203
1204 Ok(PendingModuleControlRpc {
1205 endpoint: module.endpoint,
1206 module_sink: module.sink,
1207 negotiated_ver: module.negotiated_ver,
1208 corr,
1209 receiver,
1210 })
1211 }
1212
1213 #[allow(clippy::too_many_arguments)]
1214 fn begin_route_bind_relay_inner(
1215 &self,
1216 client_connection_id: ConnectionId,
1217 client_sink: FrameSink,
1218 client_negotiated_ver: u8,
1219 client_corr: u64,
1220 expected_module_id: &str,
1221 principal: Principal,
1222 scope: Option<BoundScope>,
1223 project_root: Option<ProjectRootId>,
1224 deadline: Instant,
1225 client_permit: crate::router::EgressPermit,
1226 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1227 let mut inner = self.write_inner()?;
1228 if inner
1232 .endpoint_by_connection
1233 .contains_key(&client_connection_id)
1234 {
1235 return Err(ForwardingError::ConnectionRoleConflict {
1236 connection_id: client_connection_id,
1237 });
1238 }
1239 if inner.closing_connections.contains(&client_connection_id) {
1240 return Err(ForwardingError::ConnectionClosing {
1241 connection_id: client_connection_id,
1242 });
1243 }
1244 let module = inner
1245 .modules_by_id
1246 .get(expected_module_id)
1247 .cloned()
1248 .ok_or(ForwardingError::NoModuleConnection)?;
1249 if inner.draining_endpoints.contains_key(&module.endpoint) {
1250 return Err(ForwardingError::ModuleReloading {
1251 module_id: expected_module_id.to_string(),
1252 });
1253 }
1254 if inner
1255 .closing_connections
1256 .contains(&module.endpoint.connection_id)
1257 {
1258 return Err(ForwardingError::ConnectionClosing {
1259 connection_id: module.endpoint.connection_id,
1260 });
1261 }
1262
1263 let corr = match inner.allocate_control_corr(module.endpoint) {
1264 Ok(corr) => corr,
1265 Err(err) => {
1266 drop(inner);
1267 self.request_connection_close(
1268 module.endpoint.connection_id,
1269 CloseReason::new(
1270 "control_correlation_exhausted",
1271 "daemon-originated channel-0 correlation space exhausted",
1272 ),
1273 );
1274 return Err(err);
1275 }
1276 };
1277 let (client_channel, client_epoch, module_channel, module_epoch) =
1278 inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1279 let client_key = ClientRouteKey {
1280 connection_id: client_connection_id,
1281 channel: client_channel,
1282 };
1283 let module_key = ModuleRouteKey {
1284 endpoint: module.endpoint,
1285 channel: module_channel,
1286 };
1287 let reservation = RouteReservation {
1288 client_key,
1289 module_key,
1290 client_epoch,
1291 module_epoch,
1292 project_root,
1293 };
1294 let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
1295 route_channel: client_channel,
1296 route_epoch: client_epoch,
1297 })
1298 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1299 let route_open_frame = Frame::build_with_version(
1300 client_negotiated_ver,
1301 FrameType::Response,
1302 Flags::new(false, Priority::Passive, false),
1303 0,
1304 0,
1305 client_corr,
1306 response_body,
1307 )
1308 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1309 let (sender, receiver) = oneshot::channel();
1310 inner.reserved_client.insert(client_key, module_key);
1311 inner.reserved_module.insert(module_key, client_key);
1312 inner.pending_relays.insert(
1313 (module.endpoint, corr),
1314 PendingRouteBindRelayEntry {
1315 reservation,
1316 client_sink,
1317 client_negotiated_ver,
1318 client_permit,
1319 route_open_frame,
1320 principal,
1321 scope,
1322 deadline,
1323 relay_enqueued: false,
1324 sender,
1325 },
1326 );
1327
1328 Ok(PendingRouteBindRelay {
1329 endpoint: module.endpoint,
1330 module_sink: module.sink,
1331 negotiated_ver: module.negotiated_ver,
1332 client_channel,
1333 client_epoch,
1334 module_channel,
1335 module_epoch,
1336 corr,
1337 receiver,
1338 })
1339 }
1340
1341 pub(crate) fn mark_route_bind_relay_enqueued(
1351 &self,
1352 endpoint: ModuleEndpointId,
1353 corr: u64,
1354 ) -> Result<RouteBindRelayMark, ForwardingError> {
1355 let mut inner = self.write_inner()?;
1356 if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
1357 pending.relay_enqueued = true;
1358 return Ok(RouteBindRelayMark::Marked);
1359 }
1360 if inner
1361 .relays_released_before_enqueue
1362 .remove(&(endpoint, corr))
1363 {
1364 return Ok(RouteBindRelayMark::ReleasedWithoutReply);
1365 }
1366 Ok(RouteBindRelayMark::SettledByReply)
1367 }
1368
1369 pub(crate) fn release_client_route(
1370 &self,
1371 client_connection_id: ConnectionId,
1372 client_channel: u16,
1373 expected_epoch: u32,
1374 ) -> Result<RouteRelease, ForwardingError> {
1375 let mut inner = self.write_inner()?;
1376 let release = release_client_route_locked(
1377 &mut inner,
1378 ClientRouteKey {
1379 connection_id: client_connection_id,
1380 channel: client_channel,
1381 },
1382 expected_epoch,
1383 );
1384 self.record_route_release(&release);
1385 Ok(release)
1386 }
1387
1388 pub(crate) fn release_module_route(
1389 &self,
1390 module_connection_id: ConnectionId,
1391 module_channel: u16,
1392 expected_epoch: u32,
1393 ) -> Result<RouteRelease, ForwardingError> {
1394 let mut inner = self.write_inner()?;
1395 let Some(endpoint) = inner
1396 .endpoint_by_connection
1397 .get(&module_connection_id)
1398 .copied()
1399 else {
1400 return Ok(RouteRelease::Absent);
1401 };
1402 let release = release_module_route_locked(
1403 &mut inner,
1404 ModuleRouteKey {
1405 endpoint,
1406 channel: module_channel,
1407 },
1408 expected_epoch,
1409 );
1410 self.record_route_release(&release);
1411 Ok(release)
1412 }
1413
1414 pub(crate) fn abort_pending_relay(
1415 &self,
1416 endpoint: ModuleEndpointId,
1417 corr: u64,
1418 outcome: RouteBindRelayOutcome,
1419 ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
1420 let mut inner = self.write_inner()?;
1421 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1422 inner
1427 .relays_released_before_enqueue
1428 .remove(&(endpoint, corr));
1429 return Ok(None);
1430 };
1431 release_reserved_route_locked(
1432 &mut inner,
1433 pending.reservation.client_key,
1434 pending.reservation.module_key,
1435 );
1436 let target = pending
1437 .relay_enqueued
1438 .then(|| abandoned_route_target(&inner, &pending.reservation));
1439 let _ = pending.sender.send(outcome);
1440 Ok(target.flatten())
1441 }
1442
1443 pub(crate) fn cancel_module_control_rpc(
1444 &self,
1445 endpoint: ModuleEndpointId,
1446 corr: u64,
1447 ) -> Result<(), ForwardingError> {
1448 self.write_inner()?
1449 .pending_control_rpcs
1450 .remove(&(endpoint, corr));
1451 Ok(())
1452 }
1453
1454 pub(crate) fn tombstone_health_probe_rpc(
1455 &self,
1456 endpoint: ModuleEndpointId,
1457 corr: u64,
1458 ) -> Result<bool, ForwardingError> {
1459 let key = (endpoint, corr);
1460 let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
1461 {
1462 let mut inner = self.write_inner()?;
1463 let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
1464 return Ok(false);
1465 };
1466 let Some(probe_started_at) = pending.health_probe_started_at else {
1467 inner.pending_control_rpcs.insert(key, pending);
1468 return Ok(false);
1469 };
1470 let module_id = inner
1471 .module_id_by_endpoint
1472 .get(&endpoint)
1473 .cloned()
1474 .unwrap_or_else(|| "unknown".to_string());
1475 inner.health_probe_tombstones.insert(
1476 key,
1477 HealthProbeTombstone {
1478 expected_op: pending.expected_op,
1479 module_id,
1480 probe_started_at,
1481 expires_at,
1482 },
1483 );
1484 }
1485 self.schedule_health_probe_tombstone_expiration(key, expires_at);
1486 Ok(true)
1487 }
1488
1489 fn schedule_health_probe_tombstone_expiration(
1490 &self,
1491 key: (ModuleEndpointId, u64),
1492 expires_at: Instant,
1493 ) {
1494 let inner = Arc::downgrade(&self.inner);
1495 tokio::spawn(async move {
1496 tokio::time::sleep_until(expires_at).await;
1497 let Some(inner) = inner.upgrade() else {
1498 return;
1499 };
1500 let Ok(mut inner) = inner.write() else {
1501 return;
1502 };
1503 let expired = inner
1504 .health_probe_tombstones
1505 .get(&key)
1506 .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
1507 if expired {
1508 inner.health_probe_tombstones.remove(&key);
1509 }
1510 });
1511 }
1512
1513 pub(crate) fn complete_pending_relay(
1514 &self,
1515 connection_id: ConnectionId,
1516 corr: u64,
1517 outcome: RouteBindRelayOutcome,
1518 ) -> Result<PendingRelayCompletion, ForwardingError> {
1519 let mut inner = self.write_inner()?;
1520 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1521 return Ok(PendingRelayCompletion {
1522 settled: false,
1523 abandoned: None,
1524 });
1525 };
1526 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1527 return Ok(PendingRelayCompletion {
1528 settled: false,
1529 abandoned: None,
1530 });
1531 };
1532
1533 if Instant::now() >= pending.deadline {
1534 release_reserved_route_locked(
1535 &mut inner,
1536 pending.reservation.client_key,
1537 pending.reservation.module_key,
1538 );
1539 let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
1540 .then(|| abandoned_route_target(&inner, &pending.reservation))
1541 .flatten();
1542 let _ = pending
1543 .sender
1544 .send(RouteBindRelayOutcome::Rejected(ErrorBody {
1545 code: "module_timeout".to_string(),
1546 message: "route.bind response arrived after its daemon deadline".to_string(),
1547 detail: None,
1548 }));
1549 return Ok(PendingRelayCompletion {
1550 settled: true,
1551 abandoned,
1552 });
1553 }
1554
1555 match outcome {
1556 RouteBindRelayOutcome::Accepted
1575 if pending.client_sink.is_closed()
1576 || inner
1577 .closing_connections
1578 .contains(&pending.reservation.client_key.connection_id) =>
1579 {
1580 let reason = if pending.client_sink.is_closed() {
1581 "client egress closed before route publication"
1582 } else {
1583 "client connection is closing before route publication"
1584 };
1585 release_reserved_route_locked(
1586 &mut inner,
1587 pending.reservation.client_key,
1588 pending.reservation.module_key,
1589 );
1590 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1594 let _ = pending
1595 .sender
1596 .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
1597 return Ok(PendingRelayCompletion {
1598 settled: true,
1599 abandoned,
1600 });
1601 }
1602 RouteBindRelayOutcome::Accepted
1625 if inner.superseded_endpoints.contains_key(&endpoint)
1626 || inner.draining_endpoints.contains_key(&endpoint) =>
1627 {
1628 release_reserved_route_locked(
1629 &mut inner,
1630 pending.reservation.client_key,
1631 pending.reservation.module_key,
1632 );
1633 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1636 let module_id = inner
1637 .module_id_by_endpoint
1638 .get(&endpoint)
1639 .cloned()
1640 .unwrap_or_else(|| "unknown".to_string());
1641 let _ = pending
1642 .sender
1643 .send(RouteBindRelayOutcome::Rejected(ErrorBody::new(
1644 "module_reloading",
1645 format!("module_id '{module_id}' is reloading"),
1646 )));
1647 return Ok(PendingRelayCompletion {
1648 settled: true,
1649 abandoned,
1650 });
1651 }
1652 RouteBindRelayOutcome::Accepted
1661 if pending
1662 .scope
1663 .as_ref()
1664 .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
1665 {
1666 let refusal = pending
1667 .scope
1668 .as_ref()
1669 .and_then(|scope| scope_refusal_locked(&inner, scope))
1670 .expect("guard matched a refusal under the same lock");
1671 release_reserved_route_locked(
1672 &mut inner,
1673 pending.reservation.client_key,
1674 pending.reservation.module_key,
1675 );
1676 let abandoned = abandoned_route_target(&inner, &pending.reservation);
1677 let _ = pending
1678 .sender
1679 .send(RouteBindRelayOutcome::Rejected(refusal));
1680 return Ok(PendingRelayCompletion {
1681 settled: true,
1682 abandoned,
1683 });
1684 }
1685 RouteBindRelayOutcome::Accepted => {
1686 let abandoned = commit_route_locked(&mut inner, pending)?;
1687 return Ok(PendingRelayCompletion {
1688 settled: true,
1689 abandoned,
1690 });
1691 }
1692 terminal => {
1693 release_reserved_route_locked(
1694 &mut inner,
1695 pending.reservation.client_key,
1696 pending.reservation.module_key,
1697 );
1698 let _ = pending.sender.send(terminal);
1699 }
1700 }
1701 Ok(PendingRelayCompletion {
1702 settled: true,
1703 abandoned: None,
1704 })
1705 }
1706
1707 pub(crate) fn pending_module_control_op(
1708 &self,
1709 connection_id: ConnectionId,
1710 corr: u64,
1711 ) -> Result<Option<String>, ForwardingError> {
1712 let inner = self.read_inner()?;
1713 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1714 return Ok(None);
1715 };
1716 let key = (endpoint, corr);
1717 Ok(inner
1718 .pending_control_rpcs
1719 .get(&key)
1720 .map(|pending| pending.expected_op.clone())
1721 .or_else(|| {
1722 inner
1723 .health_probe_tombstones
1724 .get(&key)
1725 .filter(|tombstone| tombstone.expires_at > Instant::now())
1726 .map(|tombstone| tombstone.expected_op.clone())
1727 }))
1728 }
1729
1730 pub(crate) fn complete_module_control_rpc(
1731 &self,
1732 connection_id: ConnectionId,
1733 corr: u64,
1734 actual_op: Option<&str>,
1735 outcome: ModuleControlRpcOutcome,
1736 ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
1737 let now = Instant::now();
1738 let mut inner = self.write_inner()?;
1739 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1740 return Ok(ModuleControlRpcCompletion::Unknown);
1741 };
1742 let key = (endpoint, corr);
1743 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
1744 if now >= pending.deadline {
1745 let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
1746 ModuleControlRpcCompletion::LateHealthAnswer {
1747 module_id: inner
1748 .module_id_by_endpoint
1749 .get(&endpoint)
1750 .cloned()
1751 .unwrap_or_else(|| "unknown".to_string()),
1752 latency: now.saturating_duration_since(probe_started_at),
1753 }
1754 });
1755 let _ = pending
1756 .sender
1757 .send(ModuleControlRpcOutcome::DeadlineElapsed);
1758 return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
1759 }
1760 let outcome = match actual_op {
1761 Some(actual) if actual != pending.expected_op => {
1762 ModuleControlRpcOutcome::UnexpectedOp {
1763 expected: pending.expected_op,
1764 actual: actual.to_string(),
1765 }
1766 }
1767 _ => outcome,
1768 };
1769 let _ = pending.sender.send(outcome);
1770 return Ok(ModuleControlRpcCompletion::Settled);
1771 }
1772
1773 let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
1774 return Ok(ModuleControlRpcCompletion::Unknown);
1775 };
1776 if tombstone.expires_at <= now {
1777 return Ok(ModuleControlRpcCompletion::Unknown);
1778 }
1779 Ok(ModuleControlRpcCompletion::LateHealthAnswer {
1780 module_id: tombstone.module_id,
1781 latency: now.saturating_duration_since(tombstone.probe_started_at),
1782 })
1783 }
1784
1785 #[cfg(test)]
1786 pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
1787 Ok(self.read_inner()?.health_probe_tombstones.len())
1788 }
1789
1790 #[cfg(test)]
1791 pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
1792 Ok(self.read_inner()?.closing_connections.len())
1793 }
1794
1795 #[cfg(test)]
1798 pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
1799 let inner = self.read_inner()?;
1800 Ok((inner.reserved_client.len(), inner.reserved_module.len()))
1801 }
1802
1803 pub(crate) fn module_endpoint_for_connection(
1804 &self,
1805 connection_id: ConnectionId,
1806 ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
1807 Ok(self
1808 .read_inner()?
1809 .endpoint_by_connection
1810 .get(&connection_id)
1811 .copied())
1812 }
1813
1814 pub(crate) fn module_id_for_connection(
1817 &self,
1818 connection_id: ConnectionId,
1819 ) -> Result<Option<String>, ForwardingError> {
1820 let inner = self.read_inner()?;
1821 Ok(inner
1822 .endpoint_by_connection
1823 .get(&connection_id)
1824 .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
1825 .cloned())
1826 }
1827
1828 pub(crate) fn module_route_epoch_was_allocated(
1835 &self,
1836 connection_id: ConnectionId,
1837 channel: u16,
1838 epoch: u32,
1839 ) -> Result<bool, ForwardingError> {
1840 let inner = self.read_inner()?;
1841 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1842 return Ok(false);
1843 };
1844 Ok(inner
1845 .module_slot_epochs
1846 .get(&ModuleRouteKey { endpoint, channel })
1847 .is_some_and(|last| epoch != 0 && epoch <= *last))
1848 }
1849
1850 pub(crate) fn has_live_module_connection(
1851 &self,
1852 module_id: &str,
1853 ) -> Result<bool, ForwardingError> {
1854 Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
1855 }
1856
1857 pub(crate) fn lookup_data_route(
1858 &self,
1859 connection_id: ConnectionId,
1860 channel: u16,
1861 epoch: u32,
1862 ) -> Result<DataRoute, ForwardingError> {
1863 let inner = self.read_inner()?;
1864 let state = if let Some(endpoint) =
1865 inner.endpoint_by_connection.get(&connection_id).copied()
1866 {
1867 let key = ModuleRouteKey { endpoint, channel };
1868 match inner.module_to_client.get(&key) {
1869 Some(route) if route.module_epoch == epoch => {
1870 DataRouteState::Bound(Arc::clone(route))
1871 }
1872 Some(_) => DataRouteState::EpochMismatch,
1873 None if inner.reserved_module.contains_key(&key)
1874 && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
1875 {
1876 DataRouteState::Reserved
1877 }
1878 None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
1879 None => DataRouteState::Absent,
1880 }
1881 } else {
1882 let key = ClientRouteKey {
1883 connection_id,
1884 channel,
1885 };
1886 match inner.client_to_module.get(&key) {
1887 Some(route) if route.client_epoch == epoch => {
1888 DataRouteState::Bound(Arc::clone(route))
1889 }
1890 Some(_) => DataRouteState::EpochMismatch,
1891 None if inner.reserved_client.contains_key(&key)
1892 && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
1893 {
1894 DataRouteState::Reserved
1895 }
1896 None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
1897 None => DataRouteState::Absent,
1898 }
1899 };
1900 Ok(
1901 if inner.endpoint_by_connection.contains_key(&connection_id) {
1902 DataRoute::Module(state)
1903 } else {
1904 DataRoute::Client(state)
1905 },
1906 )
1907 }
1908
1909 #[cfg(test)]
1910 pub(crate) fn inject_client_slot_epoch(
1911 &self,
1912 connection_id: ConnectionId,
1913 channel: u16,
1914 last_epoch: u32,
1915 ) {
1916 let mut inner = self.write_inner().expect("forwarding lock");
1917 inner.client_slot_epochs.insert(
1918 ClientRouteKey {
1919 connection_id,
1920 channel,
1921 },
1922 last_epoch,
1923 );
1924 inner.next_client_channel.insert(connection_id, channel);
1925 }
1926
1927 #[cfg(test)]
1928 pub(crate) fn inject_module_slot_epoch(
1929 &self,
1930 endpoint: ModuleEndpointId,
1931 channel: u16,
1932 last_epoch: u32,
1933 ) {
1934 let mut inner = self.write_inner().expect("forwarding lock");
1935 inner
1936 .module_slot_epochs
1937 .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
1938 inner.next_module_channel.insert(endpoint, channel);
1939 }
1940
1941 #[cfg(test)]
1942 pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
1943 self.write_inner()
1944 .expect("forwarding lock")
1945 .next_control_corr
1946 .insert(endpoint, next_corr);
1947 }
1948
1949 pub(crate) fn cache_status(
1950 &self,
1951 endpoint: ModuleEndpointId,
1952 module_channel: u16,
1953 module_epoch: u32,
1954 status: String,
1955 ) -> Result<bool, ForwardingError> {
1956 let mut inner = self.write_inner()?;
1957 if !inner.module_id_by_endpoint.contains_key(&endpoint) {
1958 return Err(ForwardingError::StaleModuleEndpoint);
1959 }
1960
1961 let module_key = ModuleRouteKey {
1962 endpoint,
1963 channel: module_channel,
1964 };
1965 let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
1966 (route.module_epoch == module_epoch).then_some((
1967 ClientRouteKey {
1968 connection_id: route.client_connection_id,
1969 channel: route.client_channel,
1970 },
1971 route.client_epoch,
1972 ))
1973 } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
1974 (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
1975 client_key,
1976 inner
1977 .client_slot_epochs
1978 .get(&client_key)
1979 .copied()
1980 .unwrap_or(0),
1981 ))
1982 } else {
1983 None
1984 };
1985
1986 if let Some(handle) = handle {
1987 inner.status.insert(handle, status);
1988 Ok(true)
1989 } else {
1990 debug!(
1991 module_channel,
1992 module_epoch,
1993 generation = endpoint.generation,
1994 connection_id = endpoint.connection_id.get(),
1995 "dropping stale status update for module route handle"
1996 );
1997 Ok(false)
1998 }
1999 }
2000
2001 pub(crate) fn route_poll_snapshot(
2002 &self,
2003 client_connection_id: ConnectionId,
2004 client_channel: u16,
2005 client_epoch: u32,
2006 ) -> Result<RoutePollSnapshot, ForwardingError> {
2007 let inner = self.read_inner()?;
2008 let client_key = ClientRouteKey {
2009 connection_id: client_connection_id,
2010 channel: client_channel,
2011 };
2012 let Some(route) = inner.client_to_module.get(&client_key) else {
2013 return Ok(RoutePollSnapshot::Absent);
2014 };
2015 if route.client_epoch != client_epoch
2016 || !inner
2017 .module_id_by_endpoint
2018 .contains_key(&route.module_endpoint)
2019 {
2020 return Ok(RoutePollSnapshot::Absent);
2021 }
2022 Ok(RoutePollSnapshot::Bound {
2023 module_id: route.module_id.clone(),
2024 status: inner.status.get(&(client_key, client_epoch)).cloned(),
2025 })
2026 }
2027
2028 pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2029 Ok(self.read_inner()?.client_to_module.len())
2030 }
2031
2032 pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2042 let inner = self.read_inner()?;
2043 let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2044 for key in inner.client_to_module.keys() {
2045 *per_connection.entry(key.connection_id).or_insert(0) += 1;
2046 }
2047 let max = per_connection.values().copied().max().unwrap_or(0);
2048 Ok((per_connection.len(), max))
2049 }
2050
2051 pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2052 let inner = self.read_inner()?;
2053 Ok(inner
2054 .client_to_module
2055 .keys()
2056 .any(|key| key.channel == route_channel))
2057 }
2058
2059 pub(crate) fn is_daemon_draining(&self) -> Result<bool, ForwardingError> {
2061 Ok(self.read_inner()?.daemon_draining)
2062 }
2063
2064 #[cfg(unix)]
2067 pub(crate) fn begin_daemon_drain(&self) -> Result<Vec<String>, ForwardingError> {
2068 let mut inner = self.write_inner()?;
2069 inner.daemon_draining = true;
2070 let modules = inner
2071 .modules_by_id
2072 .iter()
2073 .map(|(id, module)| (id.clone(), module.endpoint))
2074 .collect::<Vec<_>>();
2075 for (_, endpoint) in &modules {
2076 inner
2077 .draining_endpoints
2078 .insert(*endpoint, RouteCloseReason::Restart);
2079 }
2080 let off_slot_endpoints = inner
2085 .candidates_by_id
2086 .values()
2087 .map(|module| module.endpoint)
2088 .chain(inner.superseded_endpoints.keys().copied())
2089 .collect::<Vec<_>>();
2090 for endpoint in off_slot_endpoints {
2091 inner
2092 .draining_endpoints
2093 .insert(endpoint, RouteCloseReason::Restart);
2094 }
2095 Ok(modules.into_iter().map(|(id, _)| id).collect())
2096 }
2097
2098 pub(crate) fn begin_module_drain(
2105 &self,
2106 module_id: &str,
2107 reason: RouteCloseReason,
2108 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2109 let mut inner = self.write_inner()?;
2110 let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2111 return Ok(None);
2112 };
2113 Ok(Some(begin_drain_locked(
2114 &mut inner, module_id, module, reason,
2115 )))
2116 }
2117
2118 pub(crate) fn begin_endpoint_drain(
2125 &self,
2126 endpoint: ModuleEndpointId,
2127 reason: RouteCloseReason,
2128 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2129 let mut inner = self.write_inner()?;
2130 let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2131 return Ok(None);
2132 };
2133 let module_id = inner
2134 .module_id_by_endpoint
2135 .get(&endpoint)
2136 .cloned()
2137 .expect("an endpoint resolved to a module connection has a module id");
2138 Ok(Some(begin_drain_locked(
2139 &mut inner, &module_id, module, reason,
2140 )))
2141 }
2142}
2143
2144fn begin_drain_locked(
2148 inner: &mut ForwardingInner,
2149 module_id: &str,
2150 module: ModuleConnection,
2151 reason: RouteCloseReason,
2152) -> ModuleDrainTarget {
2153 {
2154 let endpoint = module.endpoint;
2155 inner.draining_endpoints.insert(endpoint, reason);
2156
2157 let flows = inner
2158 .client_to_module
2159 .values()
2160 .filter(|route| route.module_endpoint == endpoint)
2161 .map(|route| Arc::clone(&route.flow))
2162 .collect::<Vec<_>>();
2163 let excluded_subscriptions = flows
2164 .into_iter()
2165 .map(|flow| flow.begin_drain())
2166 .fold(0u32, u32::saturating_add);
2167
2168 let pending_keys = inner
2169 .pending_relays
2170 .keys()
2171 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2172 .copied()
2173 .collect::<Vec<_>>();
2174 let mut abandoned_bindings = Vec::new();
2175 for key in pending_keys {
2176 let Some(pending) = inner.pending_relays.remove(&key) else {
2177 continue;
2178 };
2179 release_reserved_route_locked(
2180 inner,
2181 pending.reservation.client_key,
2182 pending.reservation.module_key,
2183 );
2184 if pending.relay_enqueued {
2185 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2186 abandoned_bindings.push(target);
2187 }
2188 } else {
2189 inner.relays_released_before_enqueue.insert(key);
2193 }
2194 let _ = pending
2195 .sender
2196 .send(RouteBindRelayOutcome::Rejected(ErrorBody::new(
2197 "module_reloading",
2198 format!("module_id '{module_id}' is reloading"),
2199 )));
2200 }
2201
2202 let pending_control_keys = inner
2203 .pending_control_rpcs
2204 .keys()
2205 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2206 .copied()
2207 .collect::<Vec<_>>();
2208 for key in pending_control_keys {
2209 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2210 let _ = pending
2211 .sender
2212 .send(ModuleControlRpcOutcome::ModuleGone(format!(
2213 "module '{module_id}' began draining during module-control RPC"
2214 )));
2215 }
2216 }
2217
2218 ModuleDrainTarget {
2219 endpoint,
2220 sink: module.sink,
2221 negotiated_ver: module.negotiated_ver,
2222 abandoned_bindings,
2223 excluded_subscriptions,
2224 }
2225 }
2226}
2227
2228#[derive(Debug, Default, PartialEq, Eq)]
2231pub(crate) struct DrainHoldouts {
2232 pub(crate) requests: usize,
2235 pub(crate) routes: usize,
2237 pub(crate) total_routes: usize,
2239 pub(crate) top_connections: Vec<(u64, usize)>,
2242 pub(crate) held: Vec<(u16, u64)>,
2251}
2252
2253pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
2255
2256impl ForwardingTable {
2257 pub(crate) fn endpoint_drain_holdouts(
2259 &self,
2260 endpoint: ModuleEndpointId,
2261 ) -> Result<DrainHoldouts, ForwardingError> {
2262 let inner = self.read_inner()?;
2263 let mut holdouts = DrainHoldouts::default();
2264 let mut by_connection: HashMap<u64, usize> = HashMap::new();
2265 for (key, route) in &inner.client_to_module {
2266 if route.module_endpoint != endpoint {
2267 continue;
2268 }
2269 holdouts.total_routes += 1;
2270 let held = route.flow.drain_in_flight();
2271 if held == 0 {
2272 continue;
2273 }
2274 holdouts.requests += held;
2275 holdouts.routes += 1;
2276 *by_connection.entry(key.connection_id.get()).or_default() += held;
2277 holdouts.held.extend(
2278 route
2279 .flow
2280 .drain_held_corrs()
2281 .into_iter()
2282 .map(|corr| (route.module_channel, corr)),
2283 );
2284 }
2285 holdouts.held.sort_unstable();
2286 holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
2287 let mut connections = by_connection.into_iter().collect::<Vec<_>>();
2288 connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
2289 connections.truncate(3);
2290 holdouts.top_connections = connections;
2291 Ok(holdouts)
2292 }
2293
2294 pub(crate) fn endpoint_in_flight_count(
2295 &self,
2296 endpoint: ModuleEndpointId,
2297 ) -> Result<usize, ForwardingError> {
2298 let inner = self.read_inner()?;
2299 Ok(inner
2300 .client_to_module
2301 .values()
2302 .filter(|route| route.module_endpoint == endpoint)
2303 .map(|route| route.flow.drain_in_flight())
2304 .sum())
2305 }
2306
2307 pub(crate) fn endpoint_is_draining(
2308 &self,
2309 endpoint: ModuleEndpointId,
2310 ) -> Result<bool, ForwardingError> {
2311 Ok(self
2312 .read_inner()?
2313 .draining_endpoints
2314 .contains_key(&endpoint))
2315 }
2316
2317 pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
2318 let inner = self.read_inner()?;
2319 Ok(inner
2320 .modules_by_id
2321 .get(module_id)
2322 .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
2323 }
2324
2325 pub(crate) fn release_module_endpoint_routes(
2326 &self,
2327 endpoint: ModuleEndpointId,
2328 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2329 let mut inner = self.write_inner()?;
2330 let routes = inner
2331 .module_to_client
2332 .iter()
2333 .filter(|(module_key, _)| module_key.endpoint == endpoint)
2334 .map(|(module_key, route)| (*module_key, route.module_epoch))
2335 .collect::<Vec<_>>();
2336 let mut released = Vec::with_capacity(routes.len());
2337 for (module_key, epoch) in routes {
2338 if let RouteRelease::Removed(target) =
2339 release_module_route_locked(&mut inner, module_key, epoch)
2340 {
2341 released.push(target);
2342 }
2343 }
2344 Ok(released)
2345 }
2346
2347 pub(crate) fn endpoint_routes(
2353 &self,
2354 endpoint: ModuleEndpointId,
2355 ) -> Result<Vec<EndpointRoute>, ForwardingError> {
2356 let inner = self.read_inner()?;
2357 Ok(endpoint_routes_locked(&inner, endpoint))
2358 }
2359
2360 pub(crate) fn route_census(
2362 &self,
2363 module_id: Option<&str>,
2364 ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
2365 let inner = self.read_inner()?;
2366 let mut endpoints = inner
2367 .modules_by_id
2368 .iter()
2369 .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
2370 .map(|(id, module)| (id.clone(), module.endpoint))
2371 .collect::<Vec<_>>();
2372 endpoints.sort_by(|left, right| left.0.cmp(&right.0));
2373 Ok(endpoints
2374 .into_iter()
2375 .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
2376 .collect())
2377 }
2378
2379 pub(crate) fn live_roots(
2381 &self,
2382 module_id: &str,
2383 ) -> Result<ModuleControlResponseToModule, ForwardingError> {
2384 let inner = self.read_inner()?;
2385 let endpoint = inner
2386 .modules_by_id
2387 .get(module_id)
2388 .map(|module| module.endpoint);
2389 let mut roots = BTreeMap::new();
2390 let mut unknown_root_bindings = 0;
2391 let mut total_bindings = 0;
2392 if let Some(endpoint) = endpoint {
2393 for binding in inner
2394 .module_to_client
2395 .values()
2396 .filter(|binding| binding.module_endpoint == endpoint)
2397 {
2398 total_bindings += 1;
2399 if let Some(root) = &binding.project_root {
2400 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2401 entry.0 += 1;
2402 } else {
2403 unknown_root_bindings += 1;
2404 }
2405 }
2406 for pending in inner
2407 .pending_relays
2408 .values()
2409 .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
2410 {
2411 total_bindings += 1;
2412 if let Some(root) = &pending.reservation.project_root {
2413 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2414 entry.1 += 1;
2415 } else {
2416 unknown_root_bindings += 1;
2417 }
2418 }
2419 }
2420 Ok(ModuleControlResponseToModule::LiveRoots {
2421 roots: roots
2422 .into_iter()
2423 .map(|(project_root, (bound, pending))| LiveRoot {
2424 project_root,
2425 bound,
2426 pending,
2427 })
2428 .collect(),
2429 unknown_root_bindings,
2430 total_bindings,
2431 })
2432 }
2433
2434 pub(crate) fn connection_has_client_routes(
2440 &self,
2441 connection_id: ConnectionId,
2442 ) -> Result<bool, ForwardingError> {
2443 let inner = self.read_inner()?;
2444 Ok(connection_has_client_routes_locked(&inner, connection_id))
2445 }
2446
2447 pub(crate) fn cleanup_connection(
2448 &self,
2449 connection_id: ConnectionId,
2450 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2451 self.cleanup_connection_counted(connection_id)
2452 .map(|cleanup| cleanup.released)
2453 }
2454
2455 pub(crate) fn cleanup_connection_counted(
2460 &self,
2461 connection_id: ConnectionId,
2462 ) -> Result<ConnectionCleanup, ForwardingError> {
2463 let mut inner = self.write_inner()?;
2464 inner.closing_connections.insert(connection_id);
2465 let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
2466 remove_module_connection_locked(&mut inner, endpoint)
2467 } else {
2468 ConnectionCleanup {
2469 released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
2470 abandoned_relays: 0,
2471 }
2472 };
2473 inner.closing_connections.remove(&connection_id);
2482 Ok(cleanup)
2483 }
2484
2485 fn cleanup_client_connection_locked(
2486 inner: &mut ForwardingInner,
2487 connection_id: ConnectionId,
2488 ) -> Vec<GoodbyeTarget> {
2489 let routes = inner
2490 .client_to_module
2491 .iter()
2492 .filter(|(key, _)| key.connection_id == connection_id)
2493 .map(|(key, route)| (*key, route.client_epoch))
2494 .collect::<Vec<_>>();
2495 let mut released = Vec::with_capacity(routes.len());
2496 for (client_key, epoch) in routes {
2497 if let RouteRelease::Removed(target) =
2498 release_client_route_locked(inner, client_key, epoch)
2499 {
2500 released.push(target);
2501 }
2502 }
2503
2504 let pending_keys = inner
2505 .pending_relays
2506 .iter()
2507 .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
2508 .map(|(key, _)| *key)
2509 .collect::<Vec<_>>();
2510 for key in pending_keys {
2511 let Some(pending) = inner.pending_relays.remove(&key) else {
2512 continue;
2513 };
2514 release_reserved_route_locked(
2515 inner,
2516 pending.reservation.client_key,
2517 pending.reservation.module_key,
2518 );
2519 if pending.relay_enqueued {
2520 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2521 released.push(target);
2522 }
2523 } else {
2524 inner.relays_released_before_enqueue.insert(key);
2528 }
2529 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
2530 "client connection closed during route.bind relay".to_string(),
2531 ));
2532 }
2533
2534 let orphaned = inner
2535 .reserved_client
2536 .iter()
2537 .filter(|(key, _)| key.connection_id == connection_id)
2538 .map(|(client, module)| (*client, *module))
2539 .collect::<Vec<_>>();
2540 for (client_key, module_key) in orphaned {
2541 release_reserved_route_locked(inner, client_key, module_key);
2542 }
2543 inner.next_client_channel.remove(&connection_id);
2544 inner
2545 .client_slot_epochs
2546 .retain(|key, _| key.connection_id != connection_id);
2547 inner
2548 .last_published_epoch
2549 .retain(|key, _| key.connection_id != connection_id);
2550 inner
2551 .status
2552 .retain(|(key, _), _| key.connection_id != connection_id);
2553
2554 released
2555 }
2556
2557 pub(crate) fn escalate_client_delivery_failure(
2566 &self,
2567 connection_id: ConnectionId,
2568 channel: u16,
2569 expected_epoch: u32,
2570 reason: CloseReason,
2571 undelivered: UndeliveredFrame<'_>,
2572 ) -> Result<bool, ForwardingError> {
2573 let principals = {
2574 let mut inner = self.write_inner()?;
2575 let key = ClientRouteKey {
2576 connection_id,
2577 channel,
2578 };
2579 if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
2580 None
2581 } else {
2582 inner.closing_connections.insert(connection_id);
2583 Some(connection_principals_locked(&inner, connection_id))
2584 }
2585 };
2586 let Some(principals) = principals else {
2587 return Ok(false);
2588 };
2589 let backlog = undelivered.sink.backlog();
2590 let close_reason = reason.to_string();
2591 if self.request_connection_close(connection_id, reason) {
2592 warn!(
2593 connection_id = connection_id.get(),
2594 principals = %principals,
2595 module_id = undelivered.module_id.unwrap_or("unknown"),
2596 client_channel = channel,
2597 queued_bytes = backlog.queued_bytes,
2598 queued_frames = backlog.queued_frames,
2599 oldest_queued_ms = backlog
2600 .oldest_age
2601 .map(|age| age.as_millis() as u64)
2602 .unwrap_or(0),
2603 close_reason = %close_reason,
2604 "closing client connection: its egress queue could not take a frame"
2605 );
2606 }
2607 Ok(true)
2608 }
2609
2610 pub(crate) fn publish_scope_changes(
2624 &self,
2625 changes: &[ScopeTagChange],
2626 ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
2627 let mut inner = self.write_inner()?;
2628 let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
2629 for change in changes {
2630 let key = (change.owner.clone(), change.scope_ref.clone());
2631 match change.after {
2632 Some(tag) => {
2633 inner.scope_tags.insert(key, tag);
2634 }
2635 None => {
2636 inner.scope_tags.remove(&key);
2637 }
2638 }
2639 by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
2640 }
2641 let mut selected = Vec::new();
2642 for (client_key, route) in &inner.client_to_module {
2643 let Some(scope) = &route.scope else {
2644 continue;
2645 };
2646 let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
2647 else {
2648 continue;
2649 };
2650 if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
2651 continue;
2652 }
2653 let reason = match &change.drain {
2654 ScopeDrain::Nothing => continue,
2655 ScopeDrain::All(reason) => *reason,
2656 ScopeDrain::Carriers(narrowed) => {
2657 let owner = Principal::Reserved {
2658 module_id: scope.owner.clone(),
2659 };
2660 let hit = route.principal != owner
2661 && narrowed.iter().any(|(principal, allowed)| {
2662 *principal == route.principal
2663 && allowed
2664 .as_ref()
2665 .is_none_or(|targets| !targets.contains(&route.module_id))
2666 });
2667 if !hit {
2668 continue;
2669 }
2670 RouteCloseReason::ScopeCarrierRemoved
2671 }
2672 };
2673 selected.push((*client_key, route.client_epoch, reason));
2674 }
2675 let mut drained = Vec::new();
2676 for (client_key, client_epoch, reason) in selected {
2677 let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
2678 continue;
2679 };
2680 let release = release_client_route_locked(&mut inner, client_key, client_epoch);
2681 self.record_route_release(&release);
2682 if let RouteRelease::Removed(module) = release {
2683 drained.push(ScopeDrainedRoute {
2684 reason,
2685 module_id: route.module_id.clone(),
2686 client: GoodbyeTarget {
2687 connection_id: route.client_connection_id,
2688 sink: route.client_sink.clone(),
2689 negotiated_ver: route.client_negotiated_ver,
2690 channel: route.client_channel,
2691 epoch: route.client_epoch,
2692 kind: GoodbyeTargetKind::Client,
2693 module_id: Some(route.module_id.clone()),
2694 },
2695 module,
2696 });
2697 }
2698 }
2699 Ok(drained)
2700 }
2701
2702 #[cfg(test)]
2704 pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
2705 self.read_inner()
2706 .ok()?
2707 .scope_tags
2708 .get(&(owner.to_string(), scope_ref.to_string()))
2709 .copied()
2710 }
2711
2712 fn record_route_release(&self, release: &RouteRelease) {
2713 match release {
2714 RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
2715 RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
2716 RouteRelease::Absent => {}
2717 }
2718 }
2719
2720 fn read_inner(&self) -> Result<RwLockReadGuard<'_, ForwardingInner>, ForwardingError> {
2721 self.inner.read().map_err(|_| ForwardingError::Poisoned)
2722 }
2723
2724 fn write_inner(&self) -> Result<RwLockWriteGuard<'_, ForwardingInner>, ForwardingError> {
2725 self.inner.write().map_err(|_| ForwardingError::Poisoned)
2726 }
2727
2728 fn lock_close_registry(
2729 &self,
2730 ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
2731 self.close_registry
2732 .lock()
2733 .unwrap_or_else(|poisoned| poisoned.into_inner())
2734 }
2735}
2736
2737impl ForwardingInner {
2738 fn allocate_route_slots(
2739 &mut self,
2740 connection_id: ConnectionId,
2741 endpoint: ModuleEndpointId,
2742 ) -> Result<(u16, u32, u16, u32), ForwardingError> {
2743 let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
2744 let mut client_channel = client_start;
2745 let client_channel = loop {
2746 let key = ClientRouteKey {
2747 connection_id,
2748 channel: client_channel,
2749 };
2750 let eligible = !self.client_to_module.contains_key(&key)
2751 && !self.reserved_client.contains_key(&key)
2752 && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2753 if eligible {
2754 break client_channel;
2755 }
2756 client_channel = next_channel(client_channel);
2757 if client_channel == client_start {
2758 return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
2759 }
2760 };
2761
2762 let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
2763 let mut module_channel = module_start;
2764 let module_channel = loop {
2765 let key = ModuleRouteKey {
2766 endpoint,
2767 channel: module_channel,
2768 };
2769 let eligible = !self.module_to_client.contains_key(&key)
2770 && !self.reserved_module.contains_key(&key)
2771 && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2772 if eligible {
2773 break module_channel;
2774 }
2775 module_channel = next_channel(module_channel);
2776 if module_channel == module_start {
2777 return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
2778 }
2779 };
2780
2781 let client_key = ClientRouteKey {
2782 connection_id,
2783 channel: client_channel,
2784 };
2785 let module_key = ModuleRouteKey {
2786 endpoint,
2787 channel: module_channel,
2788 };
2789 let client_epoch = self
2790 .client_slot_epochs
2791 .get(&client_key)
2792 .copied()
2793 .unwrap_or(0)
2794 + 1;
2795 let module_epoch = self
2796 .module_slot_epochs
2797 .get(&module_key)
2798 .copied()
2799 .unwrap_or(0)
2800 + 1;
2801 self.client_slot_epochs.insert(client_key, client_epoch);
2802 self.module_slot_epochs.insert(module_key, module_epoch);
2803 self.next_client_channel
2804 .insert(connection_id, next_channel(client_channel));
2805 self.next_module_channel
2806 .insert(endpoint, next_channel(module_channel));
2807 Ok((client_channel, client_epoch, module_channel, module_epoch))
2808 }
2809
2810 fn allocate_control_corr(
2811 &mut self,
2812 endpoint: ModuleEndpointId,
2813 ) -> Result<u64, ForwardingError> {
2814 let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
2815 if candidate == 0 {
2816 self.closing_connections.insert(endpoint.connection_id);
2817 return Err(ForwardingError::RelayCorrelationExhausted);
2818 }
2819 self.next_control_corr.insert(
2820 endpoint,
2821 if candidate == u64::MAX {
2822 0
2823 } else {
2824 candidate + 1
2825 },
2826 );
2827 Ok(candidate)
2828 }
2829}
2830
2831fn connection_has_client_routes_locked(
2832 inner: &ForwardingInner,
2833 connection_id: ConnectionId,
2834) -> bool {
2835 inner
2836 .client_to_module
2837 .keys()
2838 .any(|key| key.connection_id == connection_id)
2839 || inner
2840 .reserved_client
2841 .keys()
2842 .any(|key| key.connection_id == connection_id)
2843}
2844
2845fn check_module_connection_role_locked(
2846 inner: &ForwardingInner,
2847 connection_id: ConnectionId,
2848) -> Result<(), ForwardingError> {
2849 if inner.endpoint_by_connection.contains_key(&connection_id)
2850 || connection_has_client_routes_locked(inner, connection_id)
2851 {
2852 return Err(ForwardingError::ConnectionRoleConflict { connection_id });
2853 }
2854 Ok(())
2855}
2856
2857fn next_channel(channel: u16) -> u16 {
2858 let next = channel.wrapping_add(1);
2859 if next == 0 {
2860 1
2861 } else {
2862 next
2863 }
2864}
2865
2866fn endpoint_routes_locked(
2867 inner: &ForwardingInner,
2868 endpoint: ModuleEndpointId,
2869) -> Vec<EndpointRoute> {
2870 let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
2871 let draining = drain_reason.is_some();
2872 let mut routes = inner
2873 .module_to_client
2874 .iter()
2875 .filter(|(module_key, _)| module_key.endpoint == endpoint)
2876 .map(|(_, route)| EndpointRoute {
2877 goodbye_target: GoodbyeTarget {
2878 connection_id: route.client_connection_id,
2879 sink: route.client_sink.clone(),
2880 negotiated_ver: route.client_negotiated_ver,
2881 channel: route.client_channel,
2882 epoch: route.client_epoch,
2883 kind: GoodbyeTargetKind::Client,
2884 module_id: Some(route.module_id.clone()),
2885 },
2886 principal: route.principal.clone(),
2887 bound_at: route.bound_at,
2888 draining,
2889 drain_reason,
2890 })
2891 .collect::<Vec<_>>();
2892 routes.sort_by_key(|route| {
2893 (
2894 route.goodbye_target.connection_id.get(),
2895 route.goodbye_target.channel,
2896 route.goodbye_target.epoch,
2897 )
2898 });
2899 routes
2900}
2901
2902fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
2906 let key = (scope.owner.clone(), scope.scope_ref.clone());
2907 match inner.scope_tags.get(&key) {
2908 Some(current) if *current == scope.tag => None,
2909 Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
2910 subc_protocol::error_codes::SCOPE_CHANGED,
2911 format!(
2912 "scope '{}' of {} changed while the route was being bound; re-open it",
2913 scope.scope_ref, scope.owner
2914 ),
2915 )),
2916 _ => Some(ErrorBody::new(
2917 subc_protocol::error_codes::SCOPE_ENDED,
2918 format!(
2919 "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
2920 scope.scope_ref, scope.owner, scope.tag.scope_epoch
2921 ),
2922 )),
2923 }
2924}
2925
2926fn release_reserved_route_locked(
2927 inner: &mut ForwardingInner,
2928 client_key: ClientRouteKey,
2929 module_key: ModuleRouteKey,
2930) {
2931 if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
2932 inner.reserved_client.remove(&client_key);
2933 }
2934 if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
2935 inner.reserved_module.remove(&module_key);
2936 }
2937 inner.status.retain(|(key, _), _| *key != client_key);
2938}
2939
2940fn release_client_route_locked(
2941 inner: &mut ForwardingInner,
2942 client_key: ClientRouteKey,
2943 expected_epoch: u32,
2944) -> RouteRelease {
2945 let Some(route) = inner.client_to_module.get(&client_key) else {
2946 return RouteRelease::Absent;
2947 };
2948 if route.client_epoch != expected_epoch {
2949 return RouteRelease::Stale;
2950 }
2951 let route = inner
2952 .client_to_module
2953 .remove(&client_key)
2954 .expect("route checked under the same forwarding lock");
2955 route.flow.close();
2956 inner.module_to_client.remove(&ModuleRouteKey {
2957 endpoint: route.module_endpoint,
2958 channel: route.module_channel,
2959 });
2960 inner.status.remove(&(client_key, expected_epoch));
2961 RouteRelease::Removed(GoodbyeTarget {
2962 connection_id: route.module_endpoint.connection_id,
2963 sink: route.module_sink.clone(),
2964 negotiated_ver: route.module_negotiated_ver,
2965 channel: route.module_channel,
2966 epoch: route.module_epoch,
2967 kind: GoodbyeTargetKind::Module,
2968 module_id: Some(route.module_id.clone()),
2969 })
2970}
2971
2972fn release_module_route_locked(
2973 inner: &mut ForwardingInner,
2974 module_key: ModuleRouteKey,
2975 expected_epoch: u32,
2976) -> RouteRelease {
2977 let Some(route) = inner.module_to_client.get(&module_key) else {
2978 return RouteRelease::Absent;
2979 };
2980 if route.module_epoch != expected_epoch {
2981 return RouteRelease::Stale;
2982 }
2983 let route = inner
2984 .module_to_client
2985 .remove(&module_key)
2986 .expect("route checked under the same forwarding lock");
2987 route.flow.close();
2988 let client_key = ClientRouteKey {
2989 connection_id: route.client_connection_id,
2990 channel: route.client_channel,
2991 };
2992 inner.client_to_module.remove(&client_key);
2993 inner.status.remove(&(client_key, route.client_epoch));
2994 RouteRelease::Removed(GoodbyeTarget {
2995 connection_id: route.client_connection_id,
2996 sink: route.client_sink.clone(),
2997 negotiated_ver: route.client_negotiated_ver,
2998 channel: route.client_channel,
2999 epoch: route.client_epoch,
3000 kind: GoodbyeTargetKind::Client,
3001 module_id: Some(route.module_id.clone()),
3002 })
3003}
3004
3005fn commit_route_locked(
3006 inner: &mut ForwardingInner,
3007 pending: PendingRouteBindRelayEntry,
3008) -> Result<Option<GoodbyeTarget>, ForwardingError> {
3009 let reservation = pending.reservation;
3010 if inner
3011 .closing_connections
3012 .contains(&reservation.client_key.connection_id)
3013 {
3014 return Err(ForwardingError::ConnectionClosing {
3015 connection_id: reservation.client_key.connection_id,
3016 });
3017 }
3018 let module_id = inner
3019 .module_id_by_endpoint
3020 .get(&reservation.module_key.endpoint)
3021 .cloned()
3022 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3023 if inner
3024 .draining_endpoints
3025 .contains_key(&reservation.module_key.endpoint)
3026 {
3027 return Err(ForwardingError::ModuleReloading { module_id });
3028 }
3029 if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
3030 || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
3031 {
3032 return Err(ForwardingError::UnknownReservation {
3033 client_channel: reservation.client_key.channel,
3034 module_channel: reservation.module_key.channel,
3035 });
3036 }
3037 let module = inner
3038 .modules_by_id
3039 .get(&module_id)
3040 .filter(|module| module.endpoint == reservation.module_key.endpoint)
3041 .cloned()
3042 .ok_or(ForwardingError::StaleModuleEndpoint)?;
3043 let binding = Arc::new(RouteBinding {
3044 client_connection_id: reservation.client_key.connection_id,
3045 client_sink: pending.client_sink,
3046 client_negotiated_ver: pending.client_negotiated_ver,
3047 client_channel: reservation.client_key.channel,
3048 client_epoch: reservation.client_epoch,
3049 module_id,
3050 module_endpoint: reservation.module_key.endpoint,
3051 module_sink: module.sink,
3052 module_negotiated_ver: module.negotiated_ver,
3053 module_channel: reservation.module_key.channel,
3054 module_epoch: reservation.module_epoch,
3055 principal: pending.principal,
3056 project_root: reservation.project_root.clone(),
3057 bound_at: Instant::now(),
3058 flow: Arc::new(ChannelFlow::new(window_for(&module.concurrency))),
3059 scope: pending.scope,
3060 });
3061 inner
3062 .client_to_module
3063 .insert(reservation.client_key, Arc::clone(&binding));
3064 inner
3065 .module_to_client
3066 .insert(reservation.module_key, binding);
3067 let previous_published = inner
3068 .last_published_epoch
3069 .insert(reservation.client_key, reservation.client_epoch);
3070
3071 let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
3076 if client_writer_closed {
3077 let abandoned = abandoned_route_target(inner, &reservation);
3081 if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
3082 route.flow.close();
3083 }
3084 inner.module_to_client.remove(&reservation.module_key);
3085 inner
3086 .status
3087 .remove(&(reservation.client_key, reservation.client_epoch));
3088 match previous_published {
3089 Some(epoch) => {
3090 inner
3091 .last_published_epoch
3092 .insert(reservation.client_key, epoch);
3093 }
3094 None => {
3095 inner.last_published_epoch.remove(&reservation.client_key);
3096 }
3097 }
3098 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3099 "client egress closed during route publication".to_string(),
3100 ));
3101 return Ok(abandoned);
3102 }
3103
3104 let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
3105 Ok(None)
3106}
3107
3108fn module_connection_for_endpoint_locked(
3116 inner: &ForwardingInner,
3117 endpoint: ModuleEndpointId,
3118) -> Option<&ModuleConnection> {
3119 let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
3120 inner
3121 .modules_by_id
3122 .get(module_id)
3123 .filter(|module| module.endpoint == endpoint)
3124 .or_else(|| {
3125 inner
3126 .candidates_by_id
3127 .get(module_id)
3128 .filter(|module| module.endpoint == endpoint)
3129 })
3130 .or_else(|| inner.superseded_endpoints.get(&endpoint))
3131}
3132
3133fn abandoned_route_target(
3134 inner: &ForwardingInner,
3135 reservation: &RouteReservation,
3136) -> Option<GoodbyeTarget> {
3137 let module_id = inner
3138 .module_id_by_endpoint
3139 .get(&reservation.module_key.endpoint)?;
3140 let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
3141 (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
3142 connection_id: module.endpoint.connection_id,
3143 sink: module.sink.clone(),
3144 negotiated_ver: module.negotiated_ver,
3145 channel: reservation.module_key.channel,
3146 epoch: reservation.module_epoch,
3147 kind: GoodbyeTargetKind::Module,
3148 module_id: Some(module_id.clone()),
3149 })
3150}
3151
3152fn enqueue_hello_ack_locked(
3157 sink: &FrameSink,
3158 connection_id: ConnectionId,
3159 hello_ack: Option<Frame>,
3160) -> Result<(), ForwardingError> {
3161 let Some(hello_ack) = hello_ack else {
3162 return Ok(());
3163 };
3164 sink.try_send(hello_ack)
3165 .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
3166}
3167
3168fn remove_module_connection_locked(
3169 inner: &mut ForwardingInner,
3170 endpoint: ModuleEndpointId,
3171) -> ConnectionCleanup {
3172 inner.draining_endpoints.remove(&endpoint);
3173 let module_id = inner.module_id_by_endpoint.remove(&endpoint);
3174 if let Some(module_id) = module_id.as_ref() {
3175 if inner
3176 .modules_by_id
3177 .get(module_id)
3178 .is_some_and(|module| module.endpoint == endpoint)
3179 {
3180 inner.modules_by_id.remove(module_id);
3181 }
3182 if inner
3183 .candidates_by_id
3184 .get(module_id)
3185 .is_some_and(|module| module.endpoint == endpoint)
3186 {
3187 inner.candidates_by_id.remove(module_id);
3188 }
3189 }
3190 inner.superseded_endpoints.remove(&endpoint);
3191 inner.endpoint_by_connection.remove(&endpoint.connection_id);
3192 inner.next_module_channel.remove(&endpoint);
3193 inner.next_control_corr.remove(&endpoint);
3194 inner
3195 .health_probe_tombstones
3196 .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
3197 inner
3198 .module_slot_epochs
3199 .retain(|key, _| key.endpoint != endpoint);
3200 let reserved_module_keys: Vec<ModuleRouteKey> = inner
3201 .reserved_module
3202 .keys()
3203 .filter(|module_key| module_key.endpoint == endpoint)
3204 .copied()
3205 .collect();
3206 for module_key in reserved_module_keys {
3207 if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
3208 release_reserved_route_locked(inner, client_key, module_key);
3209 }
3210 }
3211
3212 let pending_keys: Vec<_> = inner
3213 .pending_relays
3214 .keys()
3215 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3216 .copied()
3217 .collect();
3218 let pending: Vec<_> = pending_keys
3219 .into_iter()
3220 .filter_map(|key| {
3221 inner
3222 .pending_relays
3223 .remove(&key)
3224 .map(|pending| (key, pending))
3225 })
3226 .collect();
3227 let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
3228 for (key, pending) in pending {
3229 if !pending.relay_enqueued {
3233 inner.relays_released_before_enqueue.insert(key);
3234 }
3235 let module_label = module_id.as_deref().unwrap_or("unknown");
3236 let _ = pending
3237 .sender
3238 .send(RouteBindRelayOutcome::ModuleGone(format!(
3239 "module '{module_label}' connection closed during route.bind relay"
3240 )));
3241 }
3242
3243 let pending_control_keys: Vec<_> = inner
3244 .pending_control_rpcs
3245 .keys()
3246 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3247 .copied()
3248 .collect();
3249 let pending_control: Vec<_> = pending_control_keys
3250 .into_iter()
3251 .filter_map(|key| inner.pending_control_rpcs.remove(&key))
3252 .collect();
3253 for pending in pending_control {
3254 let module_label = module_id.as_deref().unwrap_or("unknown");
3255 let _ = pending
3256 .sender
3257 .send(ModuleControlRpcOutcome::ModuleGone(format!(
3258 "module '{module_label}' connection closed during module-control RPC"
3259 )));
3260 }
3261
3262 let module_routes = inner
3263 .module_to_client
3264 .iter()
3265 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3266 .map(|(module_key, route)| (*module_key, route.module_epoch))
3267 .collect::<Vec<_>>();
3268 let mut released = Vec::with_capacity(module_routes.len());
3269 for (module_key, epoch) in module_routes {
3270 if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
3271 {
3272 released.push(target);
3273 }
3274 }
3275 ConnectionCleanup {
3276 released,
3277 abandoned_relays,
3278 }
3279}
3280
3281#[derive(Debug, Clone, Copy)]
3282struct RequestCredit {
3283 subscription: bool,
3284 excluded_from_drain: bool,
3285}
3286
3287#[derive(Debug, Default)]
3288struct CreditLedger {
3289 by_corr: HashMap<u64, Vec<RequestCredit>>,
3290}
3291
3292impl CreditLedger {
3293 fn acquire(&mut self, corr: u64, subscription: bool) {
3294 self.by_corr.entry(corr).or_default().push(RequestCredit {
3295 subscription,
3296 excluded_from_drain: false,
3297 });
3298 }
3299
3300 fn release(&mut self, corr: u64) -> bool {
3301 let Some(credits) = self.by_corr.get_mut(&corr) else {
3302 return false;
3303 };
3304 let released = credits.pop().is_some();
3305 if credits.is_empty() {
3306 self.by_corr.remove(&corr);
3307 }
3308 released
3309 }
3310
3311 fn capture_subscription_exclusions(&mut self) -> u32 {
3312 let mut excluded = 0u32;
3313 for credit in self.by_corr.values_mut().flatten() {
3314 if credit.subscription && !credit.excluded_from_drain {
3315 credit.excluded_from_drain = true;
3316 excluded = excluded.saturating_add(1);
3317 }
3318 }
3319 excluded
3320 }
3321
3322 #[cfg(test)]
3323 fn in_flight(&self) -> usize {
3324 self.by_corr.values().map(Vec::len).sum()
3325 }
3326
3327 fn drain_in_flight(&self) -> usize {
3328 self.by_corr
3329 .values()
3330 .flatten()
3331 .filter(|credit| !credit.excluded_from_drain)
3332 .count()
3333 }
3334
3335 fn drain_held_corrs(&self) -> Vec<u64> {
3338 let mut corrs = self
3339 .by_corr
3340 .iter()
3341 .flat_map(|(corr, credits)| {
3342 credits
3343 .iter()
3344 .filter(|credit| !credit.excluded_from_drain)
3345 .map(move |_| *corr)
3346 })
3347 .collect::<Vec<_>>();
3348 corrs.sort_unstable();
3349 corrs
3350 }
3351}
3352
3353#[derive(Debug, Default)]
3354struct ChannelFlowState {
3355 closed: bool,
3356 credits: CreditLedger,
3357}
3358
3359#[derive(Debug)]
3361pub(crate) struct ChannelFlow {
3362 sem: Semaphore,
3363 window: usize,
3364 state: Mutex<ChannelFlowState>,
3365}
3366
3367impl ChannelFlow {
3368 pub(crate) fn new(window: usize) -> Self {
3369 debug_assert!(window > 0, "flow-control window must be non-zero");
3370 Self {
3371 sem: Semaphore::new(window),
3372 window,
3373 state: Mutex::new(ChannelFlowState::default()),
3374 }
3375 }
3376
3377 #[cfg(test)]
3378 pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
3379 self.acquire_tagged(0, false).await
3380 }
3381
3382 pub(crate) async fn acquire_tagged(
3383 &self,
3384 corr: u64,
3385 subscription: bool,
3386 ) -> Result<(), ChannelFlowClosed> {
3387 let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
3388 let mut state = self
3389 .state
3390 .lock()
3391 .unwrap_or_else(|poisoned| poisoned.into_inner());
3392 if state.closed {
3393 return Err(ChannelFlowClosed);
3394 }
3395 state.credits.acquire(corr, subscription);
3396 permit.forget();
3397 Ok(())
3398 }
3399
3400 #[cfg(test)]
3401 pub(crate) fn release(&self) {
3402 self.release_corr(0);
3403 }
3404
3405 pub(crate) fn release_corr(&self, corr: u64) {
3406 let released = self
3407 .state
3408 .lock()
3409 .unwrap_or_else(|poisoned| poisoned.into_inner())
3410 .credits
3411 .release(corr);
3412 if !released {
3413 warn!(
3417 window = self.window,
3418 available = self.sem.available_permits(),
3419 "flow-control over-release ignored"
3420 );
3421 return;
3422 }
3423 if !self.sem.is_closed() {
3424 self.sem.add_permits(1);
3425 }
3426 }
3427
3428 #[cfg(test)]
3429 pub(crate) fn in_flight(&self) -> usize {
3430 self.state
3431 .lock()
3432 .unwrap_or_else(|poisoned| poisoned.into_inner())
3433 .credits
3434 .in_flight()
3435 }
3436
3437 pub(crate) fn drain_in_flight(&self) -> usize {
3438 self.state
3439 .lock()
3440 .unwrap_or_else(|poisoned| poisoned.into_inner())
3441 .credits
3442 .drain_in_flight()
3443 }
3444
3445 pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
3446 self.state
3447 .lock()
3448 .unwrap_or_else(|poisoned| poisoned.into_inner())
3449 .credits
3450 .drain_held_corrs()
3451 }
3452
3453 #[cfg(test)]
3454 pub(crate) fn available_permits(&self) -> usize {
3455 self.sem.available_permits()
3456 }
3457
3458 pub(crate) fn begin_drain(&self) -> u32 {
3459 let mut state = self
3460 .state
3461 .lock()
3462 .unwrap_or_else(|poisoned| poisoned.into_inner());
3463 state.closed = true;
3464 self.sem.close();
3465 state.credits.capture_subscription_exclusions()
3466 }
3467
3468 pub(crate) fn close(&self) {
3469 self.state
3470 .lock()
3471 .unwrap_or_else(|poisoned| poisoned.into_inner())
3472 .closed = true;
3473 self.sem.close();
3474 }
3475}
3476
3477#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3478pub(crate) struct ChannelFlowClosed;
3479
3480impl fmt::Display for ChannelFlowClosed {
3481 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3482 write!(f, "flow-control window closed")
3483 }
3484}
3485
3486impl Error for ChannelFlowClosed {}
3487
3488fn window_for(concurrency: &Concurrency) -> usize {
3489 match concurrency {
3490 Concurrency::Serial => 1,
3491 Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
3492 Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
3493 }
3494}
3495
3496#[derive(Debug, Clone, PartialEq, Eq)]
3497pub enum ForwardingError {
3498 ConnectionRoleConflict {
3501 connection_id: ConnectionId,
3502 },
3503 NoModuleConnection,
3504 ModuleReloading {
3505 module_id: String,
3506 },
3507 StaleModuleEndpoint,
3508 UnknownReservation {
3509 client_channel: u16,
3510 module_channel: u16,
3511 },
3512 ClientRouteChannelExhausted {
3513 connection_id: ConnectionId,
3514 },
3515 ModuleRouteChannelExhausted {
3516 endpoint: ModuleEndpointId,
3517 },
3518 RelayCorrelationExhausted,
3519 ConnectionClosing {
3520 connection_id: ConnectionId,
3521 },
3522 ClientEgressClosed {
3523 connection_id: ConnectionId,
3524 },
3525 RouteOpenBuild(String),
3526 CandidateSlotOccupied {
3528 module_id: String,
3529 },
3530 ModuleEgressUnavailable {
3533 connection_id: ConnectionId,
3534 },
3535 Poisoned,
3536}
3537
3538impl fmt::Display for ForwardingError {
3539 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3540 match self {
3541 Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
3542 Self::NoModuleConnection => write!(f, "no module connection is registered"),
3543 Self::ModuleReloading { module_id } => {
3544 write!(f, "module_id '{module_id}' is reloading")
3545 }
3546 Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
3547 Self::UnknownReservation {
3548 client_channel,
3549 module_channel,
3550 } => write!(
3551 f,
3552 "route reservation client channel {client_channel} / module channel {module_channel} was not found"
3553 ),
3554 Self::ClientRouteChannelExhausted { connection_id } => write!(
3555 f,
3556 "no client route channels are available for connection {}",
3557 connection_id.get()
3558 ),
3559 Self::ModuleRouteChannelExhausted { endpoint } => write!(
3560 f,
3561 "no module route channels are available for endpoint generation {} on connection {}",
3562 endpoint.generation,
3563 endpoint.connection_id.get()
3564 ),
3565 Self::RelayCorrelationExhausted => {
3566 write!(f, "module control correlation ids are exhausted")
3567 }
3568 Self::ConnectionClosing { connection_id } => write!(
3569 f,
3570 "connection {} is closing and cannot accept route allocation",
3571 connection_id.get()
3572 ),
3573 Self::ClientEgressClosed { connection_id } => write!(
3574 f,
3575 "client connection {} egress is closed",
3576 connection_id.get()
3577 ),
3578 Self::RouteOpenBuild(message) => {
3579 write!(f, "failed to prebuild route.open response: {message}")
3580 }
3581 Self::CandidateSlotOccupied { module_id } => write!(
3582 f,
3583 "module_id '{module_id}' already has a swap candidate registered"
3584 ),
3585 Self::ModuleEgressUnavailable { connection_id } => write!(
3586 f,
3587 "module connection {} egress is unavailable; HELLO_ACK could not be queued",
3588 connection_id.get()
3589 ),
3590 Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
3591 }
3592 }
3593}
3594
3595impl Error for ForwardingError {}
3596
3597#[cfg(test)]
3598mod tests {
3599 use std::time::Duration;
3600
3601 use super::*;
3602 use tokio::sync::mpsc;
3603
3604 #[test]
3605 fn ordinary_long_running_request_is_not_excluded_from_drain() {
3606 let mut ledger = CreditLedger::default();
3607 ledger.acquire(1, false);
3608
3609 assert_eq!(ledger.capture_subscription_exclusions(), 0);
3610 assert_eq!(ledger.drain_in_flight(), 1);
3611 }
3612
3613 #[test]
3614 fn bit_set_subscription_is_excluded_and_counted() {
3615 let mut ledger = CreditLedger::default();
3616 ledger.acquire(1, true);
3617
3618 assert_eq!(ledger.capture_subscription_exclusions(), 1);
3619 assert_eq!(ledger.drain_in_flight(), 0);
3620 }
3621
3622 #[test]
3623 fn subscription_opened_after_drain_snapshot_is_not_excluded() {
3624 let mut ledger = CreditLedger::default();
3625 ledger.acquire(1, true);
3626 assert_eq!(ledger.capture_subscription_exclusions(), 1);
3627
3628 ledger.acquire(2, true);
3629
3630 assert_eq!(ledger.drain_in_flight(), 1);
3631 }
3632
3633 #[test]
3634 fn drain_with_no_subscriptions_reports_zero_excluded() {
3635 let mut ledger = CreditLedger::default();
3636 assert_eq!(ledger.capture_subscription_exclusions(), 0);
3637 }
3638
3639 fn test_hello_ack(corr: u64) -> Frame {
3640 Frame::build(
3641 FrameType::HelloAck,
3642 Flags::new(false, Priority::Passive, false),
3643 0,
3644 0,
3645 corr,
3646 Vec::new(),
3647 )
3648 .unwrap()
3649 }
3650
3651 #[test]
3655 fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
3656 let forwarding = ForwardingTable::default();
3657
3658 let (closed_tx, closed_rx) = mpsc::channel(8);
3659 drop(closed_rx);
3660 let closed = ConnectionId::new(1);
3661 assert_eq!(
3662 forwarding.register_module_connection_acked(
3663 closed,
3664 "closed".to_string(),
3665 2,
3666 Concurrency::ModuleManaged,
3667 FrameSink::new(closed_tx),
3668 test_hello_ack(1),
3669 ),
3670 Err(ForwardingError::ModuleEgressUnavailable {
3671 connection_id: closed
3672 })
3673 );
3674
3675 let (full_tx, _full_rx) = mpsc::channel(1);
3676 let full_sink = FrameSink::new(full_tx);
3677 full_sink.try_send(test_hello_ack(99)).unwrap();
3678 let full = ConnectionId::new(2);
3679 assert_eq!(
3680 forwarding.register_module_connection_acked(
3681 full,
3682 "full".to_string(),
3683 2,
3684 Concurrency::ModuleManaged,
3685 full_sink.clone(),
3686 test_hello_ack(2),
3687 ),
3688 Err(ForwardingError::ModuleEgressUnavailable {
3689 connection_id: full
3690 })
3691 );
3692 assert_eq!(
3693 forwarding.register_candidate_module_connection_acked(
3694 full,
3695 "full".to_string(),
3696 2,
3697 Concurrency::ModuleManaged,
3698 full_sink,
3699 test_hello_ack(3),
3700 ),
3701 Err(ForwardingError::ModuleEgressUnavailable {
3702 connection_id: full
3703 })
3704 );
3705
3706 for (connection, module_id) in [(closed, "closed"), (full, "full")] {
3707 assert_eq!(
3708 forwarding
3709 .module_endpoint_for_connection(connection)
3710 .unwrap(),
3711 None
3712 );
3713 let (client_tx, _client_rx) = mpsc::channel(8);
3714 assert_eq!(
3715 forwarding
3716 .begin_route_bind_relay_for_test(
3717 ConnectionId::new(50),
3718 FrameSink::new(client_tx),
3719 1,
3720 module_id,
3721 )
3722 .err(),
3723 Some(ForwardingError::NoModuleConnection)
3724 );
3725 }
3726 assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
3727 }
3728
3729 #[test]
3732 fn acked_registration_queues_the_hello_ack_first() {
3733 let forwarding = ForwardingTable::default();
3734 let (active_tx, mut active_rx) = mpsc::channel(8);
3735 forwarding
3736 .register_module_connection_acked(
3737 ConnectionId::new(1),
3738 "acked".to_string(),
3739 2,
3740 Concurrency::ModuleManaged,
3741 FrameSink::new(active_tx),
3742 test_hello_ack(11),
3743 )
3744 .unwrap();
3745 let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
3746 forwarding
3747 .register_candidate_module_connection_acked(
3748 ConnectionId::new(2),
3749 "acked".to_string(),
3750 2,
3751 Concurrency::ModuleManaged,
3752 FrameSink::new(candidate_tx),
3753 test_hello_ack(12),
3754 )
3755 .unwrap();
3756
3757 let active_first = active_rx.try_recv().unwrap().frame;
3758 assert_eq!(active_first.header.ty, FrameType::HelloAck);
3759 assert_eq!(active_first.header.corr, 11);
3760 let candidate_first = candidate_rx.try_recv().unwrap().frame;
3761 assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
3762 assert_eq!(candidate_first.header.corr, 12);
3763 }
3764
3765 #[test]
3766 fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
3767 let forwarding = ForwardingTable::default();
3768 let module_connection = ConnectionId::new(10);
3769 let exhausted_client = ConnectionId::new(20);
3770 let second_client = ConnectionId::new(30);
3771 let (module_tx, _module_rx) = mpsc::channel(1);
3772 let endpoint = forwarding
3773 .register_module_connection(
3774 module_connection,
3775 "route-limit-provider".to_string(),
3776 1,
3777 Concurrency::ModuleManaged,
3778 FrameSink::new(module_tx),
3779 )
3780 .unwrap();
3781
3782 {
3783 let mut inner = forwarding.inner.write().unwrap();
3784 for channel in 1..=u16::MAX {
3785 inner.reserved_client.insert(
3786 ClientRouteKey {
3787 connection_id: exhausted_client,
3788 channel,
3789 },
3790 ModuleRouteKey {
3791 endpoint,
3792 channel: 1,
3793 },
3794 );
3795 }
3796 }
3797
3798 let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
3799 let err = forwarding
3800 .begin_route_bind_relay_for_test(
3801 exhausted_client,
3802 FrameSink::new(exhausted_tx),
3803 1,
3804 "route-limit-provider",
3805 )
3806 .unwrap_err();
3807 assert!(matches!(
3808 err,
3809 ForwardingError::ClientRouteChannelExhausted { connection_id }
3810 if connection_id == exhausted_client
3811 ));
3812
3813 let (second_tx, _second_rx) = mpsc::channel(1);
3814 let pending = forwarding
3815 .begin_route_bind_relay_for_test(
3816 second_client,
3817 FrameSink::new(second_tx),
3818 2,
3819 "route-limit-provider",
3820 )
3821 .unwrap();
3822 assert_eq!(pending.client_channel, 1);
3823 }
3824
3825 #[test]
3826 fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
3827 let forwarding = ForwardingTable::default();
3828 let module_connection = ConnectionId::new(40);
3829 let client = ConnectionId::new(50);
3830 let (module_tx, _module_rx) = mpsc::channel(1);
3831 forwarding
3832 .register_module_connection(
3833 module_connection,
3834 "slot-reuse-provider".to_string(),
3835 1,
3836 Concurrency::ModuleManaged,
3837 FrameSink::new(module_tx),
3838 )
3839 .unwrap();
3840
3841 let (client_tx, _client_rx) = mpsc::channel(1);
3842 let client_sink = FrameSink::new(client_tx);
3843 let mut wrapped_channel = None;
3844 for index in 0..=usize::from(u16::MAX) {
3845 let pending = forwarding
3846 .begin_route_bind_relay_for_test(
3847 client,
3848 client_sink.clone(),
3849 index as u64 + 1,
3850 "slot-reuse-provider",
3851 )
3852 .unwrap();
3853 if index == usize::from(u16::MAX) {
3854 wrapped_channel = Some(pending.module_channel);
3855 }
3856 forwarding
3857 .abort_pending_relay(
3858 pending.endpoint,
3859 pending.corr,
3860 RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
3861 )
3862 .unwrap();
3863 }
3864
3865 assert_eq!(wrapped_channel, Some(1));
3866 }
3867
3868 #[test]
3869 fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
3870 let forwarding = ForwardingTable::default();
3871 let client = ConnectionId::new(60);
3872 forwarding
3873 .inner
3874 .write()
3875 .unwrap()
3876 .next_client_channel
3877 .insert(client, 41);
3878
3879 let released = forwarding.cleanup_connection(client).unwrap();
3880
3881 assert!(released.is_empty());
3882 assert!(!forwarding
3883 .inner
3884 .read()
3885 .unwrap()
3886 .next_client_channel
3887 .contains_key(&client));
3888 }
3889
3890 #[test]
3891 fn stale_module_cleanup_preserves_fast_reconnect_successor() {
3892 let forwarding = ForwardingTable::default();
3893 let module_id = "fast-reconnect-provider";
3894 let first_connection = ConnectionId::new(70);
3895 let second_connection = ConnectionId::new(80);
3896 let (first_tx, _first_rx) = mpsc::channel(1);
3897 let first_endpoint = forwarding
3898 .register_module_connection(
3899 first_connection,
3900 module_id.to_string(),
3901 1,
3902 Concurrency::ModuleManaged,
3903 FrameSink::new(first_tx),
3904 )
3905 .unwrap();
3906 let (second_tx, _second_rx) = mpsc::channel(1);
3907 let second_endpoint = forwarding
3908 .register_module_connection(
3909 second_connection,
3910 module_id.to_string(),
3911 1,
3912 Concurrency::ModuleManaged,
3913 FrameSink::new(second_tx),
3914 )
3915 .unwrap();
3916 assert_ne!(first_endpoint, second_endpoint);
3917
3918 let released = forwarding.cleanup_connection(first_connection).unwrap();
3919
3920 assert!(released.is_empty());
3921 assert_eq!(
3922 forwarding
3923 .inner
3924 .read()
3925 .unwrap()
3926 .modules_by_id
3927 .get(module_id)
3928 .map(|module| module.endpoint),
3929 Some(second_endpoint)
3930 );
3931 assert!(forwarding.has_live_module_connection(module_id).unwrap());
3932 let control_rpc = forwarding
3933 .begin_module_control_rpc_for(
3934 module_id,
3935 "health.check",
3936 Instant::now() + Duration::from_secs(1),
3937 )
3938 .unwrap();
3939 assert_eq!(control_rpc.endpoint, second_endpoint);
3940 }
3941
3942 fn route_fixture(
3943 module_id: &str,
3944 ) -> (
3945 ForwardingTable,
3946 ConnectionId,
3947 ModuleEndpointId,
3948 ConnectionId,
3949 FrameSink,
3950 mpsc::Receiver<crate::router::OutboundFrame>,
3951 ) {
3952 let forwarding = ForwardingTable::default();
3953 let module_connection = ConnectionId::new(100);
3954 let client_connection = ConnectionId::new(200);
3955 let (module_tx, _module_rx) = mpsc::channel(8);
3956 let endpoint = forwarding
3957 .register_module_connection(
3958 module_connection,
3959 module_id.to_string(),
3960 2,
3961 Concurrency::ModuleManaged,
3962 FrameSink::new(module_tx),
3963 )
3964 .unwrap();
3965 let (client_tx, client_rx) = mpsc::channel(8);
3966 (
3967 forwarding,
3968 module_connection,
3969 endpoint,
3970 client_connection,
3971 FrameSink::new(client_tx),
3972 client_rx,
3973 )
3974 }
3975
3976 #[test]
3977 #[cfg(unix)]
3978 fn daemon_drain_gates_current_and_racing_provider_registrations() {
3979 let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
3980 assert_eq!(forwarding.begin_daemon_drain().unwrap(), ["provider"]);
3981 assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
3982 assert!(matches!(
3983 forwarding.register_module_connection(
3984 ConnectionId::new(300),
3985 "late-provider".into(),
3986 2,
3987 Concurrency::ModuleManaged,
3988 sink,
3989 ),
3990 Err(ForwardingError::ConnectionClosing { .. })
3991 ));
3992 }
3993
3994 #[test]
3995 #[cfg(unix)]
3996 fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
3997 let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
3998 route_fixture("provider");
3999 let mut pending =
4000 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
4001 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
4002 forwarding.begin_daemon_drain().unwrap();
4003 let completion = forwarding
4004 .complete_pending_relay(
4005 module_connection,
4006 pending.corr,
4007 RouteBindRelayOutcome::Accepted,
4008 )
4009 .expect("draining admission is not a fatal module error");
4010 assert!(completion.settled);
4011 let abandoned = completion
4012 .abandoned
4013 .expect("late accepted bind needs a route GOODBYE");
4014 assert_eq!(abandoned.channel, pending.module_channel);
4015 assert!(matches!(pending.receiver.try_recv().unwrap(),
4016 RouteBindRelayOutcome::Rejected(error) if error.code == "module_reloading"));
4017 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4018 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4019 assert!(
4020 rx.try_recv().is_err(),
4021 "no route.open may be published during drain"
4022 );
4023 assert_eq!(
4024 forwarding
4025 .module_endpoint_for_connection(module_connection)
4026 .unwrap(),
4027 Some(endpoint)
4028 );
4029 assert!(
4030 !forwarding
4031 .complete_pending_relay(
4032 module_connection,
4033 pending.corr,
4034 RouteBindRelayOutcome::Accepted,
4035 )
4036 .unwrap()
4037 .settled
4038 );
4039 }
4040
4041 fn test_ping(corr: u64) -> Frame {
4042 Frame::build(
4043 FrameType::Ping,
4044 Flags::new(false, Priority::Passive, false),
4045 0,
4046 0,
4047 corr,
4048 Vec::new(),
4049 )
4050 .unwrap()
4051 }
4052
4053 fn begin_test_route(
4054 forwarding: &ForwardingTable,
4055 client_connection: ConnectionId,
4056 client_sink: FrameSink,
4057 corr: u64,
4058 module_id: &str,
4059 ) -> PendingRouteBindRelay {
4060 forwarding
4061 .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
4062 .unwrap()
4063 }
4064
4065 #[test]
4066 fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
4067 for candidate in [false, true] {
4068 for register_first in [false, true] {
4069 let (forwarding, module_connection, _, client_connection, sink, _rx) =
4070 route_fixture("target");
4071 let register = || {
4072 if candidate {
4073 forwarding.register_candidate_module_connection(
4074 client_connection,
4075 "source".into(),
4076 2,
4077 Concurrency::ModuleManaged,
4078 sink.clone(),
4079 )
4080 } else {
4081 forwarding.register_module_connection(
4082 client_connection,
4083 "source".into(),
4084 2,
4085 Concurrency::ModuleManaged,
4086 sink.clone(),
4087 )
4088 }
4089 };
4090 if register_first {
4091 register().unwrap();
4092 assert!(
4093 forwarding
4094 .begin_route_bind_relay_for_test(
4095 client_connection,
4096 sink.clone(),
4097 1,
4098 "target",
4099 )
4100 .is_err(),
4101 "a deferred route.open cannot reserve after HELLO"
4102 );
4103 assert!(forwarding
4104 .cleanup_connection(client_connection)
4105 .unwrap()
4106 .is_empty());
4107 } else {
4108 let pending =
4109 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
4110 assert!(
4111 register().is_err(),
4112 "HELLO cannot register after a route reservation"
4113 );
4114 forwarding
4115 .complete_pending_relay(
4116 module_connection,
4117 pending.corr,
4118 RouteBindRelayOutcome::Accepted,
4119 )
4120 .unwrap();
4121 assert_eq!(
4122 forwarding
4123 .cleanup_connection(client_connection)
4124 .unwrap()
4125 .len(),
4126 1
4127 );
4128 }
4129 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4130 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4131 }
4132 }
4133 }
4134
4135 #[tokio::test]
4141 async fn pending_route_open_completes_behind_queued_data_frames() {
4142 assert_eq!(
4143 crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
4144 8,
4145 "the per-connection pending route.open limit is its own constant"
4146 );
4147 let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
4148 route_fixture("open-behind-data");
4149 let (sink, mut client_rx) = crate::server::connection_egress();
4150 const DATA_FRAMES: usize = 1_000;
4151 let data = |corr: u64| {
4152 Frame::build(
4153 FrameType::StreamData,
4154 Flags::new(false, Priority::Interactive, false),
4155 9,
4156 1,
4157 corr,
4158 vec![b'x'; 200],
4159 )
4160 .unwrap()
4161 };
4162 for corr in 0..DATA_FRAMES as u64 {
4163 sink.try_send(data(corr)).unwrap();
4164 }
4165 let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
4166 assert_eq!(sink.backlog().queued_bytes, data_bytes);
4167
4168 let pending = tokio::time::timeout(
4169 Duration::from_secs(5),
4170 forwarding.begin_route_bind_relay_for(
4171 client,
4172 sink.clone(),
4173 subc_protocol::PROTOCOL_VERSION,
4174 4_242,
4175 "open-behind-data",
4176 Principal::Direct,
4177 None,
4178 None,
4179 Instant::now() + Duration::from_secs(60),
4180 ),
4181 )
4182 .await
4183 .expect("reserving the route.open slot must not wait behind data frames")
4184 .unwrap();
4185 forwarding
4186 .complete_pending_relay(
4187 module_connection,
4188 pending.corr,
4189 RouteBindRelayOutcome::Accepted,
4190 )
4191 .unwrap();
4192
4193 let backlog = sink.backlog();
4194 assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
4195 assert!(
4196 backlog.queued_bytes > data_bytes,
4197 "the route.open response must be counted in queued bytes: {backlog:?}"
4198 );
4199 for corr in 0..DATA_FRAMES as u64 {
4200 assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
4201 }
4202 let open = client_rx.recv().await.unwrap();
4203 assert_eq!(open.header.corr, 4_242);
4204 assert_eq!(open.header.ty, FrameType::Response);
4205 drop(open);
4206 assert_eq!(sink.backlog().queued_bytes, 0);
4207 assert_eq!(sink.backlog().queued_frames, 0);
4208 }
4209
4210 #[tokio::test]
4215 async fn drain_holdouts_count_held_requests_and_name_the_connection() {
4216 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4217 route_fixture("holdouts");
4218 let mut bound = |corr| {
4219 let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
4220 forwarding
4221 .complete_pending_relay(
4222 module_connection,
4223 route.corr,
4224 RouteBindRelayOutcome::Accepted,
4225 )
4226 .unwrap();
4227 client_rx.try_recv().unwrap();
4228 match forwarding
4229 .lookup_data_route(client, route.client_channel, route.client_epoch)
4230 .unwrap()
4231 {
4232 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4233 other => panic!("expected live route, got {other:?}"),
4234 }
4235 };
4236 let holding = bound(61);
4237 let _idle = bound(62);
4238 holding.flow.acquire_tagged(7, false).await.unwrap();
4239 holding.flow.acquire_tagged(2, false).await.unwrap();
4240 holding.flow.acquire_tagged(3, true).await.unwrap();
4241 forwarding
4242 .begin_module_drain("holdouts", RouteCloseReason::Restart)
4243 .unwrap();
4244
4245 let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
4246 assert_eq!(
4247 holdouts,
4248 DrainHoldouts {
4249 requests: 2,
4250 routes: 1,
4251 total_routes: 2,
4252 top_connections: vec![(client.get(), 2)],
4253 held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
4256 }
4257 );
4258 }
4259
4260 #[test]
4261 fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
4262 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4263 route_fixture("census");
4264 let pending = begin_test_route(&forwarding, client, sink, 1, "census");
4265 forwarding
4266 .complete_pending_relay(
4267 module_connection,
4268 pending.corr,
4269 RouteBindRelayOutcome::Accepted,
4270 )
4271 .unwrap();
4272
4273 let routes = forwarding.endpoint_routes(endpoint).unwrap();
4274 assert_eq!(routes.len(), 1);
4275 assert!(matches!(routes[0].principal, Principal::Direct));
4276 assert_eq!(routes[0].goodbye_target.connection_id, client);
4277 assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
4278 assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
4279 assert!(!routes[0].draining);
4280
4281 forwarding
4282 .begin_module_drain("census", RouteCloseReason::Restart)
4283 .unwrap();
4284 let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
4285 assert_eq!(draining_routes.len(), 1);
4286 assert!(draining_routes[0].draining);
4287 }
4288
4289 #[test]
4290 fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
4291 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
4292 let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
4293 assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
4294 forwarding
4295 .abort_pending_relay(
4296 first.endpoint,
4297 first.corr,
4298 RouteBindRelayOutcome::ModuleGone("abort".into()),
4299 )
4300 .unwrap();
4301 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4302 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4303
4304 let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
4305 assert_eq!(second.client_channel, first.client_channel);
4306 assert_eq!(second.module_channel, first.module_channel);
4307 assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
4308 }
4309
4310 #[test]
4311 fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
4312 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4313 route_fixture("epoch-release");
4314 let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
4315 forwarding
4316 .complete_pending_relay(
4317 module_connection,
4318 first.corr,
4319 RouteBindRelayOutcome::Accepted,
4320 )
4321 .unwrap();
4322 assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
4323 assert!(matches!(
4324 forwarding
4325 .release_client_route(client, first.client_channel, first.client_epoch)
4326 .unwrap(),
4327 RouteRelease::Removed(_)
4328 ));
4329 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4330 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4331
4332 let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
4333 forwarding
4334 .complete_pending_relay(
4335 module_connection,
4336 second.corr,
4337 RouteBindRelayOutcome::Accepted,
4338 )
4339 .unwrap();
4340 assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
4341 assert!(matches!(
4342 forwarding
4343 .release_client_route(client, second.client_channel, first.client_epoch)
4344 .unwrap(),
4345 RouteRelease::Stale
4346 ));
4347 assert!(!forwarding
4348 .cache_status(
4349 endpoint,
4350 second.module_channel,
4351 first.module_epoch,
4352 "stale".into(),
4353 )
4354 .unwrap());
4355 assert!(forwarding
4356 .cache_status(
4357 endpoint,
4358 second.module_channel,
4359 second.module_epoch,
4360 "current".into(),
4361 )
4362 .unwrap());
4363 match forwarding
4364 .route_poll_snapshot(client, second.client_channel, second.client_epoch)
4365 .unwrap()
4366 {
4367 RoutePollSnapshot::Bound { status, .. } => {
4368 assert_eq!(status.as_deref(), Some("current"));
4369 }
4370 RoutePollSnapshot::Absent => panic!("successor binding was removed"),
4371 }
4372 let counters = forwarding.counters().snapshot();
4373 assert_eq!(counters["route_released_epoch_fenced"], 1);
4374 assert_eq!(counters["route_release_stale_skipped"], 1);
4375 }
4376
4377 #[test]
4378 fn max_epoch_reservation_retires_only_that_slot() {
4379 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
4380 forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
4381 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
4382 let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
4383 assert_eq!(
4384 (final_use.client_channel, final_use.client_epoch),
4385 (7, u32::MAX)
4386 );
4387 assert_eq!(
4388 (final_use.module_channel, final_use.module_epoch),
4389 (9, u32::MAX)
4390 );
4391 forwarding
4392 .abort_pending_relay(
4393 endpoint,
4394 final_use.corr,
4395 RouteBindRelayOutcome::ModuleGone("abort".into()),
4396 )
4397 .unwrap();
4398 forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
4399 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
4400 let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
4401 assert_ne!(next.client_channel, 7);
4402 assert_ne!(next.module_channel, 9);
4403 assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
4404 }
4405
4406 #[test]
4407 fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
4408 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4409 route_fixture("corr-shared");
4410 let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
4411 assert_eq!(bind.corr, 1);
4412 forwarding
4413 .abort_pending_relay(
4414 endpoint,
4415 bind.corr,
4416 RouteBindRelayOutcome::ModuleGone("abort".into()),
4417 )
4418 .unwrap();
4419 let rpc = forwarding
4420 .begin_module_control_rpc_for(
4421 "corr-shared",
4422 "health.check",
4423 Instant::now() - Duration::from_millis(1),
4424 )
4425 .unwrap();
4426 assert_eq!(rpc.corr, 2);
4427 assert_eq!(
4428 forwarding
4429 .complete_module_control_rpc(
4430 module_connection,
4431 rpc.corr,
4432 Some("health.check"),
4433 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
4434 status: subc_protocol::session::HealthStatus::Ok,
4435 detail: None,
4436 metrics: None,
4437 }),
4438 )
4439 .unwrap(),
4440 ModuleControlRpcCompletion::Settled
4441 );
4442 assert!(matches!(
4443 rpc.receiver.blocking_recv().unwrap(),
4444 ModuleControlRpcOutcome::DeadlineElapsed
4445 ));
4446 }
4447
4448 #[tokio::test(start_paused = true)]
4449 async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
4450 let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
4451 let probe_started_at = Instant::now();
4452 let rpc = forwarding
4453 .begin_health_probe_rpc_for(
4454 "tombstone-ttl",
4455 "health.check",
4456 probe_started_at,
4457 probe_started_at + Duration::from_secs(5),
4458 )
4459 .unwrap();
4460 assert!(forwarding
4461 .tombstone_health_probe_rpc(endpoint, rpc.corr)
4462 .unwrap());
4463 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
4464
4465 tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
4466 tokio::task::yield_now().await;
4467
4468 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
4469 }
4470
4471 #[test]
4472 fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
4473 let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
4474 let mut close = forwarding.register_connection_close(endpoint.connection_id);
4475 forwarding.inject_control_corr(endpoint, u64::MAX);
4476 let final_rpc = forwarding
4477 .begin_module_control_rpc_for(
4478 "corr-max",
4479 "health.check",
4480 Instant::now() + Duration::from_secs(1),
4481 )
4482 .unwrap();
4483 assert_eq!(final_rpc.corr, u64::MAX);
4484 forwarding
4485 .cancel_module_control_rpc(endpoint, final_rpc.corr)
4486 .unwrap();
4487 assert!(matches!(
4488 forwarding.begin_module_control_rpc_for(
4489 "corr-max",
4490 "health.check",
4491 Instant::now() + Duration::from_secs(1),
4492 ),
4493 Err(ForwardingError::RelayCorrelationExhausted)
4494 ));
4495 assert!(close.try_recv().is_ok());
4496 }
4497
4498 #[test]
4499 fn publication_epoch_controls_delivery_failure_escalation() {
4500 fn setup_successor(
4501 commit_successor: Option<bool>,
4502 ) -> (ForwardingTable, ConnectionId, u16, u32) {
4503 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4504 route_fixture("escalation");
4505 let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
4506 forwarding
4507 .complete_pending_relay(
4508 module_connection,
4509 first.corr,
4510 RouteBindRelayOutcome::Accepted,
4511 )
4512 .unwrap();
4513 client_rx.try_recv().unwrap();
4514 assert!(matches!(
4515 forwarding
4516 .release_client_route(client, first.client_channel, first.client_epoch)
4517 .unwrap(),
4518 RouteRelease::Removed(_)
4519 ));
4520 if let Some(commit_successor) = commit_successor {
4521 forwarding.inject_client_slot_epoch(
4522 client,
4523 first.client_channel,
4524 first.client_epoch,
4525 );
4526 forwarding.inject_module_slot_epoch(
4527 endpoint,
4528 first.module_channel,
4529 first.module_epoch,
4530 );
4531 let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
4532 if commit_successor {
4533 forwarding
4534 .complete_pending_relay(
4535 module_connection,
4536 successor.corr,
4537 RouteBindRelayOutcome::Accepted,
4538 )
4539 .unwrap();
4540 client_rx.try_recv().unwrap();
4541 } else {
4542 forwarding
4543 .abort_pending_relay(
4544 endpoint,
4545 successor.corr,
4546 RouteBindRelayOutcome::ModuleGone("abort".into()),
4547 )
4548 .unwrap();
4549 }
4550 }
4551 (forwarding, client, first.client_channel, first.client_epoch)
4552 }
4553
4554 let probe_sink = FrameSink::new(mpsc::channel(1).0);
4555 let (no_successor, client, channel, epoch) = setup_successor(None);
4556 let mut close = no_successor.register_connection_close(client);
4557 assert!(no_successor
4558 .escalate_client_delivery_failure(
4559 client,
4560 channel,
4561 epoch,
4562 CloseReason::new("delivery", "failed"),
4563 UndeliveredFrame {
4564 module_id: None,
4565 sink: &probe_sink,
4566 },
4567 )
4568 .unwrap());
4569 assert!(close.try_recv().is_ok());
4570
4571 let (aborted, client, channel, epoch) = setup_successor(Some(false));
4572 let mut close = aborted.register_connection_close(client);
4573 assert!(aborted
4574 .escalate_client_delivery_failure(
4575 client,
4576 channel,
4577 epoch,
4578 CloseReason::new("delivery", "failed"),
4579 UndeliveredFrame {
4580 module_id: None,
4581 sink: &probe_sink,
4582 },
4583 )
4584 .unwrap());
4585 assert!(close.try_recv().is_ok());
4586
4587 let (published, client, channel, epoch) = setup_successor(Some(true));
4588 let mut close = published.register_connection_close(client);
4589 assert!(!published
4590 .escalate_client_delivery_failure(
4591 client,
4592 channel,
4593 epoch,
4594 CloseReason::new("delivery", "stale failure"),
4595 UndeliveredFrame {
4596 module_id: None,
4597 sink: &probe_sink,
4598 },
4599 )
4600 .unwrap());
4601 assert!(close.try_recv().is_err());
4602 }
4603
4604 #[test]
4605 fn route_concentration_separates_client_count_from_routes_per_client() {
4606 let (forwarding, module_connection, _, client, sink, _client_rx) =
4610 route_fixture("concentration");
4611 assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
4612
4613 for corr in [70_u64, 71] {
4614 let pending =
4615 begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
4616 forwarding
4617 .complete_pending_relay(
4618 module_connection,
4619 pending.corr,
4620 RouteBindRelayOutcome::Accepted,
4621 )
4622 .unwrap();
4623 }
4624
4625 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
4627 assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
4628 }
4629
4630 #[test]
4631 fn cleanup_and_accepted_resolution_have_one_lock_winner() {
4632 let (forwarding, module_connection, _, client, sink, mut client_rx) =
4633 route_fixture("cleanup-race");
4634 let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
4635 forwarding
4636 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4637 .unwrap();
4638 let released = forwarding.cleanup_connection(client).unwrap();
4639 assert_eq!(released.len(), 1);
4640 let completion = forwarding
4641 .complete_pending_relay(
4642 module_connection,
4643 pending.corr,
4644 RouteBindRelayOutcome::Accepted,
4645 )
4646 .unwrap();
4647 assert!(!completion.settled);
4648 assert!(client_rx.try_recv().is_err());
4649 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4650
4651 let (forwarding, module_connection, _, client, sink, mut client_rx) =
4652 route_fixture("accepted-race");
4653 let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
4654 forwarding
4655 .complete_pending_relay(
4656 module_connection,
4657 pending.corr,
4658 RouteBindRelayOutcome::Accepted,
4659 )
4660 .unwrap();
4661 assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
4662 let released = forwarding.cleanup_connection(client).unwrap();
4663 assert_eq!(released.len(), 1);
4664 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4665 }
4666
4667 #[test]
4668 fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
4669 let (forwarding, module_connection, _, client, sink, mut client_rx) =
4670 route_fixture("drain-gap");
4671 let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
4672 forwarding
4673 .complete_pending_relay(
4674 module_connection,
4675 live.corr,
4676 RouteBindRelayOutcome::Accepted,
4677 )
4678 .unwrap();
4679 client_rx.try_recv().unwrap();
4680 let binding = match forwarding
4681 .lookup_data_route(client, live.client_channel, live.client_epoch)
4682 .unwrap()
4683 {
4684 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4685 other => panic!("expected live route, got {other:?}"),
4686 };
4687
4688 let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
4689 forwarding
4690 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4691 .unwrap();
4692 let control_rpc = forwarding
4693 .begin_module_control_rpc_for(
4694 "drain-gap",
4695 "health.check",
4696 Instant::now() + Duration::from_secs(1),
4697 )
4698 .unwrap();
4699 let target = forwarding
4700 .begin_module_drain("drain-gap", RouteCloseReason::Reload)
4701 .unwrap()
4702 .unwrap();
4703 assert!(matches!(
4704 control_rpc.receiver.blocking_recv().unwrap(),
4705 ModuleControlRpcOutcome::ModuleGone(_)
4706 ));
4707 assert_eq!(target.abandoned_bindings.len(), 1);
4708 assert!(binding.flow.sem.is_closed());
4709 assert!(
4710 !forwarding
4711 .complete_pending_relay(
4712 module_connection,
4713 pending.corr,
4714 RouteBindRelayOutcome::Accepted,
4715 )
4716 .unwrap()
4717 .settled
4718 );
4719 assert!(matches!(
4720 forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
4721 Err(ForwardingError::ModuleReloading { .. })
4722 ));
4723 let released = forwarding
4724 .release_module_endpoint_routes(target.endpoint)
4725 .unwrap();
4726 assert_eq!(released.len(), 1);
4727 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4728 }
4729
4730 #[test]
4738 fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
4739 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4740 route_fixture("closing-client");
4741
4742 let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
4745 forwarding
4746 .complete_pending_relay(
4747 module_connection,
4748 live.corr,
4749 RouteBindRelayOutcome::Accepted,
4750 )
4751 .unwrap();
4752 client_rx.try_recv().unwrap();
4753
4754 let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
4756 forwarding
4757 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4758 .unwrap();
4759
4760 assert!(forwarding
4762 .escalate_client_delivery_failure(
4763 client,
4764 live.client_channel,
4765 live.client_epoch,
4766 CloseReason::new(
4767 "module_to_client_delivery_failed",
4768 "client egress refused a module frame",
4769 ),
4770 UndeliveredFrame {
4771 module_id: None,
4772 sink: &sink,
4773 },
4774 )
4775 .unwrap());
4776 assert!(!sink.is_closed());
4777
4778 let completion = forwarding
4779 .complete_pending_relay(
4780 module_connection,
4781 pending.corr,
4782 RouteBindRelayOutcome::Accepted,
4783 )
4784 .expect("a closing client must not turn a module's ack into an error");
4785
4786 assert!(completion.settled);
4787 let abandoned = completion
4788 .abandoned
4789 .expect("the module must be told to drop the binding it just created");
4790 assert_eq!(abandoned.connection_id, module_connection);
4791 assert_eq!(abandoned.channel, pending.module_channel);
4792 assert_eq!(abandoned.epoch, pending.module_epoch);
4793 assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
4794 assert!(matches!(
4795 pending.receiver.blocking_recv().unwrap(),
4796 RouteBindRelayOutcome::ModuleGone(_)
4797 ));
4798 assert!(client_rx.try_recv().is_err());
4801 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
4802
4803 assert!(forwarding
4806 .has_live_module_connection("closing-client")
4807 .unwrap());
4808 let cotenant = ConnectionId::new(201);
4809 let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
4810 let cotenant_route = begin_test_route(
4811 &forwarding,
4812 cotenant,
4813 FrameSink::new(cotenant_tx),
4814 62,
4815 "closing-client",
4816 );
4817 assert_eq!(cotenant_route.endpoint, endpoint);
4818 forwarding
4819 .complete_pending_relay(
4820 module_connection,
4821 cotenant_route.corr,
4822 RouteBindRelayOutcome::Accepted,
4823 )
4824 .unwrap();
4825 assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
4826 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
4827 }
4828
4829 #[test]
4830 fn pending_route_permit_is_released_on_rejection_and_abort() {
4831 let forwarding = ForwardingTable::default();
4832 let module_connection = ConnectionId::new(300);
4833 let client = ConnectionId::new(301);
4834 let (module_tx, _module_rx) = mpsc::channel(1);
4835 let endpoint = forwarding
4836 .register_module_connection(
4837 module_connection,
4838 "permit".into(),
4839 2,
4840 Concurrency::ModuleManaged,
4841 FrameSink::new(module_tx),
4842 )
4843 .unwrap();
4844 let (client_tx, mut client_rx) = mpsc::channel(1);
4845 let sink = FrameSink::new(client_tx);
4846 let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
4847 assert!(sink.try_send(test_ping(999)).is_err());
4848 forwarding
4849 .complete_pending_relay(
4850 module_connection,
4851 rejected.corr,
4852 RouteBindRelayOutcome::Rejected(ErrorBody {
4853 code: "no".into(),
4854 message: "rejected".into(),
4855 detail: None,
4856 }),
4857 )
4858 .unwrap();
4859 sink.try_send(test_ping(1000)).unwrap();
4860 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
4861
4862 let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
4863 assert!(sink.try_send(test_ping(1001)).is_err());
4864 forwarding
4865 .abort_pending_relay(
4866 endpoint,
4867 aborted.corr,
4868 RouteBindRelayOutcome::ModuleGone("abort".into()),
4869 )
4870 .unwrap();
4871 sink.try_send(test_ping(1002)).unwrap();
4872 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
4873
4874 let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
4875 forwarding
4876 .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
4877 .unwrap();
4878 drop(client_rx);
4879 let completion = forwarding
4880 .complete_pending_relay(
4881 module_connection,
4882 receiver_closed.corr,
4883 RouteBindRelayOutcome::Accepted,
4884 )
4885 .unwrap();
4886 assert!(completion.abandoned.is_some());
4887 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4888 }
4889
4890 #[test]
4896 fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
4897 let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
4898 route_fixture("closing-set-leak");
4899
4900 const CONNECTIONS: u64 = 32;
4901 for index in 0..CONNECTIONS {
4902 let client = ConnectionId::new(1000 + index);
4903 let (client_tx, _client_rx) = mpsc::channel(8);
4904 let route = begin_test_route(
4905 &forwarding,
4906 client,
4907 FrameSink::new(client_tx),
4908 index + 1,
4909 "closing-set-leak",
4910 );
4911 forwarding
4912 .complete_pending_relay(
4913 module_connection,
4914 route.corr,
4915 RouteBindRelayOutcome::Accepted,
4916 )
4917 .unwrap();
4918 forwarding.cleanup_connection(client).unwrap();
4919 }
4920 forwarding.cleanup_connection(module_connection).unwrap();
4921
4922 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
4923 }
4924
4925 #[test]
4932 fn closing_connection_is_refused_new_work_until_cleanup_completes() {
4933 let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
4934 route_fixture("closing-gate");
4935
4936 let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
4939 forwarding
4940 .complete_pending_relay(
4941 module_connection,
4942 live.corr,
4943 RouteBindRelayOutcome::Accepted,
4944 )
4945 .unwrap();
4946 client_rx.try_recv().unwrap();
4947
4948 assert!(forwarding
4951 .escalate_client_delivery_failure(
4952 client,
4953 live.client_channel,
4954 live.client_epoch,
4955 CloseReason::new(
4956 "module_to_client_delivery_failed",
4957 "client egress refused a module frame",
4958 ),
4959 UndeliveredFrame {
4960 module_id: None,
4961 sink: &sink,
4962 },
4963 )
4964 .unwrap());
4965 assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
4966
4967 assert!(matches!(
4969 forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
4970 Err(ForwardingError::ConnectionClosing { connection_id })
4971 if connection_id == client
4972 ));
4973 let (late_tx, _late_rx) = mpsc::channel(1);
4975 assert!(matches!(
4976 forwarding.register_module_connection(
4977 client,
4978 "late-module".into(),
4979 2,
4980 Concurrency::ModuleManaged,
4981 FrameSink::new(late_tx),
4982 ),
4983 Err(ForwardingError::ConnectionClosing { connection_id })
4984 if connection_id == client
4985 ));
4986
4987 forwarding.cleanup_connection(client).unwrap();
4991 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
4992 }
4993}
4994
4995#[cfg(test)]
4998mod swap_slot_tests {
4999 use std::time::Duration;
5000
5001 use super::*;
5002 use tokio::sync::mpsc;
5003
5004 const MODULE_ID: &str = "swapped";
5005
5006 struct SwapFixture {
5007 forwarding: ForwardingTable,
5008 incumbent_connection: ConnectionId,
5009 incumbent: ModuleEndpointId,
5010 candidate_connection: ConnectionId,
5011 candidate: ModuleEndpointId,
5012 _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
5013 }
5014
5015 fn swap_fixture() -> SwapFixture {
5016 let forwarding = ForwardingTable::default();
5017 let incumbent_connection = ConnectionId::new(100);
5018 let candidate_connection = ConnectionId::new(110);
5019 let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
5020 let incumbent = forwarding
5021 .register_module_connection(
5022 incumbent_connection,
5023 MODULE_ID.to_string(),
5024 2,
5025 Concurrency::ModuleManaged,
5026 FrameSink::new(incumbent_tx),
5027 )
5028 .unwrap();
5029 let (candidate_tx, candidate_rx) = mpsc::channel(8);
5030 let candidate = forwarding
5031 .register_candidate_module_connection(
5032 candidate_connection,
5033 MODULE_ID.to_string(),
5034 2,
5035 Concurrency::ModuleManaged,
5036 FrameSink::new(candidate_tx),
5037 )
5038 .unwrap();
5039 SwapFixture {
5040 forwarding,
5041 incumbent_connection,
5042 incumbent,
5043 candidate_connection,
5044 candidate,
5045 _module_rxs: vec![incumbent_rx, candidate_rx],
5046 }
5047 }
5048
5049 fn client(
5050 raw: u64,
5051 ) -> (
5052 ConnectionId,
5053 FrameSink,
5054 mpsc::Receiver<crate::router::OutboundFrame>,
5055 ) {
5056 let (tx, rx) = mpsc::channel(8);
5057 (ConnectionId::new(raw), FrameSink::new(tx), rx)
5058 }
5059
5060 fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
5061 forwarding
5062 .read_inner()
5063 .unwrap()
5064 .client_to_module
5065 .values()
5066 .map(|route| route.module_endpoint)
5067 .collect()
5068 }
5069
5070 #[test]
5071 fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
5072 let fixture = swap_fixture();
5073 let forwarding = &fixture.forwarding;
5074 assert_ne!(fixture.incumbent, fixture.candidate);
5075
5076 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5078 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5079 let (client_connection, client_sink, _client_rx) = client(200);
5080 let pending = forwarding
5081 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5082 .unwrap();
5083 assert_eq!(pending.endpoint, fixture.incumbent);
5084 let rpc = forwarding
5085 .begin_module_control_rpc_for(
5086 MODULE_ID,
5087 "health.check",
5088 Instant::now() + Duration::from_secs(1),
5089 )
5090 .unwrap();
5091 assert_eq!(rpc.endpoint, fixture.incumbent);
5092 let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
5093 assert_eq!(census.len(), 1, "the census lists one endpoint per id");
5094
5095 assert_eq!(
5097 forwarding
5098 .module_endpoint_for_connection(fixture.candidate_connection)
5099 .unwrap(),
5100 Some(fixture.candidate)
5101 );
5102 assert_eq!(
5103 forwarding
5104 .module_id_for_connection(fixture.candidate_connection)
5105 .unwrap()
5106 .as_deref(),
5107 Some(MODULE_ID)
5108 );
5109
5110 let (other_tx, _other_rx) = mpsc::channel(1);
5112 assert_eq!(
5113 forwarding.register_candidate_module_connection(
5114 ConnectionId::new(120),
5115 MODULE_ID.to_string(),
5116 2,
5117 Concurrency::ModuleManaged,
5118 FrameSink::new(other_tx),
5119 ),
5120 Err(ForwardingError::CandidateSlotOccupied {
5121 module_id: MODULE_ID.to_string()
5122 })
5123 );
5124 }
5125
5126 #[test]
5130 fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
5131 let fixture = swap_fixture();
5132 let forwarding = &fixture.forwarding;
5133 let (early_client, early_sink, _early_rx) = client(200);
5134 let mut early = forwarding
5135 .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
5136 .unwrap();
5137 assert_eq!(early.endpoint, fixture.incumbent);
5138 assert_eq!(
5139 forwarding
5140 .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
5141 .unwrap(),
5142 RouteBindRelayMark::Marked
5143 );
5144
5145 let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5146 assert_eq!(
5147 cutover,
5148 ForwardingCutover {
5149 promoted: fixture.candidate,
5150 incumbent: Some(fixture.incumbent),
5151 }
5152 );
5153
5154 let (late_client, late_sink, _late_rx) = client(201);
5156 let late = forwarding
5157 .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
5158 .unwrap();
5159 assert_eq!(
5160 late.endpoint, fixture.candidate,
5161 "a route.open after cutover was reserved on the incumbent"
5162 );
5163
5164 let completion = forwarding
5166 .complete_pending_relay(
5167 fixture.incumbent_connection,
5168 early.corr,
5169 RouteBindRelayOutcome::Accepted,
5170 )
5171 .expect("a superseded endpoint's ack is not an error on its connection");
5172 assert!(completion.settled);
5173 assert!(
5174 !committed_endpoints(forwarding).contains(&fixture.incumbent),
5175 "a relay reserved before cutover committed a route on the incumbent"
5176 );
5177 let goodbye = completion
5178 .abandoned
5179 .expect("the incumbent is told to drop the binding it just created");
5180 assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
5181 assert_eq!(goodbye.channel, early.module_channel);
5182 assert_eq!(goodbye.epoch, early.module_epoch);
5183 assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
5184 match early.receiver.try_recv() {
5185 Ok(RouteBindRelayOutcome::Rejected(body)) => assert_eq!(body.code, "module_reloading"),
5186 other => panic!("expected a retryable module_reloading answer, got {other:?}"),
5187 }
5188 assert!(matches!(
5189 forwarding
5190 .lookup_data_route(early_client, early.client_channel, early.client_epoch)
5191 .unwrap(),
5192 DataRoute::Client(DataRouteState::Absent)
5193 ));
5194
5195 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
5198 forwarding
5199 .complete_pending_relay(
5200 fixture.candidate_connection,
5201 late.corr,
5202 RouteBindRelayOutcome::Accepted,
5203 )
5204 .unwrap();
5205 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5206 assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
5207 }
5208
5209 #[test]
5210 fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
5211 let fixture = swap_fixture();
5212 let forwarding = &fixture.forwarding;
5213 let (bound_client, bound_sink, _bound_rx) = client(200);
5215 let bound = forwarding
5216 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5217 .unwrap();
5218 forwarding
5219 .complete_pending_relay(
5220 fixture.incumbent_connection,
5221 bound.corr,
5222 RouteBindRelayOutcome::Accepted,
5223 )
5224 .unwrap();
5225 let (pending_client, pending_sink, _pending_rx) = client(201);
5226 let mut in_flight = forwarding
5227 .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
5228 .unwrap();
5229 forwarding
5230 .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
5231 .unwrap();
5232
5233 let incumbent = forwarding
5234 .cutover_candidate(MODULE_ID)
5235 .unwrap()
5236 .unwrap()
5237 .incumbent
5238 .unwrap();
5239 let target = forwarding
5240 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
5241 .unwrap()
5242 .expect("the superseded incumbent is still registered");
5243
5244 assert_eq!(target.endpoint, fixture.incumbent);
5245 assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
5246 assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
5247 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5248 assert_eq!(target.abandoned_bindings.len(), 1);
5249 assert_eq!(
5250 target.abandoned_bindings[0].channel,
5251 in_flight.module_channel
5252 );
5253 assert!(matches!(
5254 in_flight.receiver.try_recv(),
5255 Ok(RouteBindRelayOutcome::Rejected(body)) if body.code == "module_reloading"
5256 ));
5257 assert_eq!(
5258 forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
5259 1,
5260 "the incumbent's bound route stays until its drain finishes"
5261 );
5262
5263 let (next_client, next_sink, _next_rx) = client(202);
5264 let next = forwarding
5265 .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
5266 .expect("the promoted candidate keeps accepting routes");
5267 assert_eq!(next.endpoint, fixture.candidate);
5268 }
5269
5270 #[test]
5276 fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
5277 let forwarding = ForwardingTable::default();
5278 let first_connection = ConnectionId::new(70);
5279 let (first_tx, _first_rx) = mpsc::channel(8);
5280 forwarding
5281 .register_module_connection(
5282 first_connection,
5283 MODULE_ID.to_string(),
5284 2,
5285 Concurrency::ModuleManaged,
5286 FrameSink::new(first_tx),
5287 )
5288 .unwrap();
5289 let (client_connection, client_sink, _client_rx) = client(200);
5290 let mut pending = forwarding
5291 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5292 .unwrap();
5293 let (second_tx, _second_rx) = mpsc::channel(8);
5294 forwarding
5295 .register_module_connection(
5296 ConnectionId::new(80),
5297 MODULE_ID.to_string(),
5298 2,
5299 Concurrency::ModuleManaged,
5300 FrameSink::new(second_tx),
5301 )
5302 .unwrap();
5303
5304 assert_eq!(
5305 forwarding
5306 .complete_pending_relay(
5307 first_connection,
5308 pending.corr,
5309 RouteBindRelayOutcome::Accepted
5310 )
5311 .unwrap_err(),
5312 ForwardingError::StaleModuleEndpoint
5313 );
5314 assert!(committed_endpoints(&forwarding).is_empty());
5315 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5316 assert!(matches!(
5317 pending.receiver.try_recv(),
5318 Err(oneshot::error::TryRecvError::Closed)
5319 ));
5320 }
5321
5322 #[test]
5323 fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
5324 let fixture = swap_fixture();
5326 let forwarding = &fixture.forwarding;
5327 assert!(forwarding
5328 .cleanup_connection(fixture.candidate_connection)
5329 .unwrap()
5330 .is_empty());
5331 assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
5332 let (client_connection, client_sink, _client_rx) = client(200);
5333 assert_eq!(
5334 forwarding
5335 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5336 .unwrap()
5337 .endpoint,
5338 fixture.incumbent
5339 );
5340
5341 let fixture = swap_fixture();
5344 let forwarding = &fixture.forwarding;
5345 let (bound_client, bound_sink, _bound_rx) = client(200);
5346 let bound = forwarding
5347 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5348 .unwrap();
5349 forwarding
5350 .complete_pending_relay(
5351 fixture.incumbent_connection,
5352 bound.corr,
5353 RouteBindRelayOutcome::Accepted,
5354 )
5355 .unwrap();
5356 forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5357 let released = forwarding
5358 .cleanup_connection(fixture.incumbent_connection)
5359 .unwrap();
5360 assert_eq!(released.len(), 1);
5361 assert_eq!(released[0].connection_id, bound_client);
5362 assert!(forwarding
5363 .read_inner()
5364 .unwrap()
5365 .superseded_endpoints
5366 .is_empty());
5367 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5368 let (next_client, next_sink, _next_rx) = client(201);
5369 assert_eq!(
5370 forwarding
5371 .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
5372 .unwrap()
5373 .endpoint,
5374 fixture.candidate
5375 );
5376 }
5377}