1use std::{
16 collections::{BTreeMap, HashMap, HashSet},
17 error::Error,
18 fmt,
19 sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
20 time::Duration,
21};
22
23use subc_control::{
24 ClientControlResponse, RouteCloseReason, RouteOpenReplyExt, RouteOpenTarget,
25 RouteOpenTargetKind,
26};
27use subc_protocol::{
28 manifest::Concurrency,
29 session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
30 ErrorBody, Flags, FrameType, Principal, Priority,
31};
32use tokio::sync::{oneshot, Semaphore};
33use tokio::time::Instant;
34use tracing::{debug, info, warn};
35
36use crate::spawn_authority::ApprovedDeclarations;
37use crate::{
38 control::{RouteBindBreakers, RouteBindConcurrency},
39 observability::DaemonCounters,
40 registry::ConnectionId,
41 router::FrameSink,
42 scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
43 spawn_authority::{AuthorityPair, SpawnAuthorityRecord, SpawnKey},
44 Frame, ProjectRootId,
45};
46
47const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
49
50const DEFAULT_MODULE_MANAGED_WATCH_WINDOW: usize = 24;
52
53const STATELESS_PARALLEL_WINDOW: usize = 1024;
55
56const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
60
61#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
66pub struct ModuleEndpointId {
67 pub connection_id: ConnectionId,
68 pub generation: u64,
69}
70
71#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
73pub(crate) struct ClientRouteKey {
74 pub connection_id: ConnectionId,
75 pub channel: u16,
76}
77
78#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
80pub(crate) struct ModuleRouteKey {
81 pub endpoint: ModuleEndpointId,
82 pub channel: u16,
83}
84
85#[derive(Debug)]
86pub(crate) struct RouteBinding {
87 pub client_connection_id: ConnectionId,
88 pub client_sink: FrameSink,
89 pub client_negotiated_ver: u8,
90 pub client_channel: u16,
91 pub client_epoch: u32,
92 pub module_id: String,
93 pub module_endpoint: ModuleEndpointId,
94 pub module_sink: FrameSink,
95 pub module_negotiated_ver: u8,
96 pub module_channel: u16,
97 pub module_epoch: u32,
98 pub principal: Principal,
99 #[allow(dead_code)]
101 pub harness: String,
102 #[allow(dead_code)]
104 pub build_class: Option<String>,
105 #[allow(dead_code)]
107 pub operator_app: Option<crate::operator_apps::OperatorAppStamp>,
108 pub project_root: Option<ProjectRootId>,
109 pub bound_at: Instant,
110 pub flow: Arc<ChannelFlow>,
111 pub scope: Option<BoundScope>,
114 pub accepts_close_reasons: Option<Vec<String>>,
116 pub caller_authority: Option<AuthorityPair>,
119 pub target_authority: Option<AuthorityPair>,
122}
123
124#[cfg(debug_assertions)]
125thread_local! {
126 static FORWARDING_GUARDS_HELD: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
128}
129
130struct InnerGuard<G>(G);
134
135impl<G> InnerGuard<G> {
136 fn new(guard: G) -> Self {
137 #[cfg(debug_assertions)]
138 FORWARDING_GUARDS_HELD.with(|held| held.set(held.get() + 1));
139 Self(guard)
140 }
141}
142
143impl<G> Drop for InnerGuard<G> {
144 fn drop(&mut self) {
145 #[cfg(debug_assertions)]
146 FORWARDING_GUARDS_HELD.with(|held| held.set(held.get().saturating_sub(1)));
147 }
148}
149
150impl<G: std::ops::Deref> std::ops::Deref for InnerGuard<G> {
151 type Target = G::Target;
152 fn deref(&self) -> &Self::Target {
153 &self.0
154 }
155}
156
157impl<G: std::ops::DerefMut> std::ops::DerefMut for InnerGuard<G> {
158 fn deref_mut(&mut self) -> &mut Self::Target {
159 &mut self.0
160 }
161}
162
163pub(crate) fn assert_forwarding_unlocked(what: &str) {
168 #[cfg(debug_assertions)]
169 FORWARDING_GUARDS_HELD.with(|held| {
170 assert_eq!(
171 held.get(),
172 0,
173 "{what} must not be taken while this thread holds the forwarding lock"
174 );
175 });
176 #[cfg(not(debug_assertions))]
177 let _ = what;
178}
179
180#[derive(Debug, Clone, Default)]
182pub(crate) struct RouteAuthority {
183 pub(crate) incarnation: String,
185 pub(crate) caller: Option<AuthorityPair>,
187}
188
189#[derive(Debug, Clone, PartialEq, Eq)]
191pub(crate) enum TargetAuthority {
192 Prefix,
194 Unsupervised,
196 Spawn(
199 AuthorityPair,
200 crate::spawn_authority::ApprovedDeclarations,
201 bool,
202 ),
203}
204
205impl TargetAuthority {
206 fn pair(&self) -> Option<&AuthorityPair> {
207 match self {
208 Self::Spawn(pair, _, _) => Some(pair),
209 Self::Prefix | Self::Unsupervised => None,
210 }
211 }
212
213 fn reply_target(&self, incarnation: &str) -> Result<RouteOpenTarget, ForwardingError> {
216 let (kind, spawn, epoch, first_party, digest) = match self {
217 Self::Prefix => (RouteOpenTargetKind::Prefix, None, None, false, None),
218 Self::Unsupervised => (RouteOpenTargetKind::Unsupervised, None, None, false, None),
219 Self::Spawn(pair, approved, first_party) => (
220 RouteOpenTargetKind::Spawn,
221 Some(pair.key.spawn),
222 Some(pair.epoch),
223 *first_party,
224 Some(approved.digest.clone()),
225 ),
226 };
227 RouteOpenTarget::new(incarnation, kind, spawn, epoch, first_party, digest)
228 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))
229 }
230}
231
232fn route_open_reply_body(
234 reply: &ClientControlResponse,
235 target: &TargetAuthority,
236 incarnation: &str,
237) -> Result<Vec<u8>, ForwardingError> {
238 let build = |err: serde_json::Error| ForwardingError::RouteOpenBuild(err.to_string());
239 let mut body = serde_json::to_value(reply).map_err(build)?;
240 RouteOpenReplyExt::new()
241 .with_target(Some(target.reply_target(incarnation)?))
242 .write_to(&mut body)
243 .map_err(build)?;
244 serde_json::to_vec(&body).map_err(build)
245}
246
247#[derive(Debug, Clone)]
248pub(crate) enum DataRoute {
249 Client(DataRouteState),
250 Module(DataRouteState),
251}
252
253#[derive(Debug, Clone)]
254pub(crate) enum DataRouteState {
255 Bound(Arc<RouteBinding>),
256 Reserved,
257 EpochMismatch,
258 Absent,
259}
260
261#[derive(Debug, Clone, Copy, PartialEq, Eq)]
290pub(crate) enum GoodbyeTargetKind {
291 Client,
292 Module,
293}
294
295#[derive(Debug, Clone)]
296pub(crate) struct GoodbyeTarget {
297 pub connection_id: ConnectionId,
298 pub sink: FrameSink,
299 pub negotiated_ver: u8,
300 pub channel: u16,
301 pub epoch: u32,
302 pub kind: GoodbyeTargetKind,
303 pub module_id: Option<String>,
308}
309
310#[derive(Debug, Clone, Copy)]
313pub(crate) struct UndeliveredFrame<'a> {
314 pub module_id: Option<&'a str>,
316 pub sink: &'a FrameSink,
318}
319
320pub(crate) fn principal_label(principal: &Principal) -> String {
322 match principal {
323 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
324 Principal::Direct => "direct".to_string(),
325 other => format!("{other:?}"),
326 }
327}
328
329fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
332 let labels = inner
333 .client_to_module
334 .iter()
335 .filter(|(key, _)| key.connection_id == connection_id)
336 .map(|(_, route)| principal_label(&route.principal))
337 .collect::<std::collections::BTreeSet<_>>();
338 if labels.is_empty() {
339 "none".to_string()
340 } else {
341 labels.into_iter().collect::<Vec<_>>().join(",")
342 }
343}
344
345impl GoodbyeTarget {
346 pub(crate) fn close_on_delivery_failure(&self) -> bool {
349 matches!(self.kind, GoodbyeTargetKind::Client)
350 }
351}
352
353pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
366
367pub(crate) fn send_module_route_goodbye(
379 counters: &DaemonCounters,
380 sink: &FrameSink,
381 frame: Frame,
382 module_id: Option<&str>,
383 context: &'static str,
384) {
385 let channel = frame.header.channel;
386 let epoch = frame.header.epoch;
387 let Err(err) = sink.try_send(frame.clone()) else {
388 return;
389 };
390 let runtime = match tokio::runtime::Handle::try_current() {
393 Ok(runtime) if !sink.is_closed() => runtime,
394 _ => {
395 counters.increment_goodbye_relay_module_dropped(module_id);
396 warn!(
397 module_id = module_id.unwrap_or("unknown"),
398 route_channel = channel,
399 route_epoch = epoch,
400 error = %err,
401 context,
402 "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
403 );
404 return;
405 }
406 };
407 debug!(
408 module_id = module_id.unwrap_or("unknown"),
409 route_channel = channel,
410 route_epoch = epoch,
411 error = %err,
412 context,
413 "module egress queue refused route GOODBYE; delivering it once the module frees room"
414 );
415 let counters = counters.clone();
416 let sink = sink.clone();
417 let module_id = module_id.map(str::to_string);
418 runtime.spawn(async move {
419 let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
420 let why = match outcome {
421 Ok(Ok(())) => {
422 debug!(
423 module_id = module_id.as_deref().unwrap_or("unknown"),
424 route_channel = channel,
425 route_epoch = epoch,
426 context,
427 "late route GOODBYE delivered to module"
428 );
429 return;
430 }
431 Ok(Err(err)) => err.to_string(),
432 Err(_) => format!(
433 "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
434 ),
435 };
436 counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
437 warn!(
438 module_id = module_id.as_deref().unwrap_or("unknown"),
439 route_channel = channel,
440 route_epoch = epoch,
441 error = %why,
442 context,
443 "route GOODBYE to module dropped under backpressure; not closing shared module connection"
444 );
445 });
446}
447
448#[derive(Debug, Clone)]
454pub(crate) struct EndpointRoute {
455 pub goodbye_target: GoodbyeTarget,
456 pub accepts_close_reasons: Option<Vec<String>>,
457 pub principal: Principal,
458 pub bound_at: Instant,
459 pub draining: bool,
460 pub drain_reason: Option<RouteCloseReason>,
464}
465
466#[derive(Debug, Clone)]
469pub(crate) struct ScopeDrainedRoute {
470 pub reason: RouteCloseReason,
471 pub scope: BoundScope,
472 pub module_id: String,
473 pub client: GoodbyeTarget,
474 pub module: GoodbyeTarget,
475 pub accepts_close_reasons: Option<Vec<String>>,
476}
477
478#[derive(Debug, Clone)]
481pub(crate) struct AuthorityClosedRoute {
482 pub route: Arc<RouteBinding>,
483 pub client: GoodbyeTarget,
484 pub module: GoodbyeTarget,
485}
486
487#[derive(Debug)]
488pub(crate) struct PendingRouteBindRelay {
489 pub endpoint: ModuleEndpointId,
490 pub module_sink: FrameSink,
491 pub negotiated_ver: u8,
492 pub client_channel: u16,
493 pub client_epoch: u32,
494 pub module_channel: u16,
495 pub module_epoch: u32,
496 pub corr: u64,
497 pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
498}
499
500#[derive(Debug, Clone)]
501pub(crate) struct ModuleDrainTarget {
502 pub endpoint: ModuleEndpointId,
503 pub sink: FrameSink,
504 pub negotiated_ver: u8,
505 pub abandoned_bindings: Vec<GoodbyeTarget>,
506 pub excluded_subscriptions: u32,
507}
508
509#[cfg(unix)]
511#[derive(Debug, Clone)]
512pub(crate) struct ModuleConnectionTarget {
513 pub module_id: String,
514 pub endpoint: ModuleEndpointId,
515 pub sink: FrameSink,
516 pub negotiated_ver: u8,
517}
518
519#[derive(Debug, Clone)]
520pub(crate) enum RouteBindRelayOutcome {
521 Accepted,
522 Rejected(ErrorBody),
527 DaemonRejected(ErrorBody),
535 ModuleGone(String),
536}
537
538#[derive(Debug, Clone, Copy, PartialEq, Eq)]
540pub(crate) struct ForwardingCutover {
541 pub promoted: ModuleEndpointId,
543 pub incumbent: Option<ModuleEndpointId>,
546}
547
548#[derive(Debug)]
550pub(crate) struct ConnectionCleanup {
551 pub released: Vec<GoodbyeTarget>,
553 pub abandoned_relays: u32,
556}
557
558#[derive(Debug, Clone)]
559pub(crate) struct PendingRelayCompletion {
560 pub settled: bool,
561 pub abandoned: Option<GoodbyeTarget>,
562}
563
564#[derive(Debug)]
565pub(crate) struct PendingModuleControlRpc {
566 pub endpoint: ModuleEndpointId,
567 pub module_sink: FrameSink,
568 pub negotiated_ver: u8,
569 pub corr: u64,
570 pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
571}
572
573#[derive(Debug, Clone)]
574pub(crate) enum ModuleControlRpcOutcome {
575 RawResponse(Vec<u8>),
576 Response(ModuleControlResponse),
577 Rejected(ErrorBody),
578 ModuleGone(String),
579 MalformedResponse(String),
580 UnexpectedOp { expected: String, actual: String },
581 DeadlineElapsed,
582}
583
584#[derive(Debug, Clone, PartialEq, Eq)]
585pub(crate) enum ModuleControlRpcCompletion {
586 Unknown,
587 Settled,
588 LateHealthAnswer {
589 module_id: String,
590 latency: Duration,
591 },
592}
593
594#[derive(Debug)]
595struct PendingModuleControlRpcEntry {
596 expected_op: String,
597 deadline: Instant,
598 health_probe_started_at: Option<Instant>,
599 sender: oneshot::Sender<ModuleControlRpcOutcome>,
600}
601
602#[derive(Debug)]
603struct HealthProbeTombstone {
604 expected_op: String,
605 module_id: String,
606 probe_started_at: Instant,
607 expires_at: Instant,
608}
609
610#[derive(Debug, Clone)]
611struct RouteReservation {
612 client_key: ClientRouteKey,
613 module_key: ModuleRouteKey,
614 client_epoch: u32,
615 module_epoch: u32,
616 project_root: Option<ProjectRootId>,
617}
618
619#[derive(Debug)]
620struct PendingRouteBindRelayEntry {
621 reservation: RouteReservation,
622 client_sink: FrameSink,
623 client_negotiated_ver: u8,
624 client_permit: crate::router::EgressPermit,
625 route_open_frame: Frame,
626 busy_refusals: bool,
627 accepts_close_reasons: Option<Vec<String>>,
628 request_window: usize,
629 watch_window: Option<usize>,
630 principal: Principal,
631 scope: Option<BoundScope>,
634 harness: String,
635 build_class: Option<String>,
636 operator_app: Option<crate::operator_apps::OperatorAppStamp>,
637 caller_authority: Option<AuthorityPair>,
640 target_authority: TargetAuthority,
643 incarnation: String,
645 deadline: Instant,
646 relay_enqueued: bool,
657 sender: oneshot::Sender<RouteBindRelayOutcome>,
658}
659
660#[derive(Debug, Clone, Copy, PartialEq, Eq)]
663pub(crate) enum RouteBindRelayMark {
664 Marked,
666 SettledByReply,
670 ReleasedWithoutReply,
674}
675
676#[derive(Debug, Clone)]
677pub(crate) enum RouteRelease {
678 Removed(GoodbyeTarget),
679 Stale,
680 Absent,
681}
682
683#[derive(Debug, Clone)]
684pub(crate) enum RoutePollSnapshot {
685 Bound {
686 module_id: String,
687 status: Option<String>,
688 },
689 Absent,
690}
691
692#[derive(Debug, Clone)]
693struct ModuleConnection {
694 endpoint: ModuleEndpointId,
695 sink: FrameSink,
696 negotiated_ver: u8,
697 concurrency: Concurrency,
698 verified_prefix_owner: Option<String>,
700}
701
702pub(crate) struct ModuleHelloAck {
703 pub(crate) frame: Frame,
704 pub(crate) verified_prefix_owner: Option<String>,
705}
706
707#[derive(Debug, Default)]
708struct ForwardingInner {
709 operator_confirms: Arc<crate::operator_confirm::OperatorConfirms>,
710 daemon_draining: bool,
711 daemon_drain_results: HashMap<ModuleEndpointId, bool>,
716 modules_by_id: HashMap<String, ModuleConnection>,
720 candidates_by_id: HashMap<String, ModuleConnection>,
726 superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
732 endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
733 module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
734 draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
738 closing_connections: HashSet<ConnectionId>,
739 spawn_connections: HashMap<ConnectionId, std::sync::Weak<crate::supervise::RegistrationSpawn>>,
741 target_authority_by_connection: HashMap<ConnectionId, Arc<SpawnAuthorityRecord>>,
744 spawn_approved: HashMap<SpawnKey, ApprovedDeclarations>,
745 spawn_epochs: HashMap<SpawnKey, u64>,
749 spawn_marks: HashSet<SpawnKey>,
752 dead_spawns: HashSet<SpawnKey>,
756 next_generation: u64,
757 reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
758 reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
759 next_client_channel: HashMap<ConnectionId, u16>,
760 next_module_channel: HashMap<ModuleEndpointId, u16>,
761 client_slot_epochs: HashMap<ClientRouteKey, u32>,
762 module_slot_epochs: HashMap<ModuleRouteKey, u32>,
763 last_published_epoch: HashMap<ClientRouteKey, u32>,
764 client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
765 module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
766 status: HashMap<(ClientRouteKey, u32), String>,
767 pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
768 relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
773 scope_tags: HashMap<(String, String), ScopeTag>,
778 next_control_corr: HashMap<ModuleEndpointId, u64>,
779 pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
780 health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
781}
782
783#[derive(Debug, Clone)]
784pub(crate) struct CloseReason {
785 code: &'static str,
786 message: String,
787}
788
789impl CloseReason {
790 pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
791 Self {
792 code,
793 message: message.into(),
794 }
795 }
796}
797
798impl fmt::Display for CloseReason {
799 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
800 write!(f, "{}: {}", self.code, self.message)
801 }
802}
803
804pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
805
806#[derive(Debug, Default)]
808pub struct ForwardingTable {
809 #[cfg(all(test, target_os = "linux"))]
810 route_commit_pause: Mutex<Option<RouteCommitPause>>,
811 #[cfg(all(test, target_os = "linux"))]
812 cleanup_tie_observer: Mutex<Option<oneshot::Sender<(bool, usize)>>>,
813 #[cfg(all(test, target_os = "linux"))]
814 cleanup_pause: Mutex<Option<RouteCommitPause>>,
815 #[cfg(test)]
816 policy_rebind_before_write: Mutex<Option<Arc<std::sync::Barrier>>>,
817 inner: Arc<RwLock<ForwardingInner>>,
818 frame_drop_warnings: Mutex<HashMap<ConnectionId, ChannelDropWarnings>>,
819 #[cfg(feature = "test-support")]
820 route_closing_hold: Mutex<Option<oneshot::Receiver<()>>>,
821 close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
822 counters: DaemonCounters,
823 route_bind_breakers: RouteBindBreakers,
828 route_bind_concurrency: RouteBindConcurrency,
831 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
835}
836
837#[cfg(all(test, target_os = "linux"))]
838#[derive(Debug)]
839struct RouteCommitPause {
840 reached: oneshot::Sender<()>,
841 release: std::sync::mpsc::Receiver<()>,
842}
843
844pub(crate) type PolicyRevocation = (Vec<EndpointRoute>, Vec<GoodbyeTarget>);
845
846#[derive(Debug)]
847struct FrameDropWarning {
848 last_warned: tokio::time::Instant,
849 suppressed: u64,
850}
851
852type ChannelDropWarnings = HashMap<(u16, &'static str), FrameDropWarning>;
853
854impl ForwardingTable {
855 #[cfg(all(test, target_os = "linux"))]
856 pub(crate) fn pause_cleanup_for_test(
857 &self,
858 ) -> (oneshot::Receiver<()>, std::sync::mpsc::Sender<()>) {
859 let (reached, observed) = oneshot::channel();
860 let (release, paused) = std::sync::mpsc::channel();
861 *self.cleanup_pause.lock().unwrap() = Some(RouteCommitPause {
862 reached,
863 release: paused,
864 });
865 (observed, release)
866 }
867 #[cfg(all(test, target_os = "linux"))]
868 pub(crate) fn observe_cleanup_tie_for_test(&self) -> oneshot::Receiver<(bool, usize)> {
869 let (send, receive) = oneshot::channel();
870 *self.cleanup_tie_observer.lock().unwrap() = Some(send);
871 receive
872 }
873 #[cfg(all(test, target_os = "linux"))]
874 pub(crate) fn spawn_connection_recorded_for_test(&self, connection: ConnectionId) -> bool {
875 self.read_inner()
876 .unwrap()
877 .spawn_connections
878 .contains_key(&connection)
879 }
880 #[cfg(all(test, target_os = "linux"))]
881 pub(crate) fn pause_route_commit_for_test(
882 &self,
883 ) -> (oneshot::Receiver<()>, std::sync::mpsc::Sender<()>) {
884 let (reached, observed) = oneshot::channel();
885 let (release, paused) = std::sync::mpsc::channel();
886 *self.route_commit_pause.lock().unwrap() = Some(RouteCommitPause {
887 reached,
888 release: paused,
889 });
890 (observed, release)
891 }
892 pub(crate) fn bind_spawn(
893 &self,
894 connection: ConnectionId,
895 spawn: &Arc<crate::supervise::RegistrationSpawn>,
896 ) -> Result<(), ForwardingError> {
897 self.write_inner()?
898 .spawn_connections
899 .insert(connection, Arc::downgrade(spawn));
900 Ok(())
901 }
902
903 pub(crate) fn bind_target_authority(
906 &self,
907 connection: ConnectionId,
908 record: Arc<SpawnAuthorityRecord>,
909 ) -> Result<(), ForwardingError> {
910 let approved = record.approved();
911 let mut inner = self.write_inner()?;
912 inner.spawn_approved.insert(record.key(), approved);
913 inner
914 .target_authority_by_connection
915 .insert(connection, record);
916 Ok(())
917 }
918
919 pub(crate) fn unbind_target_authority(
922 &self,
923 connection: ConnectionId,
924 ) -> Result<(), ForwardingError> {
925 self.write_inner()?
926 .target_authority_by_connection
927 .remove(&connection);
928 Ok(())
929 }
930 pub(crate) fn target_authority_for_connection(
932 &self,
933 connection: ConnectionId,
934 ) -> Result<Option<Arc<SpawnAuthorityRecord>>, ForwardingError> {
935 Ok(self
936 .read_inner()?
937 .target_authority_by_connection
938 .get(&connection)
939 .cloned())
940 }
941
942 pub(crate) fn connection_for_spawn(
944 &self,
945 key: &SpawnKey,
946 ) -> Result<Option<ConnectionId>, ForwardingError> {
947 Ok(self
948 .read_inner()?
949 .target_authority_by_connection
950 .iter()
951 .find(|(_, record)| record.key() == *key)
952 .map(|(connection, _)| *connection))
953 }
954
955 pub(crate) fn active_module_endpoint(
956 &self,
957 module_id: &str,
958 ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
959 Ok(self
960 .read_inner()?
961 .modules_by_id
962 .get(module_id)
963 .map(|module| module.endpoint))
964 }
965
966 pub(crate) fn config_delivery_sink(
967 &self,
968 module: &str,
969 connection: ConnectionId,
970 ) -> Result<Option<FrameSink>, ForwardingError> {
971 let inner = self.read_inner()?;
972 Ok(inner
973 .modules_by_id
974 .get(module)
975 .filter(|target| {
976 target.endpoint.connection_id == connection
977 && !inner.closing_connections.contains(&connection)
978 })
979 .map(|target| target.sink.clone()))
980 }
981 pub(crate) fn warn_frame_drop(
984 &self,
985 connection_id: ConnectionId,
986 channel: u16,
987 epoch: u32,
988 direction: &'static str,
989 reason: &'static str,
990 ) {
991 let now = tokio::time::Instant::now();
992 let mut warnings = self
993 .frame_drop_warnings
994 .lock()
995 .unwrap_or_else(std::sync::PoisonError::into_inner);
996 let entry = warnings
997 .entry(connection_id)
998 .or_default()
999 .entry((channel, reason));
1000 let suppressed = match entry {
1001 std::collections::hash_map::Entry::Occupied(mut entry) => {
1002 let state = entry.get_mut();
1003 if now.saturating_duration_since(state.last_warned) < Duration::from_secs(60) {
1004 state.suppressed = state.suppressed.saturating_add(1);
1005 return;
1006 }
1007 state.last_warned = now;
1008 std::mem::take(&mut state.suppressed)
1009 }
1010 std::collections::hash_map::Entry::Vacant(entry) => {
1011 entry.insert(FrameDropWarning {
1012 last_warned: now,
1013 suppressed: 0,
1014 });
1015 0
1016 }
1017 };
1018 warn!(
1019 connection_id = connection_id.get(),
1020 channel, epoch, direction, reason, suppressed, "dropping data-plane frame"
1021 );
1022 }
1023
1024 #[cfg(feature = "test-support")]
1066 pub fn configure_operator_script(
1067 &self,
1068 script: &std::path::Path,
1069 events: Option<&std::path::Path>,
1070 limits: Option<crate::operator_confirm::OperatorLimits>,
1071 ) {
1072 self.operator_confirms()
1073 .configure_script(script, events, limits);
1074 }
1075
1076 #[cfg(feature = "test-support")]
1080 pub fn hold_route_closing_for_test(&self) -> oneshot::Sender<()> {
1081 let (release, hold) = oneshot::channel();
1082 *self.route_closing_hold.lock().unwrap() = Some(hold);
1083 release
1084 }
1085
1086 #[cfg(feature = "test-support")]
1087 pub(crate) async fn wait_before_route_closing_for_test(&self) {
1088 let hold = self.route_closing_hold.lock().unwrap().take();
1089 if let Some(hold) = hold {
1090 let _ = hold.await;
1091 }
1092 }
1093
1094 pub(crate) fn operator_confirms(&self) -> Arc<crate::operator_confirm::OperatorConfirms> {
1095 Arc::clone(
1096 &self
1097 .inner
1098 .read()
1099 .unwrap_or_else(|p| p.into_inner())
1100 .operator_confirms,
1101 )
1102 }
1103
1104 #[cfg(test)]
1105 pub(crate) fn inject_operator_principal(&self, key: ModuleRouteKey, principal: Principal) {
1106 self.alter_operator_binding_for_test(key, |binding| binding.principal = principal);
1107 }
1108
1109 #[cfg(test)]
1110 pub(crate) fn alter_operator_binding_for_test(
1111 &self,
1112 key: ModuleRouteKey,
1113 alter: impl FnOnce(&mut RouteBinding),
1114 ) {
1115 let mut inner = self.write_inner().unwrap();
1116 let old = inner.module_to_client.remove(&key).unwrap();
1117 let client = ClientRouteKey {
1118 connection_id: old.client_connection_id,
1119 channel: old.client_channel,
1120 };
1121 inner.client_to_module.remove(&client);
1122 let mut binding = Arc::try_unwrap(old).expect("test binding has no outstanding readers");
1123 alter(&mut binding);
1124 let binding = Arc::new(binding);
1125 inner.client_to_module.insert(client, Arc::clone(&binding));
1126 inner.module_to_client.insert(key, binding);
1127 }
1128
1129 #[cfg(test)]
1132 pub(crate) fn invalidate_operator_route_for_test(&self, key: ModuleRouteKey, stale: bool) {
1133 let mut inner = self.write_inner().unwrap();
1134 let old = inner.module_to_client.remove(&key).unwrap();
1135 let client = ClientRouteKey {
1136 connection_id: old.client_connection_id,
1137 channel: old.client_channel,
1138 };
1139 inner.client_to_module.remove(&client);
1140 if stale {
1141 let mut binding = Arc::try_unwrap(old).expect("no binding readers at the test pause");
1142 binding.module_epoch += 1;
1143 let binding = Arc::new(binding);
1144 inner.client_to_module.insert(client, binding.clone());
1145 inner.module_to_client.insert(key, binding);
1146 }
1147 }
1148
1149 pub(crate) fn with_operator_route<T>(
1153 &self,
1154 connection_id: ConnectionId,
1155 channel: u16,
1156 epoch: u32,
1157 admit: impl FnOnce(Option<&RouteBinding>) -> T,
1158 ) -> Result<T, ForwardingError> {
1159 self.with_operator_route_epoch(connection_id, channel, |binding| {
1160 admit(binding.filter(|binding| binding.module_epoch == epoch))
1161 })
1162 }
1163
1164 pub(crate) fn with_operator_route_epoch<T>(
1167 &self,
1168 connection_id: ConnectionId,
1169 channel: u16,
1170 decide: impl FnOnce(Option<&RouteBinding>) -> T,
1171 ) -> Result<T, ForwardingError> {
1172 let inner = self.read_inner()?;
1173 let binding = inner
1174 .endpoint_by_connection
1175 .get(&connection_id)
1176 .and_then(|endpoint| {
1177 inner.module_to_client.get(&ModuleRouteKey {
1178 endpoint: *endpoint,
1179 channel,
1180 })
1181 });
1182 Ok(decide(binding.map(Arc::as_ref)))
1183 }
1184
1185 pub(crate) fn counters(&self) -> DaemonCounters {
1186 self.counters.clone()
1187 }
1188
1189 pub(crate) fn counters_ref(&self) -> &DaemonCounters {
1190 &self.counters
1191 }
1192
1193 pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
1194 self.route_bind_breakers.clone()
1195 }
1196
1197 pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
1198 self.route_bind_concurrency.clone()
1199 }
1200
1201 pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
1202 Arc::clone(&self.route_outages)
1203 }
1204
1205 pub(crate) fn register_connection_close(
1206 &self,
1207 connection_id: ConnectionId,
1208 ) -> ConnectionCloseReceiver {
1209 let (sender, receiver) = oneshot::channel();
1210 let replaced = self
1211 .lock_close_registry()
1212 .insert(connection_id, sender)
1213 .is_some();
1214 if replaced {
1215 warn!(
1216 connection_id = connection_id.get(),
1217 "replaced existing connection close registration"
1218 );
1219 }
1220 receiver
1221 }
1222
1223 pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
1224 self.lock_close_registry().remove(&connection_id);
1225 if let Ok(mut inner) = self.write_inner() {
1230 inner.spawn_connections.remove(&connection_id);
1231 inner.target_authority_by_connection.remove(&connection_id);
1232 }
1233 }
1234
1235 #[cfg(unix)]
1244 pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
1245 let senders: Vec<_> = self.lock_close_registry().drain().collect();
1246 let count = senders.len();
1247 for (_, sender) in senders {
1248 let _ = sender.send(reason.clone());
1249 }
1250 count
1251 }
1252
1253 #[cfg(unix)]
1257 pub(crate) fn module_connections(
1258 &self,
1259 ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
1260 let inner = self.read_inner()?;
1261 let mut seen = HashSet::new();
1262 Ok(inner
1263 .modules_by_id
1264 .values()
1265 .chain(inner.candidates_by_id.values())
1266 .chain(inner.superseded_endpoints.values())
1267 .filter(|module| seen.insert(module.endpoint))
1268 .map(|module| ModuleConnectionTarget {
1269 module_id: inner
1270 .module_id_by_endpoint
1271 .get(&module.endpoint)
1272 .cloned()
1273 .unwrap_or_default(),
1274 endpoint: module.endpoint,
1275 sink: module.sink.clone(),
1276 negotiated_ver: module.negotiated_ver,
1277 })
1278 .collect())
1279 }
1280
1281 pub(crate) fn request_connection_close(
1285 &self,
1286 connection_id: ConnectionId,
1287 reason: CloseReason,
1288 ) -> bool {
1289 let sender = self.lock_close_registry().remove(&connection_id);
1290 if let Some(sender) = sender {
1291 debug!(
1292 connection_id = connection_id.get(),
1293 close_reason = %reason,
1294 "requesting connection close"
1295 );
1296 let _ = sender.send(reason);
1297 true
1298 } else {
1299 debug!(
1300 connection_id = connection_id.get(),
1301 close_reason = %reason,
1302 "connection close request ignored for inactive connection"
1303 );
1304 false
1305 }
1306 }
1307
1308 pub fn register_module_connection(
1309 &self,
1310 connection_id: ConnectionId,
1311 module_id: String,
1312 negotiated_ver: u8,
1313 concurrency: Concurrency,
1314 sink: FrameSink,
1315 ) -> Result<ModuleEndpointId, ForwardingError> {
1316 self.register_module_connection_inner(
1317 connection_id,
1318 module_id,
1319 negotiated_ver,
1320 concurrency,
1321 sink,
1322 None,
1323 )
1324 }
1325
1326 pub(crate) fn register_module_connection_acked(
1339 &self,
1340 connection_id: ConnectionId,
1341 module_id: String,
1342 negotiated_ver: u8,
1343 concurrency: Concurrency,
1344 sink: FrameSink,
1345 hello_ack: ModuleHelloAck,
1346 ) -> Result<ModuleEndpointId, ForwardingError> {
1347 self.register_module_connection_inner(
1348 connection_id,
1349 module_id,
1350 negotiated_ver,
1351 concurrency,
1352 sink,
1353 Some(hello_ack),
1354 )
1355 }
1356
1357 fn register_module_connection_inner(
1358 &self,
1359 connection_id: ConnectionId,
1360 module_id: String,
1361 negotiated_ver: u8,
1362 concurrency: Concurrency,
1363 sink: FrameSink,
1364 hello_ack: Option<ModuleHelloAck>,
1365 ) -> Result<ModuleEndpointId, ForwardingError> {
1366 let mut inner = self.write_inner()?;
1367 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1368 return Err(ForwardingError::ConnectionClosing { connection_id });
1369 }
1370 check_module_connection_role_locked(&inner, connection_id)?;
1371 let verified_prefix_owner = hello_ack
1374 .as_ref()
1375 .and_then(|hello| hello.verified_prefix_owner.clone());
1376 enqueue_hello_ack_locked(&sink, connection_id, hello_ack.map(|hello| hello.frame))?;
1377
1378 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1379 let endpoint = ModuleEndpointId {
1380 connection_id,
1381 generation: inner.next_generation,
1382 };
1383 inner.endpoint_by_connection.insert(connection_id, endpoint);
1384 inner
1385 .module_id_by_endpoint
1386 .insert(endpoint, module_id.clone());
1387 inner.next_module_channel.insert(endpoint, 1);
1388 inner.next_control_corr.insert(endpoint, 1);
1389 inner.modules_by_id.insert(
1390 module_id.clone(),
1391 ModuleConnection {
1392 endpoint,
1393 sink,
1394 negotiated_ver,
1395 concurrency,
1396 verified_prefix_owner,
1397 },
1398 );
1399 drop(inner);
1400
1401 if let Some(discarded) = self
1414 .route_bind_breakers
1415 .reset_for_new_module_connection(&module_id)
1416 {
1417 info!(
1418 module_id = %module_id,
1419 discarded_consecutive_timeouts = discarded,
1420 "route.bind breaker state discarded: a new module connection replaced the process it described"
1421 );
1422 }
1423 Ok(endpoint)
1424 }
1425
1426 #[cfg(test)]
1440 pub(crate) fn register_candidate_module_connection(
1441 &self,
1442 connection_id: ConnectionId,
1443 module_id: String,
1444 negotiated_ver: u8,
1445 concurrency: Concurrency,
1446 sink: FrameSink,
1447 ) -> Result<ModuleEndpointId, ForwardingError> {
1448 self.register_candidate_module_connection_inner(
1449 connection_id,
1450 module_id,
1451 negotiated_ver,
1452 concurrency,
1453 sink,
1454 None,
1455 )
1456 }
1457
1458 pub(crate) fn register_candidate_module_connection_acked(
1465 &self,
1466 connection_id: ConnectionId,
1467 module_id: String,
1468 negotiated_ver: u8,
1469 concurrency: Concurrency,
1470 sink: FrameSink,
1471 hello_ack: Frame,
1472 ) -> Result<ModuleEndpointId, ForwardingError> {
1473 self.register_candidate_module_connection_inner(
1474 connection_id,
1475 module_id,
1476 negotiated_ver,
1477 concurrency,
1478 sink,
1479 Some(hello_ack),
1480 )
1481 }
1482
1483 fn register_candidate_module_connection_inner(
1484 &self,
1485 connection_id: ConnectionId,
1486 module_id: String,
1487 negotiated_ver: u8,
1488 concurrency: Concurrency,
1489 sink: FrameSink,
1490 hello_ack: Option<Frame>,
1491 ) -> Result<ModuleEndpointId, ForwardingError> {
1492 let mut inner = self.write_inner()?;
1493 if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1494 return Err(ForwardingError::ConnectionClosing { connection_id });
1495 }
1496 check_module_connection_role_locked(&inner, connection_id)?;
1497 if inner.candidates_by_id.contains_key(&module_id) {
1498 return Err(ForwardingError::CandidateSlotOccupied { module_id });
1499 }
1500 enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1503
1504 inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1505 let endpoint = ModuleEndpointId {
1506 connection_id,
1507 generation: inner.next_generation,
1508 };
1509 inner.endpoint_by_connection.insert(connection_id, endpoint);
1510 inner
1511 .module_id_by_endpoint
1512 .insert(endpoint, module_id.clone());
1513 inner.next_module_channel.insert(endpoint, 1);
1514 inner.next_control_corr.insert(endpoint, 1);
1515 inner.candidates_by_id.insert(
1516 module_id,
1517 ModuleConnection {
1518 endpoint,
1519 sink,
1520 negotiated_ver,
1521 concurrency,
1522 verified_prefix_owner: None,
1523 },
1524 );
1525 Ok(endpoint)
1526 }
1527
1528 pub(crate) fn cutover_candidate(
1546 &self,
1547 module_id: &str,
1548 ) -> Result<Option<ForwardingCutover>, ForwardingError> {
1549 let mut inner = self.write_inner()?;
1550 if inner.daemon_draining {
1551 return Err(ForwardingError::ModuleReloading {
1552 module_id: module_id.to_string(),
1553 });
1554 }
1555 let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
1556 return Ok(None);
1557 };
1558 let promoted = candidate.endpoint;
1559 let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
1560 let incumbent = incumbent.map(|incumbent| {
1561 let endpoint = incumbent.endpoint;
1562 inner.superseded_endpoints.insert(endpoint, incumbent);
1563 endpoint
1564 });
1565 drop(inner);
1566
1567 if let Some(discarded) = self
1571 .route_bind_breakers
1572 .reset_for_new_module_connection(module_id)
1573 {
1574 info!(
1575 module_id = %module_id,
1576 discarded_consecutive_timeouts = discarded,
1577 "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
1578 );
1579 }
1580 Ok(Some(ForwardingCutover {
1581 promoted,
1582 incumbent,
1583 }))
1584 }
1585
1586 #[allow(clippy::too_many_arguments)]
1587 pub(crate) async fn begin_route_bind_relay_for(
1588 &self,
1589 client_connection_id: ConnectionId,
1590 client_sink: FrameSink,
1591 client_negotiated_ver: u8,
1592 client_corr: u64,
1593 module_id: &str,
1594 principal: Principal,
1595 scope: Option<BoundScope>,
1596 project_root: Option<ProjectRootId>,
1597 harness: String,
1598 build_class: Option<String>,
1599 operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1600 authority: RouteAuthority,
1601 deadline: Instant,
1602 busy_refusals: bool,
1603 accepts_close_reasons: Option<Vec<String>>,
1604 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1605 let client_permit =
1609 client_sink
1610 .reserve_owned()
1611 .await
1612 .map_err(|_| ForwardingError::ClientEgressClosed {
1613 connection_id: client_connection_id,
1614 })?;
1615 self.begin_route_bind_relay_inner(
1616 client_connection_id,
1617 client_sink,
1618 client_negotiated_ver,
1619 client_corr,
1620 module_id,
1621 principal,
1622 scope,
1623 project_root,
1624 harness,
1625 build_class,
1626 operator_app,
1627 authority,
1628 deadline,
1629 client_permit,
1630 busy_refusals,
1631 accepts_close_reasons,
1632 )
1633 }
1634
1635 #[cfg(test)]
1636 pub(crate) fn begin_route_bind_relay_for_test(
1637 &self,
1638 client_connection_id: ConnectionId,
1639 client_sink: FrameSink,
1640 client_corr: u64,
1641 module_id: &str,
1642 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1643 self.begin_route_bind_relay_with_busy_for_test(
1644 client_connection_id,
1645 client_sink,
1646 client_corr,
1647 module_id,
1648 false,
1649 )
1650 }
1651
1652 #[cfg(test)]
1653 pub(crate) fn begin_route_bind_relay_with_busy_for_test(
1654 &self,
1655 client_connection_id: ConnectionId,
1656 client_sink: FrameSink,
1657 client_corr: u64,
1658 module_id: &str,
1659 busy_refusals: bool,
1660 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1661 let permit =
1662 client_sink
1663 .try_reserve_owned()
1664 .map_err(|_| ForwardingError::ClientEgressClosed {
1665 connection_id: client_connection_id,
1666 })?;
1667 self.begin_route_bind_relay_inner(
1668 client_connection_id,
1669 client_sink,
1670 subc_protocol::PROTOCOL_VERSION,
1671 client_corr,
1672 module_id,
1673 Principal::Direct,
1674 None,
1675 None,
1676 String::new(),
1677 None,
1678 None,
1679 RouteAuthority::default(),
1680 Instant::now() + std::time::Duration::from_secs(60),
1681 permit,
1682 busy_refusals,
1683 None,
1684 )
1685 }
1686
1687 pub(crate) fn begin_module_control_rpc_for(
1688 &self,
1689 module_id: &str,
1690 expected_op: &str,
1691 deadline: Instant,
1692 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1693 self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1694 }
1695
1696 pub(crate) fn begin_health_probe_rpc_for(
1697 &self,
1698 module_id: &str,
1699 expected_op: &str,
1700 probe_started_at: Instant,
1701 deadline: Instant,
1702 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1703 self.begin_module_control_rpc_inner(
1704 module_id,
1705 expected_op,
1706 deadline,
1707 Some(probe_started_at),
1708 false,
1709 )
1710 }
1711
1712 pub(crate) fn begin_drain_health_probe_rpc_for(
1713 &self,
1714 module_id: &str,
1715 expected_op: &str,
1716 probe_started_at: Instant,
1717 deadline: Instant,
1718 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1719 self.begin_module_control_rpc_inner(
1720 module_id,
1721 expected_op,
1722 deadline,
1723 Some(probe_started_at),
1724 true,
1725 )
1726 }
1727
1728 pub(crate) fn begin_endpoint_health_probe_rpc_for(
1736 &self,
1737 endpoint: ModuleEndpointId,
1738 expected_op: &str,
1739 probe_started_at: Instant,
1740 deadline: Instant,
1741 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1742 let inner = self.write_inner()?;
1743 let module = module_connection_for_endpoint_locked(&inner, endpoint)
1744 .cloned()
1745 .ok_or(ForwardingError::NoModuleConnection)?;
1746 let module_id = inner
1747 .module_id_by_endpoint
1748 .get(&endpoint)
1749 .cloned()
1750 .unwrap_or_default();
1751 self.begin_control_rpc_locked(
1754 inner,
1755 &module_id,
1756 module,
1757 expected_op,
1758 deadline,
1759 Some(probe_started_at),
1760 true,
1761 )
1762 }
1763
1764 fn begin_module_control_rpc_inner(
1765 &self,
1766 module_id: &str,
1767 expected_op: &str,
1768 deadline: Instant,
1769 health_probe_started_at: Option<Instant>,
1770 allow_draining: bool,
1771 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1772 let inner = self.write_inner()?;
1773 let module = inner
1774 .modules_by_id
1775 .get(module_id)
1776 .cloned()
1777 .ok_or(ForwardingError::NoModuleConnection)?;
1778 self.begin_control_rpc_locked(
1779 inner,
1780 module_id,
1781 module,
1782 expected_op,
1783 deadline,
1784 health_probe_started_at,
1785 allow_draining,
1786 )
1787 }
1788
1789 #[allow(clippy::too_many_arguments)]
1790 fn begin_control_rpc_locked(
1791 &self,
1792 mut inner: InnerGuard<RwLockWriteGuard<'_, ForwardingInner>>,
1793 module_id: &str,
1794 module: ModuleConnection,
1795 expected_op: &str,
1796 deadline: Instant,
1797 health_probe_started_at: Option<Instant>,
1798 allow_draining: bool,
1799 ) -> Result<PendingModuleControlRpc, ForwardingError> {
1800 if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1801 return Err(ForwardingError::ModuleReloading {
1802 module_id: module_id.to_string(),
1803 });
1804 }
1805 if inner
1806 .closing_connections
1807 .contains(&module.endpoint.connection_id)
1808 {
1809 return Err(ForwardingError::ConnectionClosing {
1810 connection_id: module.endpoint.connection_id,
1811 });
1812 }
1813 if health_probe_started_at.is_some() {
1814 inner
1818 .health_probe_tombstones
1819 .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1820 }
1821 let corr = match inner.allocate_control_corr(module.endpoint) {
1822 Ok(corr) => corr,
1823 Err(err) => {
1824 drop(inner);
1825 self.request_connection_close(
1826 module.endpoint.connection_id,
1827 CloseReason::new(
1828 "control_correlation_exhausted",
1829 "daemon-originated channel-0 correlation space exhausted",
1830 ),
1831 );
1832 return Err(err);
1833 }
1834 };
1835 let (sender, receiver) = oneshot::channel();
1836 inner.pending_control_rpcs.insert(
1837 (module.endpoint, corr),
1838 PendingModuleControlRpcEntry {
1839 expected_op: expected_op.to_string(),
1840 deadline,
1841 health_probe_started_at,
1842 sender,
1843 },
1844 );
1845
1846 Ok(PendingModuleControlRpc {
1847 endpoint: module.endpoint,
1848 module_sink: module.sink,
1849 negotiated_ver: module.negotiated_ver,
1850 corr,
1851 receiver,
1852 })
1853 }
1854
1855 #[allow(clippy::too_many_arguments)]
1856 fn begin_route_bind_relay_inner(
1857 &self,
1858 client_connection_id: ConnectionId,
1859 client_sink: FrameSink,
1860 client_negotiated_ver: u8,
1861 client_corr: u64,
1862 expected_module_id: &str,
1863 principal: Principal,
1864 scope: Option<BoundScope>,
1865 project_root: Option<ProjectRootId>,
1866 harness: String,
1867 build_class: Option<String>,
1868 operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1869 authority: RouteAuthority,
1870 deadline: Instant,
1871 client_permit: crate::router::EgressPermit,
1872 busy_refusals: bool,
1873 accepts_close_reasons: Option<Vec<String>>,
1874 ) -> Result<PendingRouteBindRelay, ForwardingError> {
1875 let mut inner = self.write_inner()?;
1876 if inner
1880 .endpoint_by_connection
1881 .contains_key(&client_connection_id)
1882 {
1883 return Err(ForwardingError::ConnectionRoleConflict {
1884 connection_id: client_connection_id,
1885 });
1886 }
1887 if inner.closing_connections.contains(&client_connection_id) {
1888 return Err(ForwardingError::ConnectionClosing {
1889 connection_id: client_connection_id,
1890 });
1891 }
1892 let module = inner
1893 .modules_by_id
1894 .get(expected_module_id)
1895 .cloned()
1896 .ok_or(ForwardingError::NoModuleConnection)?;
1897 if inner.draining_endpoints.contains_key(&module.endpoint) {
1898 return Err(ForwardingError::ModuleReloading {
1899 module_id: expected_module_id.to_string(),
1900 });
1901 }
1902 if inner
1903 .closing_connections
1904 .contains(&module.endpoint.connection_id)
1905 {
1906 return Err(ForwardingError::ConnectionClosing {
1907 connection_id: module.endpoint.connection_id,
1908 });
1909 }
1910
1911 let corr = match inner.allocate_control_corr(module.endpoint) {
1912 Ok(corr) => corr,
1913 Err(err) => {
1914 drop(inner);
1915 self.request_connection_close(
1916 module.endpoint.connection_id,
1917 CloseReason::new(
1918 "control_correlation_exhausted",
1919 "daemon-originated channel-0 correlation space exhausted",
1920 ),
1921 );
1922 return Err(err);
1923 }
1924 };
1925 let (client_channel, client_epoch, module_channel, module_epoch) =
1926 inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1927 let client_key = ClientRouteKey {
1928 connection_id: client_connection_id,
1929 channel: client_channel,
1930 };
1931 let module_key = ModuleRouteKey {
1932 endpoint: module.endpoint,
1933 channel: module_channel,
1934 };
1935 let reservation = RouteReservation {
1936 client_key,
1937 module_key,
1938 client_epoch,
1939 module_epoch,
1940 project_root,
1941 };
1942 let target_authority = target_authority_locked(&inner, &module);
1945 let response_body = route_open_reply_body(
1946 &ClientControlResponse::RouteOpen {
1947 request_window: None,
1948 watch_window: None,
1949 route_channel: client_channel,
1950 route_epoch: client_epoch,
1951 },
1952 &target_authority,
1953 &authority.incarnation,
1954 )?;
1955 let route_open_frame = Frame::build_with_version(
1956 client_negotiated_ver,
1957 FrameType::Response,
1958 Flags::new(false, Priority::Passive, false),
1959 0,
1960 0,
1961 client_corr,
1962 response_body,
1963 )
1964 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1965 let (sender, receiver) = oneshot::channel();
1966 inner.reserved_client.insert(client_key, module_key);
1967 inner.reserved_module.insert(module_key, client_key);
1968 inner.pending_relays.insert(
1969 (module.endpoint, corr),
1970 PendingRouteBindRelayEntry {
1971 reservation,
1972 client_sink,
1973 client_negotiated_ver,
1974 client_permit,
1975 route_open_frame,
1976 busy_refusals,
1977 accepts_close_reasons,
1978 request_window: window_for(&module.concurrency),
1979 watch_window: (busy_refusals && module.concurrency == Concurrency::ModuleManaged)
1980 .then_some(DEFAULT_MODULE_MANAGED_WATCH_WINDOW),
1981 principal,
1982 scope,
1983 harness,
1984 build_class,
1985 operator_app,
1986 caller_authority: authority.caller,
1987 target_authority,
1988 incarnation: authority.incarnation,
1989 deadline,
1990 relay_enqueued: false,
1991 sender,
1992 },
1993 );
1994
1995 Ok(PendingRouteBindRelay {
1996 endpoint: module.endpoint,
1997 module_sink: module.sink,
1998 negotiated_ver: module.negotiated_ver,
1999 client_channel,
2000 client_epoch,
2001 module_channel,
2002 module_epoch,
2003 corr,
2004 receiver,
2005 })
2006 }
2007
2008 pub(crate) fn mark_route_bind_relay_enqueued(
2018 &self,
2019 endpoint: ModuleEndpointId,
2020 corr: u64,
2021 ) -> Result<RouteBindRelayMark, ForwardingError> {
2022 let mut inner = self.write_inner()?;
2023 if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
2024 pending.relay_enqueued = true;
2025 return Ok(RouteBindRelayMark::Marked);
2026 }
2027 if inner
2028 .relays_released_before_enqueue
2029 .remove(&(endpoint, corr))
2030 {
2031 return Ok(RouteBindRelayMark::ReleasedWithoutReply);
2032 }
2033 Ok(RouteBindRelayMark::SettledByReply)
2034 }
2035
2036 pub(crate) fn release_client_route(
2037 &self,
2038 client_connection_id: ConnectionId,
2039 client_channel: u16,
2040 expected_epoch: u32,
2041 ) -> Result<RouteRelease, ForwardingError> {
2042 let mut inner = self.write_inner()?;
2043 let release = release_client_route_locked(
2044 &mut inner,
2045 ClientRouteKey {
2046 connection_id: client_connection_id,
2047 channel: client_channel,
2048 },
2049 expected_epoch,
2050 );
2051 self.record_route_release(&release);
2052 Ok(release)
2053 }
2054
2055 pub(crate) fn release_module_route(
2056 &self,
2057 module_connection_id: ConnectionId,
2058 module_channel: u16,
2059 expected_epoch: u32,
2060 ) -> Result<RouteRelease, ForwardingError> {
2061 let mut inner = self.write_inner()?;
2062 let Some(endpoint) = inner
2063 .endpoint_by_connection
2064 .get(&module_connection_id)
2065 .copied()
2066 else {
2067 return Ok(RouteRelease::Absent);
2068 };
2069 let release = release_module_route_locked(
2070 &mut inner,
2071 ModuleRouteKey {
2072 endpoint,
2073 channel: module_channel,
2074 },
2075 expected_epoch,
2076 );
2077 self.record_route_release(&release);
2078 Ok(release)
2079 }
2080
2081 pub(crate) fn abort_pending_relay(
2082 &self,
2083 endpoint: ModuleEndpointId,
2084 corr: u64,
2085 outcome: RouteBindRelayOutcome,
2086 ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
2087 let mut inner = self.write_inner()?;
2088 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
2089 inner
2094 .relays_released_before_enqueue
2095 .remove(&(endpoint, corr));
2096 return Ok(None);
2097 };
2098 release_reserved_route_locked(
2099 &mut inner,
2100 pending.reservation.client_key,
2101 pending.reservation.module_key,
2102 );
2103 let target = pending
2104 .relay_enqueued
2105 .then(|| abandoned_route_target(&inner, &pending.reservation));
2106 let _ = pending.sender.send(outcome);
2107 Ok(target.flatten())
2108 }
2109
2110 pub(crate) fn cancel_module_control_rpc(
2111 &self,
2112 endpoint: ModuleEndpointId,
2113 corr: u64,
2114 ) -> Result<(), ForwardingError> {
2115 self.write_inner()?
2116 .pending_control_rpcs
2117 .remove(&(endpoint, corr));
2118 Ok(())
2119 }
2120
2121 pub(crate) fn tombstone_health_probe_rpc(
2122 &self,
2123 endpoint: ModuleEndpointId,
2124 corr: u64,
2125 ) -> Result<bool, ForwardingError> {
2126 let key = (endpoint, corr);
2127 let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
2128 {
2129 let mut inner = self.write_inner()?;
2130 let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
2131 return Ok(false);
2132 };
2133 let Some(probe_started_at) = pending.health_probe_started_at else {
2134 inner.pending_control_rpcs.insert(key, pending);
2135 return Ok(false);
2136 };
2137 let module_id = inner
2138 .module_id_by_endpoint
2139 .get(&endpoint)
2140 .cloned()
2141 .unwrap_or_else(|| "unknown".to_string());
2142 inner.health_probe_tombstones.insert(
2143 key,
2144 HealthProbeTombstone {
2145 expected_op: pending.expected_op,
2146 module_id,
2147 probe_started_at,
2148 expires_at,
2149 },
2150 );
2151 }
2152 self.schedule_health_probe_tombstone_expiration(key, expires_at);
2153 Ok(true)
2154 }
2155
2156 fn schedule_health_probe_tombstone_expiration(
2157 &self,
2158 key: (ModuleEndpointId, u64),
2159 expires_at: Instant,
2160 ) {
2161 let inner = Arc::downgrade(&self.inner);
2162 tokio::spawn(async move {
2163 tokio::time::sleep_until(expires_at).await;
2164 let Some(inner) = inner.upgrade() else {
2165 return;
2166 };
2167 let Ok(mut inner) = inner.write() else {
2168 return;
2169 };
2170 let expired = inner
2171 .health_probe_tombstones
2172 .get(&key)
2173 .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
2174 if expired {
2175 inner.health_probe_tombstones.remove(&key);
2176 }
2177 });
2178 }
2179
2180 pub(crate) fn complete_pending_relay(
2181 &self,
2182 connection_id: ConnectionId,
2183 corr: u64,
2184 outcome: RouteBindRelayOutcome,
2185 ) -> Result<PendingRelayCompletion, ForwardingError> {
2186 let mut inner = self.write_inner()?;
2187 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2188 return Ok(PendingRelayCompletion {
2189 settled: false,
2190 abandoned: None,
2191 });
2192 };
2193 let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
2194 return Ok(PendingRelayCompletion {
2195 settled: false,
2196 abandoned: None,
2197 });
2198 };
2199
2200 if Instant::now() >= pending.deadline {
2201 release_reserved_route_locked(
2202 &mut inner,
2203 pending.reservation.client_key,
2204 pending.reservation.module_key,
2205 );
2206 let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
2207 .then(|| abandoned_route_target(&inner, &pending.reservation))
2208 .flatten();
2209 let _ = pending
2210 .sender
2211 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody {
2212 code: "module_timeout".to_string(),
2213 message: "route.bind response arrived after its daemon deadline".to_string(),
2214 detail: None,
2215 }));
2216 return Ok(PendingRelayCompletion {
2217 settled: true,
2218 abandoned,
2219 });
2220 }
2221
2222 match outcome {
2231 RouteBindRelayOutcome::Accepted
2251 if pending.client_sink.is_closed()
2252 || spawn_connection_dead(
2253 &inner,
2254 pending.reservation.client_key.connection_id,
2255 )
2256 || inner
2257 .closing_connections
2258 .contains(&pending.reservation.client_key.connection_id) =>
2259 {
2260 let reason = if pending.client_sink.is_closed() {
2261 "client egress closed before route publication"
2262 } else {
2263 "client connection is closing before route publication"
2264 };
2265 release_reserved_route_locked(
2266 &mut inner,
2267 pending.reservation.client_key,
2268 pending.reservation.module_key,
2269 );
2270 let abandoned = abandoned_route_target(&inner, &pending.reservation);
2274 let _ = pending
2275 .sender
2276 .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
2277 return Ok(PendingRelayCompletion {
2278 settled: true,
2279 abandoned,
2280 });
2281 }
2282 RouteBindRelayOutcome::Accepted
2305 if inner.superseded_endpoints.contains_key(&endpoint)
2306 || spawn_connection_dead(&inner, connection_id)
2307 || inner.draining_endpoints.contains_key(&endpoint) =>
2308 {
2309 release_reserved_route_locked(
2310 &mut inner,
2311 pending.reservation.client_key,
2312 pending.reservation.module_key,
2313 );
2314 let abandoned = abandoned_route_target(&inner, &pending.reservation);
2317 let module_id = inner
2318 .module_id_by_endpoint
2319 .get(&endpoint)
2320 .cloned()
2321 .unwrap_or_else(|| "unknown".to_string());
2322 let _ = pending
2323 .sender
2324 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
2325 "module_reloading",
2326 format!("module_id '{module_id}' is reloading"),
2327 )));
2328 return Ok(PendingRelayCompletion {
2329 settled: true,
2330 abandoned,
2331 });
2332 }
2333 RouteBindRelayOutcome::Accepted
2342 if pending
2343 .scope
2344 .as_ref()
2345 .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
2346 {
2347 let refusal = pending
2348 .scope
2349 .as_ref()
2350 .and_then(|scope| scope_refusal_locked(&inner, scope))
2351 .expect("guard matched a refusal under the same lock");
2352 release_reserved_route_locked(
2353 &mut inner,
2354 pending.reservation.client_key,
2355 pending.reservation.module_key,
2356 );
2357 let abandoned = abandoned_route_target(&inner, &pending.reservation);
2358 let _ = pending
2359 .sender
2360 .send(RouteBindRelayOutcome::DaemonRejected(refusal));
2361 return Ok(PendingRelayCompletion {
2362 settled: true,
2363 abandoned,
2364 });
2365 }
2366 RouteBindRelayOutcome::Accepted
2373 if authority_stale_locked(&inner, pending.caller_authority.as_ref())
2374 || authority_stale_locked(&inner, pending.target_authority.pair()) =>
2375 {
2376 release_reserved_route_locked(
2377 &mut inner,
2378 pending.reservation.client_key,
2379 pending.reservation.module_key,
2380 );
2381 let abandoned = abandoned_route_target(&inner, &pending.reservation);
2382 let _ = pending
2383 .sender
2384 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
2385 subc_protocol::error_codes::STALE_AUTHORITY_EPOCH,
2386 "the caller's or target's authority changed while the route was binding",
2387 )));
2388 return Ok(PendingRelayCompletion {
2389 settled: true,
2390 abandoned,
2391 });
2392 }
2393 RouteBindRelayOutcome::Accepted => {
2394 #[cfg(all(test, target_os = "linux"))]
2395 if let Some(pause) = self.route_commit_pause.lock().unwrap().take() {
2396 let _ = pause.reached.send(());
2397 pause
2398 .release
2399 .recv_timeout(Duration::from_secs(5))
2400 .expect("test must release bind publication");
2401 }
2402 let abandoned = commit_route_locked(&mut inner, pending)?;
2403 return Ok(PendingRelayCompletion {
2404 settled: true,
2405 abandoned,
2406 });
2407 }
2408 terminal => {
2409 release_reserved_route_locked(
2410 &mut inner,
2411 pending.reservation.client_key,
2412 pending.reservation.module_key,
2413 );
2414 let _ = pending.sender.send(terminal);
2415 }
2416 }
2417 Ok(PendingRelayCompletion {
2418 settled: true,
2419 abandoned: None,
2420 })
2421 }
2422
2423 pub(crate) fn pending_module_control_op(
2424 &self,
2425 connection_id: ConnectionId,
2426 corr: u64,
2427 ) -> Result<Option<String>, ForwardingError> {
2428 let inner = self.read_inner()?;
2429 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2430 return Ok(None);
2431 };
2432 let key = (endpoint, corr);
2433 Ok(inner
2434 .pending_control_rpcs
2435 .get(&key)
2436 .map(|pending| pending.expected_op.clone())
2437 .or_else(|| {
2438 inner
2439 .health_probe_tombstones
2440 .get(&key)
2441 .filter(|tombstone| tombstone.expires_at > Instant::now())
2442 .map(|tombstone| tombstone.expected_op.clone())
2443 }))
2444 }
2445
2446 pub(crate) fn complete_module_control_rpc(
2447 &self,
2448 connection_id: ConnectionId,
2449 corr: u64,
2450 actual_op: Option<&str>,
2451 outcome: ModuleControlRpcOutcome,
2452 ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
2453 let now = Instant::now();
2454 let mut inner = self.write_inner()?;
2455 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2456 return Ok(ModuleControlRpcCompletion::Unknown);
2457 };
2458 let key = (endpoint, corr);
2459 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2460 if now >= pending.deadline {
2461 let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
2462 ModuleControlRpcCompletion::LateHealthAnswer {
2463 module_id: inner
2464 .module_id_by_endpoint
2465 .get(&endpoint)
2466 .cloned()
2467 .unwrap_or_else(|| "unknown".to_string()),
2468 latency: now.saturating_duration_since(probe_started_at),
2469 }
2470 });
2471 let _ = pending
2472 .sender
2473 .send(ModuleControlRpcOutcome::DeadlineElapsed);
2474 return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
2475 }
2476 let outcome = match actual_op {
2477 Some(actual) if actual != pending.expected_op => {
2478 ModuleControlRpcOutcome::UnexpectedOp {
2479 expected: pending.expected_op,
2480 actual: actual.to_string(),
2481 }
2482 }
2483 _ => outcome,
2484 };
2485 let _ = pending.sender.send(outcome);
2486 return Ok(ModuleControlRpcCompletion::Settled);
2487 }
2488
2489 let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
2490 return Ok(ModuleControlRpcCompletion::Unknown);
2491 };
2492 if tombstone.expires_at <= now {
2493 return Ok(ModuleControlRpcCompletion::Unknown);
2494 }
2495 Ok(ModuleControlRpcCompletion::LateHealthAnswer {
2496 module_id: tombstone.module_id,
2497 latency: now.saturating_duration_since(tombstone.probe_started_at),
2498 })
2499 }
2500
2501 #[cfg(test)]
2502 pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
2503 Ok(self.read_inner()?.health_probe_tombstones.len())
2504 }
2505
2506 #[cfg(test)]
2507 pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
2508 Ok(self.read_inner()?.closing_connections.len())
2509 }
2510
2511 #[cfg(test)]
2514 pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
2515 let inner = self.read_inner()?;
2516 Ok((inner.reserved_client.len(), inner.reserved_module.len()))
2517 }
2518
2519 pub(crate) fn module_endpoint_for_connection(
2520 &self,
2521 connection_id: ConnectionId,
2522 ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
2523 Ok(self
2524 .read_inner()?
2525 .endpoint_by_connection
2526 .get(&connection_id)
2527 .copied())
2528 }
2529
2530 pub(crate) fn module_id_for_connection(
2533 &self,
2534 connection_id: ConnectionId,
2535 ) -> Result<Option<String>, ForwardingError> {
2536 let inner = self.read_inner()?;
2537 Ok(inner
2538 .endpoint_by_connection
2539 .get(&connection_id)
2540 .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
2541 .cloned())
2542 }
2543
2544 pub(crate) fn module_route_epoch_was_allocated(
2551 &self,
2552 connection_id: ConnectionId,
2553 channel: u16,
2554 epoch: u32,
2555 ) -> Result<bool, ForwardingError> {
2556 let inner = self.read_inner()?;
2557 let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2558 return Ok(false);
2559 };
2560 Ok(inner
2561 .module_slot_epochs
2562 .get(&ModuleRouteKey { endpoint, channel })
2563 .is_some_and(|last| epoch != 0 && epoch <= *last))
2564 }
2565
2566 pub(crate) fn has_live_module_connection(
2567 &self,
2568 module_id: &str,
2569 ) -> Result<bool, ForwardingError> {
2570 Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
2571 }
2572
2573 pub(crate) fn try_admit_request(
2576 &self,
2577 route: &Arc<RouteBinding>,
2578 epoch: u32,
2579 corr: u64,
2580 subscription: bool,
2581 ) -> Result<RequestAdmission, ForwardingError> {
2582 let inner = self.read_inner()?;
2583 let key = ClientRouteKey {
2584 connection_id: route.client_connection_id,
2585 channel: route.client_channel,
2586 };
2587 if epoch != route.client_epoch
2588 || !inner
2589 .client_to_module
2590 .get(&key)
2591 .is_some_and(|current| Arc::ptr_eq(current, route))
2592 {
2593 return Ok(RequestAdmission::Stale);
2594 }
2595 let admission = route.flow.try_acquire_tagged(corr, subscription);
2596 Ok(match admission {
2597 RequestAdmission::Stale
2598 if inner
2599 .draining_endpoints
2600 .contains_key(&route.module_endpoint) =>
2601 {
2602 RequestAdmission::ModuleReloading
2603 }
2604 admission => admission,
2605 })
2606 }
2607
2608 pub(crate) fn lookup_data_route(
2609 &self,
2610 connection_id: ConnectionId,
2611 channel: u16,
2612 epoch: u32,
2613 ) -> Result<DataRoute, ForwardingError> {
2614 let inner = self.read_inner()?;
2615 let state = if let Some(endpoint) =
2616 inner.endpoint_by_connection.get(&connection_id).copied()
2617 {
2618 let key = ModuleRouteKey { endpoint, channel };
2619 match inner.module_to_client.get(&key) {
2620 Some(route) if route.module_epoch == epoch => {
2621 DataRouteState::Bound(Arc::clone(route))
2622 }
2623 Some(_) => DataRouteState::EpochMismatch,
2624 None if inner.reserved_module.contains_key(&key)
2625 && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
2626 {
2627 DataRouteState::Reserved
2628 }
2629 None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
2630 None => DataRouteState::Absent,
2631 }
2632 } else {
2633 let key = ClientRouteKey {
2634 connection_id,
2635 channel,
2636 };
2637 match inner.client_to_module.get(&key) {
2638 Some(route) if route.client_epoch == epoch => {
2639 DataRouteState::Bound(Arc::clone(route))
2640 }
2641 Some(_) => DataRouteState::EpochMismatch,
2642 None if inner.reserved_client.contains_key(&key)
2643 && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
2644 {
2645 DataRouteState::Reserved
2646 }
2647 None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
2648 None => DataRouteState::Absent,
2649 }
2650 };
2651 Ok(
2652 if inner.endpoint_by_connection.contains_key(&connection_id) {
2653 DataRoute::Module(state)
2654 } else {
2655 DataRoute::Client(state)
2656 },
2657 )
2658 }
2659
2660 #[cfg(test)]
2661 pub(crate) fn inject_client_slot_epoch(
2662 &self,
2663 connection_id: ConnectionId,
2664 channel: u16,
2665 last_epoch: u32,
2666 ) {
2667 let mut inner = self.write_inner().expect("forwarding lock");
2668 inner.client_slot_epochs.insert(
2669 ClientRouteKey {
2670 connection_id,
2671 channel,
2672 },
2673 last_epoch,
2674 );
2675 inner.next_client_channel.insert(connection_id, channel);
2676 }
2677
2678 #[cfg(test)]
2679 pub(crate) fn inject_module_slot_epoch(
2680 &self,
2681 endpoint: ModuleEndpointId,
2682 channel: u16,
2683 last_epoch: u32,
2684 ) {
2685 let mut inner = self.write_inner().expect("forwarding lock");
2686 inner
2687 .module_slot_epochs
2688 .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
2689 inner.next_module_channel.insert(endpoint, channel);
2690 }
2691
2692 #[cfg(test)]
2693 pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
2694 self.write_inner()
2695 .expect("forwarding lock")
2696 .next_control_corr
2697 .insert(endpoint, next_corr);
2698 }
2699
2700 pub(crate) fn cache_status(
2701 &self,
2702 endpoint: ModuleEndpointId,
2703 module_channel: u16,
2704 module_epoch: u32,
2705 status: String,
2706 ) -> Result<bool, ForwardingError> {
2707 let mut inner = self.write_inner()?;
2708 if !inner.module_id_by_endpoint.contains_key(&endpoint) {
2709 return Err(ForwardingError::StaleModuleEndpoint);
2710 }
2711
2712 let module_key = ModuleRouteKey {
2713 endpoint,
2714 channel: module_channel,
2715 };
2716 let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
2717 (route.module_epoch == module_epoch).then_some((
2718 ClientRouteKey {
2719 connection_id: route.client_connection_id,
2720 channel: route.client_channel,
2721 },
2722 route.client_epoch,
2723 ))
2724 } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
2725 (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
2726 client_key,
2727 inner
2728 .client_slot_epochs
2729 .get(&client_key)
2730 .copied()
2731 .unwrap_or(0),
2732 ))
2733 } else {
2734 None
2735 };
2736
2737 if let Some(handle) = handle {
2738 inner.status.insert(handle, status);
2739 Ok(true)
2740 } else {
2741 debug!(
2742 module_channel,
2743 module_epoch,
2744 generation = endpoint.generation,
2745 connection_id = endpoint.connection_id.get(),
2746 "dropping stale status update for module route handle"
2747 );
2748 Ok(false)
2749 }
2750 }
2751
2752 pub(crate) fn route_poll_snapshot(
2753 &self,
2754 client_connection_id: ConnectionId,
2755 client_channel: u16,
2756 client_epoch: u32,
2757 ) -> Result<RoutePollSnapshot, ForwardingError> {
2758 let inner = self.read_inner()?;
2759 let client_key = ClientRouteKey {
2760 connection_id: client_connection_id,
2761 channel: client_channel,
2762 };
2763 let Some(route) = inner.client_to_module.get(&client_key) else {
2764 return Ok(RoutePollSnapshot::Absent);
2765 };
2766 if route.client_epoch != client_epoch
2767 || !inner
2768 .module_id_by_endpoint
2769 .contains_key(&route.module_endpoint)
2770 {
2771 return Ok(RoutePollSnapshot::Absent);
2772 }
2773 Ok(RoutePollSnapshot::Bound {
2774 module_id: route.module_id.clone(),
2775 status: inner.status.get(&(client_key, client_epoch)).cloned(),
2776 })
2777 }
2778
2779 pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2780 Ok(self.read_inner()?.client_to_module.len())
2781 }
2782
2783 pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2793 let inner = self.read_inner()?;
2794 let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2795 for key in inner.client_to_module.keys() {
2796 *per_connection.entry(key.connection_id).or_insert(0) += 1;
2797 }
2798 let max = per_connection.values().copied().max().unwrap_or(0);
2799 Ok((per_connection.len(), max))
2800 }
2801
2802 pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2803 let inner = self.read_inner()?;
2804 Ok(inner
2805 .client_to_module
2806 .keys()
2807 .any(|key| key.channel == route_channel))
2808 }
2809
2810 pub(crate) fn daemon_drain_result_for_connection(
2814 &self,
2815 connection_id: ConnectionId,
2816 ) -> Result<Option<bool>, ForwardingError> {
2817 let inner = self.read_inner()?;
2818 Ok(inner.daemon_draining.then(|| {
2819 inner
2820 .endpoint_by_connection
2821 .get(&connection_id)
2822 .and_then(|endpoint| inner.daemon_drain_results.get(endpoint))
2823 .copied()
2824 .unwrap_or(false)
2825 }))
2826 }
2827
2828 #[cfg(unix)]
2829 pub(crate) fn record_daemon_drain_result(
2830 &self,
2831 endpoint: ModuleEndpointId,
2832 drained: bool,
2833 ) -> Result<(), ForwardingError> {
2834 let mut inner = self.write_inner()?;
2835 if let Some(result) = inner.daemon_drain_results.get_mut(&endpoint) {
2839 *result = drained;
2840 }
2841 Ok(())
2842 }
2843
2844 #[cfg(unix)]
2847 pub(crate) fn begin_daemon_drain(
2848 &self,
2849 ) -> Result<Vec<(String, ModuleEndpointId)>, ForwardingError> {
2850 let mut inner = self.write_inner()?;
2851 if inner.daemon_draining {
2852 return Ok(Vec::new());
2853 }
2854 inner.daemon_draining = true;
2855 let modules = inner
2856 .modules_by_id
2857 .iter()
2858 .map(|(id, module)| (id.clone(), module.endpoint))
2859 .collect::<Vec<_>>();
2860 for (_, endpoint) in &modules {
2861 inner.daemon_drain_results.insert(*endpoint, false);
2862 inner
2863 .draining_endpoints
2864 .insert(*endpoint, RouteCloseReason::Restart);
2865 }
2866 let off_slot_endpoints = inner
2871 .candidates_by_id
2872 .values()
2873 .map(|module| module.endpoint)
2874 .chain(inner.superseded_endpoints.keys().copied())
2875 .collect::<Vec<_>>();
2876 for endpoint in off_slot_endpoints {
2877 inner
2878 .draining_endpoints
2879 .insert(endpoint, RouteCloseReason::Restart);
2880 }
2881 Ok(modules)
2882 }
2883
2884 pub(crate) fn begin_module_drain(
2891 &self,
2892 module_id: &str,
2893 reason: RouteCloseReason,
2894 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2895 let mut inner = self.write_inner()?;
2896 let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2897 return Ok(None);
2898 };
2899 Ok(Some(begin_drain_locked(
2900 &mut inner, module_id, module, reason,
2901 )))
2902 }
2903
2904 pub(crate) fn begin_endpoint_drain(
2911 &self,
2912 endpoint: ModuleEndpointId,
2913 reason: RouteCloseReason,
2914 ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2915 let mut inner = self.write_inner()?;
2916 let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2917 return Ok(None);
2918 };
2919 let module_id = inner
2920 .module_id_by_endpoint
2921 .get(&endpoint)
2922 .cloned()
2923 .expect("an endpoint resolved to a module connection has a module id");
2924 Ok(Some(begin_drain_locked(
2925 &mut inner, &module_id, module, reason,
2926 )))
2927 }
2928}
2929
2930fn begin_drain_locked(
2934 inner: &mut ForwardingInner,
2935 module_id: &str,
2936 module: ModuleConnection,
2937 reason: RouteCloseReason,
2938) -> ModuleDrainTarget {
2939 {
2940 let endpoint = module.endpoint;
2941 inner.draining_endpoints.insert(endpoint, reason);
2942
2943 let flows = inner
2944 .client_to_module
2945 .values()
2946 .filter(|route| route.module_endpoint == endpoint)
2947 .map(|route| Arc::clone(&route.flow))
2948 .collect::<Vec<_>>();
2949 let excluded_subscriptions = flows
2950 .into_iter()
2951 .map(|flow| flow.begin_drain())
2952 .fold(0u32, u32::saturating_add);
2953
2954 let pending_keys = inner
2955 .pending_relays
2956 .keys()
2957 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2958 .copied()
2959 .collect::<Vec<_>>();
2960 let mut abandoned_bindings = Vec::new();
2961 for key in pending_keys {
2962 let Some(pending) = inner.pending_relays.remove(&key) else {
2963 continue;
2964 };
2965 release_reserved_route_locked(
2966 inner,
2967 pending.reservation.client_key,
2968 pending.reservation.module_key,
2969 );
2970 if pending.relay_enqueued {
2971 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2972 abandoned_bindings.push(target);
2973 }
2974 } else {
2975 inner.relays_released_before_enqueue.insert(key);
2979 }
2980 let _ = pending
2981 .sender
2982 .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
2983 "module_reloading",
2984 format!("module_id '{module_id}' is reloading"),
2985 )));
2986 }
2987
2988 let pending_control_keys = inner
2989 .pending_control_rpcs
2990 .keys()
2991 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2992 .copied()
2993 .collect::<Vec<_>>();
2994 for key in pending_control_keys {
2995 if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2996 let _ = pending
2997 .sender
2998 .send(ModuleControlRpcOutcome::ModuleGone(format!(
2999 "module '{module_id}' began draining during module-control RPC"
3000 )));
3001 }
3002 }
3003
3004 ModuleDrainTarget {
3005 endpoint,
3006 sink: module.sink,
3007 negotiated_ver: module.negotiated_ver,
3008 abandoned_bindings,
3009 excluded_subscriptions,
3010 }
3011 }
3012}
3013
3014#[derive(Debug, Default, PartialEq, Eq)]
3017pub(crate) struct DrainHoldouts {
3018 pub(crate) requests: usize,
3021 pub(crate) routes: usize,
3023 pub(crate) total_routes: usize,
3025 pub(crate) top_connections: Vec<(u64, usize)>,
3028 pub(crate) held: Vec<(u16, u64)>,
3037}
3038
3039pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
3041
3042impl ForwardingTable {
3043 pub(crate) fn endpoint_drain_holdouts(
3045 &self,
3046 endpoint: ModuleEndpointId,
3047 ) -> Result<DrainHoldouts, ForwardingError> {
3048 let inner = self.read_inner()?;
3049 let mut holdouts = DrainHoldouts::default();
3050 let mut by_connection: HashMap<u64, usize> = HashMap::new();
3051 for (key, route) in &inner.client_to_module {
3052 if route.module_endpoint != endpoint {
3053 continue;
3054 }
3055 holdouts.total_routes += 1;
3056 let held = route.flow.drain_in_flight();
3057 if held == 0 {
3058 continue;
3059 }
3060 holdouts.requests += held;
3061 holdouts.routes += 1;
3062 *by_connection.entry(key.connection_id.get()).or_default() += held;
3063 holdouts.held.extend(
3064 route
3065 .flow
3066 .drain_held_corrs()
3067 .into_iter()
3068 .map(|corr| (route.module_channel, corr)),
3069 );
3070 }
3071 holdouts.held.sort_unstable();
3072 holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
3073 let mut connections = by_connection.into_iter().collect::<Vec<_>>();
3074 connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
3075 connections.truncate(3);
3076 holdouts.top_connections = connections;
3077 Ok(holdouts)
3078 }
3079
3080 pub(crate) fn endpoint_in_flight_count(
3081 &self,
3082 endpoint: ModuleEndpointId,
3083 ) -> Result<usize, ForwardingError> {
3084 let inner = self.read_inner()?;
3085 Ok(inner
3086 .client_to_module
3087 .values()
3088 .filter(|route| route.module_endpoint == endpoint)
3089 .map(|route| route.flow.drain_in_flight())
3090 .sum())
3091 }
3092
3093 pub(crate) fn endpoint_is_draining(
3094 &self,
3095 endpoint: ModuleEndpointId,
3096 ) -> Result<bool, ForwardingError> {
3097 Ok(self
3098 .read_inner()?
3099 .draining_endpoints
3100 .contains_key(&endpoint))
3101 }
3102
3103 pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
3104 let inner = self.read_inner()?;
3105 Ok(inner
3106 .modules_by_id
3107 .get(module_id)
3108 .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
3109 }
3110
3111 pub(crate) fn release_module_endpoint_routes(
3112 &self,
3113 endpoint: ModuleEndpointId,
3114 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
3115 let mut inner = self.write_inner()?;
3116 let routes = inner
3117 .module_to_client
3118 .iter()
3119 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3120 .map(|(module_key, route)| (*module_key, route.module_epoch))
3121 .collect::<Vec<_>>();
3122 let mut released = Vec::with_capacity(routes.len());
3123 for (module_key, epoch) in routes {
3124 if let RouteRelease::Removed(target) =
3125 release_module_route_locked(&mut inner, module_key, epoch)
3126 {
3127 released.push(target);
3128 }
3129 }
3130 Ok(released)
3131 }
3132
3133 pub(crate) fn rebind_module_policy(
3136 &self,
3137 module_id: &str,
3138 expected_connection: Option<ConnectionId>,
3139 authorized: impl FnOnce() -> bool,
3140 ) -> Result<Option<PolicyRevocation>, ForwardingError> {
3141 #[cfg(test)]
3142 if let Some(hold) = self.policy_rebind_before_write.lock().unwrap().take() {
3143 hold.wait();
3144 hold.wait();
3145 }
3146 let mut inner = self.write_inner()?;
3147 let module = inner
3148 .modules_by_id
3149 .get(module_id)
3150 .cloned()
3151 .ok_or(ForwardingError::NoModuleConnection)?;
3152 if expected_connection.is_some_and(|connection| module.endpoint.connection_id != connection)
3153 || !authorized()
3154 {
3155 return Ok(None);
3156 }
3157 let endpoint = module.endpoint;
3158 if inner.draining_endpoints.contains_key(&endpoint) {
3159 return Err(ForwardingError::ModuleReloading {
3160 module_id: module_id.to_owned(),
3161 });
3162 }
3163 let target = begin_drain_locked(&mut inner, module_id, module, RouteCloseReason::Reload);
3164 let routes = endpoint_routes_locked(&inner, endpoint);
3165 let mut goodbyes = target.abandoned_bindings;
3166 for route in &routes {
3167 let target = &route.goodbye_target;
3168 if let RouteRelease::Removed(module) = release_client_route_locked(
3169 &mut inner,
3170 ClientRouteKey {
3171 connection_id: target.connection_id,
3172 channel: target.channel,
3173 },
3174 target.epoch,
3175 ) {
3176 goodbyes.push(module);
3177 goodbyes.push(target.clone());
3178 }
3179 }
3180 inner.draining_endpoints.remove(&endpoint);
3181 Ok(Some((routes, goodbyes)))
3182 }
3183
3184 #[cfg(test)]
3185 pub(crate) fn pause_policy_rebind_before_write(&self, hold: Arc<std::sync::Barrier>) {
3186 *self.policy_rebind_before_write.lock().unwrap() = Some(hold);
3187 }
3188
3189 #[cfg(test)]
3190 pub(crate) fn with_forwarding_write_lock_for_test(&self, work: impl FnOnce()) {
3191 let _guard = self.write_inner().unwrap();
3192 work();
3193 }
3194
3195 pub(crate) fn endpoint_routes(
3201 &self,
3202 endpoint: ModuleEndpointId,
3203 ) -> Result<Vec<EndpointRoute>, ForwardingError> {
3204 let inner = self.read_inner()?;
3205 Ok(endpoint_routes_locked(&inner, endpoint))
3206 }
3207
3208 pub(crate) fn route_census(
3210 &self,
3211 module_id: Option<&str>,
3212 ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
3213 let inner = self.read_inner()?;
3214 let mut endpoints = inner
3215 .modules_by_id
3216 .iter()
3217 .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
3218 .map(|(id, module)| (id.clone(), module.endpoint))
3219 .collect::<Vec<_>>();
3220 endpoints.sort_by(|left, right| left.0.cmp(&right.0));
3221 Ok(endpoints
3222 .into_iter()
3223 .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
3224 .collect())
3225 }
3226
3227 pub(crate) fn live_roots(
3229 &self,
3230 module_id: &str,
3231 ) -> Result<ModuleControlResponseToModule, ForwardingError> {
3232 let inner = self.read_inner()?;
3233 let endpoint = inner
3234 .modules_by_id
3235 .get(module_id)
3236 .map(|module| module.endpoint);
3237 let mut roots = BTreeMap::new();
3238 let mut unknown_root_bindings = 0;
3239 let mut total_bindings = 0;
3240 if let Some(endpoint) = endpoint {
3241 for binding in inner
3242 .module_to_client
3243 .values()
3244 .filter(|binding| binding.module_endpoint == endpoint)
3245 {
3246 total_bindings += 1;
3247 let workdir = binding
3248 .scope
3249 .as_ref()
3250 .and_then(|scope| scope.workdir.as_deref())
3251 .map(std::path::Path::new);
3252 if let Some(root) = &binding.project_root {
3253 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
3254 entry.0 += 1;
3255 } else if workdir.is_none() {
3256 unknown_root_bindings += 1;
3257 }
3258 if let Some(workdir) = workdir.filter(|workdir| {
3259 binding.project_root.as_ref().map(ProjectRootId::as_path) != Some(*workdir)
3260 }) {
3261 roots.entry(workdir.to_path_buf()).or_insert((0, 0)).0 += 1;
3262 }
3263 }
3264 for pending in inner
3265 .pending_relays
3266 .values()
3267 .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
3268 {
3269 total_bindings += 1;
3270 if let Some(root) = &pending.reservation.project_root {
3271 let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
3272 entry.1 += 1;
3273 } else {
3274 unknown_root_bindings += 1;
3275 }
3276 }
3277 }
3278 Ok(ModuleControlResponseToModule::LiveRoots {
3279 roots: roots
3280 .into_iter()
3281 .map(|(project_root, (bound, pending))| LiveRoot {
3282 project_root,
3283 bound,
3284 pending,
3285 })
3286 .collect(),
3287 unknown_root_bindings,
3288 total_bindings,
3289 })
3290 }
3291
3292 pub(crate) fn connection_has_client_routes(
3298 &self,
3299 connection_id: ConnectionId,
3300 ) -> Result<bool, ForwardingError> {
3301 let inner = self.read_inner()?;
3302 Ok(connection_has_client_routes_locked(&inner, connection_id))
3303 }
3304
3305 pub(crate) fn cleanup_connection(
3306 &self,
3307 connection_id: ConnectionId,
3308 ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
3309 self.cleanup_connection_counted(connection_id)
3310 .map(|cleanup| cleanup.released)
3311 }
3312
3313 pub(crate) fn cleanup_connection_counted(
3318 &self,
3319 connection_id: ConnectionId,
3320 ) -> Result<ConnectionCleanup, ForwardingError> {
3321 self.frame_drop_warnings
3322 .lock()
3323 .unwrap_or_else(std::sync::PoisonError::into_inner)
3324 .remove(&connection_id);
3325 let mut inner = self.write_inner()?;
3326 inner.closing_connections.insert(connection_id);
3327 #[cfg(all(test, target_os = "linux"))]
3328 if let Some(pause) = self.cleanup_pause.lock().unwrap().take() {
3329 let _ = pause.reached.send(());
3330 pause
3331 .release
3332 .recv_timeout(Duration::from_secs(5))
3333 .expect("test must release connection cleanup");
3334 }
3335 #[cfg(all(test, target_os = "linux"))]
3336 if let Some(observe) = self.cleanup_tie_observer.lock().unwrap().take() {
3337 let pending = inner
3338 .pending_relays
3339 .values()
3340 .filter(|relay| relay.reservation.client_key.connection_id == connection_id)
3341 .count();
3342 let _ = observe.send((
3343 inner.spawn_connections.contains_key(&connection_id),
3344 pending,
3345 ));
3346 }
3347 let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
3348 remove_module_connection_locked(&mut inner, endpoint)
3349 } else {
3350 ConnectionCleanup {
3351 released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
3352 abandoned_relays: 0,
3353 }
3354 };
3355 inner.closing_connections.remove(&connection_id);
3364 Ok(cleanup)
3365 }
3366
3367 fn cleanup_client_connection_locked(
3368 inner: &mut ForwardingInner,
3369 connection_id: ConnectionId,
3370 ) -> Vec<GoodbyeTarget> {
3371 let routes = inner
3372 .client_to_module
3373 .iter()
3374 .filter(|(key, _)| key.connection_id == connection_id)
3375 .map(|(key, route)| (*key, route.client_epoch))
3376 .collect::<Vec<_>>();
3377 let mut released = Vec::with_capacity(routes.len());
3378 for (client_key, epoch) in routes {
3379 if let RouteRelease::Removed(target) =
3380 release_client_route_locked(inner, client_key, epoch)
3381 {
3382 released.push(target);
3383 }
3384 }
3385
3386 let pending_keys = inner
3387 .pending_relays
3388 .iter()
3389 .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
3390 .map(|(key, _)| *key)
3391 .collect::<Vec<_>>();
3392 for key in pending_keys {
3393 let Some(pending) = inner.pending_relays.remove(&key) else {
3394 continue;
3395 };
3396 release_reserved_route_locked(
3397 inner,
3398 pending.reservation.client_key,
3399 pending.reservation.module_key,
3400 );
3401 if pending.relay_enqueued {
3402 if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
3403 released.push(target);
3404 }
3405 } else {
3406 inner.relays_released_before_enqueue.insert(key);
3410 }
3411 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3412 "client connection closed during route.bind relay".to_string(),
3413 ));
3414 }
3415
3416 let orphaned = inner
3417 .reserved_client
3418 .iter()
3419 .filter(|(key, _)| key.connection_id == connection_id)
3420 .map(|(client, module)| (*client, *module))
3421 .collect::<Vec<_>>();
3422 for (client_key, module_key) in orphaned {
3423 release_reserved_route_locked(inner, client_key, module_key);
3424 }
3425 inner.next_client_channel.remove(&connection_id);
3426 inner
3427 .client_slot_epochs
3428 .retain(|key, _| key.connection_id != connection_id);
3429 inner
3430 .last_published_epoch
3431 .retain(|key, _| key.connection_id != connection_id);
3432 inner
3433 .status
3434 .retain(|(key, _), _| key.connection_id != connection_id);
3435
3436 released
3437 }
3438
3439 pub(crate) fn escalate_client_delivery_failure(
3448 &self,
3449 connection_id: ConnectionId,
3450 channel: u16,
3451 expected_epoch: u32,
3452 reason: CloseReason,
3453 undelivered: UndeliveredFrame<'_>,
3454 ) -> Result<bool, ForwardingError> {
3455 self.escalate_client_delivery_failure_diagnosed(
3456 connection_id,
3457 channel,
3458 expected_epoch,
3459 reason,
3460 undelivered,
3461 None,
3462 )
3463 }
3464
3465 pub(crate) fn escalate_client_delivery_failure_diagnosed(
3466 &self,
3467 connection_id: ConnectionId,
3468 channel: u16,
3469 expected_epoch: u32,
3470 reason: CloseReason,
3471 undelivered: UndeliveredFrame<'_>,
3472 diagnosis: Option<&crate::router::ClientDeliveryDiagnosis>,
3473 ) -> Result<bool, ForwardingError> {
3474 let principals = {
3475 let mut inner = self.write_inner()?;
3476 let key = ClientRouteKey {
3477 connection_id,
3478 channel,
3479 };
3480 if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
3481 None
3482 } else {
3483 inner.closing_connections.insert(connection_id);
3484 Some(connection_principals_locked(&inner, connection_id))
3485 }
3486 };
3487 let Some(principals) = principals else {
3488 self.warn_frame_drop(
3492 connection_id,
3493 channel,
3494 expected_epoch,
3495 "module_to_client",
3496 diagnosis
3497 .map(|failure| failure.cause)
3498 .unwrap_or("delivery_failed"),
3499 );
3500 return Ok(false);
3501 };
3502 let backlog = undelivered.sink.backlog();
3503 let close_reason = reason.to_string();
3504 if self.request_connection_close(connection_id, reason) {
3505 warn!(
3506 connection_id = connection_id.get(),
3507 principals = %principals,
3508 module_id = undelivered.module_id.unwrap_or("unknown"),
3509 client_channel = channel,
3510 queued_bytes = backlog.queued_bytes,
3511 queued_frames = backlog.queued_frames,
3512 frame_type = ?diagnosis.map(|failure| failure.frame_type),
3513 frame_flags = ?diagnosis.map(|failure| failure.flags.0),
3514 close_cause = diagnosis.map(|failure| failure.cause).unwrap_or("delivery_failed"),
3515 oldest_queued_ms = backlog
3516 .oldest_age
3517 .map(|age| age.as_millis() as u64)
3518 .unwrap_or(0),
3519 close_reason = %close_reason,
3520 "closing client connection: its egress queue could not take a frame"
3521 );
3522 }
3523 Ok(true)
3524 }
3525
3526 pub(crate) fn publish_scope_changes(
3540 &self,
3541 changes: &[ScopeTagChange],
3542 ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
3543 let mut inner = self.write_inner()?;
3544 let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
3545 for change in changes {
3546 let key = (change.owner.clone(), change.scope_ref.clone());
3547 match change.after {
3548 Some(tag) => {
3549 inner.scope_tags.insert(key, tag);
3550 }
3551 None => {
3552 inner.scope_tags.remove(&key);
3553 }
3554 }
3555 by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
3556 }
3557 let mut selected = Vec::new();
3558 for (client_key, route) in &inner.client_to_module {
3559 let Some(scope) = &route.scope else {
3560 continue;
3561 };
3562 let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
3563 else {
3564 continue;
3565 };
3566 if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
3567 continue;
3568 }
3569 let reason = match &change.drain {
3570 ScopeDrain::Nothing => continue,
3571 ScopeDrain::All(reason) => *reason,
3572 ScopeDrain::Carriers(narrowed) => {
3573 let owner = Principal::Reserved {
3574 module_id: scope.owner.clone(),
3575 };
3576 let hit = route.principal != owner
3577 && narrowed.iter().any(|(principal, allowed)| {
3578 *principal == route.principal
3579 && allowed
3580 .as_ref()
3581 .is_none_or(|targets| !targets.contains(&route.module_id))
3582 });
3583 if !hit {
3584 continue;
3585 }
3586 RouteCloseReason::ScopeCarrierRemoved
3587 }
3588 };
3589 selected.push((*client_key, route.client_epoch, reason));
3590 }
3591 let mut drained = Vec::new();
3592 for (client_key, client_epoch, reason) in selected {
3593 let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
3594 continue;
3595 };
3596 let release = release_client_route_locked(&mut inner, client_key, client_epoch);
3597 self.record_route_release(&release);
3598 if let RouteRelease::Removed(module) = release {
3599 drained.push(ScopeDrainedRoute {
3600 reason,
3601 scope: route
3602 .scope
3603 .clone()
3604 .expect("selected scope drain has a bound scope"),
3605 module_id: route.module_id.clone(),
3606 client: GoodbyeTarget {
3607 connection_id: route.client_connection_id,
3608 sink: route.client_sink.clone(),
3609 negotiated_ver: route.client_negotiated_ver,
3610 channel: route.client_channel,
3611 epoch: route.client_epoch,
3612 kind: GoodbyeTargetKind::Client,
3613 module_id: Some(route.module_id.clone()),
3614 },
3615 module,
3616 accepts_close_reasons: route.accepts_close_reasons.clone(),
3617 });
3618 }
3619 }
3620 Ok(drained)
3621 }
3622
3623 pub(crate) fn publish_authority_change(
3642 &self,
3643 key: &SpawnKey,
3644 epoch: u64,
3645 alive: bool,
3646 first_party: bool,
3647 ) -> Result<Vec<AuthorityClosedRoute>, ForwardingError> {
3648 self.publish_approved_authority_change(key, epoch, alive, Some(first_party), None)
3649 }
3650
3651 pub(crate) fn publish_approved_authority_change(
3654 &self,
3655 key: &SpawnKey,
3656 epoch: u64,
3657 alive: bool,
3658 first_party: Option<bool>,
3659 approved: Option<&ApprovedDeclarations>,
3660 ) -> Result<Vec<AuthorityClosedRoute>, ForwardingError> {
3661 let mut inner = self.write_inner()?;
3662 if !alive {
3663 inner.spawn_approved.remove(key);
3664 inner.dead_spawns.insert(key.clone());
3665 prune_dead_spawn_tombstones_locked(&mut inner, key);
3666 } else if let Some(approved) = approved {
3667 inner.spawn_approved.insert(key.clone(), approved.clone());
3668 }
3669 let published = inner.spawn_epochs.entry(key.clone()).or_insert(epoch);
3672 *published = (*published).max(epoch);
3673 if !alive || first_party == Some(false) {
3674 inner.spawn_marks.remove(key);
3675 } else if first_party == Some(true) {
3676 inner.spawn_marks.insert(key.clone());
3677 }
3678 let older = |pair: &Option<AuthorityPair>| {
3679 pair.as_ref()
3680 .is_some_and(|pair| pair.key == *key && pair.epoch < epoch)
3681 };
3682 let selected = inner
3683 .client_to_module
3684 .iter()
3685 .filter(|(_, route)| older(&route.caller_authority) || older(&route.target_authority))
3686 .map(|(client_key, route)| (*client_key, route.client_epoch))
3687 .collect::<Vec<_>>();
3688 let mut closed = Vec::new();
3689 for (client_key, client_epoch) in selected {
3690 let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
3691 continue;
3692 };
3693 let release = release_client_route_locked(&mut inner, client_key, client_epoch);
3694 self.record_route_release(&release);
3695 if let RouteRelease::Removed(module) = release {
3696 closed.push(AuthorityClosedRoute {
3697 client: GoodbyeTarget {
3698 connection_id: route.client_connection_id,
3699 sink: route.client_sink.clone(),
3700 negotiated_ver: route.client_negotiated_ver,
3701 channel: route.client_channel,
3702 epoch: route.client_epoch,
3703 kind: GoodbyeTargetKind::Client,
3704 module_id: Some(route.module_id.clone()),
3705 },
3706 module,
3707 route,
3708 });
3709 }
3710 }
3711 Ok(closed)
3712 }
3713
3714 pub(crate) fn publish_spawn_mark(&self, key: &SpawnKey) -> Result<(), ForwardingError> {
3720 self.write_inner()?.spawn_marks.insert(key.clone());
3721 Ok(())
3722 }
3723
3724 #[cfg(all(test, unix, feature = "test-support"))]
3726 pub(crate) fn published_spawn_epoch(&self, key: &SpawnKey) -> u64 {
3727 published_epoch_locked(&self.read_inner().unwrap(), key)
3728 }
3729
3730 #[cfg(all(test, unix, feature = "test-support"))]
3733 pub(crate) fn route_authority_for_test(
3734 &self,
3735 client: ConnectionId,
3736 ) -> Vec<(Option<AuthorityPair>, Option<AuthorityPair>)> {
3737 self.read_inner()
3738 .unwrap()
3739 .client_to_module
3740 .iter()
3741 .filter(|(key, _)| key.connection_id == client)
3742 .map(|(_, route)| {
3743 (
3744 route.caller_authority.clone(),
3745 route.target_authority.clone(),
3746 )
3747 })
3748 .collect()
3749 }
3750
3751 #[cfg(test)]
3753 pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
3754 self.read_inner()
3755 .ok()?
3756 .scope_tags
3757 .get(&(owner.to_string(), scope_ref.to_string()))
3758 .copied()
3759 }
3760
3761 fn record_route_release(&self, release: &RouteRelease) {
3762 match release {
3763 RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
3764 RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
3765 RouteRelease::Absent => {}
3766 }
3767 }
3768
3769 fn read_inner(
3770 &self,
3771 ) -> Result<InnerGuard<RwLockReadGuard<'_, ForwardingInner>>, ForwardingError> {
3772 self.inner
3773 .read()
3774 .map(InnerGuard::new)
3775 .map_err(|_| ForwardingError::Poisoned)
3776 }
3777
3778 fn write_inner(
3779 &self,
3780 ) -> Result<InnerGuard<RwLockWriteGuard<'_, ForwardingInner>>, ForwardingError> {
3781 self.inner
3782 .write()
3783 .map(InnerGuard::new)
3784 .map_err(|_| ForwardingError::Poisoned)
3785 }
3786
3787 fn lock_close_registry(
3788 &self,
3789 ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
3790 self.close_registry
3791 .lock()
3792 .unwrap_or_else(|poisoned| poisoned.into_inner())
3793 }
3794}
3795
3796impl ForwardingInner {
3797 fn allocate_route_slots(
3798 &mut self,
3799 connection_id: ConnectionId,
3800 endpoint: ModuleEndpointId,
3801 ) -> Result<(u16, u32, u16, u32), ForwardingError> {
3802 let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
3803 let mut client_channel = client_start;
3804 let client_channel = loop {
3805 let key = ClientRouteKey {
3806 connection_id,
3807 channel: client_channel,
3808 };
3809 let eligible = !self.client_to_module.contains_key(&key)
3810 && !self.reserved_client.contains_key(&key)
3811 && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3812 if eligible {
3813 break client_channel;
3814 }
3815 client_channel = next_channel(client_channel);
3816 if client_channel == client_start {
3817 return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
3818 }
3819 };
3820
3821 let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
3822 let mut module_channel = module_start;
3823 let module_channel = loop {
3824 let key = ModuleRouteKey {
3825 endpoint,
3826 channel: module_channel,
3827 };
3828 let eligible = !self.module_to_client.contains_key(&key)
3829 && !self.reserved_module.contains_key(&key)
3830 && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3831 if eligible {
3832 break module_channel;
3833 }
3834 module_channel = next_channel(module_channel);
3835 if module_channel == module_start {
3836 return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
3837 }
3838 };
3839
3840 let client_key = ClientRouteKey {
3841 connection_id,
3842 channel: client_channel,
3843 };
3844 let module_key = ModuleRouteKey {
3845 endpoint,
3846 channel: module_channel,
3847 };
3848 let client_epoch = self
3849 .client_slot_epochs
3850 .get(&client_key)
3851 .copied()
3852 .unwrap_or(0)
3853 + 1;
3854 let module_epoch = self
3855 .module_slot_epochs
3856 .get(&module_key)
3857 .copied()
3858 .unwrap_or(0)
3859 + 1;
3860 self.client_slot_epochs.insert(client_key, client_epoch);
3861 self.module_slot_epochs.insert(module_key, module_epoch);
3862 self.next_client_channel
3863 .insert(connection_id, next_channel(client_channel));
3864 self.next_module_channel
3865 .insert(endpoint, next_channel(module_channel));
3866 Ok((client_channel, client_epoch, module_channel, module_epoch))
3867 }
3868
3869 fn allocate_control_corr(
3870 &mut self,
3871 endpoint: ModuleEndpointId,
3872 ) -> Result<u64, ForwardingError> {
3873 let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
3874 if candidate == 0 {
3875 self.closing_connections.insert(endpoint.connection_id);
3876 return Err(ForwardingError::RelayCorrelationExhausted);
3877 }
3878 self.next_control_corr.insert(
3879 endpoint,
3880 if candidate == u64::MAX {
3881 0
3882 } else {
3883 candidate + 1
3884 },
3885 );
3886 Ok(candidate)
3887 }
3888}
3889
3890fn connection_has_client_routes_locked(
3891 inner: &ForwardingInner,
3892 connection_id: ConnectionId,
3893) -> bool {
3894 inner
3895 .client_to_module
3896 .keys()
3897 .any(|key| key.connection_id == connection_id)
3898 || inner
3899 .reserved_client
3900 .keys()
3901 .any(|key| key.connection_id == connection_id)
3902}
3903
3904fn check_module_connection_role_locked(
3905 inner: &ForwardingInner,
3906 connection_id: ConnectionId,
3907) -> Result<(), ForwardingError> {
3908 if inner.endpoint_by_connection.contains_key(&connection_id)
3909 || connection_has_client_routes_locked(inner, connection_id)
3910 {
3911 return Err(ForwardingError::ConnectionRoleConflict { connection_id });
3912 }
3913 Ok(())
3914}
3915
3916fn next_channel(channel: u16) -> u16 {
3917 let next = channel.wrapping_add(1);
3918 if next == 0 {
3919 1
3920 } else {
3921 next
3922 }
3923}
3924
3925fn endpoint_routes_locked(
3926 inner: &ForwardingInner,
3927 endpoint: ModuleEndpointId,
3928) -> Vec<EndpointRoute> {
3929 let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
3930 let draining = drain_reason.is_some();
3931 let mut routes = inner
3932 .module_to_client
3933 .iter()
3934 .filter(|(module_key, _)| module_key.endpoint == endpoint)
3935 .map(|(_, route)| EndpointRoute {
3936 goodbye_target: GoodbyeTarget {
3937 connection_id: route.client_connection_id,
3938 sink: route.client_sink.clone(),
3939 negotiated_ver: route.client_negotiated_ver,
3940 channel: route.client_channel,
3941 epoch: route.client_epoch,
3942 kind: GoodbyeTargetKind::Client,
3943 module_id: Some(route.module_id.clone()),
3944 },
3945 principal: route.principal.clone(),
3946 accepts_close_reasons: route.accepts_close_reasons.clone(),
3947 bound_at: route.bound_at,
3948 draining,
3949 drain_reason,
3950 })
3951 .collect::<Vec<_>>();
3952 routes.sort_by_key(|route| {
3953 (
3954 route.goodbye_target.connection_id.get(),
3955 route.goodbye_target.channel,
3956 route.goodbye_target.epoch,
3957 )
3958 });
3959 routes
3960}
3961
3962fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
3966 let key = (scope.owner.clone(), scope.scope_ref.clone());
3967 match inner.scope_tags.get(&key) {
3968 Some(current) if *current == scope.tag => None,
3969 Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
3970 subc_protocol::error_codes::SCOPE_CHANGED,
3971 format!(
3972 "scope '{}' of {} changed while the route was being bound; re-open it",
3973 scope.scope_ref, scope.owner
3974 ),
3975 )),
3976 _ => Some(ErrorBody::new(
3977 subc_protocol::error_codes::SCOPE_ENDED,
3978 format!(
3979 "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
3980 scope.scope_ref, scope.owner, scope.tag.scope_epoch
3981 ),
3982 )),
3983 }
3984}
3985
3986fn release_reserved_route_locked(
3987 inner: &mut ForwardingInner,
3988 client_key: ClientRouteKey,
3989 module_key: ModuleRouteKey,
3990) {
3991 if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
3992 inner.reserved_client.remove(&client_key);
3993 }
3994 if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
3995 inner.reserved_module.remove(&module_key);
3996 }
3997 inner.status.retain(|(key, _), _| *key != client_key);
3998}
3999
4000fn release_client_route_locked(
4001 inner: &mut ForwardingInner,
4002 client_key: ClientRouteKey,
4003 expected_epoch: u32,
4004) -> RouteRelease {
4005 let Some(route) = inner.client_to_module.get(&client_key) else {
4006 return RouteRelease::Absent;
4007 };
4008 if route.client_epoch != expected_epoch {
4009 return RouteRelease::Stale;
4010 }
4011 let route = inner
4012 .client_to_module
4013 .remove(&client_key)
4014 .expect("route checked under the same forwarding lock");
4015 route.flow.close();
4016 if let Some(scope) = &route.scope {
4017 scope.release_bind();
4018 }
4019 inner.module_to_client.remove(&ModuleRouteKey {
4020 endpoint: route.module_endpoint,
4021 channel: route.module_channel,
4022 });
4023 inner.operator_confirms.route_closed(ModuleRouteKey {
4024 endpoint: route.module_endpoint,
4025 channel: route.module_channel,
4026 });
4027 inner.status.remove(&(client_key, expected_epoch));
4028 RouteRelease::Removed(GoodbyeTarget {
4029 connection_id: route.module_endpoint.connection_id,
4030 sink: route.module_sink.clone(),
4031 negotiated_ver: route.module_negotiated_ver,
4032 channel: route.module_channel,
4033 epoch: route.module_epoch,
4034 kind: GoodbyeTargetKind::Module,
4035 module_id: Some(route.module_id.clone()),
4036 })
4037}
4038
4039fn release_module_route_locked(
4040 inner: &mut ForwardingInner,
4041 module_key: ModuleRouteKey,
4042 expected_epoch: u32,
4043) -> RouteRelease {
4044 let Some(route) = inner.module_to_client.get(&module_key) else {
4045 return RouteRelease::Absent;
4046 };
4047 if route.module_epoch != expected_epoch {
4048 return RouteRelease::Stale;
4049 }
4050 let route = inner
4051 .module_to_client
4052 .remove(&module_key)
4053 .expect("route checked under the same forwarding lock");
4054 inner.operator_confirms.route_closed(module_key);
4055 route.flow.close();
4056 if let Some(scope) = &route.scope {
4057 scope.release_bind();
4058 }
4059 let client_key = ClientRouteKey {
4060 connection_id: route.client_connection_id,
4061 channel: route.client_channel,
4062 };
4063 inner.client_to_module.remove(&client_key);
4064 inner.status.remove(&(client_key, route.client_epoch));
4065 RouteRelease::Removed(GoodbyeTarget {
4066 connection_id: route.client_connection_id,
4067 sink: route.client_sink.clone(),
4068 negotiated_ver: route.client_negotiated_ver,
4069 channel: route.client_channel,
4070 epoch: route.client_epoch,
4071 kind: GoodbyeTargetKind::Client,
4072 module_id: Some(route.module_id.clone()),
4073 })
4074}
4075
4076fn commit_route_locked(
4077 inner: &mut ForwardingInner,
4078 mut pending: PendingRouteBindRelayEntry,
4079) -> Result<Option<GoodbyeTarget>, ForwardingError> {
4080 let reservation = pending.reservation;
4081 if inner
4082 .closing_connections
4083 .contains(&reservation.client_key.connection_id)
4084 {
4085 return Err(ForwardingError::ConnectionClosing {
4086 connection_id: reservation.client_key.connection_id,
4087 });
4088 }
4089 let module_id = inner
4090 .module_id_by_endpoint
4091 .get(&reservation.module_key.endpoint)
4092 .cloned()
4093 .ok_or(ForwardingError::StaleModuleEndpoint)?;
4094 if inner
4095 .draining_endpoints
4096 .contains_key(&reservation.module_key.endpoint)
4097 {
4098 return Err(ForwardingError::ModuleReloading { module_id });
4099 }
4100 if pending.busy_refusals {
4101 let body = route_open_reply_body(
4102 &ClientControlResponse::RouteOpen {
4103 route_channel: reservation.client_key.channel,
4104 route_epoch: reservation.client_epoch,
4105 request_window: Some(pending.request_window),
4106 watch_window: pending.watch_window,
4107 },
4108 &pending.target_authority,
4109 &pending.incarnation,
4110 )?;
4111 pending.route_open_frame = Frame::build_with_version(
4112 pending.client_negotiated_ver,
4113 FrameType::Response,
4114 pending.route_open_frame.header.flags,
4115 0,
4116 0,
4117 pending.route_open_frame.header.corr,
4118 body,
4119 )
4120 .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
4121 }
4122 if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
4123 || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
4124 {
4125 return Err(ForwardingError::UnknownReservation {
4126 client_channel: reservation.client_key.channel,
4127 module_channel: reservation.module_key.channel,
4128 });
4129 }
4130 let module = inner
4131 .modules_by_id
4132 .get(&module_id)
4133 .filter(|module| module.endpoint == reservation.module_key.endpoint)
4134 .cloned()
4135 .ok_or(ForwardingError::StaleModuleEndpoint)?;
4136 let binding = Arc::new(RouteBinding {
4137 client_connection_id: reservation.client_key.connection_id,
4138 client_sink: pending.client_sink,
4139 client_negotiated_ver: pending.client_negotiated_ver,
4140 client_channel: reservation.client_key.channel,
4141 client_epoch: reservation.client_epoch,
4142 module_id,
4143 module_endpoint: reservation.module_key.endpoint,
4144 module_sink: module.sink,
4145 module_negotiated_ver: module.negotiated_ver,
4146 module_channel: reservation.module_key.channel,
4147 module_epoch: reservation.module_epoch,
4148 principal: pending.principal,
4149 harness: pending.harness,
4150 build_class: pending.build_class,
4151 operator_app: pending.operator_app,
4152 project_root: reservation.project_root.clone(),
4153 bound_at: Instant::now(),
4154 flow: Arc::new(if pending.busy_refusals {
4155 ChannelFlow::with_admission(pending.request_window, true, pending.watch_window)
4156 } else {
4157 ChannelFlow::new(pending.request_window)
4158 }),
4159 scope: pending.scope,
4160 caller_authority: pending.caller_authority,
4161 target_authority: pending.target_authority.pair().cloned(),
4162 accepts_close_reasons: pending.accepts_close_reasons,
4163 });
4164 inner
4165 .client_to_module
4166 .insert(reservation.client_key, Arc::clone(&binding));
4167 inner
4168 .module_to_client
4169 .insert(reservation.module_key, Arc::clone(&binding));
4170 let previous_published = inner
4171 .last_published_epoch
4172 .insert(reservation.client_key, reservation.client_epoch);
4173
4174 if let Some(scope) = &binding.scope {
4178 scope.record_served(&binding.module_id, module.verified_prefix_owner.as_deref());
4179 }
4180
4181 let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
4186 if client_writer_closed {
4187 let abandoned = abandoned_route_target(inner, &reservation);
4191 if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
4192 route.flow.close();
4193 }
4194 inner.module_to_client.remove(&reservation.module_key);
4195 inner
4196 .status
4197 .remove(&(reservation.client_key, reservation.client_epoch));
4198 match previous_published {
4199 Some(epoch) => {
4200 inner
4201 .last_published_epoch
4202 .insert(reservation.client_key, epoch);
4203 }
4204 None => {
4205 inner.last_published_epoch.remove(&reservation.client_key);
4206 }
4207 }
4208 let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
4209 "client egress closed during route publication".to_string(),
4210 ));
4211 return Ok(abandoned);
4212 }
4213
4214 if let Some(scope) = &binding.scope {
4215 scope.commit_presence();
4216 }
4217 let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
4218 Ok(None)
4219}
4220
4221fn target_authority_locked(inner: &ForwardingInner, module: &ModuleConnection) -> TargetAuthority {
4224 if module.verified_prefix_owner.is_some() {
4225 return TargetAuthority::Prefix;
4226 }
4227 match inner
4228 .target_authority_by_connection
4229 .get(&module.endpoint.connection_id)
4230 {
4231 Some(record) => {
4232 let key = record.key();
4233 TargetAuthority::Spawn(
4234 AuthorityPair {
4235 epoch: published_epoch_locked(inner, &key),
4236 key: key.clone(),
4237 },
4238 inner
4239 .spawn_approved
4240 .get(&record.key())
4241 .cloned()
4242 .unwrap_or_else(ApprovedDeclarations::none_approved),
4243 inner.spawn_marks.contains(&key),
4244 )
4245 }
4246 None => TargetAuthority::Unsupervised,
4247 }
4248}
4249
4250fn published_epoch_locked(inner: &ForwardingInner, key: &SpawnKey) -> u64 {
4251 inner
4252 .spawn_epochs
4253 .get(key)
4254 .copied()
4255 .unwrap_or(crate::spawn_authority::INITIAL_AUTHORITY_EPOCH)
4256}
4257
4258fn prune_dead_spawn_tombstones_locked(inner: &mut ForwardingInner, newest: &SpawnKey) {
4264 let named = inner
4265 .pending_relays
4266 .values()
4267 .flat_map(|pending| {
4268 [
4269 pending.caller_authority.as_ref(),
4270 pending.target_authority.pair(),
4271 ]
4272 })
4273 .flatten()
4274 .map(|pair| pair.key.clone())
4275 .collect::<HashSet<_>>();
4276 let expired = inner
4277 .dead_spawns
4278 .iter()
4279 .filter(|dead| {
4280 dead.module_id == newest.module_id && *dead != newest && !named.contains(*dead)
4281 })
4282 .cloned()
4283 .collect::<Vec<_>>();
4284 for dead in expired {
4285 inner.dead_spawns.remove(&dead);
4286 inner.spawn_epochs.remove(&dead);
4287 }
4288}
4289
4290fn authority_stale_locked(inner: &ForwardingInner, pair: Option<&AuthorityPair>) -> bool {
4291 pair.is_some_and(|pair| published_epoch_locked(inner, &pair.key) > pair.epoch)
4292}
4293
4294fn spawn_connection_dead(inner: &ForwardingInner, connection: ConnectionId) -> bool {
4295 inner
4296 .spawn_connections
4297 .get(&connection)
4298 .is_some_and(|spawn| spawn.upgrade().is_none_or(|spawn| !spawn.launch_ready()))
4299}
4300
4301fn module_connection_for_endpoint_locked(
4309 inner: &ForwardingInner,
4310 endpoint: ModuleEndpointId,
4311) -> Option<&ModuleConnection> {
4312 let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
4313 inner
4314 .modules_by_id
4315 .get(module_id)
4316 .filter(|module| module.endpoint == endpoint)
4317 .or_else(|| {
4318 inner
4319 .candidates_by_id
4320 .get(module_id)
4321 .filter(|module| module.endpoint == endpoint)
4322 })
4323 .or_else(|| inner.superseded_endpoints.get(&endpoint))
4324}
4325
4326fn abandoned_route_target(
4327 inner: &ForwardingInner,
4328 reservation: &RouteReservation,
4329) -> Option<GoodbyeTarget> {
4330 let module_id = inner
4331 .module_id_by_endpoint
4332 .get(&reservation.module_key.endpoint)?;
4333 let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
4334 (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
4335 connection_id: module.endpoint.connection_id,
4336 sink: module.sink.clone(),
4337 negotiated_ver: module.negotiated_ver,
4338 channel: reservation.module_key.channel,
4339 epoch: reservation.module_epoch,
4340 kind: GoodbyeTargetKind::Module,
4341 module_id: Some(module_id.clone()),
4342 })
4343}
4344
4345fn enqueue_hello_ack_locked(
4350 sink: &FrameSink,
4351 connection_id: ConnectionId,
4352 hello_ack: Option<Frame>,
4353) -> Result<(), ForwardingError> {
4354 let Some(hello_ack) = hello_ack else {
4355 return Ok(());
4356 };
4357 sink.try_send(hello_ack)
4358 .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
4359}
4360
4361fn remove_module_connection_locked(
4362 inner: &mut ForwardingInner,
4363 endpoint: ModuleEndpointId,
4364) -> ConnectionCleanup {
4365 inner.operator_confirms.module_closed(endpoint);
4368 inner.draining_endpoints.remove(&endpoint);
4369 inner.daemon_drain_results.remove(&endpoint);
4370 let module_id = inner.module_id_by_endpoint.remove(&endpoint);
4371 if let Some(module_id) = module_id.as_ref() {
4372 if inner
4373 .modules_by_id
4374 .get(module_id)
4375 .is_some_and(|module| module.endpoint == endpoint)
4376 {
4377 inner.modules_by_id.remove(module_id);
4378 }
4379 if inner
4380 .candidates_by_id
4381 .get(module_id)
4382 .is_some_and(|module| module.endpoint == endpoint)
4383 {
4384 inner.candidates_by_id.remove(module_id);
4385 }
4386 }
4387 inner.superseded_endpoints.remove(&endpoint);
4388 inner.endpoint_by_connection.remove(&endpoint.connection_id);
4389 inner.next_module_channel.remove(&endpoint);
4390 inner.next_control_corr.remove(&endpoint);
4391 inner
4392 .health_probe_tombstones
4393 .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
4394 inner
4395 .module_slot_epochs
4396 .retain(|key, _| key.endpoint != endpoint);
4397 let reserved_module_keys: Vec<ModuleRouteKey> = inner
4398 .reserved_module
4399 .keys()
4400 .filter(|module_key| module_key.endpoint == endpoint)
4401 .copied()
4402 .collect();
4403 for module_key in reserved_module_keys {
4404 if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
4405 release_reserved_route_locked(inner, client_key, module_key);
4406 }
4407 }
4408
4409 let pending_keys: Vec<_> = inner
4410 .pending_relays
4411 .keys()
4412 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
4413 .copied()
4414 .collect();
4415 let pending: Vec<_> = pending_keys
4416 .into_iter()
4417 .filter_map(|key| {
4418 inner
4419 .pending_relays
4420 .remove(&key)
4421 .map(|pending| (key, pending))
4422 })
4423 .collect();
4424 let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
4425 for (key, pending) in pending {
4426 if !pending.relay_enqueued {
4430 inner.relays_released_before_enqueue.insert(key);
4431 }
4432 let module_label = module_id.as_deref().unwrap_or("unknown");
4433 let _ = pending
4434 .sender
4435 .send(RouteBindRelayOutcome::ModuleGone(format!(
4436 "module '{module_label}' connection closed during route.bind relay"
4437 )));
4438 }
4439
4440 let pending_control_keys: Vec<_> = inner
4441 .pending_control_rpcs
4442 .keys()
4443 .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
4444 .copied()
4445 .collect();
4446 let pending_control: Vec<_> = pending_control_keys
4447 .into_iter()
4448 .filter_map(|key| inner.pending_control_rpcs.remove(&key))
4449 .collect();
4450 for pending in pending_control {
4451 let module_label = module_id.as_deref().unwrap_or("unknown");
4452 let _ = pending
4453 .sender
4454 .send(ModuleControlRpcOutcome::ModuleGone(format!(
4455 "module '{module_label}' connection closed during module-control RPC"
4456 )));
4457 }
4458
4459 let module_routes = inner
4460 .module_to_client
4461 .iter()
4462 .filter(|(module_key, _)| module_key.endpoint == endpoint)
4463 .map(|(module_key, route)| (*module_key, route.module_epoch))
4464 .collect::<Vec<_>>();
4465 let mut released = Vec::with_capacity(module_routes.len());
4466 for (module_key, epoch) in module_routes {
4467 if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
4468 {
4469 released.push(target);
4470 }
4471 }
4472 ConnectionCleanup {
4473 released,
4474 abandoned_relays,
4475 }
4476}
4477
4478#[derive(Debug, Clone, Copy)]
4479struct RequestCredit {
4480 subscription: bool,
4481 watch_credit: bool,
4482 excluded_from_drain: bool,
4483}
4484
4485#[derive(Debug, Default)]
4486struct CreditLedger {
4487 by_corr: HashMap<u64, Vec<RequestCredit>>,
4488}
4489
4490impl CreditLedger {
4491 fn acquire(&mut self, corr: u64, subscription: bool, watch_credit: bool) {
4492 self.by_corr.entry(corr).or_default().push(RequestCredit {
4493 subscription,
4494 watch_credit,
4495 excluded_from_drain: false,
4496 });
4497 }
4498
4499 fn release(&mut self, corr: u64) -> Option<RequestCredit> {
4500 let credits = self.by_corr.get_mut(&corr)?;
4501 let released = credits.pop();
4502 if credits.is_empty() {
4503 self.by_corr.remove(&corr);
4504 }
4505 released
4506 }
4507
4508 fn capture_subscription_exclusions(&mut self) -> u32 {
4509 let mut excluded = 0u32;
4510 for credit in self.by_corr.values_mut().flatten() {
4511 if credit.subscription && !credit.excluded_from_drain {
4512 credit.excluded_from_drain = true;
4513 excluded = excluded.saturating_add(1);
4514 }
4515 }
4516 excluded
4517 }
4518
4519 fn in_flight(&self) -> usize {
4520 self.by_corr.values().map(Vec::len).sum()
4521 }
4522
4523 fn drain_in_flight(&self) -> usize {
4524 self.by_corr
4525 .values()
4526 .flatten()
4527 .filter(|credit| !credit.excluded_from_drain)
4528 .count()
4529 }
4530
4531 fn drain_held_corrs(&self) -> Vec<u64> {
4534 let mut corrs = self
4535 .by_corr
4536 .iter()
4537 .flat_map(|(corr, credits)| {
4538 credits
4539 .iter()
4540 .filter(|credit| !credit.excluded_from_drain)
4541 .map(move |_| *corr)
4542 })
4543 .collect::<Vec<_>>();
4544 corrs.sort_unstable();
4545 corrs
4546 }
4547}
4548
4549#[derive(Debug, Default)]
4550struct ChannelFlowState {
4551 closed: bool,
4552 credits: CreditLedger,
4553 last_busy_warn: Option<tokio::time::Instant>,
4554}
4555
4556#[derive(Debug)]
4558pub(crate) struct ChannelFlow {
4559 sem: Semaphore,
4560 window: usize,
4561 busy_refusals: bool,
4562 watch_sem: Option<Semaphore>,
4563 watch_window: Option<usize>,
4564 state: Mutex<ChannelFlowState>,
4565}
4566
4567impl ChannelFlow {
4568 pub(crate) fn new(window: usize) -> Self {
4569 Self::with_admission(window, false, None)
4570 }
4571
4572 fn with_admission(window: usize, busy_refusals: bool, watch_window: Option<usize>) -> Self {
4573 debug_assert!(window > 0, "flow-control window must be non-zero");
4574 Self {
4575 sem: Semaphore::new(window),
4576 window,
4577 busy_refusals,
4578 watch_sem: watch_window.map(Semaphore::new),
4579 watch_window,
4580 state: Mutex::new(ChannelFlowState::default()),
4581 }
4582 }
4583
4584 pub(crate) fn busy_refusals(&self) -> bool {
4585 self.busy_refusals
4586 }
4587
4588 fn try_acquire_tagged(&self, corr: u64, subscription: bool) -> RequestAdmission {
4589 let mut state = self
4590 .state
4591 .lock()
4592 .unwrap_or_else(|poisoned| poisoned.into_inner());
4593 if state.closed {
4594 return RequestAdmission::Stale;
4595 }
4596 let busy = |state: &ChannelFlowState| {
4597 RequestAdmission::Busy(subc_protocol::RouteBusyDetail {
4598 reason: "no_credit".into(),
4599 request_in_flight: state.credits.in_flight(),
4600 watch_in_flight: state
4601 .credits
4602 .by_corr
4603 .values()
4604 .flatten()
4605 .filter(|credit| credit.subscription)
4606 .count(),
4607 request_window: self.window,
4608 watch_window: self.watch_window,
4609 })
4610 };
4611 let Ok(total) = self.sem.try_acquire() else {
4612 return busy(&state);
4613 };
4614 let watch = if subscription {
4617 if let Some(sem) = &self.watch_sem {
4618 match sem.try_acquire() {
4619 Ok(permit) => Some(permit),
4620 Err(_) => return busy(&state),
4621 }
4622 } else {
4623 None
4624 }
4625 } else {
4626 None
4627 };
4628 state.credits.acquire(corr, subscription, watch.is_some());
4629 total.forget();
4630 if let Some(watch) = watch {
4631 watch.forget();
4632 }
4633 RequestAdmission::Admitted
4634 }
4635
4636 pub(crate) fn should_warn_busy(&self) -> bool {
4637 let mut state = self
4638 .state
4639 .lock()
4640 .unwrap_or_else(|poisoned| poisoned.into_inner());
4641 let now = tokio::time::Instant::now();
4642 if state
4643 .last_busy_warn
4644 .is_some_and(|last| now.duration_since(last) < std::time::Duration::from_secs(60))
4645 {
4646 return false;
4647 }
4648 state.last_busy_warn = Some(now);
4649 true
4650 }
4651
4652 #[cfg(test)]
4653 pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
4654 self.acquire_tagged(0, false).await
4655 }
4656
4657 pub(crate) async fn acquire_tagged(
4658 &self,
4659 corr: u64,
4660 subscription: bool,
4661 ) -> Result<(), ChannelFlowClosed> {
4662 let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
4663 let mut state = self
4664 .state
4665 .lock()
4666 .unwrap_or_else(|poisoned| poisoned.into_inner());
4667 if state.closed {
4668 return Err(ChannelFlowClosed);
4669 }
4670 state.credits.acquire(corr, subscription, false);
4671 permit.forget();
4672 Ok(())
4673 }
4674
4675 #[cfg(test)]
4676 pub(crate) fn release(&self) {
4677 self.release_corr(0);
4678 }
4679
4680 pub(crate) fn release_corr(&self, corr: u64) {
4681 let mut state = self
4682 .state
4683 .lock()
4684 .unwrap_or_else(|poisoned| poisoned.into_inner());
4685 let Some(released) = state.credits.release(corr) else {
4686 warn!(
4690 window = self.window,
4691 available = self.sem.available_permits(),
4692 "flow-control over-release ignored"
4693 );
4694 return;
4695 };
4696 if !self.sem.is_closed() {
4697 self.sem.add_permits(1);
4698 }
4699 if released.watch_credit {
4700 if let Some(watch_sem) = self.watch_sem.as_ref().filter(|sem| !sem.is_closed()) {
4701 watch_sem.add_permits(1);
4702 }
4703 }
4704 }
4705
4706 #[cfg(test)]
4707 pub(crate) fn in_flight(&self) -> usize {
4708 self.state
4709 .lock()
4710 .unwrap_or_else(|poisoned| poisoned.into_inner())
4711 .credits
4712 .in_flight()
4713 }
4714
4715 pub(crate) fn drain_in_flight(&self) -> usize {
4716 self.state
4717 .lock()
4718 .unwrap_or_else(|poisoned| poisoned.into_inner())
4719 .credits
4720 .drain_in_flight()
4721 }
4722
4723 pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
4724 self.state
4725 .lock()
4726 .unwrap_or_else(|poisoned| poisoned.into_inner())
4727 .credits
4728 .drain_held_corrs()
4729 }
4730
4731 #[cfg(test)]
4732 pub(crate) fn available_permits(&self) -> usize {
4733 self.sem.available_permits()
4734 }
4735
4736 pub(crate) fn begin_drain(&self) -> u32 {
4737 let mut state = self
4738 .state
4739 .lock()
4740 .unwrap_or_else(|poisoned| poisoned.into_inner());
4741 state.closed = true;
4742 self.sem.close();
4743 if let Some(sem) = &self.watch_sem {
4744 sem.close();
4745 }
4746 state.credits.capture_subscription_exclusions()
4747 }
4748
4749 pub(crate) fn close(&self) {
4750 let mut state = self
4751 .state
4752 .lock()
4753 .unwrap_or_else(|poisoned| poisoned.into_inner());
4754 state.closed = true;
4755 self.sem.close();
4756 if let Some(sem) = &self.watch_sem {
4757 sem.close();
4758 }
4759 }
4760}
4761
4762#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4763pub(crate) struct ChannelFlowClosed;
4764
4765#[derive(Debug, Clone, PartialEq, Eq)]
4766pub(crate) enum RequestAdmission {
4767 Admitted,
4768 Busy(subc_protocol::RouteBusyDetail),
4769 ModuleReloading,
4770 Stale,
4771}
4772
4773impl fmt::Display for ChannelFlowClosed {
4774 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4775 write!(f, "flow-control window closed")
4776 }
4777}
4778
4779impl Error for ChannelFlowClosed {}
4780
4781#[cfg(test)]
4782#[path = "route_busy_flow_tests.rs"]
4783mod route_busy_flow_tests;
4784
4785fn window_for(concurrency: &Concurrency) -> usize {
4786 match concurrency {
4787 Concurrency::Serial => 1,
4788 Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
4789 Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
4790 }
4791}
4792
4793#[derive(Debug, Clone, PartialEq, Eq)]
4794pub enum ForwardingError {
4795 ConnectionRoleConflict {
4798 connection_id: ConnectionId,
4799 },
4800 NoModuleConnection,
4801 ModuleReloading {
4802 module_id: String,
4803 },
4804 StaleModuleEndpoint,
4805 UnknownReservation {
4806 client_channel: u16,
4807 module_channel: u16,
4808 },
4809 ClientRouteChannelExhausted {
4810 connection_id: ConnectionId,
4811 },
4812 ModuleRouteChannelExhausted {
4813 endpoint: ModuleEndpointId,
4814 },
4815 RelayCorrelationExhausted,
4816 ConnectionClosing {
4817 connection_id: ConnectionId,
4818 },
4819 ClientEgressClosed {
4820 connection_id: ConnectionId,
4821 },
4822 RouteOpenBuild(String),
4823 CandidateSlotOccupied {
4825 module_id: String,
4826 },
4827 ModuleEgressUnavailable {
4830 connection_id: ConnectionId,
4831 },
4832 Poisoned,
4833}
4834
4835impl fmt::Display for ForwardingError {
4836 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4837 match self {
4838 Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
4839 Self::NoModuleConnection => write!(f, "no module connection is registered"),
4840 Self::ModuleReloading { module_id } => {
4841 write!(f, "module_id '{module_id}' is reloading")
4842 }
4843 Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
4844 Self::UnknownReservation {
4845 client_channel,
4846 module_channel,
4847 } => write!(
4848 f,
4849 "route reservation client channel {client_channel} / module channel {module_channel} was not found"
4850 ),
4851 Self::ClientRouteChannelExhausted { connection_id } => write!(
4852 f,
4853 "no client route channels are available for connection {}",
4854 connection_id.get()
4855 ),
4856 Self::ModuleRouteChannelExhausted { endpoint } => write!(
4857 f,
4858 "no module route channels are available for endpoint generation {} on connection {}",
4859 endpoint.generation,
4860 endpoint.connection_id.get()
4861 ),
4862 Self::RelayCorrelationExhausted => {
4863 write!(f, "module control correlation ids are exhausted")
4864 }
4865 Self::ConnectionClosing { connection_id } => write!(
4866 f,
4867 "connection {} is closing and cannot accept route allocation",
4868 connection_id.get()
4869 ),
4870 Self::ClientEgressClosed { connection_id } => write!(
4871 f,
4872 "client connection {} egress is closed",
4873 connection_id.get()
4874 ),
4875 Self::RouteOpenBuild(message) => {
4876 write!(f, "failed to prebuild route.open response: {message}")
4877 }
4878 Self::CandidateSlotOccupied { module_id } => write!(
4879 f,
4880 "module_id '{module_id}' already has a swap candidate registered"
4881 ),
4882 Self::ModuleEgressUnavailable { connection_id } => write!(
4883 f,
4884 "module connection {} egress is unavailable; HELLO_ACK could not be queued",
4885 connection_id.get()
4886 ),
4887 Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
4888 }
4889 }
4890}
4891
4892impl Error for ForwardingError {}
4893
4894#[cfg(test)]
4895mod tests {
4896 use std::time::Duration;
4897
4898 use super::*;
4899 use tokio::sync::mpsc;
4900
4901 #[test]
4902 fn ordinary_long_running_request_is_not_excluded_from_drain() {
4903 let mut ledger = CreditLedger::default();
4904 ledger.acquire(1, false, false);
4905
4906 assert_eq!(ledger.capture_subscription_exclusions(), 0);
4907 assert_eq!(ledger.drain_in_flight(), 1);
4908 }
4909
4910 #[test]
4911 fn bit_set_subscription_is_excluded_and_counted() {
4912 let mut ledger = CreditLedger::default();
4913 ledger.acquire(1, true, false);
4914
4915 assert_eq!(ledger.capture_subscription_exclusions(), 1);
4916 assert_eq!(ledger.drain_in_flight(), 0);
4917 }
4918
4919 #[test]
4920 fn subscription_opened_after_drain_snapshot_is_not_excluded() {
4921 let mut ledger = CreditLedger::default();
4922 ledger.acquire(1, true, false);
4923 assert_eq!(ledger.capture_subscription_exclusions(), 1);
4924
4925 ledger.acquire(2, true, false);
4926
4927 assert_eq!(ledger.drain_in_flight(), 1);
4928 }
4929
4930 #[test]
4931 fn drain_with_no_subscriptions_reports_zero_excluded() {
4932 let mut ledger = CreditLedger::default();
4933 assert_eq!(ledger.capture_subscription_exclusions(), 0);
4934 }
4935
4936 fn test_hello_ack(corr: u64) -> Frame {
4937 Frame::build(
4938 FrameType::HelloAck,
4939 Flags::new(false, Priority::Passive, false),
4940 0,
4941 0,
4942 corr,
4943 Vec::new(),
4944 )
4945 .unwrap()
4946 }
4947
4948 #[test]
4952 fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
4953 let forwarding = ForwardingTable::default();
4954
4955 let (closed_tx, closed_rx) = mpsc::channel(8);
4956 drop(closed_rx);
4957 let closed = ConnectionId::new(1);
4958 assert_eq!(
4959 forwarding.register_module_connection_acked(
4960 closed,
4961 "closed".to_string(),
4962 2,
4963 Concurrency::ModuleManaged,
4964 FrameSink::new(closed_tx),
4965 ModuleHelloAck {
4966 frame: test_hello_ack(1),
4967 verified_prefix_owner: None
4968 },
4969 ),
4970 Err(ForwardingError::ModuleEgressUnavailable {
4971 connection_id: closed
4972 })
4973 );
4974
4975 let (full_tx, _full_rx) = mpsc::channel(1);
4976 let full_sink = FrameSink::new(full_tx);
4977 full_sink.try_send(test_hello_ack(99)).unwrap();
4978 let full = ConnectionId::new(2);
4979 assert_eq!(
4980 forwarding.register_module_connection_acked(
4981 full,
4982 "full".to_string(),
4983 2,
4984 Concurrency::ModuleManaged,
4985 full_sink.clone(),
4986 ModuleHelloAck {
4987 frame: test_hello_ack(2),
4988 verified_prefix_owner: None
4989 },
4990 ),
4991 Err(ForwardingError::ModuleEgressUnavailable {
4992 connection_id: full
4993 })
4994 );
4995 assert_eq!(
4996 forwarding.register_candidate_module_connection_acked(
4997 full,
4998 "full".to_string(),
4999 2,
5000 Concurrency::ModuleManaged,
5001 full_sink,
5002 test_hello_ack(3),
5003 ),
5004 Err(ForwardingError::ModuleEgressUnavailable {
5005 connection_id: full
5006 })
5007 );
5008
5009 for (connection, module_id) in [(closed, "closed"), (full, "full")] {
5010 assert_eq!(
5011 forwarding
5012 .module_endpoint_for_connection(connection)
5013 .unwrap(),
5014 None
5015 );
5016 let (client_tx, _client_rx) = mpsc::channel(8);
5017 assert_eq!(
5018 forwarding
5019 .begin_route_bind_relay_for_test(
5020 ConnectionId::new(50),
5021 FrameSink::new(client_tx),
5022 1,
5023 module_id,
5024 )
5025 .err(),
5026 Some(ForwardingError::NoModuleConnection)
5027 );
5028 }
5029 assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
5030 }
5031
5032 #[test]
5035 fn acked_registration_queues_the_hello_ack_first() {
5036 let forwarding = ForwardingTable::default();
5037 let (active_tx, mut active_rx) = mpsc::channel(8);
5038 forwarding
5039 .register_module_connection_acked(
5040 ConnectionId::new(1),
5041 "acked".to_string(),
5042 2,
5043 Concurrency::ModuleManaged,
5044 FrameSink::new(active_tx),
5045 ModuleHelloAck {
5046 frame: test_hello_ack(11),
5047 verified_prefix_owner: None,
5048 },
5049 )
5050 .unwrap();
5051 let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
5052 forwarding
5053 .register_candidate_module_connection_acked(
5054 ConnectionId::new(2),
5055 "acked".to_string(),
5056 2,
5057 Concurrency::ModuleManaged,
5058 FrameSink::new(candidate_tx),
5059 test_hello_ack(12),
5060 )
5061 .unwrap();
5062
5063 let active_first = active_rx.try_recv().unwrap().frame;
5064 assert_eq!(active_first.header.ty, FrameType::HelloAck);
5065 assert_eq!(active_first.header.corr, 11);
5066 let candidate_first = candidate_rx.try_recv().unwrap().frame;
5067 assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
5068 assert_eq!(candidate_first.header.corr, 12);
5069 }
5070
5071 #[test]
5072 fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
5073 let forwarding = ForwardingTable::default();
5074 let module_connection = ConnectionId::new(10);
5075 let exhausted_client = ConnectionId::new(20);
5076 let second_client = ConnectionId::new(30);
5077 let (module_tx, _module_rx) = mpsc::channel(1);
5078 let endpoint = forwarding
5079 .register_module_connection(
5080 module_connection,
5081 "route-limit-provider".to_string(),
5082 1,
5083 Concurrency::ModuleManaged,
5084 FrameSink::new(module_tx),
5085 )
5086 .unwrap();
5087
5088 {
5089 let mut inner = forwarding.inner.write().unwrap();
5090 for channel in 1..=u16::MAX {
5091 inner.reserved_client.insert(
5092 ClientRouteKey {
5093 connection_id: exhausted_client,
5094 channel,
5095 },
5096 ModuleRouteKey {
5097 endpoint,
5098 channel: 1,
5099 },
5100 );
5101 }
5102 }
5103
5104 let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
5105 let err = forwarding
5106 .begin_route_bind_relay_for_test(
5107 exhausted_client,
5108 FrameSink::new(exhausted_tx),
5109 1,
5110 "route-limit-provider",
5111 )
5112 .unwrap_err();
5113 assert!(matches!(
5114 err,
5115 ForwardingError::ClientRouteChannelExhausted { connection_id }
5116 if connection_id == exhausted_client
5117 ));
5118
5119 let (second_tx, _second_rx) = mpsc::channel(1);
5120 let pending = forwarding
5121 .begin_route_bind_relay_for_test(
5122 second_client,
5123 FrameSink::new(second_tx),
5124 2,
5125 "route-limit-provider",
5126 )
5127 .unwrap();
5128 assert_eq!(pending.client_channel, 1);
5129 }
5130
5131 #[test]
5132 fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
5133 let forwarding = ForwardingTable::default();
5134 let module_connection = ConnectionId::new(40);
5135 let client = ConnectionId::new(50);
5136 let (module_tx, _module_rx) = mpsc::channel(1);
5137 forwarding
5138 .register_module_connection(
5139 module_connection,
5140 "slot-reuse-provider".to_string(),
5141 1,
5142 Concurrency::ModuleManaged,
5143 FrameSink::new(module_tx),
5144 )
5145 .unwrap();
5146
5147 let (client_tx, _client_rx) = mpsc::channel(1);
5148 let client_sink = FrameSink::new(client_tx);
5149 let mut wrapped_channel = None;
5150 for index in 0..=usize::from(u16::MAX) {
5151 let pending = forwarding
5152 .begin_route_bind_relay_for_test(
5153 client,
5154 client_sink.clone(),
5155 index as u64 + 1,
5156 "slot-reuse-provider",
5157 )
5158 .unwrap();
5159 if index == usize::from(u16::MAX) {
5160 wrapped_channel = Some(pending.module_channel);
5161 }
5162 forwarding
5163 .abort_pending_relay(
5164 pending.endpoint,
5165 pending.corr,
5166 RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
5167 )
5168 .unwrap();
5169 }
5170
5171 assert_eq!(wrapped_channel, Some(1));
5172 }
5173
5174 #[test]
5175 fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
5176 let forwarding = ForwardingTable::default();
5177 let client = ConnectionId::new(60);
5178 forwarding
5179 .inner
5180 .write()
5181 .unwrap()
5182 .next_client_channel
5183 .insert(client, 41);
5184
5185 let released = forwarding.cleanup_connection(client).unwrap();
5186
5187 assert!(released.is_empty());
5188 assert!(!forwarding
5189 .inner
5190 .read()
5191 .unwrap()
5192 .next_client_channel
5193 .contains_key(&client));
5194 }
5195
5196 #[test]
5197 fn stale_module_cleanup_preserves_fast_reconnect_successor() {
5198 let forwarding = ForwardingTable::default();
5199 let module_id = "fast-reconnect-provider";
5200 let first_connection = ConnectionId::new(70);
5201 let second_connection = ConnectionId::new(80);
5202 let (first_tx, _first_rx) = mpsc::channel(1);
5203 let first_endpoint = forwarding
5204 .register_module_connection(
5205 first_connection,
5206 module_id.to_string(),
5207 1,
5208 Concurrency::ModuleManaged,
5209 FrameSink::new(first_tx),
5210 )
5211 .unwrap();
5212 let (second_tx, _second_rx) = mpsc::channel(1);
5213 let second_endpoint = forwarding
5214 .register_module_connection(
5215 second_connection,
5216 module_id.to_string(),
5217 1,
5218 Concurrency::ModuleManaged,
5219 FrameSink::new(second_tx),
5220 )
5221 .unwrap();
5222 assert_ne!(first_endpoint, second_endpoint);
5223
5224 let released = forwarding.cleanup_connection(first_connection).unwrap();
5225
5226 assert!(released.is_empty());
5227 assert_eq!(
5228 forwarding
5229 .inner
5230 .read()
5231 .unwrap()
5232 .modules_by_id
5233 .get(module_id)
5234 .map(|module| module.endpoint),
5235 Some(second_endpoint)
5236 );
5237 assert!(forwarding.has_live_module_connection(module_id).unwrap());
5238 let control_rpc = forwarding
5239 .begin_module_control_rpc_for(
5240 module_id,
5241 "health.check",
5242 Instant::now() + Duration::from_secs(1),
5243 )
5244 .unwrap();
5245 assert_eq!(control_rpc.endpoint, second_endpoint);
5246 }
5247
5248 fn route_fixture(
5249 module_id: &str,
5250 ) -> (
5251 ForwardingTable,
5252 ConnectionId,
5253 ModuleEndpointId,
5254 ConnectionId,
5255 FrameSink,
5256 mpsc::Receiver<crate::router::OutboundFrame>,
5257 ) {
5258 let forwarding = ForwardingTable::default();
5259 let module_connection = ConnectionId::new(100);
5260 let client_connection = ConnectionId::new(200);
5261 let (module_tx, _module_rx) = mpsc::channel(8);
5262 let endpoint = forwarding
5263 .register_module_connection(
5264 module_connection,
5265 module_id.to_string(),
5266 2,
5267 Concurrency::ModuleManaged,
5268 FrameSink::new(module_tx),
5269 )
5270 .unwrap();
5271 let (client_tx, client_rx) = mpsc::channel(8);
5272 (
5273 forwarding,
5274 module_connection,
5275 endpoint,
5276 client_connection,
5277 FrameSink::new(client_tx),
5278 client_rx,
5279 )
5280 }
5281
5282 #[test]
5283 #[cfg(unix)]
5284 fn daemon_drain_gates_current_and_racing_provider_registrations() {
5285 let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
5286 assert_eq!(
5287 forwarding.begin_daemon_drain().unwrap(),
5288 [("provider".to_string(), endpoint)]
5289 );
5290 assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
5291 assert!(matches!(
5292 forwarding.register_module_connection(
5293 ConnectionId::new(300),
5294 "late-provider".into(),
5295 2,
5296 Concurrency::ModuleManaged,
5297 sink,
5298 ),
5299 Err(ForwardingError::ConnectionClosing { .. })
5300 ));
5301 }
5302
5303 #[test]
5304 #[cfg(unix)]
5305 fn daemon_drain_results_are_endpoint_fenced_and_removed_on_cleanup() {
5306 let (forwarding, connection, endpoint, _, _, _) = route_fixture("provider");
5307 assert_eq!(
5308 forwarding
5309 .daemon_drain_result_for_connection(connection)
5310 .unwrap(),
5311 None
5312 );
5313 forwarding
5314 .record_daemon_drain_result(endpoint, true)
5315 .unwrap();
5316 forwarding.begin_daemon_drain().unwrap();
5317 assert_eq!(
5318 forwarding
5319 .daemon_drain_result_for_connection(connection)
5320 .unwrap(),
5321 Some(false)
5322 );
5323 let replacement = ModuleEndpointId {
5324 generation: endpoint.generation + 1,
5325 ..endpoint
5326 };
5327 forwarding
5328 .record_daemon_drain_result(replacement, true)
5329 .unwrap();
5330 assert_eq!(
5331 forwarding
5332 .daemon_drain_result_for_connection(connection)
5333 .unwrap(),
5334 Some(false)
5335 );
5336 forwarding
5337 .record_daemon_drain_result(endpoint, true)
5338 .unwrap();
5339 assert_eq!(
5340 forwarding
5341 .daemon_drain_result_for_connection(connection)
5342 .unwrap(),
5343 Some(true)
5344 );
5345 assert_eq!(
5346 forwarding
5347 .daemon_drain_result_for_connection(ConnectionId::new(300))
5348 .unwrap(),
5349 Some(false)
5350 );
5351 forwarding.cleanup_connection(connection).unwrap();
5352 forwarding
5353 .record_daemon_drain_result(endpoint, true)
5354 .unwrap();
5355 assert_eq!(
5356 forwarding
5357 .daemon_drain_result_for_connection(connection)
5358 .unwrap(),
5359 Some(false)
5360 );
5361 assert!(forwarding
5362 .read_inner()
5363 .unwrap()
5364 .daemon_drain_results
5365 .is_empty());
5366 }
5367
5368 #[test]
5369 #[cfg(unix)]
5370 fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
5371 let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
5372 route_fixture("provider");
5373 let mut pending =
5374 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
5375 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
5376 forwarding.begin_daemon_drain().unwrap();
5377 let completion = forwarding
5378 .complete_pending_relay(
5379 module_connection,
5380 pending.corr,
5381 RouteBindRelayOutcome::Accepted,
5382 )
5383 .expect("draining admission is not a fatal module error");
5384 assert!(completion.settled);
5385 let abandoned = completion
5386 .abandoned
5387 .expect("late accepted bind needs a route GOODBYE");
5388 assert_eq!(abandoned.channel, pending.module_channel);
5389 assert!(matches!(pending.receiver.try_recv().unwrap(),
5390 RouteBindRelayOutcome::DaemonRejected(error) if error.code == "module_reloading"));
5391 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5392 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5393 assert!(
5394 rx.try_recv().is_err(),
5395 "no route.open may be published during drain"
5396 );
5397 assert_eq!(
5398 forwarding
5399 .module_endpoint_for_connection(module_connection)
5400 .unwrap(),
5401 Some(endpoint)
5402 );
5403 assert!(
5404 !forwarding
5405 .complete_pending_relay(
5406 module_connection,
5407 pending.corr,
5408 RouteBindRelayOutcome::Accepted,
5409 )
5410 .unwrap()
5411 .settled
5412 );
5413 }
5414
5415 fn test_ping(corr: u64) -> Frame {
5416 Frame::build(
5417 FrameType::Ping,
5418 Flags::new(false, Priority::Passive, false),
5419 0,
5420 0,
5421 corr,
5422 Vec::new(),
5423 )
5424 .unwrap()
5425 }
5426
5427 fn begin_test_route(
5428 forwarding: &ForwardingTable,
5429 client_connection: ConnectionId,
5430 client_sink: FrameSink,
5431 corr: u64,
5432 module_id: &str,
5433 ) -> PendingRouteBindRelay {
5434 forwarding
5435 .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
5436 .unwrap()
5437 }
5438
5439 #[test]
5440 fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
5441 for candidate in [false, true] {
5442 for register_first in [false, true] {
5443 let (forwarding, module_connection, _, client_connection, sink, _rx) =
5444 route_fixture("target");
5445 let register = || {
5446 if candidate {
5447 forwarding.register_candidate_module_connection(
5448 client_connection,
5449 "source".into(),
5450 2,
5451 Concurrency::ModuleManaged,
5452 sink.clone(),
5453 )
5454 } else {
5455 forwarding.register_module_connection(
5456 client_connection,
5457 "source".into(),
5458 2,
5459 Concurrency::ModuleManaged,
5460 sink.clone(),
5461 )
5462 }
5463 };
5464 if register_first {
5465 register().unwrap();
5466 assert!(
5467 forwarding
5468 .begin_route_bind_relay_for_test(
5469 client_connection,
5470 sink.clone(),
5471 1,
5472 "target",
5473 )
5474 .is_err(),
5475 "a deferred route.open cannot reserve after HELLO"
5476 );
5477 assert!(forwarding
5478 .cleanup_connection(client_connection)
5479 .unwrap()
5480 .is_empty());
5481 } else {
5482 let pending =
5483 begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
5484 assert!(
5485 register().is_err(),
5486 "HELLO cannot register after a route reservation"
5487 );
5488 forwarding
5489 .complete_pending_relay(
5490 module_connection,
5491 pending.corr,
5492 RouteBindRelayOutcome::Accepted,
5493 )
5494 .unwrap();
5495 assert_eq!(
5496 forwarding
5497 .cleanup_connection(client_connection)
5498 .unwrap()
5499 .len(),
5500 1
5501 );
5502 }
5503 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5504 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5505 }
5506 }
5507 }
5508
5509 #[tokio::test]
5515 async fn pending_route_open_completes_behind_queued_data_frames() {
5516 assert_eq!(
5517 crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
5518 8,
5519 "the per-connection pending route.open limit is its own constant"
5520 );
5521 let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
5522 route_fixture("open-behind-data");
5523 let (sink, mut client_rx) = crate::server::connection_egress();
5524 const DATA_FRAMES: usize = 1_000;
5525 let data = |corr: u64| {
5526 Frame::build(
5527 FrameType::StreamData,
5528 Flags::new(false, Priority::Interactive, false),
5529 9,
5530 1,
5531 corr,
5532 vec![b'x'; 200],
5533 )
5534 .unwrap()
5535 };
5536 for corr in 0..DATA_FRAMES as u64 {
5537 sink.try_send(data(corr)).unwrap();
5538 }
5539 let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
5540 assert_eq!(sink.backlog().queued_bytes, data_bytes);
5541
5542 let pending = tokio::time::timeout(
5543 Duration::from_secs(5),
5544 forwarding.begin_route_bind_relay_for(
5545 client,
5546 sink.clone(),
5547 subc_protocol::PROTOCOL_VERSION,
5548 4_242,
5549 "open-behind-data",
5550 Principal::Direct,
5551 None,
5552 None,
5553 "fixture".into(),
5554 None,
5555 None,
5556 RouteAuthority::default(),
5557 Instant::now() + Duration::from_secs(60),
5558 false,
5559 None,
5560 ),
5561 )
5562 .await
5563 .expect("reserving the route.open slot must not wait behind data frames")
5564 .unwrap();
5565 forwarding
5566 .complete_pending_relay(
5567 module_connection,
5568 pending.corr,
5569 RouteBindRelayOutcome::Accepted,
5570 )
5571 .unwrap();
5572
5573 let backlog = sink.backlog();
5574 assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
5575 assert!(
5576 backlog.queued_bytes > data_bytes,
5577 "the route.open response must be counted in queued bytes: {backlog:?}"
5578 );
5579 for corr in 0..DATA_FRAMES as u64 {
5580 assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
5581 }
5582 let open = client_rx.recv().await.unwrap();
5583 assert_eq!(open.header.corr, 4_242);
5584 assert_eq!(open.header.ty, FrameType::Response);
5585 drop(open);
5586 assert_eq!(sink.backlog().queued_bytes, 0);
5587 assert_eq!(sink.backlog().queued_frames, 0);
5588 }
5589
5590 #[tokio::test]
5595 async fn drain_holdouts_count_held_requests_and_name_the_connection() {
5596 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5597 route_fixture("holdouts");
5598 let mut bound = |corr| {
5599 let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
5600 forwarding
5601 .complete_pending_relay(
5602 module_connection,
5603 route.corr,
5604 RouteBindRelayOutcome::Accepted,
5605 )
5606 .unwrap();
5607 client_rx.try_recv().unwrap();
5608 match forwarding
5609 .lookup_data_route(client, route.client_channel, route.client_epoch)
5610 .unwrap()
5611 {
5612 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
5613 other => panic!("expected live route, got {other:?}"),
5614 }
5615 };
5616 let holding = bound(61);
5617 let _idle = bound(62);
5618 holding.flow.acquire_tagged(7, false).await.unwrap();
5619 holding.flow.acquire_tagged(2, false).await.unwrap();
5620 holding.flow.acquire_tagged(3, true).await.unwrap();
5621 forwarding
5622 .begin_module_drain("holdouts", RouteCloseReason::Restart)
5623 .unwrap();
5624
5625 let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
5626 assert_eq!(
5627 holdouts,
5628 DrainHoldouts {
5629 requests: 2,
5630 routes: 1,
5631 total_routes: 2,
5632 top_connections: vec![(client.get(), 2)],
5633 held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
5636 }
5637 );
5638 }
5639
5640 #[test]
5641 fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
5642 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
5643 route_fixture("census");
5644 let pending = begin_test_route(&forwarding, client, sink, 1, "census");
5645 forwarding
5646 .complete_pending_relay(
5647 module_connection,
5648 pending.corr,
5649 RouteBindRelayOutcome::Accepted,
5650 )
5651 .unwrap();
5652
5653 let routes = forwarding.endpoint_routes(endpoint).unwrap();
5654 assert_eq!(routes.len(), 1);
5655 assert!(matches!(routes[0].principal, Principal::Direct));
5656 assert_eq!(routes[0].goodbye_target.connection_id, client);
5657 assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
5658 assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
5659 assert!(!routes[0].draining);
5660
5661 forwarding
5662 .begin_module_drain("census", RouteCloseReason::Restart)
5663 .unwrap();
5664 let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
5665 assert_eq!(draining_routes.len(), 1);
5666 assert!(draining_routes[0].draining);
5667 }
5668
5669 #[test]
5670 fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
5671 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
5672 let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
5673 assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
5674 forwarding
5675 .abort_pending_relay(
5676 first.endpoint,
5677 first.corr,
5678 RouteBindRelayOutcome::ModuleGone("abort".into()),
5679 )
5680 .unwrap();
5681 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
5682 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
5683
5684 let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
5685 assert_eq!(second.client_channel, first.client_channel);
5686 assert_eq!(second.module_channel, first.module_channel);
5687 assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
5688 }
5689
5690 #[test]
5691 fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
5692 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5693 route_fixture("epoch-release");
5694 let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
5695 forwarding
5696 .complete_pending_relay(
5697 module_connection,
5698 first.corr,
5699 RouteBindRelayOutcome::Accepted,
5700 )
5701 .unwrap();
5702 assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
5703 assert!(matches!(
5704 forwarding
5705 .release_client_route(client, first.client_channel, first.client_epoch)
5706 .unwrap(),
5707 RouteRelease::Removed(_)
5708 ));
5709 forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
5710 forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
5711
5712 let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
5713 forwarding
5714 .complete_pending_relay(
5715 module_connection,
5716 second.corr,
5717 RouteBindRelayOutcome::Accepted,
5718 )
5719 .unwrap();
5720 assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
5721 assert!(matches!(
5722 forwarding
5723 .release_client_route(client, second.client_channel, first.client_epoch)
5724 .unwrap(),
5725 RouteRelease::Stale
5726 ));
5727 assert!(!forwarding
5728 .cache_status(
5729 endpoint,
5730 second.module_channel,
5731 first.module_epoch,
5732 "stale".into(),
5733 )
5734 .unwrap());
5735 assert!(forwarding
5736 .cache_status(
5737 endpoint,
5738 second.module_channel,
5739 second.module_epoch,
5740 "current".into(),
5741 )
5742 .unwrap());
5743 match forwarding
5744 .route_poll_snapshot(client, second.client_channel, second.client_epoch)
5745 .unwrap()
5746 {
5747 RoutePollSnapshot::Bound { status, .. } => {
5748 assert_eq!(status.as_deref(), Some("current"));
5749 }
5750 RoutePollSnapshot::Absent => panic!("successor binding was removed"),
5751 }
5752 let counters = forwarding.counters().snapshot();
5753 assert_eq!(counters["route_released_epoch_fenced"], 1);
5754 assert_eq!(counters["route_release_stale_skipped"], 1);
5755 }
5756
5757 #[test]
5758 fn max_epoch_reservation_retires_only_that_slot() {
5759 let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
5760 forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
5761 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
5762 let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
5763 assert_eq!(
5764 (final_use.client_channel, final_use.client_epoch),
5765 (7, u32::MAX)
5766 );
5767 assert_eq!(
5768 (final_use.module_channel, final_use.module_epoch),
5769 (9, u32::MAX)
5770 );
5771 forwarding
5772 .abort_pending_relay(
5773 endpoint,
5774 final_use.corr,
5775 RouteBindRelayOutcome::ModuleGone("abort".into()),
5776 )
5777 .unwrap();
5778 forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
5779 forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
5780 let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
5781 assert_ne!(next.client_channel, 7);
5782 assert_ne!(next.module_channel, 9);
5783 assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
5784 }
5785
5786 #[test]
5787 fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
5788 let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
5789 route_fixture("corr-shared");
5790 let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
5791 assert_eq!(bind.corr, 1);
5792 forwarding
5793 .abort_pending_relay(
5794 endpoint,
5795 bind.corr,
5796 RouteBindRelayOutcome::ModuleGone("abort".into()),
5797 )
5798 .unwrap();
5799 let rpc = forwarding
5800 .begin_module_control_rpc_for(
5801 "corr-shared",
5802 "health.check",
5803 Instant::now() - Duration::from_millis(1),
5804 )
5805 .unwrap();
5806 assert_eq!(rpc.corr, 2);
5807 assert_eq!(
5808 forwarding
5809 .complete_module_control_rpc(
5810 module_connection,
5811 rpc.corr,
5812 Some("health.check"),
5813 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
5814 status: subc_protocol::session::HealthStatus::Ok,
5815 detail: None,
5816 metrics: None,
5817 }),
5818 )
5819 .unwrap(),
5820 ModuleControlRpcCompletion::Settled
5821 );
5822 assert!(matches!(
5823 rpc.receiver.blocking_recv().unwrap(),
5824 ModuleControlRpcOutcome::DeadlineElapsed
5825 ));
5826 }
5827
5828 #[tokio::test(start_paused = true)]
5829 async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
5830 let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
5831 let probe_started_at = Instant::now();
5832 let rpc = forwarding
5833 .begin_health_probe_rpc_for(
5834 "tombstone-ttl",
5835 "health.check",
5836 probe_started_at,
5837 probe_started_at + Duration::from_secs(5),
5838 )
5839 .unwrap();
5840 assert!(forwarding
5841 .tombstone_health_probe_rpc(endpoint, rpc.corr)
5842 .unwrap());
5843 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
5844
5845 tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
5846 tokio::task::yield_now().await;
5847
5848 assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
5849 }
5850
5851 #[test]
5852 fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
5853 let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
5854 let mut close = forwarding.register_connection_close(endpoint.connection_id);
5855 forwarding.inject_control_corr(endpoint, u64::MAX);
5856 let final_rpc = forwarding
5857 .begin_module_control_rpc_for(
5858 "corr-max",
5859 "health.check",
5860 Instant::now() + Duration::from_secs(1),
5861 )
5862 .unwrap();
5863 assert_eq!(final_rpc.corr, u64::MAX);
5864 forwarding
5865 .cancel_module_control_rpc(endpoint, final_rpc.corr)
5866 .unwrap();
5867 assert!(matches!(
5868 forwarding.begin_module_control_rpc_for(
5869 "corr-max",
5870 "health.check",
5871 Instant::now() + Duration::from_secs(1),
5872 ),
5873 Err(ForwardingError::RelayCorrelationExhausted)
5874 ));
5875 assert!(close.try_recv().is_ok());
5876 }
5877
5878 #[test]
5879 fn publication_epoch_controls_delivery_failure_escalation() {
5880 fn setup_successor(
5881 commit_successor: Option<bool>,
5882 ) -> (ForwardingTable, ConnectionId, u16, u32) {
5883 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5884 route_fixture("escalation");
5885 let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
5886 forwarding
5887 .complete_pending_relay(
5888 module_connection,
5889 first.corr,
5890 RouteBindRelayOutcome::Accepted,
5891 )
5892 .unwrap();
5893 client_rx.try_recv().unwrap();
5894 assert!(matches!(
5895 forwarding
5896 .release_client_route(client, first.client_channel, first.client_epoch)
5897 .unwrap(),
5898 RouteRelease::Removed(_)
5899 ));
5900 if let Some(commit_successor) = commit_successor {
5901 forwarding.inject_client_slot_epoch(
5902 client,
5903 first.client_channel,
5904 first.client_epoch,
5905 );
5906 forwarding.inject_module_slot_epoch(
5907 endpoint,
5908 first.module_channel,
5909 first.module_epoch,
5910 );
5911 let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
5912 if commit_successor {
5913 forwarding
5914 .complete_pending_relay(
5915 module_connection,
5916 successor.corr,
5917 RouteBindRelayOutcome::Accepted,
5918 )
5919 .unwrap();
5920 client_rx.try_recv().unwrap();
5921 } else {
5922 forwarding
5923 .abort_pending_relay(
5924 endpoint,
5925 successor.corr,
5926 RouteBindRelayOutcome::ModuleGone("abort".into()),
5927 )
5928 .unwrap();
5929 }
5930 }
5931 (forwarding, client, first.client_channel, first.client_epoch)
5932 }
5933
5934 let probe_sink = FrameSink::new(mpsc::channel(1).0);
5935 let (no_successor, client, channel, epoch) = setup_successor(None);
5936 let mut close = no_successor.register_connection_close(client);
5937 assert!(no_successor
5938 .escalate_client_delivery_failure(
5939 client,
5940 channel,
5941 epoch,
5942 CloseReason::new("delivery", "failed"),
5943 UndeliveredFrame {
5944 module_id: None,
5945 sink: &probe_sink,
5946 },
5947 )
5948 .unwrap());
5949 assert!(close.try_recv().is_ok());
5950
5951 let (aborted, client, channel, epoch) = setup_successor(Some(false));
5952 let mut close = aborted.register_connection_close(client);
5953 assert!(aborted
5954 .escalate_client_delivery_failure(
5955 client,
5956 channel,
5957 epoch,
5958 CloseReason::new("delivery", "failed"),
5959 UndeliveredFrame {
5960 module_id: None,
5961 sink: &probe_sink,
5962 },
5963 )
5964 .unwrap());
5965 assert!(close.try_recv().is_ok());
5966
5967 let (published, client, channel, epoch) = setup_successor(Some(true));
5968 let mut close = published.register_connection_close(client);
5969 assert!(!published
5970 .escalate_client_delivery_failure(
5971 client,
5972 channel,
5973 epoch,
5974 CloseReason::new("delivery", "stale failure"),
5975 UndeliveredFrame {
5976 module_id: None,
5977 sink: &probe_sink,
5978 },
5979 )
5980 .unwrap());
5981 assert!(close.try_recv().is_err());
5982 }
5983
5984 #[test]
5985 fn route_concentration_separates_client_count_from_routes_per_client() {
5986 let (forwarding, module_connection, _, client, sink, _client_rx) =
5990 route_fixture("concentration");
5991 assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
5992
5993 for corr in [70_u64, 71] {
5994 let pending =
5995 begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
5996 forwarding
5997 .complete_pending_relay(
5998 module_connection,
5999 pending.corr,
6000 RouteBindRelayOutcome::Accepted,
6001 )
6002 .unwrap();
6003 }
6004
6005 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
6007 assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
6008 }
6009
6010 #[test]
6011 fn cleanup_and_accepted_resolution_have_one_lock_winner() {
6012 let (forwarding, module_connection, _, client, sink, mut client_rx) =
6013 route_fixture("cleanup-race");
6014 let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
6015 forwarding
6016 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
6017 .unwrap();
6018 let released = forwarding.cleanup_connection(client).unwrap();
6019 assert_eq!(released.len(), 1);
6020 let completion = forwarding
6021 .complete_pending_relay(
6022 module_connection,
6023 pending.corr,
6024 RouteBindRelayOutcome::Accepted,
6025 )
6026 .unwrap();
6027 assert!(!completion.settled);
6028 assert!(client_rx.try_recv().is_err());
6029 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
6030
6031 let (forwarding, module_connection, _, client, sink, mut client_rx) =
6032 route_fixture("accepted-race");
6033 let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
6034 forwarding
6035 .complete_pending_relay(
6036 module_connection,
6037 pending.corr,
6038 RouteBindRelayOutcome::Accepted,
6039 )
6040 .unwrap();
6041 assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
6042 let released = forwarding.cleanup_connection(client).unwrap();
6043 assert_eq!(released.len(), 1);
6044 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
6045 }
6046
6047 #[test]
6048 fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
6049 let (forwarding, module_connection, _, client, sink, mut client_rx) =
6050 route_fixture("drain-gap");
6051 let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
6052 forwarding
6053 .complete_pending_relay(
6054 module_connection,
6055 live.corr,
6056 RouteBindRelayOutcome::Accepted,
6057 )
6058 .unwrap();
6059 client_rx.try_recv().unwrap();
6060 let binding = match forwarding
6061 .lookup_data_route(client, live.client_channel, live.client_epoch)
6062 .unwrap()
6063 {
6064 DataRoute::Client(DataRouteState::Bound(binding)) => binding,
6065 other => panic!("expected live route, got {other:?}"),
6066 };
6067
6068 let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
6069 forwarding
6070 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
6071 .unwrap();
6072 let control_rpc = forwarding
6073 .begin_module_control_rpc_for(
6074 "drain-gap",
6075 "health.check",
6076 Instant::now() + Duration::from_secs(1),
6077 )
6078 .unwrap();
6079 let target = forwarding
6080 .begin_module_drain("drain-gap", RouteCloseReason::Reload)
6081 .unwrap()
6082 .unwrap();
6083 assert!(matches!(
6084 control_rpc.receiver.blocking_recv().unwrap(),
6085 ModuleControlRpcOutcome::ModuleGone(_)
6086 ));
6087 assert_eq!(target.abandoned_bindings.len(), 1);
6088 assert!(binding.flow.sem.is_closed());
6089 assert!(
6090 !forwarding
6091 .complete_pending_relay(
6092 module_connection,
6093 pending.corr,
6094 RouteBindRelayOutcome::Accepted,
6095 )
6096 .unwrap()
6097 .settled
6098 );
6099 assert!(matches!(
6100 forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
6101 Err(ForwardingError::ModuleReloading { .. })
6102 ));
6103 let released = forwarding
6104 .release_module_endpoint_routes(target.endpoint)
6105 .unwrap();
6106 assert_eq!(released.len(), 1);
6107 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
6108 }
6109
6110 #[test]
6118 fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
6119 let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
6120 route_fixture("closing-client");
6121
6122 let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
6125 forwarding
6126 .complete_pending_relay(
6127 module_connection,
6128 live.corr,
6129 RouteBindRelayOutcome::Accepted,
6130 )
6131 .unwrap();
6132 client_rx.try_recv().unwrap();
6133
6134 let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
6136 forwarding
6137 .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
6138 .unwrap();
6139
6140 assert!(forwarding
6142 .escalate_client_delivery_failure(
6143 client,
6144 live.client_channel,
6145 live.client_epoch,
6146 CloseReason::new(
6147 "module_to_client_delivery_failed",
6148 "client egress refused a module frame",
6149 ),
6150 UndeliveredFrame {
6151 module_id: None,
6152 sink: &sink,
6153 },
6154 )
6155 .unwrap());
6156 assert!(!sink.is_closed());
6157
6158 let completion = forwarding
6159 .complete_pending_relay(
6160 module_connection,
6161 pending.corr,
6162 RouteBindRelayOutcome::Accepted,
6163 )
6164 .expect("a closing client must not turn a module's ack into an error");
6165
6166 assert!(completion.settled);
6167 let abandoned = completion
6168 .abandoned
6169 .expect("the module must be told to drop the binding it just created");
6170 assert_eq!(abandoned.connection_id, module_connection);
6171 assert_eq!(abandoned.channel, pending.module_channel);
6172 assert_eq!(abandoned.epoch, pending.module_epoch);
6173 assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
6174 assert!(matches!(
6175 pending.receiver.blocking_recv().unwrap(),
6176 RouteBindRelayOutcome::ModuleGone(_)
6177 ));
6178 assert!(client_rx.try_recv().is_err());
6181 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
6182
6183 assert!(forwarding
6186 .has_live_module_connection("closing-client")
6187 .unwrap());
6188 let cotenant = ConnectionId::new(201);
6189 let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
6190 let cotenant_route = begin_test_route(
6191 &forwarding,
6192 cotenant,
6193 FrameSink::new(cotenant_tx),
6194 62,
6195 "closing-client",
6196 );
6197 assert_eq!(cotenant_route.endpoint, endpoint);
6198 forwarding
6199 .complete_pending_relay(
6200 module_connection,
6201 cotenant_route.corr,
6202 RouteBindRelayOutcome::Accepted,
6203 )
6204 .unwrap();
6205 assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
6206 assert_eq!(forwarding.active_binding_count().unwrap(), 2);
6207 }
6208
6209 #[test]
6210 fn pending_route_permit_is_released_on_rejection_and_abort() {
6211 let forwarding = ForwardingTable::default();
6212 let module_connection = ConnectionId::new(300);
6213 let client = ConnectionId::new(301);
6214 let (module_tx, _module_rx) = mpsc::channel(1);
6215 let endpoint = forwarding
6216 .register_module_connection(
6217 module_connection,
6218 "permit".into(),
6219 2,
6220 Concurrency::ModuleManaged,
6221 FrameSink::new(module_tx),
6222 )
6223 .unwrap();
6224 let (client_tx, mut client_rx) = mpsc::channel(1);
6225 let sink = FrameSink::new(client_tx);
6226 let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
6227 assert!(sink.try_send(test_ping(999)).is_err());
6228 forwarding
6229 .complete_pending_relay(
6230 module_connection,
6231 rejected.corr,
6232 RouteBindRelayOutcome::Rejected(ErrorBody {
6233 code: "no".into(),
6234 message: "rejected".into(),
6235 detail: None,
6236 }),
6237 )
6238 .unwrap();
6239 sink.try_send(test_ping(1000)).unwrap();
6240 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
6241
6242 let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
6243 assert!(sink.try_send(test_ping(1001)).is_err());
6244 forwarding
6245 .abort_pending_relay(
6246 endpoint,
6247 aborted.corr,
6248 RouteBindRelayOutcome::ModuleGone("abort".into()),
6249 )
6250 .unwrap();
6251 sink.try_send(test_ping(1002)).unwrap();
6252 assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
6253
6254 let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
6255 forwarding
6256 .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
6257 .unwrap();
6258 drop(client_rx);
6259 let completion = forwarding
6260 .complete_pending_relay(
6261 module_connection,
6262 receiver_closed.corr,
6263 RouteBindRelayOutcome::Accepted,
6264 )
6265 .unwrap();
6266 assert!(completion.abandoned.is_some());
6267 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
6268 }
6269
6270 #[test]
6276 fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
6277 let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
6278 route_fixture("closing-set-leak");
6279
6280 const CONNECTIONS: u64 = 32;
6281 for index in 0..CONNECTIONS {
6282 let client = ConnectionId::new(1000 + index);
6283 let (client_tx, _client_rx) = mpsc::channel(8);
6284 let route = begin_test_route(
6285 &forwarding,
6286 client,
6287 FrameSink::new(client_tx),
6288 index + 1,
6289 "closing-set-leak",
6290 );
6291 forwarding
6292 .complete_pending_relay(
6293 module_connection,
6294 route.corr,
6295 RouteBindRelayOutcome::Accepted,
6296 )
6297 .unwrap();
6298 forwarding.cleanup_connection(client).unwrap();
6299 }
6300 forwarding.cleanup_connection(module_connection).unwrap();
6301
6302 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
6303 }
6304
6305 #[test]
6312 fn closing_connection_is_refused_new_work_until_cleanup_completes() {
6313 let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
6314 route_fixture("closing-gate");
6315
6316 let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
6319 forwarding
6320 .complete_pending_relay(
6321 module_connection,
6322 live.corr,
6323 RouteBindRelayOutcome::Accepted,
6324 )
6325 .unwrap();
6326 client_rx.try_recv().unwrap();
6327
6328 assert!(forwarding
6331 .escalate_client_delivery_failure(
6332 client,
6333 live.client_channel,
6334 live.client_epoch,
6335 CloseReason::new(
6336 "module_to_client_delivery_failed",
6337 "client egress refused a module frame",
6338 ),
6339 UndeliveredFrame {
6340 module_id: None,
6341 sink: &sink,
6342 },
6343 )
6344 .unwrap());
6345 assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
6346
6347 assert!(matches!(
6349 forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
6350 Err(ForwardingError::ConnectionClosing { connection_id })
6351 if connection_id == client
6352 ));
6353 let (late_tx, _late_rx) = mpsc::channel(1);
6355 assert!(matches!(
6356 forwarding.register_module_connection(
6357 client,
6358 "late-module".into(),
6359 2,
6360 Concurrency::ModuleManaged,
6361 FrameSink::new(late_tx),
6362 ),
6363 Err(ForwardingError::ConnectionClosing { connection_id })
6364 if connection_id == client
6365 ));
6366
6367 forwarding.cleanup_connection(client).unwrap();
6371 assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
6372 }
6373}
6374
6375#[cfg(test)]
6378mod swap_slot_tests {
6379 use std::time::Duration;
6380
6381 use super::*;
6382 use tokio::sync::mpsc;
6383
6384 const MODULE_ID: &str = "swapped";
6385
6386 struct SwapFixture {
6387 forwarding: ForwardingTable,
6388 incumbent_connection: ConnectionId,
6389 incumbent: ModuleEndpointId,
6390 candidate_connection: ConnectionId,
6391 candidate: ModuleEndpointId,
6392 _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
6393 }
6394
6395 fn swap_fixture() -> SwapFixture {
6396 let forwarding = ForwardingTable::default();
6397 let incumbent_connection = ConnectionId::new(100);
6398 let candidate_connection = ConnectionId::new(110);
6399 let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
6400 let incumbent = forwarding
6401 .register_module_connection(
6402 incumbent_connection,
6403 MODULE_ID.to_string(),
6404 2,
6405 Concurrency::ModuleManaged,
6406 FrameSink::new(incumbent_tx),
6407 )
6408 .unwrap();
6409 let (candidate_tx, candidate_rx) = mpsc::channel(8);
6410 let candidate = forwarding
6411 .register_candidate_module_connection(
6412 candidate_connection,
6413 MODULE_ID.to_string(),
6414 2,
6415 Concurrency::ModuleManaged,
6416 FrameSink::new(candidate_tx),
6417 )
6418 .unwrap();
6419 SwapFixture {
6420 forwarding,
6421 incumbent_connection,
6422 incumbent,
6423 candidate_connection,
6424 candidate,
6425 _module_rxs: vec![incumbent_rx, candidate_rx],
6426 }
6427 }
6428
6429 fn client(
6430 raw: u64,
6431 ) -> (
6432 ConnectionId,
6433 FrameSink,
6434 mpsc::Receiver<crate::router::OutboundFrame>,
6435 ) {
6436 let (tx, rx) = mpsc::channel(8);
6437 (ConnectionId::new(raw), FrameSink::new(tx), rx)
6438 }
6439
6440 fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
6441 forwarding
6442 .read_inner()
6443 .unwrap()
6444 .client_to_module
6445 .values()
6446 .map(|route| route.module_endpoint)
6447 .collect()
6448 }
6449
6450 #[test]
6451 fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
6452 let fixture = swap_fixture();
6453 let forwarding = &fixture.forwarding;
6454 assert_ne!(fixture.incumbent, fixture.candidate);
6455
6456 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
6458 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
6459 let (client_connection, client_sink, _client_rx) = client(200);
6460 let pending = forwarding
6461 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
6462 .unwrap();
6463 assert_eq!(pending.endpoint, fixture.incumbent);
6464 let rpc = forwarding
6465 .begin_module_control_rpc_for(
6466 MODULE_ID,
6467 "health.check",
6468 Instant::now() + Duration::from_secs(1),
6469 )
6470 .unwrap();
6471 assert_eq!(rpc.endpoint, fixture.incumbent);
6472 let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
6473 assert_eq!(census.len(), 1, "the census lists one endpoint per id");
6474
6475 assert_eq!(
6477 forwarding
6478 .module_endpoint_for_connection(fixture.candidate_connection)
6479 .unwrap(),
6480 Some(fixture.candidate)
6481 );
6482 assert_eq!(
6483 forwarding
6484 .module_id_for_connection(fixture.candidate_connection)
6485 .unwrap()
6486 .as_deref(),
6487 Some(MODULE_ID)
6488 );
6489
6490 let (other_tx, _other_rx) = mpsc::channel(1);
6492 assert_eq!(
6493 forwarding.register_candidate_module_connection(
6494 ConnectionId::new(120),
6495 MODULE_ID.to_string(),
6496 2,
6497 Concurrency::ModuleManaged,
6498 FrameSink::new(other_tx),
6499 ),
6500 Err(ForwardingError::CandidateSlotOccupied {
6501 module_id: MODULE_ID.to_string()
6502 })
6503 );
6504 }
6505
6506 #[test]
6510 fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
6511 let fixture = swap_fixture();
6512 let forwarding = &fixture.forwarding;
6513 let (early_client, early_sink, _early_rx) = client(200);
6514 let mut early = forwarding
6515 .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
6516 .unwrap();
6517 assert_eq!(early.endpoint, fixture.incumbent);
6518 assert_eq!(
6519 forwarding
6520 .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
6521 .unwrap(),
6522 RouteBindRelayMark::Marked
6523 );
6524
6525 let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
6526 assert_eq!(
6527 cutover,
6528 ForwardingCutover {
6529 promoted: fixture.candidate,
6530 incumbent: Some(fixture.incumbent),
6531 }
6532 );
6533
6534 let (late_client, late_sink, _late_rx) = client(201);
6536 let late = forwarding
6537 .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
6538 .unwrap();
6539 assert_eq!(
6540 late.endpoint, fixture.candidate,
6541 "a route.open after cutover was reserved on the incumbent"
6542 );
6543
6544 let completion = forwarding
6546 .complete_pending_relay(
6547 fixture.incumbent_connection,
6548 early.corr,
6549 RouteBindRelayOutcome::Accepted,
6550 )
6551 .expect("a superseded endpoint's ack is not an error on its connection");
6552 assert!(completion.settled);
6553 assert!(
6554 !committed_endpoints(forwarding).contains(&fixture.incumbent),
6555 "a relay reserved before cutover committed a route on the incumbent"
6556 );
6557 let goodbye = completion
6558 .abandoned
6559 .expect("the incumbent is told to drop the binding it just created");
6560 assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
6561 assert_eq!(goodbye.channel, early.module_channel);
6562 assert_eq!(goodbye.epoch, early.module_epoch);
6563 assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
6564 match early.receiver.try_recv() {
6565 Ok(RouteBindRelayOutcome::DaemonRejected(body)) => {
6566 assert_eq!(body.code, "module_reloading")
6567 }
6568 other => panic!("expected a retryable module_reloading answer, got {other:?}"),
6569 }
6570 assert!(matches!(
6571 forwarding
6572 .lookup_data_route(early_client, early.client_channel, early.client_epoch)
6573 .unwrap(),
6574 DataRoute::Client(DataRouteState::Absent)
6575 ));
6576
6577 assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
6580 forwarding
6581 .complete_pending_relay(
6582 fixture.candidate_connection,
6583 late.corr,
6584 RouteBindRelayOutcome::Accepted,
6585 )
6586 .unwrap();
6587 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
6588 assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
6589 }
6590
6591 #[test]
6592 fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
6593 let fixture = swap_fixture();
6594 let forwarding = &fixture.forwarding;
6595 let (bound_client, bound_sink, _bound_rx) = client(200);
6597 let bound = forwarding
6598 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
6599 .unwrap();
6600 forwarding
6601 .complete_pending_relay(
6602 fixture.incumbent_connection,
6603 bound.corr,
6604 RouteBindRelayOutcome::Accepted,
6605 )
6606 .unwrap();
6607 let (pending_client, pending_sink, _pending_rx) = client(201);
6608 let mut in_flight = forwarding
6609 .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
6610 .unwrap();
6611 forwarding
6612 .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
6613 .unwrap();
6614
6615 let incumbent = forwarding
6616 .cutover_candidate(MODULE_ID)
6617 .unwrap()
6618 .unwrap()
6619 .incumbent
6620 .unwrap();
6621 let target = forwarding
6622 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
6623 .unwrap()
6624 .expect("the superseded incumbent is still registered");
6625
6626 assert_eq!(target.endpoint, fixture.incumbent);
6627 assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
6628 assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
6629 assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
6630 assert_eq!(target.abandoned_bindings.len(), 1);
6631 assert_eq!(
6632 target.abandoned_bindings[0].channel,
6633 in_flight.module_channel
6634 );
6635 assert!(matches!(
6636 in_flight.receiver.try_recv(),
6637 Ok(RouteBindRelayOutcome::DaemonRejected(body)) if body.code == "module_reloading"
6638 ));
6639 assert_eq!(
6640 forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
6641 1,
6642 "the incumbent's bound route stays until its drain finishes"
6643 );
6644
6645 let (next_client, next_sink, _next_rx) = client(202);
6646 let next = forwarding
6647 .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
6648 .expect("the promoted candidate keeps accepting routes");
6649 assert_eq!(next.endpoint, fixture.candidate);
6650 }
6651
6652 #[test]
6658 fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
6659 let forwarding = ForwardingTable::default();
6660 let first_connection = ConnectionId::new(70);
6661 let (first_tx, _first_rx) = mpsc::channel(8);
6662 forwarding
6663 .register_module_connection(
6664 first_connection,
6665 MODULE_ID.to_string(),
6666 2,
6667 Concurrency::ModuleManaged,
6668 FrameSink::new(first_tx),
6669 )
6670 .unwrap();
6671 let (client_connection, client_sink, _client_rx) = client(200);
6672 let mut pending = forwarding
6673 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
6674 .unwrap();
6675 let (second_tx, _second_rx) = mpsc::channel(8);
6676 forwarding
6677 .register_module_connection(
6678 ConnectionId::new(80),
6679 MODULE_ID.to_string(),
6680 2,
6681 Concurrency::ModuleManaged,
6682 FrameSink::new(second_tx),
6683 )
6684 .unwrap();
6685
6686 assert_eq!(
6687 forwarding
6688 .complete_pending_relay(
6689 first_connection,
6690 pending.corr,
6691 RouteBindRelayOutcome::Accepted
6692 )
6693 .unwrap_err(),
6694 ForwardingError::StaleModuleEndpoint
6695 );
6696 assert!(committed_endpoints(&forwarding).is_empty());
6697 assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
6698 assert!(matches!(
6699 pending.receiver.try_recv(),
6700 Err(oneshot::error::TryRecvError::Closed)
6701 ));
6702 }
6703
6704 #[test]
6705 fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
6706 let fixture = swap_fixture();
6708 let forwarding = &fixture.forwarding;
6709 assert!(forwarding
6710 .cleanup_connection(fixture.candidate_connection)
6711 .unwrap()
6712 .is_empty());
6713 assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
6714 let (client_connection, client_sink, _client_rx) = client(200);
6715 assert_eq!(
6716 forwarding
6717 .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
6718 .unwrap()
6719 .endpoint,
6720 fixture.incumbent
6721 );
6722
6723 let fixture = swap_fixture();
6726 let forwarding = &fixture.forwarding;
6727 let (bound_client, bound_sink, _bound_rx) = client(200);
6728 let bound = forwarding
6729 .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
6730 .unwrap();
6731 forwarding
6732 .complete_pending_relay(
6733 fixture.incumbent_connection,
6734 bound.corr,
6735 RouteBindRelayOutcome::Accepted,
6736 )
6737 .unwrap();
6738 forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
6739 let released = forwarding
6740 .cleanup_connection(fixture.incumbent_connection)
6741 .unwrap();
6742 assert_eq!(released.len(), 1);
6743 assert_eq!(released[0].connection_id, bound_client);
6744 assert!(forwarding
6745 .read_inner()
6746 .unwrap()
6747 .superseded_endpoints
6748 .is_empty());
6749 assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
6750 let (next_client, next_sink, _next_rx) = client(201);
6751 assert_eq!(
6752 forwarding
6753 .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
6754 .unwrap()
6755 .endpoint,
6756 fixture.candidate
6757 );
6758 }
6759}