Skip to main content

subc_daemon/
forwarding.rs

1use std::{
2    collections::{BTreeMap, HashMap, HashSet},
3    error::Error,
4    fmt,
5    sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
6    time::Duration,
7};
8
9use subc_control::{ClientControlResponse, RouteCloseReason};
10use subc_protocol::{
11    manifest::Concurrency,
12    session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
13    ErrorBody, Flags, FrameType, Principal, Priority,
14};
15use tokio::sync::{oneshot, Semaphore};
16use tokio::time::Instant;
17use tracing::{debug, info, warn};
18
19use crate::{
20    control::{RouteBindBreakers, RouteBindConcurrency},
21    observability::DaemonCounters,
22    registry::ConnectionId,
23    router::FrameSink,
24    scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
25    Frame, ProjectRootId,
26};
27
28/// Default per-channel request-credit window for modules that schedule internally.
29const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
30
31/// High per-channel cap for stateless modules; this is an OOM guard, not scheduling policy.
32const STATELESS_PARALLEL_WINDOW: usize = 1024;
33
34/// A stopped probe cycle cannot retain its last unanswered correlation forever.
35/// Active endpoints replace the tombstone on their next serial health probe;
36/// this backstop covers endpoints that stop probing altogether.
37const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
38
39/// Module connection identity used in forwarding keys.
40///
41/// The generation is bumped every time a module connection is registered so a future restart cannot
42/// accidentally reuse a stale `(connection_id, route_channel)` binding.
43#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
44pub struct ModuleEndpointId {
45    pub connection_id: ConnectionId,
46    pub generation: u64,
47}
48
49/// Client-local route key. A route channel is unique only within one client connection.
50#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
51pub(crate) struct ClientRouteKey {
52    pub connection_id: ConnectionId,
53    pub channel: u16,
54}
55
56/// Module-local route key. A route channel is unique only within one live module endpoint.
57#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
58pub(crate) struct ModuleRouteKey {
59    pub endpoint: ModuleEndpointId,
60    pub channel: u16,
61}
62
63#[derive(Debug)]
64pub(crate) struct RouteBinding {
65    pub client_connection_id: ConnectionId,
66    pub client_sink: FrameSink,
67    pub client_negotiated_ver: u8,
68    pub client_channel: u16,
69    pub client_epoch: u32,
70    pub module_id: String,
71    pub module_endpoint: ModuleEndpointId,
72    pub module_sink: FrameSink,
73    pub module_negotiated_ver: u8,
74    pub module_channel: u16,
75    pub module_epoch: u32,
76    pub principal: Principal,
77    pub project_root: Option<ProjectRootId>,
78    pub bound_at: Instant,
79    pub flow: Arc<ChannelFlow>,
80    /// The scope the route was admitted under, with the tag its bind was
81    /// stamped at. A scope change finds the routes to close by it.
82    pub scope: Option<BoundScope>,
83}
84
85#[derive(Debug, Clone)]
86pub(crate) enum DataRoute {
87    Client(DataRouteState),
88    Module(DataRouteState),
89}
90
91#[derive(Debug, Clone)]
92pub(crate) enum DataRouteState {
93    Bound(Arc<RouteBinding>),
94    Reserved,
95    EpochMismatch,
96    Absent,
97}
98
99/// Which kind of peer a route GOODBYE is being delivered to. This decides what
100/// happens when the GOODBYE cannot be enqueued (egress full/closed):
101/// - `Client`: escalate to closing that client connection (a socket close is a
102///   stronger teardown signal, and a full client egress means it is the slow
103///   client we would drop anyway).
104/// - `Module`: never close; deliver late instead (see
105///   [`send_module_route_goodbye`]), and drop only if the module still has no
106///   room after [`LATE_MODULE_GOODBYE_DEADLINE`]. A client-disconnect notifies the
107///   SHARED module that one client's route is gone; closing the module on its
108///   egress backpressure would tear down every co-tenant client (the exact
109///   cross-tenant blast radius this never-close rule exists to prevent — observed when a
110///   flooding dead client filled BOTH its own and the module's egress, so its
111///   route-gone GOODBYE to the module failed and closed the shared connection).
112///   subc has already removed the route from its forwarding state and drops
113///   stale module frames for the released channel (see router.rs), so subc's
114///   own routing is correct. The residual: under SUSTAINED module-egress
115///   backpressure a module-targeted route-gone notification can be lost, which a
116///   module using it for client-refcounting (e.g. AFT's session accounting)
117///   would miss. This is INTENTIONALLY ACCEPTED, not a gap. A consuming module
118///   must bound stale bindings with its own idle-activity reaper (last-touched
119///   TTL) independent of route-gone signals — AFT does exactly this, so a lost
120///   GOODBYE degrades to "the
121///   binding stays warm until its idle TTL" (bounded wasted resources), never an
122///   unbounded leak; disk-durable replay is unaffected. A dedicated reliable
123///   module control lane was evaluated and deliberately NOT
124///   built: it would add starvation-avoidance machinery to the thin core for a
125///   bounded warm-resource window that is not a correctness issue. never-close
126///   is the invariant that matters here.
127#[derive(Debug, Clone, Copy, PartialEq, Eq)]
128pub(crate) enum GoodbyeTargetKind {
129    Client,
130    Module,
131}
132
133#[derive(Debug, Clone)]
134pub(crate) struct GoodbyeTarget {
135    pub connection_id: ConnectionId,
136    pub sink: FrameSink,
137    pub negotiated_ver: u8,
138    pub channel: u16,
139    pub epoch: u32,
140    pub kind: GoodbyeTargetKind,
141    /// The module on the other end of the route: for a module-targeted relay
142    /// the receiving module (attributing a dropped relay), for a client target
143    /// the module whose route is going away (naming it when an undeliverable
144    /// relay closes the client).
145    pub module_id: Option<String>,
146}
147
148/// The frame a client connection's egress queue refused, for the diagnosis
149/// logged when that refusal closes the connection.
150#[derive(Debug, Clone, Copy)]
151pub(crate) struct UndeliveredFrame<'a> {
152    /// The module on the other end of the route the frame belonged to, when known.
153    pub module_id: Option<&'a str>,
154    /// The refusing connection's sink, read for what its queue held.
155    pub sink: &'a FrameSink,
156}
157
158/// How a route's principal appears in logs: `direct`, or `reserved:<module>`.
159fn principal_label(principal: &Principal) -> String {
160    match principal {
161        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
162        Principal::Direct => "direct".to_string(),
163        other => format!("{other:?}"),
164    }
165}
166
167/// The distinct principals of the routes bound on one client connection,
168/// sorted and comma-joined, or `none` when it has no bound route.
169fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
170    let labels = inner
171        .client_to_module
172        .iter()
173        .filter(|(key, _)| key.connection_id == connection_id)
174        .map(|(_, route)| principal_label(&route.principal))
175        .collect::<std::collections::BTreeSet<_>>();
176    if labels.is_empty() {
177        "none".to_string()
178    } else {
179        labels.into_iter().collect::<Vec<_>>().join(",")
180    }
181}
182
183impl GoodbyeTarget {
184    /// True only when an undeliverable GOODBYE should escalate to closing the
185    /// target connection. Never escalate for module recipients.
186    pub(crate) fn close_on_delivery_failure(&self) -> bool {
187        matches!(self.kind, GoodbyeTargetKind::Client)
188    }
189}
190
191/// How long a route GOODBYE that a module's egress queue refused keeps waiting
192/// for room before it is given up and counted as dropped.
193///
194/// A module that stops reading for a moment (a GC pause, a long synchronous
195/// handler, a reconnect herd it is still working through) should still learn
196/// that the route is gone, because a module that never hears the GOODBYE keeps
197/// the route and keeps sending on it for as long as its connection lives. The
198/// value is the default module drain timeout: the daemon already treats that as
199/// the longest it is reasonable to wait on a module that is busy but alive, and
200/// a module that cannot free 21 bytes of egress in that time is wedged rather
201/// than slow. Resolving each module's own configured drain timeout here would
202/// need a registry lookup on a path that holds only the module's sink.
203pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
204
205/// Send a route GOODBYE to a module without ever closing the module's shared
206/// connection, which would sever every other route it serves.
207///
208/// The frame is enqueued at once when the module's egress queue has room. When
209/// it does not, a detached task waits for room (bounded by
210/// [`LATE_MODULE_GOODBYE_DEADLINE`]) and sends it then. Arriving late is safe:
211/// the daemon has already released the route, and module SDKs tear a route
212/// down only when both the channel AND the epoch match the route installed on
213/// that channel, so a late GOODBYE for a channel since reused at a newer epoch
214/// is ignored. The GOODBYE is counted in `goodbye_relay_module_dropped` only
215/// when the connection is closed or the deadline passes.
216pub(crate) fn send_module_route_goodbye(
217    counters: &DaemonCounters,
218    sink: &FrameSink,
219    frame: Frame,
220    module_id: Option<&str>,
221    context: &'static str,
222) {
223    let channel = frame.header.channel;
224    let epoch = frame.header.epoch;
225    let Err(err) = sink.try_send(frame.clone()) else {
226        return;
227    };
228    // Waiting is pointless once the module's writer is gone, and impossible
229    // outside a Tokio runtime (cleanup that runs from a destructor at shutdown).
230    let runtime = match tokio::runtime::Handle::try_current() {
231        Ok(runtime) if !sink.is_closed() => runtime,
232        _ => {
233            counters.increment_goodbye_relay_module_dropped(module_id);
234            warn!(
235            module_id = module_id.unwrap_or("unknown"),
236            route_channel = channel,
237            route_epoch = epoch,
238            error = %err,
239            context,
240            "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
241            );
242            return;
243        }
244    };
245    debug!(
246        module_id = module_id.unwrap_or("unknown"),
247        route_channel = channel,
248        route_epoch = epoch,
249        error = %err,
250        context,
251        "module egress queue refused route GOODBYE; delivering it once the module frees room"
252    );
253    let counters = counters.clone();
254    let sink = sink.clone();
255    let module_id = module_id.map(str::to_string);
256    runtime.spawn(async move {
257        let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
258        let why = match outcome {
259            Ok(Ok(())) => {
260                debug!(
261                    module_id = module_id.as_deref().unwrap_or("unknown"),
262                    route_channel = channel,
263                    route_epoch = epoch,
264                    context,
265                    "late route GOODBYE delivered to module"
266                );
267                return;
268            }
269            Ok(Err(err)) => err.to_string(),
270            Err(_) => format!(
271                "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
272            ),
273        };
274        counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
275        warn!(
276            module_id = module_id.as_deref().unwrap_or("unknown"),
277            route_channel = channel,
278            route_epoch = epoch,
279            error = %why,
280            context,
281            "route GOODBYE to module dropped under backpressure; not closing shared module connection"
282        );
283    });
284}
285
286/// One route currently served by a module endpoint.
287///
288/// `goodbye_target` is deliberately retained alongside the census projection so
289/// a draining caller can address exactly the same route set that this read
290/// reports, without a second forwarding-table pass.
291#[derive(Debug, Clone)]
292pub(crate) struct EndpointRoute {
293    pub goodbye_target: GoodbyeTarget,
294    pub principal: Principal,
295    pub bound_at: Instant,
296    pub draining: bool,
297    /// WHY the endpoint is draining, when it is. Carried per-route so the
298    /// census can answer "closing because of what" without a second lookup;
299    /// `None` exactly when `draining` is false (one source: the drain map).
300    pub drain_reason: Option<RouteCloseReason>,
301}
302
303/// One live route a scope change closed: both ends to send GOODBYE to, and the
304/// reason for the client's `route.closed`.
305#[derive(Debug, Clone)]
306pub(crate) struct ScopeDrainedRoute {
307    pub reason: RouteCloseReason,
308    pub module_id: String,
309    pub client: GoodbyeTarget,
310    pub module: GoodbyeTarget,
311}
312
313#[derive(Debug)]
314pub(crate) struct PendingRouteBindRelay {
315    pub endpoint: ModuleEndpointId,
316    pub module_sink: FrameSink,
317    pub negotiated_ver: u8,
318    pub client_channel: u16,
319    pub client_epoch: u32,
320    pub module_channel: u16,
321    pub module_epoch: u32,
322    pub corr: u64,
323    pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
324}
325
326#[derive(Debug, Clone)]
327pub(crate) struct ModuleDrainTarget {
328    pub endpoint: ModuleEndpointId,
329    pub sink: FrameSink,
330    pub negotiated_ver: u8,
331    pub abandoned_bindings: Vec<GoodbyeTarget>,
332    pub excluded_subscriptions: u32,
333}
334
335/// One registered module connection, as [`ForwardingTable::module_connections`]
336/// reports it: enough to send it a frame and then close it.
337#[cfg(unix)]
338#[derive(Debug, Clone)]
339pub(crate) struct ModuleConnectionTarget {
340    pub module_id: String,
341    pub endpoint: ModuleEndpointId,
342    pub sink: FrameSink,
343    pub negotiated_ver: u8,
344}
345
346#[derive(Debug, Clone)]
347pub(crate) enum RouteBindRelayOutcome {
348    Accepted,
349    Rejected(ErrorBody),
350    ModuleGone(String),
351}
352
353/// What [`ForwardingTable::cutover_candidate`] did.
354#[derive(Debug, Clone, Copy, PartialEq, Eq)]
355pub(crate) struct ForwardingCutover {
356    /// The endpoint now in the active slot (the former candidate).
357    pub promoted: ModuleEndpointId,
358    /// The endpoint demoted out of the active slot, to be drained by endpoint.
359    /// `None` when the incumbent's connection was already gone.
360    pub incumbent: Option<ModuleEndpointId>,
361}
362
363/// What tearing down one connection released.
364#[derive(Debug)]
365pub(crate) struct ConnectionCleanup {
366    /// Routes whose other end must be sent GOODBYE.
367    pub released: Vec<GoodbyeTarget>,
368    /// Pending route.bind relays to the closed module that were aborted: opens
369    /// that had not been bound yet. Zero when a client connection closed.
370    pub abandoned_relays: u32,
371}
372
373#[derive(Debug, Clone)]
374pub(crate) struct PendingRelayCompletion {
375    pub settled: bool,
376    pub abandoned: Option<GoodbyeTarget>,
377}
378
379#[derive(Debug)]
380pub(crate) struct PendingModuleControlRpc {
381    pub endpoint: ModuleEndpointId,
382    pub module_sink: FrameSink,
383    pub negotiated_ver: u8,
384    pub corr: u64,
385    pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
386}
387
388#[derive(Debug, Clone)]
389pub(crate) enum ModuleControlRpcOutcome {
390    Response(ModuleControlResponse),
391    Rejected(ErrorBody),
392    ModuleGone(String),
393    MalformedResponse(String),
394    UnexpectedOp { expected: String, actual: String },
395    DeadlineElapsed,
396}
397
398#[derive(Debug, Clone, PartialEq, Eq)]
399pub(crate) enum ModuleControlRpcCompletion {
400    Unknown,
401    Settled,
402    LateHealthAnswer {
403        module_id: String,
404        latency: Duration,
405    },
406}
407
408#[derive(Debug)]
409struct PendingModuleControlRpcEntry {
410    expected_op: String,
411    deadline: Instant,
412    health_probe_started_at: Option<Instant>,
413    sender: oneshot::Sender<ModuleControlRpcOutcome>,
414}
415
416#[derive(Debug)]
417struct HealthProbeTombstone {
418    expected_op: String,
419    module_id: String,
420    probe_started_at: Instant,
421    expires_at: Instant,
422}
423
424#[derive(Debug, Clone)]
425struct RouteReservation {
426    client_key: ClientRouteKey,
427    module_key: ModuleRouteKey,
428    client_epoch: u32,
429    module_epoch: u32,
430    project_root: Option<ProjectRootId>,
431}
432
433#[derive(Debug)]
434struct PendingRouteBindRelayEntry {
435    reservation: RouteReservation,
436    client_sink: FrameSink,
437    client_negotiated_ver: u8,
438    client_permit: crate::router::EgressPermit,
439    route_open_frame: Frame,
440    principal: Principal,
441    /// The scope admitted at `route.open`, with the tag captured then. Commit
442    /// compares it with the published tag.
443    scope: Option<BoundScope>,
444    deadline: Instant,
445    /// Set once `route.open` has finished putting the route.bind on the
446    /// module's egress queue. A path that drops this entry WITHOUT a module
447    /// reply (client teardown, module drain, the open's own abort) sends the
448    /// module a GOODBYE for the bind only when this is set, because only then
449    /// is the GOODBYE guaranteed to queue behind the bind. When it is not set,
450    /// the removal is recorded in `relays_released_before_enqueue` instead and
451    /// `route.open` sends that GOODBYE itself once its send has completed.
452    ///
453    /// A module reply does not consult it: an ack proves the module received
454    /// the bind, so the reply path decides about the GOODBYE on its own.
455    relay_enqueued: bool,
456    sender: oneshot::Sender<RouteBindRelayOutcome>,
457}
458
459/// What `mark_route_bind_relay_enqueued` found once `route.open` had put the
460/// route.bind on the module's queue.
461#[derive(Debug, Clone, Copy, PartialEq, Eq)]
462pub(crate) enum RouteBindRelayMark {
463    /// The relay is still pending; later removals handle their own GOODBYE.
464    Marked,
465    /// The module already answered and the reply path settled the relay:
466    /// it committed the route, or it refused it and sent any GOODBYE the
467    /// module needed. `route.open` must send nothing.
468    SettledByReply,
469    /// The relay was dropped without a module reply while the bind was still
470    /// being queued, so nothing has told the module to drop the binding it
471    /// is about to create. `route.open` must send that GOODBYE.
472    ReleasedWithoutReply,
473}
474
475#[derive(Debug, Clone)]
476pub(crate) enum RouteRelease {
477    Removed(GoodbyeTarget),
478    Stale,
479    Absent,
480}
481
482#[derive(Debug, Clone)]
483pub(crate) enum RoutePollSnapshot {
484    Bound {
485        module_id: String,
486        status: Option<String>,
487    },
488    Absent,
489}
490
491#[derive(Debug, Clone)]
492struct ModuleConnection {
493    endpoint: ModuleEndpointId,
494    sink: FrameSink,
495    negotiated_ver: u8,
496    concurrency: Concurrency,
497}
498
499#[derive(Debug, Default)]
500struct ForwardingInner {
501    daemon_draining: bool,
502    /// The ACTIVE slot: the one endpoint per module id that routing resolves.
503    /// Every by-id lookup (relay reservation, drain-by-id, liveness, census,
504    /// live roots, module-control RPCs) reads this map and nothing else.
505    modules_by_id: HashMap<String, ModuleConnection>,
506    /// The CANDIDATE slot of a blue/green swap: a second process registered
507    /// under an id that already has an active endpoint. It has a full endpoint
508    /// identity (so its own connection can be looked up and torn down) but no
509    /// by-id lookup sees it, so nothing is routed to it until `cutover_candidate`
510    /// promotes it.
511    candidates_by_id: HashMap<String, ModuleConnection>,
512    /// Former active endpoints demoted by `cutover_candidate`, until their
513    /// connection is removed. Membership is what distinguishes an endpoint that
514    /// was SUPERSEDED by a promotion (still alive, still carrying bound routes,
515    /// about to be drained) from one that is merely STALE (replaced some other
516    /// way, which is treated as a fault on the acking connection).
517    superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
518    endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
519    module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
520    /// Endpoints mid-drain, keyed to the reason the drain was begun with. The
521    /// value serves the census ("draining because restart"); membership alone
522    /// still answers every admission-gate check.
523    draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
524    closing_connections: HashSet<ConnectionId>,
525    next_generation: u64,
526    reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
527    reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
528    next_client_channel: HashMap<ConnectionId, u16>,
529    next_module_channel: HashMap<ModuleEndpointId, u16>,
530    client_slot_epochs: HashMap<ClientRouteKey, u32>,
531    module_slot_epochs: HashMap<ModuleRouteKey, u32>,
532    last_published_epoch: HashMap<ClientRouteKey, u32>,
533    client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
534    module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
535    status: HashMap<(ClientRouteKey, u32), String>,
536    pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
537    /// Pending relays removed without a module reply before `route.open`
538    /// marked them enqueued. `route.open` consumes its key when it marks the
539    /// relay (and sends the GOODBYE the removal could not send safely) or when
540    /// it aborts, so an entry lives only while that open is mid-send.
541    relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
542    /// Each scope's current `(scope_epoch, version)`, keyed `(owner, ref)`,
543    /// published by a scope sync in the same step that changes the record. A
544    /// bind commit reads it here so it never takes the scope table's lock
545    /// while holding this one. Absent means the scope is not live.
546    scope_tags: HashMap<(String, String), ScopeTag>,
547    next_control_corr: HashMap<ModuleEndpointId, u64>,
548    pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
549    health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
550}
551
552#[derive(Debug, Clone)]
553pub(crate) struct CloseReason {
554    code: &'static str,
555    message: String,
556}
557
558impl CloseReason {
559    pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
560        Self {
561            code,
562            message: message.into(),
563        }
564    }
565}
566
567impl fmt::Display for CloseReason {
568    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
569        write!(f, "{}: {}", self.code, self.message)
570    }
571}
572
573pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
574
575/// Dynamic forwarding state shared by the control plane and data-plane router.
576#[derive(Debug, Default)]
577pub struct ForwardingTable {
578    inner: Arc<RwLock<ForwardingInner>>,
579    close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
580    counters: DaemonCounters,
581    /// Per-target-module bind-relay breaker state. It lives here, beside the
582    /// module connections it describes, because this is where a module
583    /// connection's identity is established and therefore where a stale
584    /// verdict has to be discarded.
585    route_bind_breakers: RouteBindBreakers,
586    /// Current route.bind relays keyed by target module. Admission is shared
587    /// across every client connection that points at the same endpoint.
588    route_bind_concurrency: RouteBindConcurrency,
589    /// Start and end of each module's `route.open` outage. Held here for the
590    /// same reason as the breakers: every handler built over this table must
591    /// see one outage per module, or each would log its own opening line.
592    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
593}
594
595impl ForwardingTable {
596    pub(crate) fn counters(&self) -> DaemonCounters {
597        self.counters.clone()
598    }
599
600    pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
601        self.route_bind_breakers.clone()
602    }
603
604    pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
605        self.route_bind_concurrency.clone()
606    }
607
608    pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
609        Arc::clone(&self.route_outages)
610    }
611
612    pub(crate) fn register_connection_close(
613        &self,
614        connection_id: ConnectionId,
615    ) -> ConnectionCloseReceiver {
616        let (sender, receiver) = oneshot::channel();
617        let replaced = self
618            .lock_close_registry()
619            .insert(connection_id, sender)
620            .is_some();
621        if replaced {
622            warn!(
623                connection_id = connection_id.get(),
624                "replaced existing connection close registration"
625            );
626        }
627        receiver
628    }
629
630    pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
631        self.lock_close_registry().remove(&connection_id);
632    }
633
634    /// Close every established connection, modules and clients alike, as the
635    /// last step of an announced daemon shutdown. By the time this runs each
636    /// registered module has already been sent a module GOODBYE (see
637    /// `Supervisor::end_children_for_daemon_shutdown`), so the EOF that follows
638    /// is a planned stop; EOF with no GOODBYE before it stays reserved for a
639    /// daemon that went away unannounced. Closing while the daemon is still
640    /// alive lets it wait for the modules' own teardowns before it ends
641    /// whatever is left.
642    #[cfg(unix)]
643    pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
644        let senders: Vec<_> = self.lock_close_registry().drain().collect();
645        let count = senders.len();
646        for (_, sender) in senders {
647            let _ = sender.send(reason.clone());
648        }
649        count
650    }
651
652    /// Every registered module connection, whichever slot it occupies (active,
653    /// swap candidate, or superseded incumbent), once each. Daemon shutdown
654    /// uses this to tell each of them it is a planned stop before closing it.
655    #[cfg(unix)]
656    pub(crate) fn module_connections(
657        &self,
658    ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
659        let inner = self.read_inner()?;
660        let mut seen = HashSet::new();
661        Ok(inner
662            .modules_by_id
663            .values()
664            .chain(inner.candidates_by_id.values())
665            .chain(inner.superseded_endpoints.values())
666            .filter(|module| seen.insert(module.endpoint))
667            .map(|module| ModuleConnectionTarget {
668                module_id: inner
669                    .module_id_by_endpoint
670                    .get(&module.endpoint)
671                    .cloned()
672                    .unwrap_or_default(),
673                endpoint: module.endpoint,
674                sink: module.sink.clone(),
675                negotiated_ver: module.negotiated_ver,
676            })
677            .collect())
678    }
679
680    /// Ask a registered connection to close. Returns true only for the request
681    /// that actually reached it; later requests for the same connection, and
682    /// requests for a connection that is not registered, return false.
683    pub(crate) fn request_connection_close(
684        &self,
685        connection_id: ConnectionId,
686        reason: CloseReason,
687    ) -> bool {
688        let sender = self.lock_close_registry().remove(&connection_id);
689        if let Some(sender) = sender {
690            debug!(
691                connection_id = connection_id.get(),
692                close_reason = %reason,
693                "requesting connection close"
694            );
695            let _ = sender.send(reason);
696            true
697        } else {
698            debug!(
699                connection_id = connection_id.get(),
700                close_reason = %reason,
701                "connection close request ignored for inactive connection"
702            );
703            false
704        }
705    }
706
707    pub fn register_module_connection(
708        &self,
709        connection_id: ConnectionId,
710        module_id: String,
711        negotiated_ver: u8,
712        concurrency: Concurrency,
713        sink: FrameSink,
714    ) -> Result<ModuleEndpointId, ForwardingError> {
715        self.register_module_connection_inner(
716            connection_id,
717            module_id,
718            negotiated_ver,
719            concurrency,
720            sink,
721            None,
722        )
723    }
724
725    /// Register a module connection and queue its HELLO_ACK as the first frame
726    /// on its sink, in the same write-lock critical section that makes the
727    /// endpoint visible.
728    ///
729    /// A module reads HELLO_ACK as the first frame after its HELLO and exits on
730    /// anything else. Once the endpoint is in `modules_by_id`, a `route.open`
731    /// on any other connection can queue a `route.bind` request onto this sink,
732    /// so the ack must already be queued by then. Every lookup that can queue a
733    /// frame for the module takes this lock, so nothing can get in ahead of it.
734    ///
735    /// If the ack cannot be queued (the sink is closed or full), the
736    /// registration fails with nothing inserted.
737    pub(crate) fn register_module_connection_acked(
738        &self,
739        connection_id: ConnectionId,
740        module_id: String,
741        negotiated_ver: u8,
742        concurrency: Concurrency,
743        sink: FrameSink,
744        hello_ack: Frame,
745    ) -> Result<ModuleEndpointId, ForwardingError> {
746        self.register_module_connection_inner(
747            connection_id,
748            module_id,
749            negotiated_ver,
750            concurrency,
751            sink,
752            Some(hello_ack),
753        )
754    }
755
756    fn register_module_connection_inner(
757        &self,
758        connection_id: ConnectionId,
759        module_id: String,
760        negotiated_ver: u8,
761        concurrency: Concurrency,
762        sink: FrameSink,
763        hello_ack: Option<Frame>,
764    ) -> Result<ModuleEndpointId, ForwardingError> {
765        let mut inner = self.write_inner()?;
766        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
767            return Err(ForwardingError::ConnectionClosing { connection_id });
768        }
769        check_module_connection_role_locked(&inner, connection_id)?;
770        // Every refusal check has passed and nothing has been mutated yet, so
771        // a failed enqueue leaves the table exactly as it was.
772        enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
773
774        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
775        let endpoint = ModuleEndpointId {
776            connection_id,
777            generation: inner.next_generation,
778        };
779        inner.endpoint_by_connection.insert(connection_id, endpoint);
780        inner
781            .module_id_by_endpoint
782            .insert(endpoint, module_id.clone());
783        inner.next_module_channel.insert(endpoint, 1);
784        inner.next_control_corr.insert(endpoint, 1);
785        inner.modules_by_id.insert(
786            module_id.clone(),
787            ModuleConnection {
788                endpoint,
789                sink,
790                negotiated_ver,
791                concurrency,
792            },
793        );
794        drop(inner);
795
796        // A new module connection has arrived under this id, so anything the
797        // bind-relay breaker learned was learned about a process that is no
798        // longer the one behind this name. See
799        // `RouteBindBreakers::reset_for_new_module_connection`.
800        //
801        // Keyed on ARRIVAL rather than on teardown deliberately: a connection
802        // going away is not evidence about anything, and a module whose
803        // connection drops without coming back should keep its verdict until
804        // something actually registers in its place. This also covers the
805        // unclean replacements -- a module killed mid-bind, or one whose
806        // connection was closing -- because registration is the single path by
807        // which any module connection becomes usable.
808        if let Some(discarded) = self
809            .route_bind_breakers
810            .reset_for_new_module_connection(&module_id)
811        {
812            info!(
813                module_id = %module_id,
814                discarded_consecutive_timeouts = discarded,
815                "route.bind breaker state discarded: a new module connection replaced the process it described"
816            );
817        }
818        Ok(endpoint)
819    }
820
821    /// Register a blue/green swap candidate for `module_id` into the candidate
822    /// slot, alongside whatever endpoint is active for the id.
823    ///
824    /// Unlike [`Self::register_module_connection`], this never touches the
825    /// active slot and never resets the bind-relay breaker: the incumbent is
826    /// still the process serving the id, so what the breaker learned about it
827    /// is still true. The breaker is reset at cutover instead, when the process
828    /// behind the name actually changes. A second candidate for the same id is
829    /// refused.
830    ///
831    /// Production registers candidates through
832    /// [`Self::register_candidate_module_connection_acked`]; this ack-less form
833    /// is for tests that build forwarding state directly.
834    #[cfg(test)]
835    pub(crate) fn register_candidate_module_connection(
836        &self,
837        connection_id: ConnectionId,
838        module_id: String,
839        negotiated_ver: u8,
840        concurrency: Concurrency,
841        sink: FrameSink,
842    ) -> Result<ModuleEndpointId, ForwardingError> {
843        self.register_candidate_module_connection_inner(
844            connection_id,
845            module_id,
846            negotiated_ver,
847            concurrency,
848            sink,
849            None,
850        )
851    }
852
853    /// The swap-candidate counterpart of
854    /// [`Self::register_module_connection_acked`]: the HELLO_ACK is queued on
855    /// the candidate's sink before its endpoint is inserted. No by-id lookup
856    /// sees a candidate, but its own connection's endpoint does become
857    /// resolvable here, and at cutover it becomes routable; the ack has to be
858    /// ahead of anything either can queue.
859    pub(crate) fn register_candidate_module_connection_acked(
860        &self,
861        connection_id: ConnectionId,
862        module_id: String,
863        negotiated_ver: u8,
864        concurrency: Concurrency,
865        sink: FrameSink,
866        hello_ack: Frame,
867    ) -> Result<ModuleEndpointId, ForwardingError> {
868        self.register_candidate_module_connection_inner(
869            connection_id,
870            module_id,
871            negotiated_ver,
872            concurrency,
873            sink,
874            Some(hello_ack),
875        )
876    }
877
878    fn register_candidate_module_connection_inner(
879        &self,
880        connection_id: ConnectionId,
881        module_id: String,
882        negotiated_ver: u8,
883        concurrency: Concurrency,
884        sink: FrameSink,
885        hello_ack: Option<Frame>,
886    ) -> Result<ModuleEndpointId, ForwardingError> {
887        let mut inner = self.write_inner()?;
888        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
889            return Err(ForwardingError::ConnectionClosing { connection_id });
890        }
891        check_module_connection_role_locked(&inner, connection_id)?;
892        if inner.candidates_by_id.contains_key(&module_id) {
893            return Err(ForwardingError::CandidateSlotOccupied { module_id });
894        }
895        // As in `register_module_connection_inner`: after every refusal check,
896        // before any mutation.
897        enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
898
899        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
900        let endpoint = ModuleEndpointId {
901            connection_id,
902            generation: inner.next_generation,
903        };
904        inner.endpoint_by_connection.insert(connection_id, endpoint);
905        inner
906            .module_id_by_endpoint
907            .insert(endpoint, module_id.clone());
908        inner.next_module_channel.insert(endpoint, 1);
909        inner.next_control_corr.insert(endpoint, 1);
910        inner.candidates_by_id.insert(
911            module_id,
912            ModuleConnection {
913                endpoint,
914                sink,
915                negotiated_ver,
916                concurrency,
917            },
918        );
919        Ok(endpoint)
920    }
921
922    /// Promote `module_id`'s candidate to the active slot, in ONE forwarding
923    /// write-lock critical section.
924    ///
925    /// That single section is the linearization point of a swap. Every relay
926    /// reservation resolves the active slot under the same lock, so each one
927    /// lands wholly before cutover (on the incumbent) or wholly after it (on the
928    /// promoted candidate), never on a mix. A relay reserved on the incumbent
929    /// before cutover can no longer commit: `commit_route_locked` requires the
930    /// reservation's endpoint to be the active one, and an ack for it arriving
931    /// later is answered by the superseded-endpoint arm of
932    /// `complete_pending_relay` instead of ending the incumbent's connection.
933    ///
934    /// Both endpoints keep their identities: nothing is re-keyed, so the
935    /// incumbent's bound routes and its pending correlation keys stay exactly
936    /// where they are until it is drained with [`Self::begin_endpoint_drain`],
937    /// using the incumbent endpoint this returns. Returns `Ok(None)` when there
938    /// is no candidate for the id.
939    pub(crate) fn cutover_candidate(
940        &self,
941        module_id: &str,
942    ) -> Result<Option<ForwardingCutover>, ForwardingError> {
943        let mut inner = self.write_inner()?;
944        if inner.daemon_draining {
945            return Err(ForwardingError::ModuleReloading {
946                module_id: module_id.to_string(),
947            });
948        }
949        let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
950            return Ok(None);
951        };
952        let promoted = candidate.endpoint;
953        let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
954        let incumbent = incumbent.map(|incumbent| {
955            let endpoint = incumbent.endpoint;
956            inner.superseded_endpoints.insert(endpoint, incumbent);
957            endpoint
958        });
959        drop(inner);
960
961        // A different process now answers for this id; see the matching reset
962        // in `register_module_connection` for why a verdict about the old one
963        // must not carry over.
964        if let Some(discarded) = self
965            .route_bind_breakers
966            .reset_for_new_module_connection(module_id)
967        {
968            info!(
969                module_id = %module_id,
970                discarded_consecutive_timeouts = discarded,
971                "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
972            );
973        }
974        Ok(Some(ForwardingCutover {
975            promoted,
976            incumbent,
977        }))
978    }
979
980    #[allow(clippy::too_many_arguments)]
981    pub(crate) async fn begin_route_bind_relay_for(
982        &self,
983        client_connection_id: ConnectionId,
984        client_sink: FrameSink,
985        client_negotiated_ver: u8,
986        client_corr: u64,
987        module_id: &str,
988        principal: Principal,
989        scope: Option<BoundScope>,
990        project_root: Option<ProjectRootId>,
991        deadline: Instant,
992    ) -> Result<PendingRouteBindRelay, ForwardingError> {
993        // Reserve egress capacity before taking the forwarding lock. The permit is
994        // held until the bind reaches one terminal state, so an accepted bind can
995        // publish its table entry and RouteOpen response in one critical section.
996        let client_permit =
997            client_sink
998                .reserve_owned()
999                .await
1000                .map_err(|_| ForwardingError::ClientEgressClosed {
1001                    connection_id: client_connection_id,
1002                })?;
1003        self.begin_route_bind_relay_inner(
1004            client_connection_id,
1005            client_sink,
1006            client_negotiated_ver,
1007            client_corr,
1008            module_id,
1009            principal,
1010            scope,
1011            project_root,
1012            deadline,
1013            client_permit,
1014        )
1015    }
1016
1017    #[cfg(test)]
1018    pub(crate) fn begin_route_bind_relay_for_test(
1019        &self,
1020        client_connection_id: ConnectionId,
1021        client_sink: FrameSink,
1022        client_corr: u64,
1023        module_id: &str,
1024    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1025        let permit =
1026            client_sink
1027                .try_reserve_owned()
1028                .map_err(|_| ForwardingError::ClientEgressClosed {
1029                    connection_id: client_connection_id,
1030                })?;
1031        self.begin_route_bind_relay_inner(
1032            client_connection_id,
1033            client_sink,
1034            subc_protocol::PROTOCOL_VERSION,
1035            client_corr,
1036            module_id,
1037            Principal::Direct,
1038            None,
1039            None,
1040            Instant::now() + std::time::Duration::from_secs(60),
1041            permit,
1042        )
1043    }
1044
1045    pub(crate) fn begin_module_control_rpc_for(
1046        &self,
1047        module_id: &str,
1048        expected_op: &str,
1049        deadline: Instant,
1050    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1051        self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1052    }
1053
1054    pub(crate) fn begin_health_probe_rpc_for(
1055        &self,
1056        module_id: &str,
1057        expected_op: &str,
1058        probe_started_at: Instant,
1059        deadline: Instant,
1060    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1061        self.begin_module_control_rpc_inner(
1062            module_id,
1063            expected_op,
1064            deadline,
1065            Some(probe_started_at),
1066            false,
1067        )
1068    }
1069
1070    pub(crate) fn begin_drain_health_probe_rpc_for(
1071        &self,
1072        module_id: &str,
1073        expected_op: &str,
1074        probe_started_at: Instant,
1075        deadline: Instant,
1076    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1077        self.begin_module_control_rpc_inner(
1078            module_id,
1079            expected_op,
1080            deadline,
1081            Some(probe_started_at),
1082            true,
1083        )
1084    }
1085
1086    /// A health probe addressed to one endpoint, in whichever slot it is.
1087    ///
1088    /// Every other control RPC resolves the id's ACTIVE endpoint, which is how
1089    /// a swap candidate (not active until cutover) and a superseded incumbent
1090    /// (not active after it) are unreachable by them. A swap needs to probe
1091    /// exactly those two: the candidate before promoting it, and the incumbent
1092    /// for its busy gauges while it drains.
1093    pub(crate) fn begin_endpoint_health_probe_rpc_for(
1094        &self,
1095        endpoint: ModuleEndpointId,
1096        expected_op: &str,
1097        probe_started_at: Instant,
1098        deadline: Instant,
1099    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1100        let inner = self.write_inner()?;
1101        let module = module_connection_for_endpoint_locked(&inner, endpoint)
1102            .cloned()
1103            .ok_or(ForwardingError::NoModuleConnection)?;
1104        let module_id = inner
1105            .module_id_by_endpoint
1106            .get(&endpoint)
1107            .cloned()
1108            .unwrap_or_default();
1109        // A draining incumbent is exactly what this probes for busy gauges, so
1110        // draining is allowed, as it is for the by-id drain probe.
1111        self.begin_control_rpc_locked(
1112            inner,
1113            &module_id,
1114            module,
1115            expected_op,
1116            deadline,
1117            Some(probe_started_at),
1118            true,
1119        )
1120    }
1121
1122    fn begin_module_control_rpc_inner(
1123        &self,
1124        module_id: &str,
1125        expected_op: &str,
1126        deadline: Instant,
1127        health_probe_started_at: Option<Instant>,
1128        allow_draining: bool,
1129    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1130        let inner = self.write_inner()?;
1131        let module = inner
1132            .modules_by_id
1133            .get(module_id)
1134            .cloned()
1135            .ok_or(ForwardingError::NoModuleConnection)?;
1136        self.begin_control_rpc_locked(
1137            inner,
1138            module_id,
1139            module,
1140            expected_op,
1141            deadline,
1142            health_probe_started_at,
1143            allow_draining,
1144        )
1145    }
1146
1147    #[allow(clippy::too_many_arguments)]
1148    fn begin_control_rpc_locked(
1149        &self,
1150        mut inner: RwLockWriteGuard<'_, ForwardingInner>,
1151        module_id: &str,
1152        module: ModuleConnection,
1153        expected_op: &str,
1154        deadline: Instant,
1155        health_probe_started_at: Option<Instant>,
1156        allow_draining: bool,
1157    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1158        if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1159            return Err(ForwardingError::ModuleReloading {
1160                module_id: module_id.to_string(),
1161            });
1162        }
1163        if inner
1164            .closing_connections
1165            .contains(&module.endpoint.connection_id)
1166        {
1167            return Err(ForwardingError::ConnectionClosing {
1168                connection_id: module.endpoint.connection_id,
1169            });
1170        }
1171        if health_probe_started_at.is_some() {
1172            // Recurring health probes are serial per endpoint. Once the next one
1173            // starts, an older answer can no longer improve the current snapshot,
1174            // so retaining more than the newest unanswered probe has no value.
1175            inner
1176                .health_probe_tombstones
1177                .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1178        }
1179        let corr = match inner.allocate_control_corr(module.endpoint) {
1180            Ok(corr) => corr,
1181            Err(err) => {
1182                drop(inner);
1183                self.request_connection_close(
1184                    module.endpoint.connection_id,
1185                    CloseReason::new(
1186                        "control_correlation_exhausted",
1187                        "daemon-originated channel-0 correlation space exhausted",
1188                    ),
1189                );
1190                return Err(err);
1191            }
1192        };
1193        let (sender, receiver) = oneshot::channel();
1194        inner.pending_control_rpcs.insert(
1195            (module.endpoint, corr),
1196            PendingModuleControlRpcEntry {
1197                expected_op: expected_op.to_string(),
1198                deadline,
1199                health_probe_started_at,
1200                sender,
1201            },
1202        );
1203
1204        Ok(PendingModuleControlRpc {
1205            endpoint: module.endpoint,
1206            module_sink: module.sink,
1207            negotiated_ver: module.negotiated_ver,
1208            corr,
1209            receiver,
1210        })
1211    }
1212
1213    #[allow(clippy::too_many_arguments)]
1214    fn begin_route_bind_relay_inner(
1215        &self,
1216        client_connection_id: ConnectionId,
1217        client_sink: FrameSink,
1218        client_negotiated_ver: u8,
1219        client_corr: u64,
1220        expected_module_id: &str,
1221        principal: Principal,
1222        scope: Option<BoundScope>,
1223        project_root: Option<ProjectRootId>,
1224        deadline: Instant,
1225        client_permit: crate::router::EgressPermit,
1226    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1227        let mut inner = self.write_inner()?;
1228        // HELLO and route.open can run on different tasks. Decide the role under
1229        // the same lock that installs endpoints and reserves client channels so
1230        // neither task can turn one connection into both kinds of owner.
1231        if inner
1232            .endpoint_by_connection
1233            .contains_key(&client_connection_id)
1234        {
1235            return Err(ForwardingError::ConnectionRoleConflict {
1236                connection_id: client_connection_id,
1237            });
1238        }
1239        if inner.closing_connections.contains(&client_connection_id) {
1240            return Err(ForwardingError::ConnectionClosing {
1241                connection_id: client_connection_id,
1242            });
1243        }
1244        let module = inner
1245            .modules_by_id
1246            .get(expected_module_id)
1247            .cloned()
1248            .ok_or(ForwardingError::NoModuleConnection)?;
1249        if inner.draining_endpoints.contains_key(&module.endpoint) {
1250            return Err(ForwardingError::ModuleReloading {
1251                module_id: expected_module_id.to_string(),
1252            });
1253        }
1254        if inner
1255            .closing_connections
1256            .contains(&module.endpoint.connection_id)
1257        {
1258            return Err(ForwardingError::ConnectionClosing {
1259                connection_id: module.endpoint.connection_id,
1260            });
1261        }
1262
1263        let corr = match inner.allocate_control_corr(module.endpoint) {
1264            Ok(corr) => corr,
1265            Err(err) => {
1266                drop(inner);
1267                self.request_connection_close(
1268                    module.endpoint.connection_id,
1269                    CloseReason::new(
1270                        "control_correlation_exhausted",
1271                        "daemon-originated channel-0 correlation space exhausted",
1272                    ),
1273                );
1274                return Err(err);
1275            }
1276        };
1277        let (client_channel, client_epoch, module_channel, module_epoch) =
1278            inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1279        let client_key = ClientRouteKey {
1280            connection_id: client_connection_id,
1281            channel: client_channel,
1282        };
1283        let module_key = ModuleRouteKey {
1284            endpoint: module.endpoint,
1285            channel: module_channel,
1286        };
1287        let reservation = RouteReservation {
1288            client_key,
1289            module_key,
1290            client_epoch,
1291            module_epoch,
1292            project_root,
1293        };
1294        let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
1295            route_channel: client_channel,
1296            route_epoch: client_epoch,
1297        })
1298        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1299        let route_open_frame = Frame::build_with_version(
1300            client_negotiated_ver,
1301            FrameType::Response,
1302            Flags::new(false, Priority::Passive, false),
1303            0,
1304            0,
1305            client_corr,
1306            response_body,
1307        )
1308        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1309        let (sender, receiver) = oneshot::channel();
1310        inner.reserved_client.insert(client_key, module_key);
1311        inner.reserved_module.insert(module_key, client_key);
1312        inner.pending_relays.insert(
1313            (module.endpoint, corr),
1314            PendingRouteBindRelayEntry {
1315                reservation,
1316                client_sink,
1317                client_negotiated_ver,
1318                client_permit,
1319                route_open_frame,
1320                principal,
1321                scope,
1322                deadline,
1323                relay_enqueued: false,
1324                sender,
1325            },
1326        );
1327
1328        Ok(PendingRouteBindRelay {
1329            endpoint: module.endpoint,
1330            module_sink: module.sink,
1331            negotiated_ver: module.negotiated_ver,
1332            client_channel,
1333            client_epoch,
1334            module_channel,
1335            module_epoch,
1336            corr,
1337            receiver,
1338        })
1339    }
1340
1341    /// Called by `route.open` right after the route.bind entered the module's
1342    /// egress queue.
1343    ///
1344    /// The module reader runs concurrently with `route.open`, so a fast module
1345    /// can receive the bind, ack it, and have the reader commit the route
1346    /// before this runs. A missing entry therefore does not mean the bind was
1347    /// abandoned: only a removal recorded in `relays_released_before_enqueue`
1348    /// does. Telling the two apart is what stops `route.open` from sending a
1349    /// GOODBYE for a route that was just committed.
1350    pub(crate) fn mark_route_bind_relay_enqueued(
1351        &self,
1352        endpoint: ModuleEndpointId,
1353        corr: u64,
1354    ) -> Result<RouteBindRelayMark, ForwardingError> {
1355        let mut inner = self.write_inner()?;
1356        if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
1357            pending.relay_enqueued = true;
1358            return Ok(RouteBindRelayMark::Marked);
1359        }
1360        if inner
1361            .relays_released_before_enqueue
1362            .remove(&(endpoint, corr))
1363        {
1364            return Ok(RouteBindRelayMark::ReleasedWithoutReply);
1365        }
1366        Ok(RouteBindRelayMark::SettledByReply)
1367    }
1368
1369    pub(crate) fn release_client_route(
1370        &self,
1371        client_connection_id: ConnectionId,
1372        client_channel: u16,
1373        expected_epoch: u32,
1374    ) -> Result<RouteRelease, ForwardingError> {
1375        let mut inner = self.write_inner()?;
1376        let release = release_client_route_locked(
1377            &mut inner,
1378            ClientRouteKey {
1379                connection_id: client_connection_id,
1380                channel: client_channel,
1381            },
1382            expected_epoch,
1383        );
1384        self.record_route_release(&release);
1385        Ok(release)
1386    }
1387
1388    pub(crate) fn release_module_route(
1389        &self,
1390        module_connection_id: ConnectionId,
1391        module_channel: u16,
1392        expected_epoch: u32,
1393    ) -> Result<RouteRelease, ForwardingError> {
1394        let mut inner = self.write_inner()?;
1395        let Some(endpoint) = inner
1396            .endpoint_by_connection
1397            .get(&module_connection_id)
1398            .copied()
1399        else {
1400            return Ok(RouteRelease::Absent);
1401        };
1402        let release = release_module_route_locked(
1403            &mut inner,
1404            ModuleRouteKey {
1405                endpoint,
1406                channel: module_channel,
1407            },
1408            expected_epoch,
1409        );
1410        self.record_route_release(&release);
1411        Ok(release)
1412    }
1413
1414    pub(crate) fn abort_pending_relay(
1415        &self,
1416        endpoint: ModuleEndpointId,
1417        corr: u64,
1418        outcome: RouteBindRelayOutcome,
1419    ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
1420        let mut inner = self.write_inner()?;
1421        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1422            // If another path dropped the relay while `route.open` was still
1423            // sending the bind, and that open is now giving up without marking
1424            // it (its send failed or it was cancelled), forget the record so it
1425            // cannot outlive the open.
1426            inner
1427                .relays_released_before_enqueue
1428                .remove(&(endpoint, corr));
1429            return Ok(None);
1430        };
1431        release_reserved_route_locked(
1432            &mut inner,
1433            pending.reservation.client_key,
1434            pending.reservation.module_key,
1435        );
1436        let target = pending
1437            .relay_enqueued
1438            .then(|| abandoned_route_target(&inner, &pending.reservation));
1439        let _ = pending.sender.send(outcome);
1440        Ok(target.flatten())
1441    }
1442
1443    pub(crate) fn cancel_module_control_rpc(
1444        &self,
1445        endpoint: ModuleEndpointId,
1446        corr: u64,
1447    ) -> Result<(), ForwardingError> {
1448        self.write_inner()?
1449            .pending_control_rpcs
1450            .remove(&(endpoint, corr));
1451        Ok(())
1452    }
1453
1454    pub(crate) fn tombstone_health_probe_rpc(
1455        &self,
1456        endpoint: ModuleEndpointId,
1457        corr: u64,
1458    ) -> Result<bool, ForwardingError> {
1459        let key = (endpoint, corr);
1460        let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
1461        {
1462            let mut inner = self.write_inner()?;
1463            let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
1464                return Ok(false);
1465            };
1466            let Some(probe_started_at) = pending.health_probe_started_at else {
1467                inner.pending_control_rpcs.insert(key, pending);
1468                return Ok(false);
1469            };
1470            let module_id = inner
1471                .module_id_by_endpoint
1472                .get(&endpoint)
1473                .cloned()
1474                .unwrap_or_else(|| "unknown".to_string());
1475            inner.health_probe_tombstones.insert(
1476                key,
1477                HealthProbeTombstone {
1478                    expected_op: pending.expected_op,
1479                    module_id,
1480                    probe_started_at,
1481                    expires_at,
1482                },
1483            );
1484        }
1485        self.schedule_health_probe_tombstone_expiration(key, expires_at);
1486        Ok(true)
1487    }
1488
1489    fn schedule_health_probe_tombstone_expiration(
1490        &self,
1491        key: (ModuleEndpointId, u64),
1492        expires_at: Instant,
1493    ) {
1494        let inner = Arc::downgrade(&self.inner);
1495        tokio::spawn(async move {
1496            tokio::time::sleep_until(expires_at).await;
1497            let Some(inner) = inner.upgrade() else {
1498                return;
1499            };
1500            let Ok(mut inner) = inner.write() else {
1501                return;
1502            };
1503            let expired = inner
1504                .health_probe_tombstones
1505                .get(&key)
1506                .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
1507            if expired {
1508                inner.health_probe_tombstones.remove(&key);
1509            }
1510        });
1511    }
1512
1513    pub(crate) fn complete_pending_relay(
1514        &self,
1515        connection_id: ConnectionId,
1516        corr: u64,
1517        outcome: RouteBindRelayOutcome,
1518    ) -> Result<PendingRelayCompletion, ForwardingError> {
1519        let mut inner = self.write_inner()?;
1520        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1521            return Ok(PendingRelayCompletion {
1522                settled: false,
1523                abandoned: None,
1524            });
1525        };
1526        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1527            return Ok(PendingRelayCompletion {
1528                settled: false,
1529                abandoned: None,
1530            });
1531        };
1532
1533        if Instant::now() >= pending.deadline {
1534            release_reserved_route_locked(
1535                &mut inner,
1536                pending.reservation.client_key,
1537                pending.reservation.module_key,
1538            );
1539            let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
1540                .then(|| abandoned_route_target(&inner, &pending.reservation))
1541                .flatten();
1542            let _ = pending
1543                .sender
1544                .send(RouteBindRelayOutcome::Rejected(ErrorBody {
1545                    code: "module_timeout".to_string(),
1546                    message: "route.bind response arrived after its daemon deadline".to_string(),
1547                    detail: None,
1548                }));
1549            return Ok(PendingRelayCompletion {
1550                settled: true,
1551                abandoned,
1552            });
1553        }
1554
1555        match outcome {
1556            // Two shapes of "the client is not there to receive this route" that
1557            // must resolve identically: its egress is already closed, or it is
1558            // marked closing (its connection loop has been asked to end, but has
1559            // not drained yet, so the sink is still open).
1560            //
1561            // Only the first used to be caught here. The second fell through to
1562            // `commit_route_locked`, which refuses a closing client with
1563            // `ConnectionClosing` -- and this function is called from the MODULE
1564            // connection's frame handler, so that refusal ended the module's
1565            // connection instead of this one client's route. One dying client's
1566            // route.open then took down a connection carrying every other
1567            // client's routes to that module.
1568            //
1569            // The remedy for both is the same, which is why they share an arm:
1570            // give back the reserved handle pair, tell the waiting route.open the
1571            // route is gone, and report the module-side channel so the caller can
1572            // send a channel-scoped GOODBYE for the binding the module just
1573            // created. Nothing here touches the module connection.
1574            RouteBindRelayOutcome::Accepted
1575                if pending.client_sink.is_closed()
1576                    || inner
1577                        .closing_connections
1578                        .contains(&pending.reservation.client_key.connection_id) =>
1579            {
1580                let reason = if pending.client_sink.is_closed() {
1581                    "client egress closed before route publication"
1582                } else {
1583                    "client connection is closing before route publication"
1584                };
1585                release_reserved_route_locked(
1586                    &mut inner,
1587                    pending.reservation.client_key,
1588                    pending.reservation.module_key,
1589                );
1590                // Not gated on `relay_enqueued`: an ack proves the module
1591                // received the bind, and `route.open` sends nothing for a
1592                // relay a reply already settled, so this is the only GOODBYE.
1593                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1594                let _ = pending
1595                    .sender
1596                    .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
1597                return Ok(PendingRelayCompletion {
1598                    settled: true,
1599                    abandoned,
1600                });
1601            }
1602            // The endpoint that acked this bind is no longer accepting new
1603            // routes, for one of two reasons. Either it was the module's active
1604            // endpoint when this relay was reserved and a blue/green swap has
1605            // since promoted a candidate (the replacement process registered
1606            // beside it) into the active slot, superseding it; or a daemon
1607            // drain has gated it, and the ack arrived before the ordered module
1608            // drain settled pending relays. The module did nothing wrong: it
1609            // bound the route it was asked to bind, and the endpoint still
1610            // carries every other client's routes until it is drained. So the
1611            // ack is settled here, while the pending entry still holds the
1612            // client's sender and the reserved handle pair: release the pair,
1613            // refuse the waiting route.open with `module_reloading`, which
1614            // callers retry (after a swap the retry reserves on the promoted
1615            // endpoint), and hand back the module-side channel so the caller
1616            // sends one channel-scoped GOODBYE for the binding the module just
1617            // created. Returning an error instead would end the module
1618            // connection all those other routes share; nothing here touches it.
1619            //
1620            // An endpoint that stopped being active any other way is stale
1621            // (replaced, but not by a swap and not draining). It still falls
1622            // through to `commit_route_locked`, which refuses it with
1623            // `StaleModuleEndpoint` exactly as before swaps existed.
1624            RouteBindRelayOutcome::Accepted
1625                if inner.superseded_endpoints.contains_key(&endpoint)
1626                    || inner.draining_endpoints.contains_key(&endpoint) =>
1627            {
1628                release_reserved_route_locked(
1629                    &mut inner,
1630                    pending.reservation.client_key,
1631                    pending.reservation.module_key,
1632                );
1633                // Not gated on `relay_enqueued`: an ack proves the module
1634                // received the bind, whether or not the enqueue mark was set.
1635                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1636                let module_id = inner
1637                    .module_id_by_endpoint
1638                    .get(&endpoint)
1639                    .cloned()
1640                    .unwrap_or_else(|| "unknown".to_string());
1641                let _ = pending
1642                    .sender
1643                    .send(RouteBindRelayOutcome::Rejected(ErrorBody::new(
1644                        "module_reloading",
1645                        format!("module_id '{module_id}' is reloading"),
1646                    )));
1647                return Ok(PendingRelayCompletion {
1648                    settled: true,
1649                    abandoned,
1650                });
1651            }
1652            // The scope the open was admitted under ended or changed after
1653            // admission. The module bound a route under a stamp that is no
1654            // longer true, so it must not become routable. Settled here like
1655            // the superseded arm above, never as an error from
1656            // `commit_route_locked`, which would close the module's whole
1657            // connection and every other client's routes to it: release the
1658            // pair, answer the waiting route.open by name, and hand back the
1659            // module-side channel for one channel-scoped GOODBYE.
1660            RouteBindRelayOutcome::Accepted
1661                if pending
1662                    .scope
1663                    .as_ref()
1664                    .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
1665            {
1666                let refusal = pending
1667                    .scope
1668                    .as_ref()
1669                    .and_then(|scope| scope_refusal_locked(&inner, scope))
1670                    .expect("guard matched a refusal under the same lock");
1671                release_reserved_route_locked(
1672                    &mut inner,
1673                    pending.reservation.client_key,
1674                    pending.reservation.module_key,
1675                );
1676                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1677                let _ = pending
1678                    .sender
1679                    .send(RouteBindRelayOutcome::Rejected(refusal));
1680                return Ok(PendingRelayCompletion {
1681                    settled: true,
1682                    abandoned,
1683                });
1684            }
1685            RouteBindRelayOutcome::Accepted => {
1686                let abandoned = commit_route_locked(&mut inner, pending)?;
1687                return Ok(PendingRelayCompletion {
1688                    settled: true,
1689                    abandoned,
1690                });
1691            }
1692            terminal => {
1693                release_reserved_route_locked(
1694                    &mut inner,
1695                    pending.reservation.client_key,
1696                    pending.reservation.module_key,
1697                );
1698                let _ = pending.sender.send(terminal);
1699            }
1700        }
1701        Ok(PendingRelayCompletion {
1702            settled: true,
1703            abandoned: None,
1704        })
1705    }
1706
1707    pub(crate) fn pending_module_control_op(
1708        &self,
1709        connection_id: ConnectionId,
1710        corr: u64,
1711    ) -> Result<Option<String>, ForwardingError> {
1712        let inner = self.read_inner()?;
1713        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1714            return Ok(None);
1715        };
1716        let key = (endpoint, corr);
1717        Ok(inner
1718            .pending_control_rpcs
1719            .get(&key)
1720            .map(|pending| pending.expected_op.clone())
1721            .or_else(|| {
1722                inner
1723                    .health_probe_tombstones
1724                    .get(&key)
1725                    .filter(|tombstone| tombstone.expires_at > Instant::now())
1726                    .map(|tombstone| tombstone.expected_op.clone())
1727            }))
1728    }
1729
1730    pub(crate) fn complete_module_control_rpc(
1731        &self,
1732        connection_id: ConnectionId,
1733        corr: u64,
1734        actual_op: Option<&str>,
1735        outcome: ModuleControlRpcOutcome,
1736    ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
1737        let now = Instant::now();
1738        let mut inner = self.write_inner()?;
1739        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1740            return Ok(ModuleControlRpcCompletion::Unknown);
1741        };
1742        let key = (endpoint, corr);
1743        if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
1744            if now >= pending.deadline {
1745                let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
1746                    ModuleControlRpcCompletion::LateHealthAnswer {
1747                        module_id: inner
1748                            .module_id_by_endpoint
1749                            .get(&endpoint)
1750                            .cloned()
1751                            .unwrap_or_else(|| "unknown".to_string()),
1752                        latency: now.saturating_duration_since(probe_started_at),
1753                    }
1754                });
1755                let _ = pending
1756                    .sender
1757                    .send(ModuleControlRpcOutcome::DeadlineElapsed);
1758                return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
1759            }
1760            let outcome = match actual_op {
1761                Some(actual) if actual != pending.expected_op => {
1762                    ModuleControlRpcOutcome::UnexpectedOp {
1763                        expected: pending.expected_op,
1764                        actual: actual.to_string(),
1765                    }
1766                }
1767                _ => outcome,
1768            };
1769            let _ = pending.sender.send(outcome);
1770            return Ok(ModuleControlRpcCompletion::Settled);
1771        }
1772
1773        let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
1774            return Ok(ModuleControlRpcCompletion::Unknown);
1775        };
1776        if tombstone.expires_at <= now {
1777            return Ok(ModuleControlRpcCompletion::Unknown);
1778        }
1779        Ok(ModuleControlRpcCompletion::LateHealthAnswer {
1780            module_id: tombstone.module_id,
1781            latency: now.saturating_duration_since(tombstone.probe_started_at),
1782        })
1783    }
1784
1785    #[cfg(test)]
1786    pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
1787        Ok(self.read_inner()?.health_probe_tombstones.len())
1788    }
1789
1790    #[cfg(test)]
1791    pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
1792        Ok(self.read_inner()?.closing_connections.len())
1793    }
1794
1795    /// Reserved-but-uncommitted route handles, counted on both index sides, so
1796    /// a test can assert a reservation pair was actually given back.
1797    #[cfg(test)]
1798    pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
1799        let inner = self.read_inner()?;
1800        Ok((inner.reserved_client.len(), inner.reserved_module.len()))
1801    }
1802
1803    pub(crate) fn module_endpoint_for_connection(
1804        &self,
1805        connection_id: ConnectionId,
1806    ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
1807        Ok(self
1808            .read_inner()?
1809            .endpoint_by_connection
1810            .get(&connection_id)
1811            .copied())
1812    }
1813
1814    /// Looks up the module registered on a data-plane connection so route-drop
1815    /// diagnostics name the emitter instead of only reporting a daemon total.
1816    pub(crate) fn module_id_for_connection(
1817        &self,
1818        connection_id: ConnectionId,
1819    ) -> Result<Option<String>, ForwardingError> {
1820        let inner = self.read_inner()?;
1821        Ok(inner
1822            .endpoint_by_connection
1823            .get(&connection_id)
1824            .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
1825            .cloned())
1826    }
1827
1828    /// Whether the daemon ever allocated `(channel, epoch)` on this module
1829    /// connection. Epochs on a module channel are handed out as 1, 2, 3, ...
1830    /// and the last one handed out is remembered until the connection ends, so
1831    /// every epoch from 1 up to that one was a real route at some point. Used
1832    /// only on the drop path, to tell a module still sending on a route the
1833    /// daemon released from one sending on a route that never existed.
1834    pub(crate) fn module_route_epoch_was_allocated(
1835        &self,
1836        connection_id: ConnectionId,
1837        channel: u16,
1838        epoch: u32,
1839    ) -> Result<bool, ForwardingError> {
1840        let inner = self.read_inner()?;
1841        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1842            return Ok(false);
1843        };
1844        Ok(inner
1845            .module_slot_epochs
1846            .get(&ModuleRouteKey { endpoint, channel })
1847            .is_some_and(|last| epoch != 0 && epoch <= *last))
1848    }
1849
1850    pub(crate) fn has_live_module_connection(
1851        &self,
1852        module_id: &str,
1853    ) -> Result<bool, ForwardingError> {
1854        Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
1855    }
1856
1857    pub(crate) fn lookup_data_route(
1858        &self,
1859        connection_id: ConnectionId,
1860        channel: u16,
1861        epoch: u32,
1862    ) -> Result<DataRoute, ForwardingError> {
1863        let inner = self.read_inner()?;
1864        let state = if let Some(endpoint) =
1865            inner.endpoint_by_connection.get(&connection_id).copied()
1866        {
1867            let key = ModuleRouteKey { endpoint, channel };
1868            match inner.module_to_client.get(&key) {
1869                Some(route) if route.module_epoch == epoch => {
1870                    DataRouteState::Bound(Arc::clone(route))
1871                }
1872                Some(_) => DataRouteState::EpochMismatch,
1873                None if inner.reserved_module.contains_key(&key)
1874                    && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
1875                {
1876                    DataRouteState::Reserved
1877                }
1878                None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
1879                None => DataRouteState::Absent,
1880            }
1881        } else {
1882            let key = ClientRouteKey {
1883                connection_id,
1884                channel,
1885            };
1886            match inner.client_to_module.get(&key) {
1887                Some(route) if route.client_epoch == epoch => {
1888                    DataRouteState::Bound(Arc::clone(route))
1889                }
1890                Some(_) => DataRouteState::EpochMismatch,
1891                None if inner.reserved_client.contains_key(&key)
1892                    && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
1893                {
1894                    DataRouteState::Reserved
1895                }
1896                None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
1897                None => DataRouteState::Absent,
1898            }
1899        };
1900        Ok(
1901            if inner.endpoint_by_connection.contains_key(&connection_id) {
1902                DataRoute::Module(state)
1903            } else {
1904                DataRoute::Client(state)
1905            },
1906        )
1907    }
1908
1909    #[cfg(test)]
1910    pub(crate) fn inject_client_slot_epoch(
1911        &self,
1912        connection_id: ConnectionId,
1913        channel: u16,
1914        last_epoch: u32,
1915    ) {
1916        let mut inner = self.write_inner().expect("forwarding lock");
1917        inner.client_slot_epochs.insert(
1918            ClientRouteKey {
1919                connection_id,
1920                channel,
1921            },
1922            last_epoch,
1923        );
1924        inner.next_client_channel.insert(connection_id, channel);
1925    }
1926
1927    #[cfg(test)]
1928    pub(crate) fn inject_module_slot_epoch(
1929        &self,
1930        endpoint: ModuleEndpointId,
1931        channel: u16,
1932        last_epoch: u32,
1933    ) {
1934        let mut inner = self.write_inner().expect("forwarding lock");
1935        inner
1936            .module_slot_epochs
1937            .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
1938        inner.next_module_channel.insert(endpoint, channel);
1939    }
1940
1941    #[cfg(test)]
1942    pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
1943        self.write_inner()
1944            .expect("forwarding lock")
1945            .next_control_corr
1946            .insert(endpoint, next_corr);
1947    }
1948
1949    pub(crate) fn cache_status(
1950        &self,
1951        endpoint: ModuleEndpointId,
1952        module_channel: u16,
1953        module_epoch: u32,
1954        status: String,
1955    ) -> Result<bool, ForwardingError> {
1956        let mut inner = self.write_inner()?;
1957        if !inner.module_id_by_endpoint.contains_key(&endpoint) {
1958            return Err(ForwardingError::StaleModuleEndpoint);
1959        }
1960
1961        let module_key = ModuleRouteKey {
1962            endpoint,
1963            channel: module_channel,
1964        };
1965        let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
1966            (route.module_epoch == module_epoch).then_some((
1967                ClientRouteKey {
1968                    connection_id: route.client_connection_id,
1969                    channel: route.client_channel,
1970                },
1971                route.client_epoch,
1972            ))
1973        } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
1974            (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
1975                client_key,
1976                inner
1977                    .client_slot_epochs
1978                    .get(&client_key)
1979                    .copied()
1980                    .unwrap_or(0),
1981            ))
1982        } else {
1983            None
1984        };
1985
1986        if let Some(handle) = handle {
1987            inner.status.insert(handle, status);
1988            Ok(true)
1989        } else {
1990            debug!(
1991                module_channel,
1992                module_epoch,
1993                generation = endpoint.generation,
1994                connection_id = endpoint.connection_id.get(),
1995                "dropping stale status update for module route handle"
1996            );
1997            Ok(false)
1998        }
1999    }
2000
2001    pub(crate) fn route_poll_snapshot(
2002        &self,
2003        client_connection_id: ConnectionId,
2004        client_channel: u16,
2005        client_epoch: u32,
2006    ) -> Result<RoutePollSnapshot, ForwardingError> {
2007        let inner = self.read_inner()?;
2008        let client_key = ClientRouteKey {
2009            connection_id: client_connection_id,
2010            channel: client_channel,
2011        };
2012        let Some(route) = inner.client_to_module.get(&client_key) else {
2013            return Ok(RoutePollSnapshot::Absent);
2014        };
2015        if route.client_epoch != client_epoch
2016            || !inner
2017                .module_id_by_endpoint
2018                .contains_key(&route.module_endpoint)
2019        {
2020            return Ok(RoutePollSnapshot::Absent);
2021        }
2022        Ok(RoutePollSnapshot::Bound {
2023            module_id: route.module_id.clone(),
2024            status: inner.status.get(&(client_key, client_epoch)).cloned(),
2025        })
2026    }
2027
2028    pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2029        Ok(self.read_inner()?.client_to_module.len())
2030    }
2031
2032    /// How many distinct client connections hold at least one committed route,
2033    /// alongside the largest number of routes any single connection holds.
2034    ///
2035    /// `connected_clients` alone cannot distinguish many clients with a route
2036    /// each from one client accumulating hundreds, and those have opposite
2037    /// causes. Reading it required an out-of-band `lsof` during a live
2038    /// investigation, and the count of connections was mistaken for a count of
2039    /// client processes — which sent two of us after cleanup paths that were
2040    /// working correctly.
2041    pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2042        let inner = self.read_inner()?;
2043        let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2044        for key in inner.client_to_module.keys() {
2045            *per_connection.entry(key.connection_id).or_insert(0) += 1;
2046        }
2047        let max = per_connection.values().copied().max().unwrap_or(0);
2048        Ok((per_connection.len(), max))
2049    }
2050
2051    pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2052        let inner = self.read_inner()?;
2053        Ok(inner
2054            .client_to_module
2055            .keys()
2056            .any(|key| key.channel == route_channel))
2057    }
2058
2059    /// Whether daemon-wide shutdown has begun draining all providers.
2060    pub(crate) fn is_daemon_draining(&self) -> Result<bool, ForwardingError> {
2061        Ok(self.read_inner()?.daemon_draining)
2062    }
2063
2064    /// Gate every provider atomically, including registrations racing shutdown.
2065    /// No supervisor lock is held while taking the forwarding lock.
2066    #[cfg(unix)]
2067    pub(crate) fn begin_daemon_drain(&self) -> Result<Vec<String>, ForwardingError> {
2068        let mut inner = self.write_inner()?;
2069        inner.daemon_draining = true;
2070        let modules = inner
2071            .modules_by_id
2072            .iter()
2073            .map(|(id, module)| (id.clone(), module.endpoint))
2074            .collect::<Vec<_>>();
2075        for (_, endpoint) in &modules {
2076            inner
2077                .draining_endpoints
2078                .insert(*endpoint, RouteCloseReason::Restart);
2079        }
2080        // Swap candidates and superseded incumbents are not routable, but they
2081        // are live endpoints that could still be sent module-control work, so
2082        // the daemon-wide gate covers them too. The returned ids are unchanged:
2083        // they name modules for the supervisor to drain, one per id.
2084        let off_slot_endpoints = inner
2085            .candidates_by_id
2086            .values()
2087            .map(|module| module.endpoint)
2088            .chain(inner.superseded_endpoints.keys().copied())
2089            .collect::<Vec<_>>();
2090        for endpoint in off_slot_endpoints {
2091            inner
2092                .draining_endpoints
2093                .insert(endpoint, RouteCloseReason::Restart);
2094        }
2095        Ok(modules.into_iter().map(|(id, _)| id).collect())
2096    }
2097
2098    /// Begin draining whatever endpoint is ACTIVE for `module_id`.
2099    ///
2100    /// After a swap's cutover the active endpoint is the promoted candidate, so
2101    /// this must not be used to drain the incumbent; use
2102    /// [`Self::begin_endpoint_drain`] with the endpoint `cutover_candidate`
2103    /// returned.
2104    pub(crate) fn begin_module_drain(
2105        &self,
2106        module_id: &str,
2107        reason: RouteCloseReason,
2108    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2109        let mut inner = self.write_inner()?;
2110        let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2111            return Ok(None);
2112        };
2113        Ok(Some(begin_drain_locked(
2114            &mut inner, module_id, module, reason,
2115        )))
2116    }
2117
2118    /// Begin draining one specific endpoint, whichever slot it is in.
2119    ///
2120    /// This is how a swap drains its incumbent after cutover: the incumbent's
2121    /// endpoint is captured by `cutover_candidate`, and resolving it by module id
2122    /// instead would find the promoted candidate and leave neither process
2123    /// routable. Returns `Ok(None)` when the endpoint is no longer registered.
2124    pub(crate) fn begin_endpoint_drain(
2125        &self,
2126        endpoint: ModuleEndpointId,
2127        reason: RouteCloseReason,
2128    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2129        let mut inner = self.write_inner()?;
2130        let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2131            return Ok(None);
2132        };
2133        let module_id = inner
2134            .module_id_by_endpoint
2135            .get(&endpoint)
2136            .cloned()
2137            .expect("an endpoint resolved to a module connection has a module id");
2138        Ok(Some(begin_drain_locked(
2139            &mut inner, &module_id, module, reason,
2140        )))
2141    }
2142}
2143
2144/// Mark `module`'s endpoint draining and settle everything still pending on it.
2145/// Shared by the by-id and by-endpoint drain entry points, which differ only in
2146/// how they find the endpoint.
2147fn begin_drain_locked(
2148    inner: &mut ForwardingInner,
2149    module_id: &str,
2150    module: ModuleConnection,
2151    reason: RouteCloseReason,
2152) -> ModuleDrainTarget {
2153    {
2154        let endpoint = module.endpoint;
2155        inner.draining_endpoints.insert(endpoint, reason);
2156
2157        let flows = inner
2158            .client_to_module
2159            .values()
2160            .filter(|route| route.module_endpoint == endpoint)
2161            .map(|route| Arc::clone(&route.flow))
2162            .collect::<Vec<_>>();
2163        let excluded_subscriptions = flows
2164            .into_iter()
2165            .map(|flow| flow.begin_drain())
2166            .fold(0u32, u32::saturating_add);
2167
2168        let pending_keys = inner
2169            .pending_relays
2170            .keys()
2171            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2172            .copied()
2173            .collect::<Vec<_>>();
2174        let mut abandoned_bindings = Vec::new();
2175        for key in pending_keys {
2176            let Some(pending) = inner.pending_relays.remove(&key) else {
2177                continue;
2178            };
2179            release_reserved_route_locked(
2180                inner,
2181                pending.reservation.client_key,
2182                pending.reservation.module_key,
2183            );
2184            if pending.relay_enqueued {
2185                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2186                    abandoned_bindings.push(target);
2187                }
2188            } else {
2189                // The bind may still be on its way into the module's queue;
2190                // a GOODBYE sent now could overtake it. `route.open` sends
2191                // it once its send completes.
2192                inner.relays_released_before_enqueue.insert(key);
2193            }
2194            let _ = pending
2195                .sender
2196                .send(RouteBindRelayOutcome::Rejected(ErrorBody::new(
2197                    "module_reloading",
2198                    format!("module_id '{module_id}' is reloading"),
2199                )));
2200        }
2201
2202        let pending_control_keys = inner
2203            .pending_control_rpcs
2204            .keys()
2205            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2206            .copied()
2207            .collect::<Vec<_>>();
2208        for key in pending_control_keys {
2209            if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2210                let _ = pending
2211                    .sender
2212                    .send(ModuleControlRpcOutcome::ModuleGone(format!(
2213                        "module '{module_id}' began draining during module-control RPC"
2214                    )));
2215            }
2216        }
2217
2218        ModuleDrainTarget {
2219            endpoint,
2220            sink: module.sink,
2221            negotiated_ver: module.negotiated_ver,
2222            abandoned_bindings,
2223            excluded_subscriptions,
2224        }
2225    }
2226}
2227
2228/// What a drain that timed out was still waiting on, for the log line that
2229/// reports the timeout. Per-request ages are not tracked.
2230#[derive(Debug, Default, PartialEq, Eq)]
2231pub(crate) struct DrainHoldouts {
2232    /// Requests still counted against the drain (subscriptions flagged as
2233    /// such are excluded from the drain and not counted here).
2234    pub(crate) requests: usize,
2235    /// Routes holding at least one of those requests.
2236    pub(crate) routes: usize,
2237    /// Every route on the endpoint, for scale.
2238    pub(crate) total_routes: usize,
2239    /// The client connections holding the most requests, largest first, at
2240    /// most three: enough to name the consumer without listing every route.
2241    pub(crate) top_connections: Vec<(u64, usize)>,
2242    /// The held requests themselves, as the module saw them, so its own log
2243    /// can say what each one was: `(module channel, corr)`, ordered by channel
2244    /// then corr, at most [`DRAIN_HELD_REQUESTS_LISTED`]. The daemon never reads
2245    /// request bodies, so it cannot name a request's method; the module can,
2246    /// from the channel and corr. A request is released only when the module
2247    /// sends a terminal frame (Response, Error or StreamEnd) with its corr on
2248    /// its route, so every pair listed here is one the module ended, if at all,
2249    /// without sending that frame.
2250    pub(crate) held: Vec<(u16, u64)>,
2251}
2252
2253/// How many held requests the drain-timeout line lists by channel and corr.
2254pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
2255
2256impl ForwardingTable {
2257    /// Summarise the requests one endpoint's drain is still waiting on.
2258    pub(crate) fn endpoint_drain_holdouts(
2259        &self,
2260        endpoint: ModuleEndpointId,
2261    ) -> Result<DrainHoldouts, ForwardingError> {
2262        let inner = self.read_inner()?;
2263        let mut holdouts = DrainHoldouts::default();
2264        let mut by_connection: HashMap<u64, usize> = HashMap::new();
2265        for (key, route) in &inner.client_to_module {
2266            if route.module_endpoint != endpoint {
2267                continue;
2268            }
2269            holdouts.total_routes += 1;
2270            let held = route.flow.drain_in_flight();
2271            if held == 0 {
2272                continue;
2273            }
2274            holdouts.requests += held;
2275            holdouts.routes += 1;
2276            *by_connection.entry(key.connection_id.get()).or_default() += held;
2277            holdouts.held.extend(
2278                route
2279                    .flow
2280                    .drain_held_corrs()
2281                    .into_iter()
2282                    .map(|corr| (route.module_channel, corr)),
2283            );
2284        }
2285        holdouts.held.sort_unstable();
2286        holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
2287        let mut connections = by_connection.into_iter().collect::<Vec<_>>();
2288        connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
2289        connections.truncate(3);
2290        holdouts.top_connections = connections;
2291        Ok(holdouts)
2292    }
2293
2294    pub(crate) fn endpoint_in_flight_count(
2295        &self,
2296        endpoint: ModuleEndpointId,
2297    ) -> Result<usize, ForwardingError> {
2298        let inner = self.read_inner()?;
2299        Ok(inner
2300            .client_to_module
2301            .values()
2302            .filter(|route| route.module_endpoint == endpoint)
2303            .map(|route| route.flow.drain_in_flight())
2304            .sum())
2305    }
2306
2307    pub(crate) fn endpoint_is_draining(
2308        &self,
2309        endpoint: ModuleEndpointId,
2310    ) -> Result<bool, ForwardingError> {
2311        Ok(self
2312            .read_inner()?
2313            .draining_endpoints
2314            .contains_key(&endpoint))
2315    }
2316
2317    pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
2318        let inner = self.read_inner()?;
2319        Ok(inner
2320            .modules_by_id
2321            .get(module_id)
2322            .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
2323    }
2324
2325    pub(crate) fn release_module_endpoint_routes(
2326        &self,
2327        endpoint: ModuleEndpointId,
2328    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2329        let mut inner = self.write_inner()?;
2330        let routes = inner
2331            .module_to_client
2332            .iter()
2333            .filter(|(module_key, _)| module_key.endpoint == endpoint)
2334            .map(|(module_key, route)| (*module_key, route.module_epoch))
2335            .collect::<Vec<_>>();
2336        let mut released = Vec::with_capacity(routes.len());
2337        for (module_key, epoch) in routes {
2338            if let RouteRelease::Removed(target) =
2339                release_module_route_locked(&mut inner, module_key, epoch)
2340            {
2341                released.push(target);
2342            }
2343        }
2344        Ok(released)
2345    }
2346
2347    /// Enumerate one endpoint's current routes without contacting the module.
2348    ///
2349    /// The read lock makes this safe while the endpoint drains: the returned
2350    /// `draining` marker describes the same table state that owns the route,
2351    /// rather than inferring liveness from a module that may be stopping.
2352    pub(crate) fn endpoint_routes(
2353        &self,
2354        endpoint: ModuleEndpointId,
2355    ) -> Result<Vec<EndpointRoute>, ForwardingError> {
2356        let inner = self.read_inner()?;
2357        Ok(endpoint_routes_locked(&inner, endpoint))
2358    }
2359
2360    /// Snapshot all live endpoint route sets under one forwarding-table read lock.
2361    pub(crate) fn route_census(
2362        &self,
2363        module_id: Option<&str>,
2364    ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
2365        let inner = self.read_inner()?;
2366        let mut endpoints = inner
2367            .modules_by_id
2368            .iter()
2369            .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
2370            .map(|(id, module)| (id.clone(), module.endpoint))
2371            .collect::<Vec<_>>();
2372        endpoints.sort_by(|left, right| left.0.cmp(&right.0));
2373        Ok(endpoints
2374            .into_iter()
2375            .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
2376            .collect())
2377    }
2378
2379    /// Snapshot only the endpoint currently routable under this module id.
2380    pub(crate) fn live_roots(
2381        &self,
2382        module_id: &str,
2383    ) -> Result<ModuleControlResponseToModule, ForwardingError> {
2384        let inner = self.read_inner()?;
2385        let endpoint = inner
2386            .modules_by_id
2387            .get(module_id)
2388            .map(|module| module.endpoint);
2389        let mut roots = BTreeMap::new();
2390        let mut unknown_root_bindings = 0;
2391        let mut total_bindings = 0;
2392        if let Some(endpoint) = endpoint {
2393            for binding in inner
2394                .module_to_client
2395                .values()
2396                .filter(|binding| binding.module_endpoint == endpoint)
2397            {
2398                total_bindings += 1;
2399                if let Some(root) = &binding.project_root {
2400                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2401                    entry.0 += 1;
2402                } else {
2403                    unknown_root_bindings += 1;
2404                }
2405            }
2406            for pending in inner
2407                .pending_relays
2408                .values()
2409                .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
2410            {
2411                total_bindings += 1;
2412                if let Some(root) = &pending.reservation.project_root {
2413                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2414                    entry.1 += 1;
2415                } else {
2416                    unknown_root_bindings += 1;
2417                }
2418            }
2419        }
2420        Ok(ModuleControlResponseToModule::LiveRoots {
2421            roots: roots
2422                .into_iter()
2423                .map(|(project_root, (bound, pending))| LiveRoot {
2424                    project_root,
2425                    bound,
2426                    pending,
2427                })
2428                .collect(),
2429            unknown_root_bindings,
2430            total_bindings,
2431        })
2432    }
2433
2434    /// True if this connection already owns committed or reserved CLIENT routes.
2435    /// A module registers (HELLO) before serving and never opens client routes, so
2436    /// a connection that has client routes must not also become a module endpoint
2437    /// — otherwise one connection holds both client and module state and cleanup
2438    /// only releases one side.
2439    pub(crate) fn connection_has_client_routes(
2440        &self,
2441        connection_id: ConnectionId,
2442    ) -> Result<bool, ForwardingError> {
2443        let inner = self.read_inner()?;
2444        Ok(connection_has_client_routes_locked(&inner, connection_id))
2445    }
2446
2447    pub(crate) fn cleanup_connection(
2448        &self,
2449        connection_id: ConnectionId,
2450    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2451        self.cleanup_connection_counted(connection_id)
2452            .map(|cleanup| cleanup.released)
2453    }
2454
2455    /// [`Self::cleanup_connection`], also reporting how many pending
2456    /// route.bind relays to the closed module were aborted. That count is the
2457    /// `abandoned` figure of the `route.closed` push sent for a lost module
2458    /// connection; it is always zero for a client connection.
2459    pub(crate) fn cleanup_connection_counted(
2460        &self,
2461        connection_id: ConnectionId,
2462    ) -> Result<ConnectionCleanup, ForwardingError> {
2463        let mut inner = self.write_inner()?;
2464        inner.closing_connections.insert(connection_id);
2465        let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
2466            remove_module_connection_locked(&mut inner, endpoint)
2467        } else {
2468            ConnectionCleanup {
2469                released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
2470                abandoned_relays: 0,
2471            }
2472        };
2473        // The closing mark refuses new work for a connection whose teardown is
2474        // still pending. This is the latest point at which lifting it is safe:
2475        // teardown has just removed every per-connection entry above, under
2476        // this same write lock, so no lookup can still find live state for the
2477        // id; and ids come from a monotonic counter that never reissues one,
2478        // so the id can never name a future connection either. Keeping the
2479        // mark past this point only grew the set by one entry per connection
2480        // for the life of the daemon.
2481        inner.closing_connections.remove(&connection_id);
2482        Ok(cleanup)
2483    }
2484
2485    fn cleanup_client_connection_locked(
2486        inner: &mut ForwardingInner,
2487        connection_id: ConnectionId,
2488    ) -> Vec<GoodbyeTarget> {
2489        let routes = inner
2490            .client_to_module
2491            .iter()
2492            .filter(|(key, _)| key.connection_id == connection_id)
2493            .map(|(key, route)| (*key, route.client_epoch))
2494            .collect::<Vec<_>>();
2495        let mut released = Vec::with_capacity(routes.len());
2496        for (client_key, epoch) in routes {
2497            if let RouteRelease::Removed(target) =
2498                release_client_route_locked(inner, client_key, epoch)
2499            {
2500                released.push(target);
2501            }
2502        }
2503
2504        let pending_keys = inner
2505            .pending_relays
2506            .iter()
2507            .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
2508            .map(|(key, _)| *key)
2509            .collect::<Vec<_>>();
2510        for key in pending_keys {
2511            let Some(pending) = inner.pending_relays.remove(&key) else {
2512                continue;
2513            };
2514            release_reserved_route_locked(
2515                inner,
2516                pending.reservation.client_key,
2517                pending.reservation.module_key,
2518            );
2519            if pending.relay_enqueued {
2520                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2521                    released.push(target);
2522                }
2523            } else {
2524                // The bind may still be on its way into the module's queue;
2525                // a GOODBYE sent now could overtake it. `route.open` sends
2526                // it once its send completes.
2527                inner.relays_released_before_enqueue.insert(key);
2528            }
2529            let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
2530                "client connection closed during route.bind relay".to_string(),
2531            ));
2532        }
2533
2534        let orphaned = inner
2535            .reserved_client
2536            .iter()
2537            .filter(|(key, _)| key.connection_id == connection_id)
2538            .map(|(client, module)| (*client, *module))
2539            .collect::<Vec<_>>();
2540        for (client_key, module_key) in orphaned {
2541            release_reserved_route_locked(inner, client_key, module_key);
2542        }
2543        inner.next_client_channel.remove(&connection_id);
2544        inner
2545            .client_slot_epochs
2546            .retain(|key, _| key.connection_id != connection_id);
2547        inner
2548            .last_published_epoch
2549            .retain(|key, _| key.connection_id != connection_id);
2550        inner
2551            .status
2552            .retain(|(key, _), _| key.connection_id != connection_id);
2553
2554        released
2555    }
2556
2557    /// Close a client connection whose egress queue refused a frame for the
2558    /// route `(connection_id, channel)` at `expected_epoch`. The whole
2559    /// connection closes because its queue is shared by every route on it.
2560    ///
2561    /// The first request that actually closes the connection logs one WARN with
2562    /// the diagnosis: which principals the connection's routes belonged to,
2563    /// which module's frame did not fit and on which client channel, and what
2564    /// the queue held at that moment.
2565    pub(crate) fn escalate_client_delivery_failure(
2566        &self,
2567        connection_id: ConnectionId,
2568        channel: u16,
2569        expected_epoch: u32,
2570        reason: CloseReason,
2571        undelivered: UndeliveredFrame<'_>,
2572    ) -> Result<bool, ForwardingError> {
2573        let principals = {
2574            let mut inner = self.write_inner()?;
2575            let key = ClientRouteKey {
2576                connection_id,
2577                channel,
2578            };
2579            if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
2580                None
2581            } else {
2582                inner.closing_connections.insert(connection_id);
2583                Some(connection_principals_locked(&inner, connection_id))
2584            }
2585        };
2586        let Some(principals) = principals else {
2587            return Ok(false);
2588        };
2589        let backlog = undelivered.sink.backlog();
2590        let close_reason = reason.to_string();
2591        if self.request_connection_close(connection_id, reason) {
2592            warn!(
2593                connection_id = connection_id.get(),
2594                principals = %principals,
2595                module_id = undelivered.module_id.unwrap_or("unknown"),
2596                client_channel = channel,
2597                queued_bytes = backlog.queued_bytes,
2598                queued_frames = backlog.queued_frames,
2599                oldest_queued_ms = backlog
2600                    .oldest_age
2601                    .map(|age| age.as_millis() as u64)
2602                    .unwrap_or(0),
2603                close_reason = %close_reason,
2604                "closing client connection: its egress queue could not take a frame"
2605            );
2606        }
2607        Ok(true)
2608    }
2609
2610    /// Publish the tags a scope sync changed and close the live routes its
2611    /// drain table selects, in one critical section.
2612    ///
2613    /// The caller holds the scope table's write lock (scope table, then this
2614    /// table, always). Publishing and selecting under one lock is what closes
2615    /// the race with a bind commit: a commit before this sees the old tag and
2616    /// its route is selected here like any other; a commit after it sees the
2617    /// new tag and is refused.
2618    ///
2619    /// Every client route index entry is scanned, which includes routes on a
2620    /// swap's superseded endpoint (the old process of a module being replaced
2621    /// blue/green, which keeps its existing routes until they drain): those
2622    /// stay indexed until drained, so ending a scope reaches them too.
2623    pub(crate) fn publish_scope_changes(
2624        &self,
2625        changes: &[ScopeTagChange],
2626    ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
2627        let mut inner = self.write_inner()?;
2628        let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
2629        for change in changes {
2630            let key = (change.owner.clone(), change.scope_ref.clone());
2631            match change.after {
2632                Some(tag) => {
2633                    inner.scope_tags.insert(key, tag);
2634                }
2635                None => {
2636                    inner.scope_tags.remove(&key);
2637                }
2638            }
2639            by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
2640        }
2641        let mut selected = Vec::new();
2642        for (client_key, route) in &inner.client_to_module {
2643            let Some(scope) = &route.scope else {
2644                continue;
2645            };
2646            let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
2647            else {
2648                continue;
2649            };
2650            if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
2651                continue;
2652            }
2653            let reason = match &change.drain {
2654                ScopeDrain::Nothing => continue,
2655                ScopeDrain::All(reason) => *reason,
2656                ScopeDrain::Carriers(narrowed) => {
2657                    let owner = Principal::Reserved {
2658                        module_id: scope.owner.clone(),
2659                    };
2660                    let hit = route.principal != owner
2661                        && narrowed.iter().any(|(principal, allowed)| {
2662                            *principal == route.principal
2663                                && allowed
2664                                    .as_ref()
2665                                    .is_none_or(|targets| !targets.contains(&route.module_id))
2666                        });
2667                    if !hit {
2668                        continue;
2669                    }
2670                    RouteCloseReason::ScopeCarrierRemoved
2671                }
2672            };
2673            selected.push((*client_key, route.client_epoch, reason));
2674        }
2675        let mut drained = Vec::new();
2676        for (client_key, client_epoch, reason) in selected {
2677            let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
2678                continue;
2679            };
2680            let release = release_client_route_locked(&mut inner, client_key, client_epoch);
2681            self.record_route_release(&release);
2682            if let RouteRelease::Removed(module) = release {
2683                drained.push(ScopeDrainedRoute {
2684                    reason,
2685                    module_id: route.module_id.clone(),
2686                    client: GoodbyeTarget {
2687                        connection_id: route.client_connection_id,
2688                        sink: route.client_sink.clone(),
2689                        negotiated_ver: route.client_negotiated_ver,
2690                        channel: route.client_channel,
2691                        epoch: route.client_epoch,
2692                        kind: GoodbyeTargetKind::Client,
2693                        module_id: Some(route.module_id.clone()),
2694                    },
2695                    module,
2696                });
2697            }
2698        }
2699        Ok(drained)
2700    }
2701
2702    /// The published tag of one scope, for tests of the commit re-check.
2703    #[cfg(test)]
2704    pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
2705        self.read_inner()
2706            .ok()?
2707            .scope_tags
2708            .get(&(owner.to_string(), scope_ref.to_string()))
2709            .copied()
2710    }
2711
2712    fn record_route_release(&self, release: &RouteRelease) {
2713        match release {
2714            RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
2715            RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
2716            RouteRelease::Absent => {}
2717        }
2718    }
2719
2720    fn read_inner(&self) -> Result<RwLockReadGuard<'_, ForwardingInner>, ForwardingError> {
2721        self.inner.read().map_err(|_| ForwardingError::Poisoned)
2722    }
2723
2724    fn write_inner(&self) -> Result<RwLockWriteGuard<'_, ForwardingInner>, ForwardingError> {
2725        self.inner.write().map_err(|_| ForwardingError::Poisoned)
2726    }
2727
2728    fn lock_close_registry(
2729        &self,
2730    ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
2731        self.close_registry
2732            .lock()
2733            .unwrap_or_else(|poisoned| poisoned.into_inner())
2734    }
2735}
2736
2737impl ForwardingInner {
2738    fn allocate_route_slots(
2739        &mut self,
2740        connection_id: ConnectionId,
2741        endpoint: ModuleEndpointId,
2742    ) -> Result<(u16, u32, u16, u32), ForwardingError> {
2743        let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
2744        let mut client_channel = client_start;
2745        let client_channel = loop {
2746            let key = ClientRouteKey {
2747                connection_id,
2748                channel: client_channel,
2749            };
2750            let eligible = !self.client_to_module.contains_key(&key)
2751                && !self.reserved_client.contains_key(&key)
2752                && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2753            if eligible {
2754                break client_channel;
2755            }
2756            client_channel = next_channel(client_channel);
2757            if client_channel == client_start {
2758                return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
2759            }
2760        };
2761
2762        let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
2763        let mut module_channel = module_start;
2764        let module_channel = loop {
2765            let key = ModuleRouteKey {
2766                endpoint,
2767                channel: module_channel,
2768            };
2769            let eligible = !self.module_to_client.contains_key(&key)
2770                && !self.reserved_module.contains_key(&key)
2771                && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2772            if eligible {
2773                break module_channel;
2774            }
2775            module_channel = next_channel(module_channel);
2776            if module_channel == module_start {
2777                return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
2778            }
2779        };
2780
2781        let client_key = ClientRouteKey {
2782            connection_id,
2783            channel: client_channel,
2784        };
2785        let module_key = ModuleRouteKey {
2786            endpoint,
2787            channel: module_channel,
2788        };
2789        let client_epoch = self
2790            .client_slot_epochs
2791            .get(&client_key)
2792            .copied()
2793            .unwrap_or(0)
2794            + 1;
2795        let module_epoch = self
2796            .module_slot_epochs
2797            .get(&module_key)
2798            .copied()
2799            .unwrap_or(0)
2800            + 1;
2801        self.client_slot_epochs.insert(client_key, client_epoch);
2802        self.module_slot_epochs.insert(module_key, module_epoch);
2803        self.next_client_channel
2804            .insert(connection_id, next_channel(client_channel));
2805        self.next_module_channel
2806            .insert(endpoint, next_channel(module_channel));
2807        Ok((client_channel, client_epoch, module_channel, module_epoch))
2808    }
2809
2810    fn allocate_control_corr(
2811        &mut self,
2812        endpoint: ModuleEndpointId,
2813    ) -> Result<u64, ForwardingError> {
2814        let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
2815        if candidate == 0 {
2816            self.closing_connections.insert(endpoint.connection_id);
2817            return Err(ForwardingError::RelayCorrelationExhausted);
2818        }
2819        self.next_control_corr.insert(
2820            endpoint,
2821            if candidate == u64::MAX {
2822                0
2823            } else {
2824                candidate + 1
2825            },
2826        );
2827        Ok(candidate)
2828    }
2829}
2830
2831fn connection_has_client_routes_locked(
2832    inner: &ForwardingInner,
2833    connection_id: ConnectionId,
2834) -> bool {
2835    inner
2836        .client_to_module
2837        .keys()
2838        .any(|key| key.connection_id == connection_id)
2839        || inner
2840            .reserved_client
2841            .keys()
2842            .any(|key| key.connection_id == connection_id)
2843}
2844
2845fn check_module_connection_role_locked(
2846    inner: &ForwardingInner,
2847    connection_id: ConnectionId,
2848) -> Result<(), ForwardingError> {
2849    if inner.endpoint_by_connection.contains_key(&connection_id)
2850        || connection_has_client_routes_locked(inner, connection_id)
2851    {
2852        return Err(ForwardingError::ConnectionRoleConflict { connection_id });
2853    }
2854    Ok(())
2855}
2856
2857fn next_channel(channel: u16) -> u16 {
2858    let next = channel.wrapping_add(1);
2859    if next == 0 {
2860        1
2861    } else {
2862        next
2863    }
2864}
2865
2866fn endpoint_routes_locked(
2867    inner: &ForwardingInner,
2868    endpoint: ModuleEndpointId,
2869) -> Vec<EndpointRoute> {
2870    let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
2871    let draining = drain_reason.is_some();
2872    let mut routes = inner
2873        .module_to_client
2874        .iter()
2875        .filter(|(module_key, _)| module_key.endpoint == endpoint)
2876        .map(|(_, route)| EndpointRoute {
2877            goodbye_target: GoodbyeTarget {
2878                connection_id: route.client_connection_id,
2879                sink: route.client_sink.clone(),
2880                negotiated_ver: route.client_negotiated_ver,
2881                channel: route.client_channel,
2882                epoch: route.client_epoch,
2883                kind: GoodbyeTargetKind::Client,
2884                module_id: Some(route.module_id.clone()),
2885            },
2886            principal: route.principal.clone(),
2887            bound_at: route.bound_at,
2888            draining,
2889            drain_reason,
2890        })
2891        .collect::<Vec<_>>();
2892    routes.sort_by_key(|route| {
2893        (
2894            route.goodbye_target.connection_id.get(),
2895            route.goodbye_target.channel,
2896            route.goodbye_target.epoch,
2897        )
2898    });
2899    routes
2900}
2901
2902/// Why a bind admitted under `scope` may no longer commit, read from the
2903/// published tags: `scope_ended` when the scope is gone or at another epoch,
2904/// `scope_changed` (retryable) when only its content moved.
2905fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
2906    let key = (scope.owner.clone(), scope.scope_ref.clone());
2907    match inner.scope_tags.get(&key) {
2908        Some(current) if *current == scope.tag => None,
2909        Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
2910            subc_protocol::error_codes::SCOPE_CHANGED,
2911            format!(
2912                "scope '{}' of {} changed while the route was being bound; re-open it",
2913                scope.scope_ref, scope.owner
2914            ),
2915        )),
2916        _ => Some(ErrorBody::new(
2917            subc_protocol::error_codes::SCOPE_ENDED,
2918            format!(
2919                "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
2920                scope.scope_ref, scope.owner, scope.tag.scope_epoch
2921            ),
2922        )),
2923    }
2924}
2925
2926fn release_reserved_route_locked(
2927    inner: &mut ForwardingInner,
2928    client_key: ClientRouteKey,
2929    module_key: ModuleRouteKey,
2930) {
2931    if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
2932        inner.reserved_client.remove(&client_key);
2933    }
2934    if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
2935        inner.reserved_module.remove(&module_key);
2936    }
2937    inner.status.retain(|(key, _), _| *key != client_key);
2938}
2939
2940fn release_client_route_locked(
2941    inner: &mut ForwardingInner,
2942    client_key: ClientRouteKey,
2943    expected_epoch: u32,
2944) -> RouteRelease {
2945    let Some(route) = inner.client_to_module.get(&client_key) else {
2946        return RouteRelease::Absent;
2947    };
2948    if route.client_epoch != expected_epoch {
2949        return RouteRelease::Stale;
2950    }
2951    let route = inner
2952        .client_to_module
2953        .remove(&client_key)
2954        .expect("route checked under the same forwarding lock");
2955    route.flow.close();
2956    inner.module_to_client.remove(&ModuleRouteKey {
2957        endpoint: route.module_endpoint,
2958        channel: route.module_channel,
2959    });
2960    inner.status.remove(&(client_key, expected_epoch));
2961    RouteRelease::Removed(GoodbyeTarget {
2962        connection_id: route.module_endpoint.connection_id,
2963        sink: route.module_sink.clone(),
2964        negotiated_ver: route.module_negotiated_ver,
2965        channel: route.module_channel,
2966        epoch: route.module_epoch,
2967        kind: GoodbyeTargetKind::Module,
2968        module_id: Some(route.module_id.clone()),
2969    })
2970}
2971
2972fn release_module_route_locked(
2973    inner: &mut ForwardingInner,
2974    module_key: ModuleRouteKey,
2975    expected_epoch: u32,
2976) -> RouteRelease {
2977    let Some(route) = inner.module_to_client.get(&module_key) else {
2978        return RouteRelease::Absent;
2979    };
2980    if route.module_epoch != expected_epoch {
2981        return RouteRelease::Stale;
2982    }
2983    let route = inner
2984        .module_to_client
2985        .remove(&module_key)
2986        .expect("route checked under the same forwarding lock");
2987    route.flow.close();
2988    let client_key = ClientRouteKey {
2989        connection_id: route.client_connection_id,
2990        channel: route.client_channel,
2991    };
2992    inner.client_to_module.remove(&client_key);
2993    inner.status.remove(&(client_key, route.client_epoch));
2994    RouteRelease::Removed(GoodbyeTarget {
2995        connection_id: route.client_connection_id,
2996        sink: route.client_sink.clone(),
2997        negotiated_ver: route.client_negotiated_ver,
2998        channel: route.client_channel,
2999        epoch: route.client_epoch,
3000        kind: GoodbyeTargetKind::Client,
3001        module_id: Some(route.module_id.clone()),
3002    })
3003}
3004
3005fn commit_route_locked(
3006    inner: &mut ForwardingInner,
3007    pending: PendingRouteBindRelayEntry,
3008) -> Result<Option<GoodbyeTarget>, ForwardingError> {
3009    let reservation = pending.reservation;
3010    if inner
3011        .closing_connections
3012        .contains(&reservation.client_key.connection_id)
3013    {
3014        return Err(ForwardingError::ConnectionClosing {
3015            connection_id: reservation.client_key.connection_id,
3016        });
3017    }
3018    let module_id = inner
3019        .module_id_by_endpoint
3020        .get(&reservation.module_key.endpoint)
3021        .cloned()
3022        .ok_or(ForwardingError::StaleModuleEndpoint)?;
3023    if inner
3024        .draining_endpoints
3025        .contains_key(&reservation.module_key.endpoint)
3026    {
3027        return Err(ForwardingError::ModuleReloading { module_id });
3028    }
3029    if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
3030        || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
3031    {
3032        return Err(ForwardingError::UnknownReservation {
3033            client_channel: reservation.client_key.channel,
3034            module_channel: reservation.module_key.channel,
3035        });
3036    }
3037    let module = inner
3038        .modules_by_id
3039        .get(&module_id)
3040        .filter(|module| module.endpoint == reservation.module_key.endpoint)
3041        .cloned()
3042        .ok_or(ForwardingError::StaleModuleEndpoint)?;
3043    let binding = Arc::new(RouteBinding {
3044        client_connection_id: reservation.client_key.connection_id,
3045        client_sink: pending.client_sink,
3046        client_negotiated_ver: pending.client_negotiated_ver,
3047        client_channel: reservation.client_key.channel,
3048        client_epoch: reservation.client_epoch,
3049        module_id,
3050        module_endpoint: reservation.module_key.endpoint,
3051        module_sink: module.sink,
3052        module_negotiated_ver: module.negotiated_ver,
3053        module_channel: reservation.module_key.channel,
3054        module_epoch: reservation.module_epoch,
3055        principal: pending.principal,
3056        project_root: reservation.project_root.clone(),
3057        bound_at: Instant::now(),
3058        flow: Arc::new(ChannelFlow::new(window_for(&module.concurrency))),
3059        scope: pending.scope,
3060    });
3061    inner
3062        .client_to_module
3063        .insert(reservation.client_key, Arc::clone(&binding));
3064    inner
3065        .module_to_client
3066        .insert(reservation.module_key, binding);
3067    let previous_published = inner
3068        .last_published_epoch
3069        .insert(reservation.client_key, reservation.client_epoch);
3070
3071    // Sending through the reserved slot cannot fail, but it reports a receiver
3072    // that closed after reservation and before this locked publication point.
3073    // The frame is stamped and charged to the queued-byte count here, when it
3074    // actually enters the queue, not when the slot was reserved.
3075    let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
3076    if client_writer_closed {
3077        // Not gated on `relay_enqueued`: only a module ack reaches this
3078        // commit, which proves the module holds the binding, and `route.open`
3079        // sends nothing for a relay a reply already settled.
3080        let abandoned = abandoned_route_target(inner, &reservation);
3081        if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
3082            route.flow.close();
3083        }
3084        inner.module_to_client.remove(&reservation.module_key);
3085        inner
3086            .status
3087            .remove(&(reservation.client_key, reservation.client_epoch));
3088        match previous_published {
3089            Some(epoch) => {
3090                inner
3091                    .last_published_epoch
3092                    .insert(reservation.client_key, epoch);
3093            }
3094            None => {
3095                inner.last_published_epoch.remove(&reservation.client_key);
3096            }
3097        }
3098        let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3099            "client egress closed during route publication".to_string(),
3100        ));
3101        return Ok(abandoned);
3102    }
3103
3104    let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
3105    Ok(None)
3106}
3107
3108/// The live module connection registered under exactly `endpoint`, whichever
3109/// slot holds it: active, swap candidate, or superseded incumbent.
3110///
3111/// Resolving by endpoint rather than by module id is what keeps an incumbent
3112/// addressable after a swap promoted a candidate over its id. An endpoint that
3113/// is in none of the three (a stale one, replaced without a promotion) resolves
3114/// to nothing, as it always has.
3115fn module_connection_for_endpoint_locked(
3116    inner: &ForwardingInner,
3117    endpoint: ModuleEndpointId,
3118) -> Option<&ModuleConnection> {
3119    let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
3120    inner
3121        .modules_by_id
3122        .get(module_id)
3123        .filter(|module| module.endpoint == endpoint)
3124        .or_else(|| {
3125            inner
3126                .candidates_by_id
3127                .get(module_id)
3128                .filter(|module| module.endpoint == endpoint)
3129        })
3130        .or_else(|| inner.superseded_endpoints.get(&endpoint))
3131}
3132
3133fn abandoned_route_target(
3134    inner: &ForwardingInner,
3135    reservation: &RouteReservation,
3136) -> Option<GoodbyeTarget> {
3137    let module_id = inner
3138        .module_id_by_endpoint
3139        .get(&reservation.module_key.endpoint)?;
3140    let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
3141    (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
3142        connection_id: module.endpoint.connection_id,
3143        sink: module.sink.clone(),
3144        negotiated_ver: module.negotiated_ver,
3145        channel: reservation.module_key.channel,
3146        epoch: reservation.module_epoch,
3147        kind: GoodbyeTargetKind::Module,
3148        module_id: Some(module_id.clone()),
3149    })
3150}
3151
3152/// Queue a registering module's HELLO_ACK on its sink. Called with the
3153/// forwarding write lock held, before the endpoint is inserted, which is what
3154/// puts the ack ahead of any `route.bind` or control RPC routed to the module.
3155/// `try_send` never waits, so holding the lock across it is safe.
3156fn enqueue_hello_ack_locked(
3157    sink: &FrameSink,
3158    connection_id: ConnectionId,
3159    hello_ack: Option<Frame>,
3160) -> Result<(), ForwardingError> {
3161    let Some(hello_ack) = hello_ack else {
3162        return Ok(());
3163    };
3164    sink.try_send(hello_ack)
3165        .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
3166}
3167
3168fn remove_module_connection_locked(
3169    inner: &mut ForwardingInner,
3170    endpoint: ModuleEndpointId,
3171) -> ConnectionCleanup {
3172    inner.draining_endpoints.remove(&endpoint);
3173    let module_id = inner.module_id_by_endpoint.remove(&endpoint);
3174    if let Some(module_id) = module_id.as_ref() {
3175        if inner
3176            .modules_by_id
3177            .get(module_id)
3178            .is_some_and(|module| module.endpoint == endpoint)
3179        {
3180            inner.modules_by_id.remove(module_id);
3181        }
3182        if inner
3183            .candidates_by_id
3184            .get(module_id)
3185            .is_some_and(|module| module.endpoint == endpoint)
3186        {
3187            inner.candidates_by_id.remove(module_id);
3188        }
3189    }
3190    inner.superseded_endpoints.remove(&endpoint);
3191    inner.endpoint_by_connection.remove(&endpoint.connection_id);
3192    inner.next_module_channel.remove(&endpoint);
3193    inner.next_control_corr.remove(&endpoint);
3194    inner
3195        .health_probe_tombstones
3196        .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
3197    inner
3198        .module_slot_epochs
3199        .retain(|key, _| key.endpoint != endpoint);
3200    let reserved_module_keys: Vec<ModuleRouteKey> = inner
3201        .reserved_module
3202        .keys()
3203        .filter(|module_key| module_key.endpoint == endpoint)
3204        .copied()
3205        .collect();
3206    for module_key in reserved_module_keys {
3207        if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
3208            release_reserved_route_locked(inner, client_key, module_key);
3209        }
3210    }
3211
3212    let pending_keys: Vec<_> = inner
3213        .pending_relays
3214        .keys()
3215        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3216        .copied()
3217        .collect();
3218    let pending: Vec<_> = pending_keys
3219        .into_iter()
3220        .filter_map(|key| {
3221            inner
3222                .pending_relays
3223                .remove(&key)
3224                .map(|pending| (key, pending))
3225        })
3226        .collect();
3227    let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
3228    for (key, pending) in pending {
3229        // Recorded like every other removal without a reply, so a `route.open`
3230        // still sending the bind learns the relay was released rather than
3231        // settled by the module.
3232        if !pending.relay_enqueued {
3233            inner.relays_released_before_enqueue.insert(key);
3234        }
3235        let module_label = module_id.as_deref().unwrap_or("unknown");
3236        let _ = pending
3237            .sender
3238            .send(RouteBindRelayOutcome::ModuleGone(format!(
3239                "module '{module_label}' connection closed during route.bind relay"
3240            )));
3241    }
3242
3243    let pending_control_keys: Vec<_> = inner
3244        .pending_control_rpcs
3245        .keys()
3246        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3247        .copied()
3248        .collect();
3249    let pending_control: Vec<_> = pending_control_keys
3250        .into_iter()
3251        .filter_map(|key| inner.pending_control_rpcs.remove(&key))
3252        .collect();
3253    for pending in pending_control {
3254        let module_label = module_id.as_deref().unwrap_or("unknown");
3255        let _ = pending
3256            .sender
3257            .send(ModuleControlRpcOutcome::ModuleGone(format!(
3258                "module '{module_label}' connection closed during module-control RPC"
3259            )));
3260    }
3261
3262    let module_routes = inner
3263        .module_to_client
3264        .iter()
3265        .filter(|(module_key, _)| module_key.endpoint == endpoint)
3266        .map(|(module_key, route)| (*module_key, route.module_epoch))
3267        .collect::<Vec<_>>();
3268    let mut released = Vec::with_capacity(module_routes.len());
3269    for (module_key, epoch) in module_routes {
3270        if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
3271        {
3272            released.push(target);
3273        }
3274    }
3275    ConnectionCleanup {
3276        released,
3277        abandoned_relays,
3278    }
3279}
3280
3281#[derive(Debug, Clone, Copy)]
3282struct RequestCredit {
3283    subscription: bool,
3284    excluded_from_drain: bool,
3285}
3286
3287#[derive(Debug, Default)]
3288struct CreditLedger {
3289    by_corr: HashMap<u64, Vec<RequestCredit>>,
3290}
3291
3292impl CreditLedger {
3293    fn acquire(&mut self, corr: u64, subscription: bool) {
3294        self.by_corr.entry(corr).or_default().push(RequestCredit {
3295            subscription,
3296            excluded_from_drain: false,
3297        });
3298    }
3299
3300    fn release(&mut self, corr: u64) -> bool {
3301        let Some(credits) = self.by_corr.get_mut(&corr) else {
3302            return false;
3303        };
3304        let released = credits.pop().is_some();
3305        if credits.is_empty() {
3306            self.by_corr.remove(&corr);
3307        }
3308        released
3309    }
3310
3311    fn capture_subscription_exclusions(&mut self) -> u32 {
3312        let mut excluded = 0u32;
3313        for credit in self.by_corr.values_mut().flatten() {
3314            if credit.subscription && !credit.excluded_from_drain {
3315                credit.excluded_from_drain = true;
3316                excluded = excluded.saturating_add(1);
3317            }
3318        }
3319        excluded
3320    }
3321
3322    #[cfg(test)]
3323    fn in_flight(&self) -> usize {
3324        self.by_corr.values().map(Vec::len).sum()
3325    }
3326
3327    fn drain_in_flight(&self) -> usize {
3328        self.by_corr
3329            .values()
3330            .flatten()
3331            .filter(|credit| !credit.excluded_from_drain)
3332            .count()
3333    }
3334
3335    /// The correlation ids of the requests a drain still waits on, one entry
3336    /// per held credit, in ascending order.
3337    fn drain_held_corrs(&self) -> Vec<u64> {
3338        let mut corrs = self
3339            .by_corr
3340            .iter()
3341            .flat_map(|(corr, credits)| {
3342                credits
3343                    .iter()
3344                    .filter(|credit| !credit.excluded_from_drain)
3345                    .map(move |_| *corr)
3346            })
3347            .collect::<Vec<_>>();
3348        corrs.sort_unstable();
3349        corrs
3350    }
3351}
3352
3353#[derive(Debug, Default)]
3354struct ChannelFlowState {
3355    closed: bool,
3356    credits: CreditLedger,
3357}
3358
3359/// Per-channel request-credit accounting shared by the client and module route halves.
3360#[derive(Debug)]
3361pub(crate) struct ChannelFlow {
3362    sem: Semaphore,
3363    window: usize,
3364    state: Mutex<ChannelFlowState>,
3365}
3366
3367impl ChannelFlow {
3368    pub(crate) fn new(window: usize) -> Self {
3369        debug_assert!(window > 0, "flow-control window must be non-zero");
3370        Self {
3371            sem: Semaphore::new(window),
3372            window,
3373            state: Mutex::new(ChannelFlowState::default()),
3374        }
3375    }
3376
3377    #[cfg(test)]
3378    pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
3379        self.acquire_tagged(0, false).await
3380    }
3381
3382    pub(crate) async fn acquire_tagged(
3383        &self,
3384        corr: u64,
3385        subscription: bool,
3386    ) -> Result<(), ChannelFlowClosed> {
3387        let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
3388        let mut state = self
3389            .state
3390            .lock()
3391            .unwrap_or_else(|poisoned| poisoned.into_inner());
3392        if state.closed {
3393            return Err(ChannelFlowClosed);
3394        }
3395        state.credits.acquire(corr, subscription);
3396        permit.forget();
3397        Ok(())
3398    }
3399
3400    #[cfg(test)]
3401    pub(crate) fn release(&self) {
3402        self.release_corr(0);
3403    }
3404
3405    pub(crate) fn release_corr(&self, corr: u64) {
3406        let released = self
3407            .state
3408            .lock()
3409            .unwrap_or_else(|poisoned| poisoned.into_inner())
3410            .credits
3411            .release(corr);
3412        if !released {
3413            // Protocol-conforming modules emit exactly one terminal per request.
3414            // This guard is a best-effort safety net against window growth, not a
3415            // security boundary against malicious peers.
3416            warn!(
3417                window = self.window,
3418                available = self.sem.available_permits(),
3419                "flow-control over-release ignored"
3420            );
3421            return;
3422        }
3423        if !self.sem.is_closed() {
3424            self.sem.add_permits(1);
3425        }
3426    }
3427
3428    #[cfg(test)]
3429    pub(crate) fn in_flight(&self) -> usize {
3430        self.state
3431            .lock()
3432            .unwrap_or_else(|poisoned| poisoned.into_inner())
3433            .credits
3434            .in_flight()
3435    }
3436
3437    pub(crate) fn drain_in_flight(&self) -> usize {
3438        self.state
3439            .lock()
3440            .unwrap_or_else(|poisoned| poisoned.into_inner())
3441            .credits
3442            .drain_in_flight()
3443    }
3444
3445    pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
3446        self.state
3447            .lock()
3448            .unwrap_or_else(|poisoned| poisoned.into_inner())
3449            .credits
3450            .drain_held_corrs()
3451    }
3452
3453    #[cfg(test)]
3454    pub(crate) fn available_permits(&self) -> usize {
3455        self.sem.available_permits()
3456    }
3457
3458    pub(crate) fn begin_drain(&self) -> u32 {
3459        let mut state = self
3460            .state
3461            .lock()
3462            .unwrap_or_else(|poisoned| poisoned.into_inner());
3463        state.closed = true;
3464        self.sem.close();
3465        state.credits.capture_subscription_exclusions()
3466    }
3467
3468    pub(crate) fn close(&self) {
3469        self.state
3470            .lock()
3471            .unwrap_or_else(|poisoned| poisoned.into_inner())
3472            .closed = true;
3473        self.sem.close();
3474    }
3475}
3476
3477#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3478pub(crate) struct ChannelFlowClosed;
3479
3480impl fmt::Display for ChannelFlowClosed {
3481    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3482        write!(f, "flow-control window closed")
3483    }
3484}
3485
3486impl Error for ChannelFlowClosed {}
3487
3488fn window_for(concurrency: &Concurrency) -> usize {
3489    match concurrency {
3490        Concurrency::Serial => 1,
3491        Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
3492        Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
3493    }
3494}
3495
3496#[derive(Debug, Clone, PartialEq, Eq)]
3497pub enum ForwardingError {
3498    /// A connection cannot hold both client routes and a module endpoint, or
3499    /// replace a module identity while that endpoint is still registered.
3500    ConnectionRoleConflict {
3501        connection_id: ConnectionId,
3502    },
3503    NoModuleConnection,
3504    ModuleReloading {
3505        module_id: String,
3506    },
3507    StaleModuleEndpoint,
3508    UnknownReservation {
3509        client_channel: u16,
3510        module_channel: u16,
3511    },
3512    ClientRouteChannelExhausted {
3513        connection_id: ConnectionId,
3514    },
3515    ModuleRouteChannelExhausted {
3516        endpoint: ModuleEndpointId,
3517    },
3518    RelayCorrelationExhausted,
3519    ConnectionClosing {
3520        connection_id: ConnectionId,
3521    },
3522    ClientEgressClosed {
3523        connection_id: ConnectionId,
3524    },
3525    RouteOpenBuild(String),
3526    /// A swap candidate is already registered for this module id.
3527    CandidateSlotOccupied {
3528        module_id: String,
3529    },
3530    /// A registering module's HELLO_ACK could not be queued because its
3531    /// outbound queue is closed or full.
3532    ModuleEgressUnavailable {
3533        connection_id: ConnectionId,
3534    },
3535    Poisoned,
3536}
3537
3538impl fmt::Display for ForwardingError {
3539    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3540        match self {
3541            Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
3542            Self::NoModuleConnection => write!(f, "no module connection is registered"),
3543            Self::ModuleReloading { module_id } => {
3544                write!(f, "module_id '{module_id}' is reloading")
3545            }
3546            Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
3547            Self::UnknownReservation {
3548                client_channel,
3549                module_channel,
3550            } => write!(
3551                f,
3552                "route reservation client channel {client_channel} / module channel {module_channel} was not found"
3553            ),
3554            Self::ClientRouteChannelExhausted { connection_id } => write!(
3555                f,
3556                "no client route channels are available for connection {}",
3557                connection_id.get()
3558            ),
3559            Self::ModuleRouteChannelExhausted { endpoint } => write!(
3560                f,
3561                "no module route channels are available for endpoint generation {} on connection {}",
3562                endpoint.generation,
3563                endpoint.connection_id.get()
3564            ),
3565            Self::RelayCorrelationExhausted => {
3566                write!(f, "module control correlation ids are exhausted")
3567            }
3568            Self::ConnectionClosing { connection_id } => write!(
3569                f,
3570                "connection {} is closing and cannot accept route allocation",
3571                connection_id.get()
3572            ),
3573            Self::ClientEgressClosed { connection_id } => write!(
3574                f,
3575                "client connection {} egress is closed",
3576                connection_id.get()
3577            ),
3578            Self::RouteOpenBuild(message) => {
3579                write!(f, "failed to prebuild route.open response: {message}")
3580            }
3581            Self::CandidateSlotOccupied { module_id } => write!(
3582                f,
3583                "module_id '{module_id}' already has a swap candidate registered"
3584            ),
3585            Self::ModuleEgressUnavailable { connection_id } => write!(
3586                f,
3587                "module connection {} egress is unavailable; HELLO_ACK could not be queued",
3588                connection_id.get()
3589            ),
3590            Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
3591        }
3592    }
3593}
3594
3595impl Error for ForwardingError {}
3596
3597#[cfg(test)]
3598mod tests {
3599    use std::time::Duration;
3600
3601    use super::*;
3602    use tokio::sync::mpsc;
3603
3604    #[test]
3605    fn ordinary_long_running_request_is_not_excluded_from_drain() {
3606        let mut ledger = CreditLedger::default();
3607        ledger.acquire(1, false);
3608
3609        assert_eq!(ledger.capture_subscription_exclusions(), 0);
3610        assert_eq!(ledger.drain_in_flight(), 1);
3611    }
3612
3613    #[test]
3614    fn bit_set_subscription_is_excluded_and_counted() {
3615        let mut ledger = CreditLedger::default();
3616        ledger.acquire(1, true);
3617
3618        assert_eq!(ledger.capture_subscription_exclusions(), 1);
3619        assert_eq!(ledger.drain_in_flight(), 0);
3620    }
3621
3622    #[test]
3623    fn subscription_opened_after_drain_snapshot_is_not_excluded() {
3624        let mut ledger = CreditLedger::default();
3625        ledger.acquire(1, true);
3626        assert_eq!(ledger.capture_subscription_exclusions(), 1);
3627
3628        ledger.acquire(2, true);
3629
3630        assert_eq!(ledger.drain_in_flight(), 1);
3631    }
3632
3633    #[test]
3634    fn drain_with_no_subscriptions_reports_zero_excluded() {
3635        let mut ledger = CreditLedger::default();
3636        assert_eq!(ledger.capture_subscription_exclusions(), 0);
3637    }
3638
3639    fn test_hello_ack(corr: u64) -> Frame {
3640        Frame::build(
3641            FrameType::HelloAck,
3642            Flags::new(false, Priority::Passive, false),
3643            0,
3644            0,
3645            corr,
3646            Vec::new(),
3647        )
3648        .unwrap()
3649    }
3650
3651    /// An acked registration whose HELLO_ACK cannot be queued must leave no
3652    /// endpoint behind: a routable module that never got its ack would read a
3653    /// route.bind first and exit. A closed queue and a full one both refuse.
3654    #[test]
3655    fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
3656        let forwarding = ForwardingTable::default();
3657
3658        let (closed_tx, closed_rx) = mpsc::channel(8);
3659        drop(closed_rx);
3660        let closed = ConnectionId::new(1);
3661        assert_eq!(
3662            forwarding.register_module_connection_acked(
3663                closed,
3664                "closed".to_string(),
3665                2,
3666                Concurrency::ModuleManaged,
3667                FrameSink::new(closed_tx),
3668                test_hello_ack(1),
3669            ),
3670            Err(ForwardingError::ModuleEgressUnavailable {
3671                connection_id: closed
3672            })
3673        );
3674
3675        let (full_tx, _full_rx) = mpsc::channel(1);
3676        let full_sink = FrameSink::new(full_tx);
3677        full_sink.try_send(test_hello_ack(99)).unwrap();
3678        let full = ConnectionId::new(2);
3679        assert_eq!(
3680            forwarding.register_module_connection_acked(
3681                full,
3682                "full".to_string(),
3683                2,
3684                Concurrency::ModuleManaged,
3685                full_sink.clone(),
3686                test_hello_ack(2),
3687            ),
3688            Err(ForwardingError::ModuleEgressUnavailable {
3689                connection_id: full
3690            })
3691        );
3692        assert_eq!(
3693            forwarding.register_candidate_module_connection_acked(
3694                full,
3695                "full".to_string(),
3696                2,
3697                Concurrency::ModuleManaged,
3698                full_sink,
3699                test_hello_ack(3),
3700            ),
3701            Err(ForwardingError::ModuleEgressUnavailable {
3702                connection_id: full
3703            })
3704        );
3705
3706        for (connection, module_id) in [(closed, "closed"), (full, "full")] {
3707            assert_eq!(
3708                forwarding
3709                    .module_endpoint_for_connection(connection)
3710                    .unwrap(),
3711                None
3712            );
3713            let (client_tx, _client_rx) = mpsc::channel(8);
3714            assert_eq!(
3715                forwarding
3716                    .begin_route_bind_relay_for_test(
3717                        ConnectionId::new(50),
3718                        FrameSink::new(client_tx),
3719                        1,
3720                        module_id,
3721                    )
3722                    .err(),
3723                Some(ForwardingError::NoModuleConnection)
3724            );
3725        }
3726        assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
3727    }
3728
3729    /// Both acked registration forms put the HELLO_ACK on the module's queue
3730    /// by the time the endpoint can be resolved.
3731    #[test]
3732    fn acked_registration_queues_the_hello_ack_first() {
3733        let forwarding = ForwardingTable::default();
3734        let (active_tx, mut active_rx) = mpsc::channel(8);
3735        forwarding
3736            .register_module_connection_acked(
3737                ConnectionId::new(1),
3738                "acked".to_string(),
3739                2,
3740                Concurrency::ModuleManaged,
3741                FrameSink::new(active_tx),
3742                test_hello_ack(11),
3743            )
3744            .unwrap();
3745        let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
3746        forwarding
3747            .register_candidate_module_connection_acked(
3748                ConnectionId::new(2),
3749                "acked".to_string(),
3750                2,
3751                Concurrency::ModuleManaged,
3752                FrameSink::new(candidate_tx),
3753                test_hello_ack(12),
3754            )
3755            .unwrap();
3756
3757        let active_first = active_rx.try_recv().unwrap().frame;
3758        assert_eq!(active_first.header.ty, FrameType::HelloAck);
3759        assert_eq!(active_first.header.corr, 11);
3760        let candidate_first = candidate_rx.try_recv().unwrap().frame;
3761        assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
3762        assert_eq!(candidate_first.header.corr, 12);
3763    }
3764
3765    #[test]
3766    fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
3767        let forwarding = ForwardingTable::default();
3768        let module_connection = ConnectionId::new(10);
3769        let exhausted_client = ConnectionId::new(20);
3770        let second_client = ConnectionId::new(30);
3771        let (module_tx, _module_rx) = mpsc::channel(1);
3772        let endpoint = forwarding
3773            .register_module_connection(
3774                module_connection,
3775                "route-limit-provider".to_string(),
3776                1,
3777                Concurrency::ModuleManaged,
3778                FrameSink::new(module_tx),
3779            )
3780            .unwrap();
3781
3782        {
3783            let mut inner = forwarding.inner.write().unwrap();
3784            for channel in 1..=u16::MAX {
3785                inner.reserved_client.insert(
3786                    ClientRouteKey {
3787                        connection_id: exhausted_client,
3788                        channel,
3789                    },
3790                    ModuleRouteKey {
3791                        endpoint,
3792                        channel: 1,
3793                    },
3794                );
3795            }
3796        }
3797
3798        let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
3799        let err = forwarding
3800            .begin_route_bind_relay_for_test(
3801                exhausted_client,
3802                FrameSink::new(exhausted_tx),
3803                1,
3804                "route-limit-provider",
3805            )
3806            .unwrap_err();
3807        assert!(matches!(
3808            err,
3809            ForwardingError::ClientRouteChannelExhausted { connection_id }
3810                if connection_id == exhausted_client
3811        ));
3812
3813        let (second_tx, _second_rx) = mpsc::channel(1);
3814        let pending = forwarding
3815            .begin_route_bind_relay_for_test(
3816                second_client,
3817                FrameSink::new(second_tx),
3818                2,
3819                "route-limit-provider",
3820            )
3821            .unwrap();
3822        assert_eq!(pending.client_channel, 1);
3823    }
3824
3825    #[test]
3826    fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
3827        let forwarding = ForwardingTable::default();
3828        let module_connection = ConnectionId::new(40);
3829        let client = ConnectionId::new(50);
3830        let (module_tx, _module_rx) = mpsc::channel(1);
3831        forwarding
3832            .register_module_connection(
3833                module_connection,
3834                "slot-reuse-provider".to_string(),
3835                1,
3836                Concurrency::ModuleManaged,
3837                FrameSink::new(module_tx),
3838            )
3839            .unwrap();
3840
3841        let (client_tx, _client_rx) = mpsc::channel(1);
3842        let client_sink = FrameSink::new(client_tx);
3843        let mut wrapped_channel = None;
3844        for index in 0..=usize::from(u16::MAX) {
3845            let pending = forwarding
3846                .begin_route_bind_relay_for_test(
3847                    client,
3848                    client_sink.clone(),
3849                    index as u64 + 1,
3850                    "slot-reuse-provider",
3851                )
3852                .unwrap();
3853            if index == usize::from(u16::MAX) {
3854                wrapped_channel = Some(pending.module_channel);
3855            }
3856            forwarding
3857                .abort_pending_relay(
3858                    pending.endpoint,
3859                    pending.corr,
3860                    RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
3861                )
3862                .unwrap();
3863        }
3864
3865        assert_eq!(wrapped_channel, Some(1));
3866    }
3867
3868    #[test]
3869    fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
3870        let forwarding = ForwardingTable::default();
3871        let client = ConnectionId::new(60);
3872        forwarding
3873            .inner
3874            .write()
3875            .unwrap()
3876            .next_client_channel
3877            .insert(client, 41);
3878
3879        let released = forwarding.cleanup_connection(client).unwrap();
3880
3881        assert!(released.is_empty());
3882        assert!(!forwarding
3883            .inner
3884            .read()
3885            .unwrap()
3886            .next_client_channel
3887            .contains_key(&client));
3888    }
3889
3890    #[test]
3891    fn stale_module_cleanup_preserves_fast_reconnect_successor() {
3892        let forwarding = ForwardingTable::default();
3893        let module_id = "fast-reconnect-provider";
3894        let first_connection = ConnectionId::new(70);
3895        let second_connection = ConnectionId::new(80);
3896        let (first_tx, _first_rx) = mpsc::channel(1);
3897        let first_endpoint = forwarding
3898            .register_module_connection(
3899                first_connection,
3900                module_id.to_string(),
3901                1,
3902                Concurrency::ModuleManaged,
3903                FrameSink::new(first_tx),
3904            )
3905            .unwrap();
3906        let (second_tx, _second_rx) = mpsc::channel(1);
3907        let second_endpoint = forwarding
3908            .register_module_connection(
3909                second_connection,
3910                module_id.to_string(),
3911                1,
3912                Concurrency::ModuleManaged,
3913                FrameSink::new(second_tx),
3914            )
3915            .unwrap();
3916        assert_ne!(first_endpoint, second_endpoint);
3917
3918        let released = forwarding.cleanup_connection(first_connection).unwrap();
3919
3920        assert!(released.is_empty());
3921        assert_eq!(
3922            forwarding
3923                .inner
3924                .read()
3925                .unwrap()
3926                .modules_by_id
3927                .get(module_id)
3928                .map(|module| module.endpoint),
3929            Some(second_endpoint)
3930        );
3931        assert!(forwarding.has_live_module_connection(module_id).unwrap());
3932        let control_rpc = forwarding
3933            .begin_module_control_rpc_for(
3934                module_id,
3935                "health.check",
3936                Instant::now() + Duration::from_secs(1),
3937            )
3938            .unwrap();
3939        assert_eq!(control_rpc.endpoint, second_endpoint);
3940    }
3941
3942    fn route_fixture(
3943        module_id: &str,
3944    ) -> (
3945        ForwardingTable,
3946        ConnectionId,
3947        ModuleEndpointId,
3948        ConnectionId,
3949        FrameSink,
3950        mpsc::Receiver<crate::router::OutboundFrame>,
3951    ) {
3952        let forwarding = ForwardingTable::default();
3953        let module_connection = ConnectionId::new(100);
3954        let client_connection = ConnectionId::new(200);
3955        let (module_tx, _module_rx) = mpsc::channel(8);
3956        let endpoint = forwarding
3957            .register_module_connection(
3958                module_connection,
3959                module_id.to_string(),
3960                2,
3961                Concurrency::ModuleManaged,
3962                FrameSink::new(module_tx),
3963            )
3964            .unwrap();
3965        let (client_tx, client_rx) = mpsc::channel(8);
3966        (
3967            forwarding,
3968            module_connection,
3969            endpoint,
3970            client_connection,
3971            FrameSink::new(client_tx),
3972            client_rx,
3973        )
3974    }
3975
3976    #[test]
3977    #[cfg(unix)]
3978    fn daemon_drain_gates_current_and_racing_provider_registrations() {
3979        let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
3980        assert_eq!(forwarding.begin_daemon_drain().unwrap(), ["provider"]);
3981        assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
3982        assert!(matches!(
3983            forwarding.register_module_connection(
3984                ConnectionId::new(300),
3985                "late-provider".into(),
3986                2,
3987                Concurrency::ModuleManaged,
3988                sink,
3989            ),
3990            Err(ForwardingError::ConnectionClosing { .. })
3991        ));
3992    }
3993
3994    #[test]
3995    #[cfg(unix)]
3996    fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
3997        let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
3998            route_fixture("provider");
3999        let mut pending =
4000            begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
4001        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
4002        forwarding.begin_daemon_drain().unwrap();
4003        let completion = forwarding
4004            .complete_pending_relay(
4005                module_connection,
4006                pending.corr,
4007                RouteBindRelayOutcome::Accepted,
4008            )
4009            .expect("draining admission is not a fatal module error");
4010        assert!(completion.settled);
4011        let abandoned = completion
4012            .abandoned
4013            .expect("late accepted bind needs a route GOODBYE");
4014        assert_eq!(abandoned.channel, pending.module_channel);
4015        assert!(matches!(pending.receiver.try_recv().unwrap(),
4016            RouteBindRelayOutcome::Rejected(error) if error.code == "module_reloading"));
4017        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4018        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4019        assert!(
4020            rx.try_recv().is_err(),
4021            "no route.open may be published during drain"
4022        );
4023        assert_eq!(
4024            forwarding
4025                .module_endpoint_for_connection(module_connection)
4026                .unwrap(),
4027            Some(endpoint)
4028        );
4029        assert!(
4030            !forwarding
4031                .complete_pending_relay(
4032                    module_connection,
4033                    pending.corr,
4034                    RouteBindRelayOutcome::Accepted,
4035                )
4036                .unwrap()
4037                .settled
4038        );
4039    }
4040
4041    fn test_ping(corr: u64) -> Frame {
4042        Frame::build(
4043            FrameType::Ping,
4044            Flags::new(false, Priority::Passive, false),
4045            0,
4046            0,
4047            corr,
4048            Vec::new(),
4049        )
4050        .unwrap()
4051    }
4052
4053    fn begin_test_route(
4054        forwarding: &ForwardingTable,
4055        client_connection: ConnectionId,
4056        client_sink: FrameSink,
4057        corr: u64,
4058        module_id: &str,
4059    ) -> PendingRouteBindRelay {
4060        forwarding
4061            .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
4062            .unwrap()
4063    }
4064
4065    #[test]
4066    fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
4067        for candidate in [false, true] {
4068            for register_first in [false, true] {
4069                let (forwarding, module_connection, _, client_connection, sink, _rx) =
4070                    route_fixture("target");
4071                let register = || {
4072                    if candidate {
4073                        forwarding.register_candidate_module_connection(
4074                            client_connection,
4075                            "source".into(),
4076                            2,
4077                            Concurrency::ModuleManaged,
4078                            sink.clone(),
4079                        )
4080                    } else {
4081                        forwarding.register_module_connection(
4082                            client_connection,
4083                            "source".into(),
4084                            2,
4085                            Concurrency::ModuleManaged,
4086                            sink.clone(),
4087                        )
4088                    }
4089                };
4090                if register_first {
4091                    register().unwrap();
4092                    assert!(
4093                        forwarding
4094                            .begin_route_bind_relay_for_test(
4095                                client_connection,
4096                                sink.clone(),
4097                                1,
4098                                "target",
4099                            )
4100                            .is_err(),
4101                        "a deferred route.open cannot reserve after HELLO"
4102                    );
4103                    assert!(forwarding
4104                        .cleanup_connection(client_connection)
4105                        .unwrap()
4106                        .is_empty());
4107                } else {
4108                    let pending =
4109                        begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
4110                    assert!(
4111                        register().is_err(),
4112                        "HELLO cannot register after a route reservation"
4113                    );
4114                    forwarding
4115                        .complete_pending_relay(
4116                            module_connection,
4117                            pending.corr,
4118                            RouteBindRelayOutcome::Accepted,
4119                        )
4120                        .unwrap();
4121                    assert_eq!(
4122                        forwarding
4123                            .cleanup_connection(client_connection)
4124                            .unwrap()
4125                            .len(),
4126                        1
4127                    );
4128                }
4129                assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4130                assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4131            }
4132        }
4133    }
4134
4135    /// A pending route.open reserves its queue slot with `reserve_owned` and
4136    /// sends its prebuilt response through that slot when the module accepts.
4137    /// A queue already holding many data frames (far more than the old 64-frame
4138    /// queue) must not stop the open from reserving or completing, and the
4139    /// response must be charged to the queue's byte count when it is sent.
4140    #[tokio::test]
4141    async fn pending_route_open_completes_behind_queued_data_frames() {
4142        assert_eq!(
4143            crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
4144            8,
4145            "the per-connection pending route.open limit is its own constant"
4146        );
4147        let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
4148            route_fixture("open-behind-data");
4149        let (sink, mut client_rx) = crate::server::connection_egress();
4150        const DATA_FRAMES: usize = 1_000;
4151        let data = |corr: u64| {
4152            Frame::build(
4153                FrameType::StreamData,
4154                Flags::new(false, Priority::Interactive, false),
4155                9,
4156                1,
4157                corr,
4158                vec![b'x'; 200],
4159            )
4160            .unwrap()
4161        };
4162        for corr in 0..DATA_FRAMES as u64 {
4163            sink.try_send(data(corr)).unwrap();
4164        }
4165        let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
4166        assert_eq!(sink.backlog().queued_bytes, data_bytes);
4167
4168        let pending = tokio::time::timeout(
4169            Duration::from_secs(5),
4170            forwarding.begin_route_bind_relay_for(
4171                client,
4172                sink.clone(),
4173                subc_protocol::PROTOCOL_VERSION,
4174                4_242,
4175                "open-behind-data",
4176                Principal::Direct,
4177                None,
4178                None,
4179                Instant::now() + Duration::from_secs(60),
4180            ),
4181        )
4182        .await
4183        .expect("reserving the route.open slot must not wait behind data frames")
4184        .unwrap();
4185        forwarding
4186            .complete_pending_relay(
4187                module_connection,
4188                pending.corr,
4189                RouteBindRelayOutcome::Accepted,
4190            )
4191            .unwrap();
4192
4193        let backlog = sink.backlog();
4194        assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
4195        assert!(
4196            backlog.queued_bytes > data_bytes,
4197            "the route.open response must be counted in queued bytes: {backlog:?}"
4198        );
4199        for corr in 0..DATA_FRAMES as u64 {
4200            assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
4201        }
4202        let open = client_rx.recv().await.unwrap();
4203        assert_eq!(open.header.corr, 4_242);
4204        assert_eq!(open.header.ty, FrameType::Response);
4205        drop(open);
4206        assert_eq!(sink.backlog().queued_bytes, 0);
4207        assert_eq!(sink.backlog().queued_frames, 0);
4208    }
4209
4210    /// The drain-timeout line reports these numbers, so they must count the
4211    /// requests the drain is waiting on and no others: a route with nothing in
4212    /// flight is not a holdout, and a flagged subscription is excluded from the
4213    /// drain and so from the count.
4214    #[tokio::test]
4215    async fn drain_holdouts_count_held_requests_and_name_the_connection() {
4216        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4217            route_fixture("holdouts");
4218        let mut bound = |corr| {
4219            let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
4220            forwarding
4221                .complete_pending_relay(
4222                    module_connection,
4223                    route.corr,
4224                    RouteBindRelayOutcome::Accepted,
4225                )
4226                .unwrap();
4227            client_rx.try_recv().unwrap();
4228            match forwarding
4229                .lookup_data_route(client, route.client_channel, route.client_epoch)
4230                .unwrap()
4231            {
4232                DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4233                other => panic!("expected live route, got {other:?}"),
4234            }
4235        };
4236        let holding = bound(61);
4237        let _idle = bound(62);
4238        holding.flow.acquire_tagged(7, false).await.unwrap();
4239        holding.flow.acquire_tagged(2, false).await.unwrap();
4240        holding.flow.acquire_tagged(3, true).await.unwrap();
4241        forwarding
4242            .begin_module_drain("holdouts", RouteCloseReason::Restart)
4243            .unwrap();
4244
4245        let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
4246        assert_eq!(
4247            holdouts,
4248            DrainHoldouts {
4249                requests: 2,
4250                routes: 1,
4251                total_routes: 2,
4252                top_connections: vec![(client.get(), 2)],
4253                // The held requests by the module's channel and corr, ascending,
4254                // without the excluded subscription (corr 3).
4255                held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
4256            }
4257        );
4258    }
4259
4260    #[test]
4261    fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
4262        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4263            route_fixture("census");
4264        let pending = begin_test_route(&forwarding, client, sink, 1, "census");
4265        forwarding
4266            .complete_pending_relay(
4267                module_connection,
4268                pending.corr,
4269                RouteBindRelayOutcome::Accepted,
4270            )
4271            .unwrap();
4272
4273        let routes = forwarding.endpoint_routes(endpoint).unwrap();
4274        assert_eq!(routes.len(), 1);
4275        assert!(matches!(routes[0].principal, Principal::Direct));
4276        assert_eq!(routes[0].goodbye_target.connection_id, client);
4277        assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
4278        assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
4279        assert!(!routes[0].draining);
4280
4281        forwarding
4282            .begin_module_drain("census", RouteCloseReason::Restart)
4283            .unwrap();
4284        let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
4285        assert_eq!(draining_routes.len(), 1);
4286        assert!(draining_routes[0].draining);
4287    }
4288
4289    #[test]
4290    fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
4291        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
4292        let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
4293        assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
4294        forwarding
4295            .abort_pending_relay(
4296                first.endpoint,
4297                first.corr,
4298                RouteBindRelayOutcome::ModuleGone("abort".into()),
4299            )
4300            .unwrap();
4301        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4302        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4303
4304        let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
4305        assert_eq!(second.client_channel, first.client_channel);
4306        assert_eq!(second.module_channel, first.module_channel);
4307        assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
4308    }
4309
4310    #[test]
4311    fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
4312        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4313            route_fixture("epoch-release");
4314        let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
4315        forwarding
4316            .complete_pending_relay(
4317                module_connection,
4318                first.corr,
4319                RouteBindRelayOutcome::Accepted,
4320            )
4321            .unwrap();
4322        assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
4323        assert!(matches!(
4324            forwarding
4325                .release_client_route(client, first.client_channel, first.client_epoch)
4326                .unwrap(),
4327            RouteRelease::Removed(_)
4328        ));
4329        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4330        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4331
4332        let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
4333        forwarding
4334            .complete_pending_relay(
4335                module_connection,
4336                second.corr,
4337                RouteBindRelayOutcome::Accepted,
4338            )
4339            .unwrap();
4340        assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
4341        assert!(matches!(
4342            forwarding
4343                .release_client_route(client, second.client_channel, first.client_epoch)
4344                .unwrap(),
4345            RouteRelease::Stale
4346        ));
4347        assert!(!forwarding
4348            .cache_status(
4349                endpoint,
4350                second.module_channel,
4351                first.module_epoch,
4352                "stale".into(),
4353            )
4354            .unwrap());
4355        assert!(forwarding
4356            .cache_status(
4357                endpoint,
4358                second.module_channel,
4359                second.module_epoch,
4360                "current".into(),
4361            )
4362            .unwrap());
4363        match forwarding
4364            .route_poll_snapshot(client, second.client_channel, second.client_epoch)
4365            .unwrap()
4366        {
4367            RoutePollSnapshot::Bound { status, .. } => {
4368                assert_eq!(status.as_deref(), Some("current"));
4369            }
4370            RoutePollSnapshot::Absent => panic!("successor binding was removed"),
4371        }
4372        let counters = forwarding.counters().snapshot();
4373        assert_eq!(counters["route_released_epoch_fenced"], 1);
4374        assert_eq!(counters["route_release_stale_skipped"], 1);
4375    }
4376
4377    #[test]
4378    fn max_epoch_reservation_retires_only_that_slot() {
4379        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
4380        forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
4381        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
4382        let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
4383        assert_eq!(
4384            (final_use.client_channel, final_use.client_epoch),
4385            (7, u32::MAX)
4386        );
4387        assert_eq!(
4388            (final_use.module_channel, final_use.module_epoch),
4389            (9, u32::MAX)
4390        );
4391        forwarding
4392            .abort_pending_relay(
4393                endpoint,
4394                final_use.corr,
4395                RouteBindRelayOutcome::ModuleGone("abort".into()),
4396            )
4397            .unwrap();
4398        forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
4399        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
4400        let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
4401        assert_ne!(next.client_channel, 7);
4402        assert_ne!(next.module_channel, 9);
4403        assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
4404    }
4405
4406    #[test]
4407    fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
4408        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4409            route_fixture("corr-shared");
4410        let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
4411        assert_eq!(bind.corr, 1);
4412        forwarding
4413            .abort_pending_relay(
4414                endpoint,
4415                bind.corr,
4416                RouteBindRelayOutcome::ModuleGone("abort".into()),
4417            )
4418            .unwrap();
4419        let rpc = forwarding
4420            .begin_module_control_rpc_for(
4421                "corr-shared",
4422                "health.check",
4423                Instant::now() - Duration::from_millis(1),
4424            )
4425            .unwrap();
4426        assert_eq!(rpc.corr, 2);
4427        assert_eq!(
4428            forwarding
4429                .complete_module_control_rpc(
4430                    module_connection,
4431                    rpc.corr,
4432                    Some("health.check"),
4433                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
4434                        status: subc_protocol::session::HealthStatus::Ok,
4435                        detail: None,
4436                        metrics: None,
4437                    }),
4438                )
4439                .unwrap(),
4440            ModuleControlRpcCompletion::Settled
4441        );
4442        assert!(matches!(
4443            rpc.receiver.blocking_recv().unwrap(),
4444            ModuleControlRpcOutcome::DeadlineElapsed
4445        ));
4446    }
4447
4448    #[tokio::test(start_paused = true)]
4449    async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
4450        let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
4451        let probe_started_at = Instant::now();
4452        let rpc = forwarding
4453            .begin_health_probe_rpc_for(
4454                "tombstone-ttl",
4455                "health.check",
4456                probe_started_at,
4457                probe_started_at + Duration::from_secs(5),
4458            )
4459            .unwrap();
4460        assert!(forwarding
4461            .tombstone_health_probe_rpc(endpoint, rpc.corr)
4462            .unwrap());
4463        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
4464
4465        tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
4466        tokio::task::yield_now().await;
4467
4468        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
4469    }
4470
4471    #[test]
4472    fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
4473        let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
4474        let mut close = forwarding.register_connection_close(endpoint.connection_id);
4475        forwarding.inject_control_corr(endpoint, u64::MAX);
4476        let final_rpc = forwarding
4477            .begin_module_control_rpc_for(
4478                "corr-max",
4479                "health.check",
4480                Instant::now() + Duration::from_secs(1),
4481            )
4482            .unwrap();
4483        assert_eq!(final_rpc.corr, u64::MAX);
4484        forwarding
4485            .cancel_module_control_rpc(endpoint, final_rpc.corr)
4486            .unwrap();
4487        assert!(matches!(
4488            forwarding.begin_module_control_rpc_for(
4489                "corr-max",
4490                "health.check",
4491                Instant::now() + Duration::from_secs(1),
4492            ),
4493            Err(ForwardingError::RelayCorrelationExhausted)
4494        ));
4495        assert!(close.try_recv().is_ok());
4496    }
4497
4498    #[test]
4499    fn publication_epoch_controls_delivery_failure_escalation() {
4500        fn setup_successor(
4501            commit_successor: Option<bool>,
4502        ) -> (ForwardingTable, ConnectionId, u16, u32) {
4503            let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4504                route_fixture("escalation");
4505            let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
4506            forwarding
4507                .complete_pending_relay(
4508                    module_connection,
4509                    first.corr,
4510                    RouteBindRelayOutcome::Accepted,
4511                )
4512                .unwrap();
4513            client_rx.try_recv().unwrap();
4514            assert!(matches!(
4515                forwarding
4516                    .release_client_route(client, first.client_channel, first.client_epoch)
4517                    .unwrap(),
4518                RouteRelease::Removed(_)
4519            ));
4520            if let Some(commit_successor) = commit_successor {
4521                forwarding.inject_client_slot_epoch(
4522                    client,
4523                    first.client_channel,
4524                    first.client_epoch,
4525                );
4526                forwarding.inject_module_slot_epoch(
4527                    endpoint,
4528                    first.module_channel,
4529                    first.module_epoch,
4530                );
4531                let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
4532                if commit_successor {
4533                    forwarding
4534                        .complete_pending_relay(
4535                            module_connection,
4536                            successor.corr,
4537                            RouteBindRelayOutcome::Accepted,
4538                        )
4539                        .unwrap();
4540                    client_rx.try_recv().unwrap();
4541                } else {
4542                    forwarding
4543                        .abort_pending_relay(
4544                            endpoint,
4545                            successor.corr,
4546                            RouteBindRelayOutcome::ModuleGone("abort".into()),
4547                        )
4548                        .unwrap();
4549                }
4550            }
4551            (forwarding, client, first.client_channel, first.client_epoch)
4552        }
4553
4554        let probe_sink = FrameSink::new(mpsc::channel(1).0);
4555        let (no_successor, client, channel, epoch) = setup_successor(None);
4556        let mut close = no_successor.register_connection_close(client);
4557        assert!(no_successor
4558            .escalate_client_delivery_failure(
4559                client,
4560                channel,
4561                epoch,
4562                CloseReason::new("delivery", "failed"),
4563                UndeliveredFrame {
4564                    module_id: None,
4565                    sink: &probe_sink,
4566                },
4567            )
4568            .unwrap());
4569        assert!(close.try_recv().is_ok());
4570
4571        let (aborted, client, channel, epoch) = setup_successor(Some(false));
4572        let mut close = aborted.register_connection_close(client);
4573        assert!(aborted
4574            .escalate_client_delivery_failure(
4575                client,
4576                channel,
4577                epoch,
4578                CloseReason::new("delivery", "failed"),
4579                UndeliveredFrame {
4580                    module_id: None,
4581                    sink: &probe_sink,
4582                },
4583            )
4584            .unwrap());
4585        assert!(close.try_recv().is_ok());
4586
4587        let (published, client, channel, epoch) = setup_successor(Some(true));
4588        let mut close = published.register_connection_close(client);
4589        assert!(!published
4590            .escalate_client_delivery_failure(
4591                client,
4592                channel,
4593                epoch,
4594                CloseReason::new("delivery", "stale failure"),
4595                UndeliveredFrame {
4596                    module_id: None,
4597                    sink: &probe_sink,
4598                },
4599            )
4600            .unwrap());
4601        assert!(close.try_recv().is_err());
4602    }
4603
4604    #[test]
4605    fn route_concentration_separates_client_count_from_routes_per_client() {
4606        // The distinction this asserts is the one a bare connection count cannot
4607        // make: two connections holding one route each and one connection
4608        // holding two are the same total, and have opposite causes.
4609        let (forwarding, module_connection, _, client, sink, _client_rx) =
4610            route_fixture("concentration");
4611        assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
4612
4613        for corr in [70_u64, 71] {
4614            let pending =
4615                begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
4616            forwarding
4617                .complete_pending_relay(
4618                    module_connection,
4619                    pending.corr,
4620                    RouteBindRelayOutcome::Accepted,
4621                )
4622                .unwrap();
4623        }
4624
4625        // One connection, two routes — not two connections with a route each.
4626        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
4627        assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
4628    }
4629
4630    #[test]
4631    fn cleanup_and_accepted_resolution_have_one_lock_winner() {
4632        let (forwarding, module_connection, _, client, sink, mut client_rx) =
4633            route_fixture("cleanup-race");
4634        let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
4635        forwarding
4636            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4637            .unwrap();
4638        let released = forwarding.cleanup_connection(client).unwrap();
4639        assert_eq!(released.len(), 1);
4640        let completion = forwarding
4641            .complete_pending_relay(
4642                module_connection,
4643                pending.corr,
4644                RouteBindRelayOutcome::Accepted,
4645            )
4646            .unwrap();
4647        assert!(!completion.settled);
4648        assert!(client_rx.try_recv().is_err());
4649        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4650
4651        let (forwarding, module_connection, _, client, sink, mut client_rx) =
4652            route_fixture("accepted-race");
4653        let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
4654        forwarding
4655            .complete_pending_relay(
4656                module_connection,
4657                pending.corr,
4658                RouteBindRelayOutcome::Accepted,
4659            )
4660            .unwrap();
4661        assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
4662        let released = forwarding.cleanup_connection(client).unwrap();
4663        assert_eq!(released.len(), 1);
4664        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4665    }
4666
4667    #[test]
4668    fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
4669        let (forwarding, module_connection, _, client, sink, mut client_rx) =
4670            route_fixture("drain-gap");
4671        let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
4672        forwarding
4673            .complete_pending_relay(
4674                module_connection,
4675                live.corr,
4676                RouteBindRelayOutcome::Accepted,
4677            )
4678            .unwrap();
4679        client_rx.try_recv().unwrap();
4680        let binding = match forwarding
4681            .lookup_data_route(client, live.client_channel, live.client_epoch)
4682            .unwrap()
4683        {
4684            DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4685            other => panic!("expected live route, got {other:?}"),
4686        };
4687
4688        let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
4689        forwarding
4690            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4691            .unwrap();
4692        let control_rpc = forwarding
4693            .begin_module_control_rpc_for(
4694                "drain-gap",
4695                "health.check",
4696                Instant::now() + Duration::from_secs(1),
4697            )
4698            .unwrap();
4699        let target = forwarding
4700            .begin_module_drain("drain-gap", RouteCloseReason::Reload)
4701            .unwrap()
4702            .unwrap();
4703        assert!(matches!(
4704            control_rpc.receiver.blocking_recv().unwrap(),
4705            ModuleControlRpcOutcome::ModuleGone(_)
4706        ));
4707        assert_eq!(target.abandoned_bindings.len(), 1);
4708        assert!(binding.flow.sem.is_closed());
4709        assert!(
4710            !forwarding
4711                .complete_pending_relay(
4712                    module_connection,
4713                    pending.corr,
4714                    RouteBindRelayOutcome::Accepted,
4715                )
4716                .unwrap()
4717                .settled
4718        );
4719        assert!(matches!(
4720            forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
4721            Err(ForwardingError::ModuleReloading { .. })
4722        ));
4723        let released = forwarding
4724            .release_module_endpoint_routes(target.endpoint)
4725            .unwrap();
4726        assert_eq!(released.len(), 1);
4727        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4728    }
4729
4730    /// A client can be marked closing while its egress is still open: the daemon
4731    /// asks a connection to close (here through the production path, a module
4732    /// frame that its egress refused) and the connection loop tears down a moment
4733    /// later. A route.bind ack that lands inside that window is answered on the
4734    /// MODULE connection's frame handler, so resolving it must not produce an
4735    /// error -- an error there ends the module connection, and that connection
4736    /// carries every other client's routes to the module.
4737    #[test]
4738    fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
4739        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4740            route_fixture("closing-client");
4741
4742        // A published route on this client: escalate_client_delivery_failure only
4743        // marks a connection closing for a route it has already published.
4744        let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
4745        forwarding
4746            .complete_pending_relay(
4747                module_connection,
4748                live.corr,
4749                RouteBindRelayOutcome::Accepted,
4750            )
4751            .unwrap();
4752        client_rx.try_recv().unwrap();
4753
4754        // A second route.open from the same client, relayed and awaiting its ack.
4755        let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
4756        forwarding
4757            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4758            .unwrap();
4759
4760        // The window: closing, but the sink is still open.
4761        assert!(forwarding
4762            .escalate_client_delivery_failure(
4763                client,
4764                live.client_channel,
4765                live.client_epoch,
4766                CloseReason::new(
4767                    "module_to_client_delivery_failed",
4768                    "client egress refused a module frame",
4769                ),
4770                UndeliveredFrame {
4771                    module_id: None,
4772                    sink: &sink,
4773                },
4774            )
4775            .unwrap());
4776        assert!(!sink.is_closed());
4777
4778        let completion = forwarding
4779            .complete_pending_relay(
4780                module_connection,
4781                pending.corr,
4782                RouteBindRelayOutcome::Accepted,
4783            )
4784            .expect("a closing client must not turn a module's ack into an error");
4785
4786        assert!(completion.settled);
4787        let abandoned = completion
4788            .abandoned
4789            .expect("the module must be told to drop the binding it just created");
4790        assert_eq!(abandoned.connection_id, module_connection);
4791        assert_eq!(abandoned.channel, pending.module_channel);
4792        assert_eq!(abandoned.epoch, pending.module_epoch);
4793        assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
4794        assert!(matches!(
4795            pending.receiver.blocking_recv().unwrap(),
4796            RouteBindRelayOutcome::ModuleGone(_)
4797        ));
4798        // No route was published to a client that is on its way out, and the
4799        // reserved handle pair went back.
4800        assert!(client_rx.try_recv().is_err());
4801        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
4802
4803        // The module endpoint is untouched: still live, and still able to take a
4804        // route from another client.
4805        assert!(forwarding
4806            .has_live_module_connection("closing-client")
4807            .unwrap());
4808        let cotenant = ConnectionId::new(201);
4809        let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
4810        let cotenant_route = begin_test_route(
4811            &forwarding,
4812            cotenant,
4813            FrameSink::new(cotenant_tx),
4814            62,
4815            "closing-client",
4816        );
4817        assert_eq!(cotenant_route.endpoint, endpoint);
4818        forwarding
4819            .complete_pending_relay(
4820                module_connection,
4821                cotenant_route.corr,
4822                RouteBindRelayOutcome::Accepted,
4823            )
4824            .unwrap();
4825        assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
4826        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
4827    }
4828
4829    #[test]
4830    fn pending_route_permit_is_released_on_rejection_and_abort() {
4831        let forwarding = ForwardingTable::default();
4832        let module_connection = ConnectionId::new(300);
4833        let client = ConnectionId::new(301);
4834        let (module_tx, _module_rx) = mpsc::channel(1);
4835        let endpoint = forwarding
4836            .register_module_connection(
4837                module_connection,
4838                "permit".into(),
4839                2,
4840                Concurrency::ModuleManaged,
4841                FrameSink::new(module_tx),
4842            )
4843            .unwrap();
4844        let (client_tx, mut client_rx) = mpsc::channel(1);
4845        let sink = FrameSink::new(client_tx);
4846        let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
4847        assert!(sink.try_send(test_ping(999)).is_err());
4848        forwarding
4849            .complete_pending_relay(
4850                module_connection,
4851                rejected.corr,
4852                RouteBindRelayOutcome::Rejected(ErrorBody {
4853                    code: "no".into(),
4854                    message: "rejected".into(),
4855                    detail: None,
4856                }),
4857            )
4858            .unwrap();
4859        sink.try_send(test_ping(1000)).unwrap();
4860        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
4861
4862        let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
4863        assert!(sink.try_send(test_ping(1001)).is_err());
4864        forwarding
4865            .abort_pending_relay(
4866                endpoint,
4867                aborted.corr,
4868                RouteBindRelayOutcome::ModuleGone("abort".into()),
4869            )
4870            .unwrap();
4871        sink.try_send(test_ping(1002)).unwrap();
4872        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
4873
4874        let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
4875        forwarding
4876            .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
4877            .unwrap();
4878        drop(client_rx);
4879        let completion = forwarding
4880            .complete_pending_relay(
4881                module_connection,
4882                receiver_closed.corr,
4883                RouteBindRelayOutcome::Accepted,
4884            )
4885            .unwrap();
4886        assert!(completion.abandoned.is_some());
4887        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4888    }
4889
4890    /// Every connection teardown passes through `cleanup_connection`, and
4891    /// connection ids come from a monotonic counter that never hands an id out
4892    /// twice. If the closing mark survives teardown, the set grows by one entry
4893    /// per connection for the life of the daemon -- the self-watchdog alone
4894    /// reconnects once a minute.
4895    #[test]
4896    fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
4897        let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
4898            route_fixture("closing-set-leak");
4899
4900        const CONNECTIONS: u64 = 32;
4901        for index in 0..CONNECTIONS {
4902            let client = ConnectionId::new(1000 + index);
4903            let (client_tx, _client_rx) = mpsc::channel(8);
4904            let route = begin_test_route(
4905                &forwarding,
4906                client,
4907                FrameSink::new(client_tx),
4908                index + 1,
4909                "closing-set-leak",
4910            );
4911            forwarding
4912                .complete_pending_relay(
4913                    module_connection,
4914                    route.corr,
4915                    RouteBindRelayOutcome::Accepted,
4916                )
4917                .unwrap();
4918            forwarding.cleanup_connection(client).unwrap();
4919        }
4920        forwarding.cleanup_connection(module_connection).unwrap();
4921
4922        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
4923    }
4924
4925    /// The closing mark exists to refuse new work for a connection that is on
4926    /// its way out but whose teardown has not run yet: the daemon asks the
4927    /// connection loop to end, and only when the loop reacts does
4928    /// `cleanup_connection` strip the connection's state. Inside that window an
4929    /// operation for the dying connection must still be refused; only after
4930    /// cleanup completes may the mark go.
4931    #[test]
4932    fn closing_connection_is_refused_new_work_until_cleanup_completes() {
4933        let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
4934            route_fixture("closing-gate");
4935
4936        // A published route: escalate_client_delivery_failure only marks a
4937        // connection closing for a route it has already published.
4938        let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
4939        forwarding
4940            .complete_pending_relay(
4941                module_connection,
4942                live.corr,
4943                RouteBindRelayOutcome::Accepted,
4944            )
4945            .unwrap();
4946        client_rx.try_recv().unwrap();
4947
4948        // Mark the connection closing through the production path without
4949        // running teardown, pinning the window open.
4950        assert!(forwarding
4951            .escalate_client_delivery_failure(
4952                client,
4953                live.client_channel,
4954                live.client_epoch,
4955                CloseReason::new(
4956                    "module_to_client_delivery_failed",
4957                    "client egress refused a module frame",
4958                ),
4959                UndeliveredFrame {
4960                    module_id: None,
4961                    sink: &sink,
4962                },
4963            )
4964            .unwrap());
4965        assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
4966
4967        // A late route.open for the closing client is refused ...
4968        assert!(matches!(
4969            forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
4970            Err(ForwardingError::ConnectionClosing { connection_id })
4971                if connection_id == client
4972        ));
4973        // ... and so is a late attempt to register the connection as a module.
4974        let (late_tx, _late_rx) = mpsc::channel(1);
4975        assert!(matches!(
4976            forwarding.register_module_connection(
4977                client,
4978                "late-module".into(),
4979                2,
4980                Concurrency::ModuleManaged,
4981                FrameSink::new(late_tx),
4982            ),
4983            Err(ForwardingError::ConnectionClosing { connection_id })
4984                if connection_id == client
4985        ));
4986
4987        // Teardown is the point that lifts the mark: it has just removed every
4988        // per-connection entry under the same lock, so the gate has nothing
4989        // left to protect for this id.
4990        forwarding.cleanup_connection(client).unwrap();
4991        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
4992    }
4993}
4994
4995/// Blue/green swap slots: a candidate registered beside the active endpoint,
4996/// promoted by `cutover_candidate`, with the old incumbent drained by endpoint.
4997#[cfg(test)]
4998mod swap_slot_tests {
4999    use std::time::Duration;
5000
5001    use super::*;
5002    use tokio::sync::mpsc;
5003
5004    const MODULE_ID: &str = "swapped";
5005
5006    struct SwapFixture {
5007        forwarding: ForwardingTable,
5008        incumbent_connection: ConnectionId,
5009        incumbent: ModuleEndpointId,
5010        candidate_connection: ConnectionId,
5011        candidate: ModuleEndpointId,
5012        _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
5013    }
5014
5015    fn swap_fixture() -> SwapFixture {
5016        let forwarding = ForwardingTable::default();
5017        let incumbent_connection = ConnectionId::new(100);
5018        let candidate_connection = ConnectionId::new(110);
5019        let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
5020        let incumbent = forwarding
5021            .register_module_connection(
5022                incumbent_connection,
5023                MODULE_ID.to_string(),
5024                2,
5025                Concurrency::ModuleManaged,
5026                FrameSink::new(incumbent_tx),
5027            )
5028            .unwrap();
5029        let (candidate_tx, candidate_rx) = mpsc::channel(8);
5030        let candidate = forwarding
5031            .register_candidate_module_connection(
5032                candidate_connection,
5033                MODULE_ID.to_string(),
5034                2,
5035                Concurrency::ModuleManaged,
5036                FrameSink::new(candidate_tx),
5037            )
5038            .unwrap();
5039        SwapFixture {
5040            forwarding,
5041            incumbent_connection,
5042            incumbent,
5043            candidate_connection,
5044            candidate,
5045            _module_rxs: vec![incumbent_rx, candidate_rx],
5046        }
5047    }
5048
5049    fn client(
5050        raw: u64,
5051    ) -> (
5052        ConnectionId,
5053        FrameSink,
5054        mpsc::Receiver<crate::router::OutboundFrame>,
5055    ) {
5056        let (tx, rx) = mpsc::channel(8);
5057        (ConnectionId::new(raw), FrameSink::new(tx), rx)
5058    }
5059
5060    fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
5061        forwarding
5062            .read_inner()
5063            .unwrap()
5064            .client_to_module
5065            .values()
5066            .map(|route| route.module_endpoint)
5067            .collect()
5068    }
5069
5070    #[test]
5071    fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
5072        let fixture = swap_fixture();
5073        let forwarding = &fixture.forwarding;
5074        assert_ne!(fixture.incumbent, fixture.candidate);
5075
5076        // Every by-id consumer still resolves the incumbent.
5077        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5078        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5079        let (client_connection, client_sink, _client_rx) = client(200);
5080        let pending = forwarding
5081            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5082            .unwrap();
5083        assert_eq!(pending.endpoint, fixture.incumbent);
5084        let rpc = forwarding
5085            .begin_module_control_rpc_for(
5086                MODULE_ID,
5087                "health.check",
5088                Instant::now() + Duration::from_secs(1),
5089            )
5090            .unwrap();
5091        assert_eq!(rpc.endpoint, fixture.incumbent);
5092        let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
5093        assert_eq!(census.len(), 1, "the census lists one endpoint per id");
5094
5095        // Connection-keyed lookups see the candidate, so its own frames resolve.
5096        assert_eq!(
5097            forwarding
5098                .module_endpoint_for_connection(fixture.candidate_connection)
5099                .unwrap(),
5100            Some(fixture.candidate)
5101        );
5102        assert_eq!(
5103            forwarding
5104                .module_id_for_connection(fixture.candidate_connection)
5105                .unwrap()
5106                .as_deref(),
5107            Some(MODULE_ID)
5108        );
5109
5110        // One candidate per id.
5111        let (other_tx, _other_rx) = mpsc::channel(1);
5112        assert_eq!(
5113            forwarding.register_candidate_module_connection(
5114                ConnectionId::new(120),
5115                MODULE_ID.to_string(),
5116                2,
5117                Concurrency::ModuleManaged,
5118                FrameSink::new(other_tx),
5119            ),
5120            Err(ForwardingError::CandidateSlotOccupied {
5121                module_id: MODULE_ID.to_string()
5122            })
5123        );
5124    }
5125
5126    /// The cutover linearization point. A relay reserved on the incumbent before
5127    /// cutover must never become a route on the incumbent, and every relay
5128    /// reserved after it must land on the promoted candidate.
5129    #[test]
5130    fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
5131        let fixture = swap_fixture();
5132        let forwarding = &fixture.forwarding;
5133        let (early_client, early_sink, _early_rx) = client(200);
5134        let mut early = forwarding
5135            .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
5136            .unwrap();
5137        assert_eq!(early.endpoint, fixture.incumbent);
5138        assert_eq!(
5139            forwarding
5140                .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
5141                .unwrap(),
5142            RouteBindRelayMark::Marked
5143        );
5144
5145        let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5146        assert_eq!(
5147            cutover,
5148            ForwardingCutover {
5149                promoted: fixture.candidate,
5150                incumbent: Some(fixture.incumbent),
5151            }
5152        );
5153
5154        // A route.open reserved after cutover goes to the promoted candidate.
5155        let (late_client, late_sink, _late_rx) = client(201);
5156        let late = forwarding
5157            .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
5158            .unwrap();
5159        assert_eq!(
5160            late.endpoint, fixture.candidate,
5161            "a route.open after cutover was reserved on the incumbent"
5162        );
5163
5164        // The incumbent acks the early relay after cutover.
5165        let completion = forwarding
5166            .complete_pending_relay(
5167                fixture.incumbent_connection,
5168                early.corr,
5169                RouteBindRelayOutcome::Accepted,
5170            )
5171            .expect("a superseded endpoint's ack is not an error on its connection");
5172        assert!(completion.settled);
5173        assert!(
5174            !committed_endpoints(forwarding).contains(&fixture.incumbent),
5175            "a relay reserved before cutover committed a route on the incumbent"
5176        );
5177        let goodbye = completion
5178            .abandoned
5179            .expect("the incumbent is told to drop the binding it just created");
5180        assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
5181        assert_eq!(goodbye.channel, early.module_channel);
5182        assert_eq!(goodbye.epoch, early.module_epoch);
5183        assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
5184        match early.receiver.try_recv() {
5185            Ok(RouteBindRelayOutcome::Rejected(body)) => assert_eq!(body.code, "module_reloading"),
5186            other => panic!("expected a retryable module_reloading answer, got {other:?}"),
5187        }
5188        assert!(matches!(
5189            forwarding
5190                .lookup_data_route(early_client, early.client_channel, early.client_epoch)
5191                .unwrap(),
5192            DataRoute::Client(DataRouteState::Absent)
5193        ));
5194
5195        // The late relay commits on the candidate; only its pair stays reserved
5196        // until then.
5197        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
5198        forwarding
5199            .complete_pending_relay(
5200                fixture.candidate_connection,
5201                late.corr,
5202                RouteBindRelayOutcome::Accepted,
5203            )
5204            .unwrap();
5205        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5206        assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
5207    }
5208
5209    #[test]
5210    fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
5211        let fixture = swap_fixture();
5212        let forwarding = &fixture.forwarding;
5213        // One bound route and one in-flight relay on the incumbent.
5214        let (bound_client, bound_sink, _bound_rx) = client(200);
5215        let bound = forwarding
5216            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5217            .unwrap();
5218        forwarding
5219            .complete_pending_relay(
5220                fixture.incumbent_connection,
5221                bound.corr,
5222                RouteBindRelayOutcome::Accepted,
5223            )
5224            .unwrap();
5225        let (pending_client, pending_sink, _pending_rx) = client(201);
5226        let mut in_flight = forwarding
5227            .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
5228            .unwrap();
5229        forwarding
5230            .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
5231            .unwrap();
5232
5233        let incumbent = forwarding
5234            .cutover_candidate(MODULE_ID)
5235            .unwrap()
5236            .unwrap()
5237            .incumbent
5238            .unwrap();
5239        let target = forwarding
5240            .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
5241            .unwrap()
5242            .expect("the superseded incumbent is still registered");
5243
5244        assert_eq!(target.endpoint, fixture.incumbent);
5245        assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
5246        assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
5247        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5248        assert_eq!(target.abandoned_bindings.len(), 1);
5249        assert_eq!(
5250            target.abandoned_bindings[0].channel,
5251            in_flight.module_channel
5252        );
5253        assert!(matches!(
5254            in_flight.receiver.try_recv(),
5255            Ok(RouteBindRelayOutcome::Rejected(body)) if body.code == "module_reloading"
5256        ));
5257        assert_eq!(
5258            forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
5259            1,
5260            "the incumbent's bound route stays until its drain finishes"
5261        );
5262
5263        let (next_client, next_sink, _next_rx) = client(202);
5264        let next = forwarding
5265            .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
5266            .expect("the promoted candidate keeps accepting routes");
5267        assert_eq!(next.endpoint, fixture.candidate);
5268    }
5269
5270    /// An endpoint replaced WITHOUT a promotion (a successor registered over it
5271    /// as an ordinary active HELLO) is stale, not superseded, and its ack still
5272    /// fails the acking connection exactly as it did before swap slots existed:
5273    /// `StaleModuleEndpoint`, the reservation indexes already stripped, and the
5274    /// waiting client's sender dropped unanswered.
5275    #[test]
5276    fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
5277        let forwarding = ForwardingTable::default();
5278        let first_connection = ConnectionId::new(70);
5279        let (first_tx, _first_rx) = mpsc::channel(8);
5280        forwarding
5281            .register_module_connection(
5282                first_connection,
5283                MODULE_ID.to_string(),
5284                2,
5285                Concurrency::ModuleManaged,
5286                FrameSink::new(first_tx),
5287            )
5288            .unwrap();
5289        let (client_connection, client_sink, _client_rx) = client(200);
5290        let mut pending = forwarding
5291            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5292            .unwrap();
5293        let (second_tx, _second_rx) = mpsc::channel(8);
5294        forwarding
5295            .register_module_connection(
5296                ConnectionId::new(80),
5297                MODULE_ID.to_string(),
5298                2,
5299                Concurrency::ModuleManaged,
5300                FrameSink::new(second_tx),
5301            )
5302            .unwrap();
5303
5304        assert_eq!(
5305            forwarding
5306                .complete_pending_relay(
5307                    first_connection,
5308                    pending.corr,
5309                    RouteBindRelayOutcome::Accepted
5310                )
5311                .unwrap_err(),
5312            ForwardingError::StaleModuleEndpoint
5313        );
5314        assert!(committed_endpoints(&forwarding).is_empty());
5315        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5316        assert!(matches!(
5317            pending.receiver.try_recv(),
5318            Err(oneshot::error::TryRecvError::Closed)
5319        ));
5320    }
5321
5322    #[test]
5323    fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
5324        // A candidate whose connection drops leaves the incumbent routable.
5325        let fixture = swap_fixture();
5326        let forwarding = &fixture.forwarding;
5327        assert!(forwarding
5328            .cleanup_connection(fixture.candidate_connection)
5329            .unwrap()
5330            .is_empty());
5331        assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
5332        let (client_connection, client_sink, _client_rx) = client(200);
5333        assert_eq!(
5334            forwarding
5335                .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5336                .unwrap()
5337                .endpoint,
5338            fixture.incumbent
5339        );
5340
5341        // After a cutover, the incumbent's teardown releases its own routes and
5342        // leaves the promoted candidate in place.
5343        let fixture = swap_fixture();
5344        let forwarding = &fixture.forwarding;
5345        let (bound_client, bound_sink, _bound_rx) = client(200);
5346        let bound = forwarding
5347            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5348            .unwrap();
5349        forwarding
5350            .complete_pending_relay(
5351                fixture.incumbent_connection,
5352                bound.corr,
5353                RouteBindRelayOutcome::Accepted,
5354            )
5355            .unwrap();
5356        forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5357        let released = forwarding
5358            .cleanup_connection(fixture.incumbent_connection)
5359            .unwrap();
5360        assert_eq!(released.len(), 1);
5361        assert_eq!(released[0].connection_id, bound_client);
5362        assert!(forwarding
5363            .read_inner()
5364            .unwrap()
5365            .superseded_endpoints
5366            .is_empty());
5367        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5368        let (next_client, next_sink, _next_rx) = client(201);
5369        assert_eq!(
5370            forwarding
5371                .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
5372                .unwrap()
5373                .endpoint,
5374            fixture.candidate
5375        );
5376    }
5377}