Skip to main content

subc_daemon/
forwarding.rs

1use std::{
2    collections::{BTreeMap, HashMap, HashSet},
3    error::Error,
4    fmt,
5    sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
6    time::Duration,
7};
8
9use subc_control::{ClientControlResponse, RouteCloseReason};
10use subc_protocol::{
11    manifest::Concurrency,
12    session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
13    ErrorBody, Flags, FrameType, Principal, Priority,
14};
15use tokio::sync::{oneshot, Semaphore};
16use tokio::time::Instant;
17use tracing::{debug, info, warn};
18
19use crate::{
20    control::{RouteBindBreakers, RouteBindConcurrency},
21    observability::DaemonCounters,
22    registry::ConnectionId,
23    router::FrameSink,
24    scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
25    Frame, ProjectRootId,
26};
27
28/// Default per-channel request-credit window for modules that schedule internally.
29const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
30
31/// Watches leave eight request credits available for ordinary calls.
32const DEFAULT_MODULE_MANAGED_WATCH_WINDOW: usize = 24;
33
34/// High per-channel cap for stateless modules; this is an OOM guard, not scheduling policy.
35const STATELESS_PARALLEL_WINDOW: usize = 1024;
36
37/// A stopped probe cycle cannot retain its last unanswered correlation forever.
38/// Active endpoints replace the tombstone on their next serial health probe;
39/// this backstop covers endpoints that stop probing altogether.
40const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
41
42/// Module connection identity used in forwarding keys.
43///
44/// The generation is bumped every time a module connection is registered so a future restart cannot
45/// accidentally reuse a stale `(connection_id, route_channel)` binding.
46#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
47pub struct ModuleEndpointId {
48    pub connection_id: ConnectionId,
49    pub generation: u64,
50}
51
52/// Client-local route key. A route channel is unique only within one client connection.
53#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
54pub(crate) struct ClientRouteKey {
55    pub connection_id: ConnectionId,
56    pub channel: u16,
57}
58
59/// Module-local route key. A route channel is unique only within one live module endpoint.
60#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
61pub(crate) struct ModuleRouteKey {
62    pub endpoint: ModuleEndpointId,
63    pub channel: u16,
64}
65
66#[derive(Debug)]
67pub(crate) struct RouteBinding {
68    pub client_connection_id: ConnectionId,
69    pub client_sink: FrameSink,
70    pub client_negotiated_ver: u8,
71    pub client_channel: u16,
72    pub client_epoch: u32,
73    pub module_id: String,
74    pub module_endpoint: ModuleEndpointId,
75    pub module_sink: FrameSink,
76    pub module_negotiated_ver: u8,
77    pub module_channel: u16,
78    pub module_epoch: u32,
79    pub principal: Principal,
80    /// Opener attributes captured at admission for daemon-side prompt policy.
81    #[allow(dead_code)]
82    pub harness: String,
83    /// Only Callosum's exact production claim is retained; never relayed.
84    #[allow(dead_code)]
85    pub build_class: Option<String>,
86    /// Captured at connect, carried through admission; not standing authority.
87    #[allow(dead_code)]
88    pub operator_app: Option<crate::operator_apps::OperatorAppStamp>,
89    pub project_root: Option<ProjectRootId>,
90    pub bound_at: Instant,
91    pub flow: Arc<ChannelFlow>,
92    /// The scope the route was admitted under, with the tag its bind was
93    /// stamped at. A scope change finds the routes to close by it.
94    pub scope: Option<BoundScope>,
95}
96
97#[derive(Debug, Clone)]
98pub(crate) enum DataRoute {
99    Client(DataRouteState),
100    Module(DataRouteState),
101}
102
103#[derive(Debug, Clone)]
104pub(crate) enum DataRouteState {
105    Bound(Arc<RouteBinding>),
106    Reserved,
107    EpochMismatch,
108    Absent,
109}
110
111/// Which kind of peer a route GOODBYE is being delivered to. This decides what
112/// happens when the GOODBYE cannot be enqueued (egress full/closed):
113/// - `Client`: escalate to closing that client connection (a socket close is a
114///   stronger teardown signal, and a full client egress means it is the slow
115///   client we would drop anyway).
116/// - `Module`: never close; deliver late instead (see
117///   [`send_module_route_goodbye`]), and drop only if the module still has no
118///   room after [`LATE_MODULE_GOODBYE_DEADLINE`]. A client-disconnect notifies the
119///   SHARED module that one client's route is gone; closing the module on its
120///   egress backpressure would tear down every co-tenant client (the exact
121///   cross-tenant blast radius this never-close rule exists to prevent — observed when a
122///   flooding dead client filled BOTH its own and the module's egress, so its
123///   route-gone GOODBYE to the module failed and closed the shared connection).
124///   subc has already removed the route from its forwarding state and drops
125///   stale module frames for the released channel (see router.rs), so subc's
126///   own routing is correct. The residual: under SUSTAINED module-egress
127///   backpressure a module-targeted route-gone notification can be lost, which a
128///   module using it for client-refcounting (e.g. AFT's session accounting)
129///   would miss. This is INTENTIONALLY ACCEPTED, not a gap. A consuming module
130///   must bound stale bindings with its own idle-activity reaper (last-touched
131///   TTL) independent of route-gone signals — AFT does exactly this, so a lost
132///   GOODBYE degrades to "the
133///   binding stays warm until its idle TTL" (bounded wasted resources), never an
134///   unbounded leak; disk-durable replay is unaffected. A dedicated reliable
135///   module control lane was evaluated and deliberately NOT
136///   built: it would add starvation-avoidance machinery to the thin core for a
137///   bounded warm-resource window that is not a correctness issue. never-close
138///   is the invariant that matters here.
139#[derive(Debug, Clone, Copy, PartialEq, Eq)]
140pub(crate) enum GoodbyeTargetKind {
141    Client,
142    Module,
143}
144
145#[derive(Debug, Clone)]
146pub(crate) struct GoodbyeTarget {
147    pub connection_id: ConnectionId,
148    pub sink: FrameSink,
149    pub negotiated_ver: u8,
150    pub channel: u16,
151    pub epoch: u32,
152    pub kind: GoodbyeTargetKind,
153    /// The module on the other end of the route: for a module-targeted relay
154    /// the receiving module (attributing a dropped relay), for a client target
155    /// the module whose route is going away (naming it when an undeliverable
156    /// relay closes the client).
157    pub module_id: Option<String>,
158}
159
160/// The frame a client connection's egress queue refused, for the diagnosis
161/// logged when that refusal closes the connection.
162#[derive(Debug, Clone, Copy)]
163pub(crate) struct UndeliveredFrame<'a> {
164    /// The module on the other end of the route the frame belonged to, when known.
165    pub module_id: Option<&'a str>,
166    /// The refusing connection's sink, read for what its queue held.
167    pub sink: &'a FrameSink,
168}
169
170/// How a route's principal appears in logs: `direct`, or `reserved:<module>`.
171pub(crate) fn principal_label(principal: &Principal) -> String {
172    match principal {
173        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
174        Principal::Direct => "direct".to_string(),
175        other => format!("{other:?}"),
176    }
177}
178
179/// The distinct principals of the routes bound on one client connection,
180/// sorted and comma-joined, or `none` when it has no bound route.
181fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
182    let labels = inner
183        .client_to_module
184        .iter()
185        .filter(|(key, _)| key.connection_id == connection_id)
186        .map(|(_, route)| principal_label(&route.principal))
187        .collect::<std::collections::BTreeSet<_>>();
188    if labels.is_empty() {
189        "none".to_string()
190    } else {
191        labels.into_iter().collect::<Vec<_>>().join(",")
192    }
193}
194
195impl GoodbyeTarget {
196    /// True only when an undeliverable GOODBYE should escalate to closing the
197    /// target connection. Never escalate for module recipients.
198    pub(crate) fn close_on_delivery_failure(&self) -> bool {
199        matches!(self.kind, GoodbyeTargetKind::Client)
200    }
201}
202
203/// How long a route GOODBYE that a module's egress queue refused keeps waiting
204/// for room before it is given up and counted as dropped.
205///
206/// A module that stops reading for a moment (a GC pause, a long synchronous
207/// handler, a reconnect herd it is still working through) should still learn
208/// that the route is gone, because a module that never hears the GOODBYE keeps
209/// the route and keeps sending on it for as long as its connection lives. The
210/// value is the default module drain timeout: the daemon already treats that as
211/// the longest it is reasonable to wait on a module that is busy but alive, and
212/// a module that cannot free 21 bytes of egress in that time is wedged rather
213/// than slow. Resolving each module's own configured drain timeout here would
214/// need a registry lookup on a path that holds only the module's sink.
215pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
216
217/// Send a route GOODBYE to a module without ever closing the module's shared
218/// connection, which would sever every other route it serves.
219///
220/// The frame is enqueued at once when the module's egress queue has room. When
221/// it does not, a detached task waits for room (bounded by
222/// [`LATE_MODULE_GOODBYE_DEADLINE`]) and sends it then. Arriving late is safe:
223/// the daemon has already released the route, and module SDKs tear a route
224/// down only when both the channel AND the epoch match the route installed on
225/// that channel, so a late GOODBYE for a channel since reused at a newer epoch
226/// is ignored. The GOODBYE is counted in `goodbye_relay_module_dropped` only
227/// when the connection is closed or the deadline passes.
228pub(crate) fn send_module_route_goodbye(
229    counters: &DaemonCounters,
230    sink: &FrameSink,
231    frame: Frame,
232    module_id: Option<&str>,
233    context: &'static str,
234) {
235    let channel = frame.header.channel;
236    let epoch = frame.header.epoch;
237    let Err(err) = sink.try_send(frame.clone()) else {
238        return;
239    };
240    // Waiting is pointless once the module's writer is gone, and impossible
241    // outside a Tokio runtime (cleanup that runs from a destructor at shutdown).
242    let runtime = match tokio::runtime::Handle::try_current() {
243        Ok(runtime) if !sink.is_closed() => runtime,
244        _ => {
245            counters.increment_goodbye_relay_module_dropped(module_id);
246            warn!(
247            module_id = module_id.unwrap_or("unknown"),
248            route_channel = channel,
249            route_epoch = epoch,
250            error = %err,
251            context,
252            "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
253            );
254            return;
255        }
256    };
257    debug!(
258        module_id = module_id.unwrap_or("unknown"),
259        route_channel = channel,
260        route_epoch = epoch,
261        error = %err,
262        context,
263        "module egress queue refused route GOODBYE; delivering it once the module frees room"
264    );
265    let counters = counters.clone();
266    let sink = sink.clone();
267    let module_id = module_id.map(str::to_string);
268    runtime.spawn(async move {
269        let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
270        let why = match outcome {
271            Ok(Ok(())) => {
272                debug!(
273                    module_id = module_id.as_deref().unwrap_or("unknown"),
274                    route_channel = channel,
275                    route_epoch = epoch,
276                    context,
277                    "late route GOODBYE delivered to module"
278                );
279                return;
280            }
281            Ok(Err(err)) => err.to_string(),
282            Err(_) => format!(
283                "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
284            ),
285        };
286        counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
287        warn!(
288            module_id = module_id.as_deref().unwrap_or("unknown"),
289            route_channel = channel,
290            route_epoch = epoch,
291            error = %why,
292            context,
293            "route GOODBYE to module dropped under backpressure; not closing shared module connection"
294        );
295    });
296}
297
298/// One route currently served by a module endpoint.
299///
300/// `goodbye_target` is deliberately retained alongside the census projection so
301/// a draining caller can address exactly the same route set that this read
302/// reports, without a second forwarding-table pass.
303#[derive(Debug, Clone)]
304pub(crate) struct EndpointRoute {
305    pub goodbye_target: GoodbyeTarget,
306    pub principal: Principal,
307    pub bound_at: Instant,
308    pub draining: bool,
309    /// WHY the endpoint is draining, when it is. Carried per-route so the
310    /// census can answer "closing because of what" without a second lookup;
311    /// `None` exactly when `draining` is false (one source: the drain map).
312    pub drain_reason: Option<RouteCloseReason>,
313}
314
315/// One live route a scope change closed: both ends to send GOODBYE to, and the
316/// reason for the client's `route.closed`.
317#[derive(Debug, Clone)]
318pub(crate) struct ScopeDrainedRoute {
319    pub reason: RouteCloseReason,
320    pub scope: BoundScope,
321    pub module_id: String,
322    pub client: GoodbyeTarget,
323    pub module: GoodbyeTarget,
324}
325
326#[derive(Debug)]
327pub(crate) struct PendingRouteBindRelay {
328    pub endpoint: ModuleEndpointId,
329    pub module_sink: FrameSink,
330    pub negotiated_ver: u8,
331    pub client_channel: u16,
332    pub client_epoch: u32,
333    pub module_channel: u16,
334    pub module_epoch: u32,
335    pub corr: u64,
336    pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
337}
338
339#[derive(Debug, Clone)]
340pub(crate) struct ModuleDrainTarget {
341    pub endpoint: ModuleEndpointId,
342    pub sink: FrameSink,
343    pub negotiated_ver: u8,
344    pub abandoned_bindings: Vec<GoodbyeTarget>,
345    pub excluded_subscriptions: u32,
346}
347
348/// One registered module connection, as [`ForwardingTable::module_connections`]
349/// reports it: enough to send it a frame and then close it.
350#[cfg(unix)]
351#[derive(Debug, Clone)]
352pub(crate) struct ModuleConnectionTarget {
353    pub module_id: String,
354    pub endpoint: ModuleEndpointId,
355    pub sink: FrameSink,
356    pub negotiated_ver: u8,
357}
358
359#[derive(Debug, Clone)]
360pub(crate) enum RouteBindRelayOutcome {
361    Accepted,
362    /// The module refused the bind with its own body. Relayed to the client
363    /// without the daemon-origin bit: the code and message are module-chosen,
364    /// so the frame must not claim daemon authorship, even when the module
365    /// picks a daemon code like `module_reloading`.
366    Rejected(ErrorBody),
367    /// The daemon refused the bind itself — a drain or superseded endpoint
368    /// (`module_reloading`), a reply that arrived after the deadline
369    /// (`module_timeout`), or a scope that ended mid-bind. Carries the
370    /// daemon-origin bit because the daemon wrote it. The module may still
371    /// have received the bind (a late reply, or a drain that began after it
372    /// accepted); the daemon then ends that binding with a GOODBYE, so no
373    /// call runs on it.
374    DaemonRejected(ErrorBody),
375    ModuleGone(String),
376}
377
378/// What [`ForwardingTable::cutover_candidate`] did.
379#[derive(Debug, Clone, Copy, PartialEq, Eq)]
380pub(crate) struct ForwardingCutover {
381    /// The endpoint now in the active slot (the former candidate).
382    pub promoted: ModuleEndpointId,
383    /// The endpoint demoted out of the active slot, to be drained by endpoint.
384    /// `None` when the incumbent's connection was already gone.
385    pub incumbent: Option<ModuleEndpointId>,
386}
387
388/// What tearing down one connection released.
389#[derive(Debug)]
390pub(crate) struct ConnectionCleanup {
391    /// Routes whose other end must be sent GOODBYE.
392    pub released: Vec<GoodbyeTarget>,
393    /// Pending route.bind relays to the closed module that were aborted: opens
394    /// that had not been bound yet. Zero when a client connection closed.
395    pub abandoned_relays: u32,
396}
397
398#[derive(Debug, Clone)]
399pub(crate) struct PendingRelayCompletion {
400    pub settled: bool,
401    pub abandoned: Option<GoodbyeTarget>,
402}
403
404#[derive(Debug)]
405pub(crate) struct PendingModuleControlRpc {
406    pub endpoint: ModuleEndpointId,
407    pub module_sink: FrameSink,
408    pub negotiated_ver: u8,
409    pub corr: u64,
410    pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
411}
412
413#[derive(Debug, Clone)]
414pub(crate) enum ModuleControlRpcOutcome {
415    Response(ModuleControlResponse),
416    Rejected(ErrorBody),
417    ModuleGone(String),
418    MalformedResponse(String),
419    UnexpectedOp { expected: String, actual: String },
420    DeadlineElapsed,
421}
422
423#[derive(Debug, Clone, PartialEq, Eq)]
424pub(crate) enum ModuleControlRpcCompletion {
425    Unknown,
426    Settled,
427    LateHealthAnswer {
428        module_id: String,
429        latency: Duration,
430    },
431}
432
433#[derive(Debug)]
434struct PendingModuleControlRpcEntry {
435    expected_op: String,
436    deadline: Instant,
437    health_probe_started_at: Option<Instant>,
438    sender: oneshot::Sender<ModuleControlRpcOutcome>,
439}
440
441#[derive(Debug)]
442struct HealthProbeTombstone {
443    expected_op: String,
444    module_id: String,
445    probe_started_at: Instant,
446    expires_at: Instant,
447}
448
449#[derive(Debug, Clone)]
450struct RouteReservation {
451    client_key: ClientRouteKey,
452    module_key: ModuleRouteKey,
453    client_epoch: u32,
454    module_epoch: u32,
455    project_root: Option<ProjectRootId>,
456}
457
458#[derive(Debug)]
459struct PendingRouteBindRelayEntry {
460    reservation: RouteReservation,
461    client_sink: FrameSink,
462    client_negotiated_ver: u8,
463    client_permit: crate::router::EgressPermit,
464    route_open_frame: Frame,
465    busy_refusals: bool,
466    request_window: usize,
467    watch_window: Option<usize>,
468    principal: Principal,
469    /// The scope admitted at `route.open`, with the tag captured then. Commit
470    /// compares it with the published tag.
471    scope: Option<BoundScope>,
472    harness: String,
473    build_class: Option<String>,
474    operator_app: Option<crate::operator_apps::OperatorAppStamp>,
475    deadline: Instant,
476    /// Set once `route.open` has finished putting the route.bind on the
477    /// module's egress queue. A path that drops this entry WITHOUT a module
478    /// reply (client teardown, module drain, the open's own abort) sends the
479    /// module a GOODBYE for the bind only when this is set, because only then
480    /// is the GOODBYE guaranteed to queue behind the bind. When it is not set,
481    /// the removal is recorded in `relays_released_before_enqueue` instead and
482    /// `route.open` sends that GOODBYE itself once its send has completed.
483    ///
484    /// A module reply does not consult it: an ack proves the module received
485    /// the bind, so the reply path decides about the GOODBYE on its own.
486    relay_enqueued: bool,
487    sender: oneshot::Sender<RouteBindRelayOutcome>,
488}
489
490/// What `mark_route_bind_relay_enqueued` found once `route.open` had put the
491/// route.bind on the module's queue.
492#[derive(Debug, Clone, Copy, PartialEq, Eq)]
493pub(crate) enum RouteBindRelayMark {
494    /// The relay is still pending; later removals handle their own GOODBYE.
495    Marked,
496    /// The module already answered and the reply path settled the relay:
497    /// it committed the route, or it refused it and sent any GOODBYE the
498    /// module needed. `route.open` must send nothing.
499    SettledByReply,
500    /// The relay was dropped without a module reply while the bind was still
501    /// being queued, so nothing has told the module to drop the binding it
502    /// is about to create. `route.open` must send that GOODBYE.
503    ReleasedWithoutReply,
504}
505
506#[derive(Debug, Clone)]
507pub(crate) enum RouteRelease {
508    Removed(GoodbyeTarget),
509    Stale,
510    Absent,
511}
512
513#[derive(Debug, Clone)]
514pub(crate) enum RoutePollSnapshot {
515    Bound {
516        module_id: String,
517        status: Option<String>,
518    },
519    Absent,
520}
521
522#[derive(Debug, Clone)]
523struct ModuleConnection {
524    endpoint: ModuleEndpointId,
525    sink: FrameSink,
526    negotiated_ver: u8,
527    concurrency: Concurrency,
528    // Captured at HELLO, not inferred from the id or a later prefix policy.
529    verified_prefix_owner: Option<String>,
530}
531
532pub(crate) struct ModuleHelloAck {
533    pub(crate) frame: Frame,
534    pub(crate) verified_prefix_owner: Option<String>,
535}
536
537#[derive(Debug, Default)]
538struct ForwardingInner {
539    operator_confirms: Arc<crate::operator_confirm::OperatorConfirms>,
540    daemon_draining: bool,
541    /// Whether forwarding reached quiescence during daemon shutdown, for each
542    /// active endpoint present when shutdown began. A pending wait is false
543    /// because it has not proved quiescence. Endpoint generations prevent a
544    /// result from describing a replacement connection with the same module id.
545    daemon_drain_results: HashMap<ModuleEndpointId, bool>,
546    /// The ACTIVE slot: the one endpoint per module id that routing resolves.
547    /// Every by-id lookup (relay reservation, drain-by-id, liveness, census,
548    /// live roots, module-control RPCs) reads this map and nothing else.
549    modules_by_id: HashMap<String, ModuleConnection>,
550    /// The CANDIDATE slot of a blue/green swap: a second process registered
551    /// under an id that already has an active endpoint. It has a full endpoint
552    /// identity (so its own connection can be looked up and torn down) but no
553    /// by-id lookup sees it, so nothing is routed to it until `cutover_candidate`
554    /// promotes it.
555    candidates_by_id: HashMap<String, ModuleConnection>,
556    /// Former active endpoints demoted by `cutover_candidate`, until their
557    /// connection is removed. Membership is what distinguishes an endpoint that
558    /// was SUPERSEDED by a promotion (still alive, still carrying bound routes,
559    /// about to be drained) from one that is merely STALE (replaced some other
560    /// way, which is treated as a fault on the acking connection).
561    superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
562    endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
563    module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
564    /// Endpoints mid-drain, keyed to the reason the drain was begun with. The
565    /// value serves the census ("draining because restart"); membership alone
566    /// still answers every admission-gate check.
567    draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
568    closing_connections: HashSet<ConnectionId>,
569    next_generation: u64,
570    reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
571    reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
572    next_client_channel: HashMap<ConnectionId, u16>,
573    next_module_channel: HashMap<ModuleEndpointId, u16>,
574    client_slot_epochs: HashMap<ClientRouteKey, u32>,
575    module_slot_epochs: HashMap<ModuleRouteKey, u32>,
576    last_published_epoch: HashMap<ClientRouteKey, u32>,
577    client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
578    module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
579    status: HashMap<(ClientRouteKey, u32), String>,
580    pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
581    /// Pending relays removed without a module reply before `route.open`
582    /// marked them enqueued. `route.open` consumes its key when it marks the
583    /// relay (and sends the GOODBYE the removal could not send safely) or when
584    /// it aborts, so an entry lives only while that open is mid-send.
585    relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
586    /// Each scope's current `(scope_epoch, version)`, keyed `(owner, ref)`,
587    /// published by a scope sync in the same step that changes the record. A
588    /// bind commit reads it here so it never takes the scope table's lock
589    /// while holding this one. Absent means the scope is not live.
590    scope_tags: HashMap<(String, String), ScopeTag>,
591    next_control_corr: HashMap<ModuleEndpointId, u64>,
592    pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
593    health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
594}
595
596#[derive(Debug, Clone)]
597pub(crate) struct CloseReason {
598    code: &'static str,
599    message: String,
600}
601
602impl CloseReason {
603    pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
604        Self {
605            code,
606            message: message.into(),
607        }
608    }
609}
610
611impl fmt::Display for CloseReason {
612    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
613        write!(f, "{}: {}", self.code, self.message)
614    }
615}
616
617pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
618
619/// Dynamic forwarding state shared by the control plane and data-plane router.
620#[derive(Debug, Default)]
621pub struct ForwardingTable {
622    inner: Arc<RwLock<ForwardingInner>>,
623    frame_drop_warnings: Mutex<HashMap<ConnectionId, ChannelDropWarnings>>,
624    #[cfg(feature = "test-support")]
625    route_closing_hold: Mutex<Option<oneshot::Receiver<()>>>,
626    close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
627    counters: DaemonCounters,
628    /// Per-target-module bind-relay breaker state. It lives here, beside the
629    /// module connections it describes, because this is where a module
630    /// connection's identity is established and therefore where a stale
631    /// verdict has to be discarded.
632    route_bind_breakers: RouteBindBreakers,
633    /// Current route.bind relays keyed by target module. Admission is shared
634    /// across every client connection that points at the same endpoint.
635    route_bind_concurrency: RouteBindConcurrency,
636    /// Start and end of each module's `route.open` outage. Held here for the
637    /// same reason as the breakers: every handler built over this table must
638    /// see one outage per module, or each would log its own opening line.
639    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
640}
641
642#[derive(Debug)]
643struct FrameDropWarning {
644    last_warned: tokio::time::Instant,
645    suppressed: u64,
646}
647
648type ChannelDropWarnings = HashMap<(u16, &'static str), FrameDropWarning>;
649
650impl ForwardingTable {
651    /// Keep a silent route diagnosable without letting a broken peer flood logs.
652    /// Epoch and direction describe the dropped frame, but do not split its limit.
653    pub(crate) fn warn_frame_drop(
654        &self,
655        connection_id: ConnectionId,
656        channel: u16,
657        epoch: u32,
658        direction: &'static str,
659        reason: &'static str,
660    ) {
661        let now = tokio::time::Instant::now();
662        let mut warnings = self
663            .frame_drop_warnings
664            .lock()
665            .unwrap_or_else(std::sync::PoisonError::into_inner);
666        let entry = warnings
667            .entry(connection_id)
668            .or_default()
669            .entry((channel, reason));
670        let suppressed = match entry {
671            std::collections::hash_map::Entry::Occupied(mut entry) => {
672                let state = entry.get_mut();
673                if now.saturating_duration_since(state.last_warned) < Duration::from_secs(60) {
674                    state.suppressed = state.suppressed.saturating_add(1);
675                    return;
676                }
677                state.last_warned = now;
678                std::mem::take(&mut state.suppressed)
679            }
680            std::collections::hash_map::Entry::Vacant(entry) => {
681                entry.insert(FrameDropWarning {
682                    last_warned: now,
683                    suppressed: 0,
684                });
685                0
686            }
687        };
688        warn!(
689            connection_id = connection_id.get(),
690            channel, epoch, direction, reason, suppressed, "dropping data-plane frame"
691        );
692    }
693
694    /// Install scripted operator confirmation for an in-process integration harness,
695    /// without bootstrapping a daemon process. Only available with `test-support`.
696    ///
697    /// The script has one response per line: `approve`, `decline`, `unavailable`,
698    /// `hang` (until withdrawn), or `approve when <path>` (wait for the path to
699    /// exist). Unreadable or exhausted scripts fail closed. If supplied, `events`
700    /// receives the provider's JSON-lines event log. `limits: None` uses
701    /// [`OperatorLimits::default`](crate::operator_confirm::OperatorLimits); use
702    /// `Some(OperatorLimits { queue_wait, prompt_timeout, stuck_grace })` to
703    /// override the queue wait, prompt timeout and stuck-provider grace periods.
704    ///
705    /// The current executable's file name **must start with `ckdev-`**. Otherwise
706    /// this logs a warning and keeps the OS provider, just like bootstrap.
707    /// `cortexkit-test-support`'s `ckdev_binary` copies a harness executable under
708    /// a qualifying name; run that copy to enable scripted confirmation. See
709    /// `crates/subc-client-rs/tests/real_daemon/operator_confirm.rs` for the
710    /// reference end-to-end harness.
711    ///
712    /// Configure the table shared by the harness's `ControlHandler` and `Router`
713    /// before serving any requests. This also configures privileged confirmations
714    /// that use the same table.
715    ///
716    /// # Panics
717    ///
718    /// Panics if operator-confirm processing has already started.
719    ///
720    /// # Example
721    ///
722    /// ```no_run
723    /// use std::{path::Path, sync::Arc};
724    /// use subc_daemon::{ControlHandler, ForwardingTable, Registry};
725    ///
726    /// let registry = Arc::new(Registry::default());
727    /// let forwarding = Arc::new(ForwardingTable::default());
728    /// forwarding.configure_operator_script(
729    ///     Path::new("operator.script"),
730    ///     Some(Path::new("operator.events")),
731    ///     None,
732    /// );
733    /// let handler = ControlHandler::with_forwarding(registry, forwarding);
734    /// ```
735    #[cfg(feature = "test-support")]
736    pub fn configure_operator_script(
737        &self,
738        script: &std::path::Path,
739        events: Option<&std::path::Path>,
740        limits: Option<crate::operator_confirm::OperatorLimits>,
741    ) {
742        self.operator_confirms()
743            .configure_script(script, events, limits);
744    }
745
746    /// Hold the next supervisor drain between enqueueing `module.draining` and
747    /// `route.closing`, so a test can forward a response in that window. Dropping
748    /// the returned sender also releases the hold, including on a test panic.
749    #[cfg(feature = "test-support")]
750    pub fn hold_route_closing_for_test(&self) -> oneshot::Sender<()> {
751        let (release, hold) = oneshot::channel();
752        *self.route_closing_hold.lock().unwrap() = Some(hold);
753        release
754    }
755
756    #[cfg(feature = "test-support")]
757    pub(crate) async fn wait_before_route_closing_for_test(&self) {
758        let hold = self.route_closing_hold.lock().unwrap().take();
759        if let Some(hold) = hold {
760            let _ = hold.await;
761        }
762    }
763
764    pub(crate) fn operator_confirms(&self) -> Arc<crate::operator_confirm::OperatorConfirms> {
765        Arc::clone(
766            &self
767                .inner
768                .read()
769                .unwrap_or_else(|p| p.into_inner())
770                .operator_confirms,
771        )
772    }
773
774    #[cfg(test)]
775    pub(crate) fn inject_operator_principal(&self, key: ModuleRouteKey, principal: Principal) {
776        self.alter_operator_binding_for_test(key, |binding| binding.principal = principal);
777    }
778
779    #[cfg(test)]
780    pub(crate) fn alter_operator_binding_for_test(
781        &self,
782        key: ModuleRouteKey,
783        alter: impl FnOnce(&mut RouteBinding),
784    ) {
785        let mut inner = self.write_inner().unwrap();
786        let old = inner.module_to_client.remove(&key).unwrap();
787        let client = ClientRouteKey {
788            connection_id: old.client_connection_id,
789            channel: old.client_channel,
790        };
791        inner.client_to_module.remove(&client);
792        let mut binding = Arc::try_unwrap(old).expect("test binding has no outstanding readers");
793        alter(&mut binding);
794        let binding = Arc::new(binding);
795        inner.client_to_module.insert(client, Arc::clone(&binding));
796        inner.module_to_client.insert(key, binding);
797    }
798
799    /// Mutate the route without delivering its withdrawal notice, so tests can
800    /// exercise the commit-point route check rather than the withdrawal path.
801    #[cfg(test)]
802    pub(crate) fn invalidate_operator_route_for_test(&self, key: ModuleRouteKey, stale: bool) {
803        let mut inner = self.write_inner().unwrap();
804        let old = inner.module_to_client.remove(&key).unwrap();
805        let client = ClientRouteKey {
806            connection_id: old.client_connection_id,
807            channel: old.client_channel,
808        };
809        inner.client_to_module.remove(&client);
810        if stale {
811            let mut binding = Arc::try_unwrap(old).expect("no binding readers at the test pause");
812            binding.module_epoch += 1;
813            let binding = Arc::new(binding);
814            inner.client_to_module.insert(client, binding.clone());
815            inner.module_to_client.insert(key, binding);
816        }
817    }
818
819    /// Keep the binding protected through confirmation admission. Route release
820    /// takes the write lock and then the same confirm-state lock, so it cannot
821    /// miss an admitted request or leave it waiting on a vanished route.
822    pub(crate) fn with_operator_route<T>(
823        &self,
824        connection_id: ConnectionId,
825        channel: u16,
826        epoch: u32,
827        admit: impl FnOnce(Option<&RouteBinding>) -> T,
828    ) -> Result<T, ForwardingError> {
829        self.with_operator_route_epoch(connection_id, channel, |binding| {
830            admit(binding.filter(|binding| binding.module_epoch == epoch))
831        })
832    }
833
834    /// Like admission, but retains a mismatched epoch so a shortcut can audit
835    /// a stale channel separately from a closed one, under the same read lock.
836    pub(crate) fn with_operator_route_epoch<T>(
837        &self,
838        connection_id: ConnectionId,
839        channel: u16,
840        decide: impl FnOnce(Option<&RouteBinding>) -> T,
841    ) -> Result<T, ForwardingError> {
842        let inner = self.read_inner()?;
843        let binding = inner
844            .endpoint_by_connection
845            .get(&connection_id)
846            .and_then(|endpoint| {
847                inner.module_to_client.get(&ModuleRouteKey {
848                    endpoint: *endpoint,
849                    channel,
850                })
851            });
852        Ok(decide(binding.map(Arc::as_ref)))
853    }
854
855    pub(crate) fn counters(&self) -> DaemonCounters {
856        self.counters.clone()
857    }
858
859    pub(crate) fn counters_ref(&self) -> &DaemonCounters {
860        &self.counters
861    }
862
863    pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
864        self.route_bind_breakers.clone()
865    }
866
867    pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
868        self.route_bind_concurrency.clone()
869    }
870
871    pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
872        Arc::clone(&self.route_outages)
873    }
874
875    pub(crate) fn register_connection_close(
876        &self,
877        connection_id: ConnectionId,
878    ) -> ConnectionCloseReceiver {
879        let (sender, receiver) = oneshot::channel();
880        let replaced = self
881            .lock_close_registry()
882            .insert(connection_id, sender)
883            .is_some();
884        if replaced {
885            warn!(
886                connection_id = connection_id.get(),
887                "replaced existing connection close registration"
888            );
889        }
890        receiver
891    }
892
893    pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
894        self.lock_close_registry().remove(&connection_id);
895    }
896
897    /// Close every established connection, modules and clients alike, as the
898    /// last step of an announced daemon shutdown. By the time this runs each
899    /// registered module has already been sent a module GOODBYE (see
900    /// `Supervisor::end_children_for_daemon_shutdown`), so the EOF that follows
901    /// is a planned stop; EOF with no GOODBYE before it stays reserved for a
902    /// daemon that went away unannounced. Closing while the daemon is still
903    /// alive lets it wait for the modules' own teardowns before it ends
904    /// whatever is left.
905    #[cfg(unix)]
906    pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
907        let senders: Vec<_> = self.lock_close_registry().drain().collect();
908        let count = senders.len();
909        for (_, sender) in senders {
910            let _ = sender.send(reason.clone());
911        }
912        count
913    }
914
915    /// Every registered module connection, whichever slot it occupies (active,
916    /// swap candidate, or superseded incumbent), once each. Daemon shutdown
917    /// uses this to tell each of them it is a planned stop before closing it.
918    #[cfg(unix)]
919    pub(crate) fn module_connections(
920        &self,
921    ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
922        let inner = self.read_inner()?;
923        let mut seen = HashSet::new();
924        Ok(inner
925            .modules_by_id
926            .values()
927            .chain(inner.candidates_by_id.values())
928            .chain(inner.superseded_endpoints.values())
929            .filter(|module| seen.insert(module.endpoint))
930            .map(|module| ModuleConnectionTarget {
931                module_id: inner
932                    .module_id_by_endpoint
933                    .get(&module.endpoint)
934                    .cloned()
935                    .unwrap_or_default(),
936                endpoint: module.endpoint,
937                sink: module.sink.clone(),
938                negotiated_ver: module.negotiated_ver,
939            })
940            .collect())
941    }
942
943    /// Ask a registered connection to close. Returns true only for the request
944    /// that actually reached it; later requests for the same connection, and
945    /// requests for a connection that is not registered, return false.
946    pub(crate) fn request_connection_close(
947        &self,
948        connection_id: ConnectionId,
949        reason: CloseReason,
950    ) -> bool {
951        let sender = self.lock_close_registry().remove(&connection_id);
952        if let Some(sender) = sender {
953            debug!(
954                connection_id = connection_id.get(),
955                close_reason = %reason,
956                "requesting connection close"
957            );
958            let _ = sender.send(reason);
959            true
960        } else {
961            debug!(
962                connection_id = connection_id.get(),
963                close_reason = %reason,
964                "connection close request ignored for inactive connection"
965            );
966            false
967        }
968    }
969
970    pub fn register_module_connection(
971        &self,
972        connection_id: ConnectionId,
973        module_id: String,
974        negotiated_ver: u8,
975        concurrency: Concurrency,
976        sink: FrameSink,
977    ) -> Result<ModuleEndpointId, ForwardingError> {
978        self.register_module_connection_inner(
979            connection_id,
980            module_id,
981            negotiated_ver,
982            concurrency,
983            sink,
984            None,
985        )
986    }
987
988    /// Register a module connection and queue its HELLO_ACK as the first frame
989    /// on its sink, in the same write-lock critical section that makes the
990    /// endpoint visible.
991    ///
992    /// A module reads HELLO_ACK as the first frame after its HELLO and exits on
993    /// anything else. Once the endpoint is in `modules_by_id`, a `route.open`
994    /// on any other connection can queue a `route.bind` request onto this sink,
995    /// so the ack must already be queued by then. Every lookup that can queue a
996    /// frame for the module takes this lock, so nothing can get in ahead of it.
997    ///
998    /// If the ack cannot be queued (the sink is closed or full), the
999    /// registration fails with nothing inserted.
1000    pub(crate) fn register_module_connection_acked(
1001        &self,
1002        connection_id: ConnectionId,
1003        module_id: String,
1004        negotiated_ver: u8,
1005        concurrency: Concurrency,
1006        sink: FrameSink,
1007        hello_ack: ModuleHelloAck,
1008    ) -> Result<ModuleEndpointId, ForwardingError> {
1009        self.register_module_connection_inner(
1010            connection_id,
1011            module_id,
1012            negotiated_ver,
1013            concurrency,
1014            sink,
1015            Some(hello_ack),
1016        )
1017    }
1018
1019    fn register_module_connection_inner(
1020        &self,
1021        connection_id: ConnectionId,
1022        module_id: String,
1023        negotiated_ver: u8,
1024        concurrency: Concurrency,
1025        sink: FrameSink,
1026        hello_ack: Option<ModuleHelloAck>,
1027    ) -> Result<ModuleEndpointId, ForwardingError> {
1028        let mut inner = self.write_inner()?;
1029        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1030            return Err(ForwardingError::ConnectionClosing { connection_id });
1031        }
1032        check_module_connection_role_locked(&inner, connection_id)?;
1033        // Every refusal check has passed and nothing has been mutated yet, so
1034        // a failed enqueue leaves the table exactly as it was.
1035        let verified_prefix_owner = hello_ack
1036            .as_ref()
1037            .and_then(|hello| hello.verified_prefix_owner.clone());
1038        enqueue_hello_ack_locked(&sink, connection_id, hello_ack.map(|hello| hello.frame))?;
1039
1040        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1041        let endpoint = ModuleEndpointId {
1042            connection_id,
1043            generation: inner.next_generation,
1044        };
1045        inner.endpoint_by_connection.insert(connection_id, endpoint);
1046        inner
1047            .module_id_by_endpoint
1048            .insert(endpoint, module_id.clone());
1049        inner.next_module_channel.insert(endpoint, 1);
1050        inner.next_control_corr.insert(endpoint, 1);
1051        inner.modules_by_id.insert(
1052            module_id.clone(),
1053            ModuleConnection {
1054                endpoint,
1055                sink,
1056                negotiated_ver,
1057                concurrency,
1058                verified_prefix_owner,
1059            },
1060        );
1061        drop(inner);
1062
1063        // A new module connection has arrived under this id, so anything the
1064        // bind-relay breaker learned was learned about a process that is no
1065        // longer the one behind this name. See
1066        // `RouteBindBreakers::reset_for_new_module_connection`.
1067        //
1068        // Keyed on ARRIVAL rather than on teardown deliberately: a connection
1069        // going away is not evidence about anything, and a module whose
1070        // connection drops without coming back should keep its verdict until
1071        // something actually registers in its place. This also covers the
1072        // unclean replacements -- a module killed mid-bind, or one whose
1073        // connection was closing -- because registration is the single path by
1074        // which any module connection becomes usable.
1075        if let Some(discarded) = self
1076            .route_bind_breakers
1077            .reset_for_new_module_connection(&module_id)
1078        {
1079            info!(
1080                module_id = %module_id,
1081                discarded_consecutive_timeouts = discarded,
1082                "route.bind breaker state discarded: a new module connection replaced the process it described"
1083            );
1084        }
1085        Ok(endpoint)
1086    }
1087
1088    /// Register a blue/green swap candidate for `module_id` into the candidate
1089    /// slot, alongside whatever endpoint is active for the id.
1090    ///
1091    /// Unlike [`Self::register_module_connection`], this never touches the
1092    /// active slot and never resets the bind-relay breaker: the incumbent is
1093    /// still the process serving the id, so what the breaker learned about it
1094    /// is still true. The breaker is reset at cutover instead, when the process
1095    /// behind the name actually changes. A second candidate for the same id is
1096    /// refused.
1097    ///
1098    /// Production registers candidates through
1099    /// [`Self::register_candidate_module_connection_acked`]; this ack-less form
1100    /// is for tests that build forwarding state directly.
1101    #[cfg(test)]
1102    pub(crate) fn register_candidate_module_connection(
1103        &self,
1104        connection_id: ConnectionId,
1105        module_id: String,
1106        negotiated_ver: u8,
1107        concurrency: Concurrency,
1108        sink: FrameSink,
1109    ) -> Result<ModuleEndpointId, ForwardingError> {
1110        self.register_candidate_module_connection_inner(
1111            connection_id,
1112            module_id,
1113            negotiated_ver,
1114            concurrency,
1115            sink,
1116            None,
1117        )
1118    }
1119
1120    /// The swap-candidate counterpart of
1121    /// [`Self::register_module_connection_acked`]: the HELLO_ACK is queued on
1122    /// the candidate's sink before its endpoint is inserted. No by-id lookup
1123    /// sees a candidate, but its own connection's endpoint does become
1124    /// resolvable here, and at cutover it becomes routable; the ack has to be
1125    /// ahead of anything either can queue.
1126    pub(crate) fn register_candidate_module_connection_acked(
1127        &self,
1128        connection_id: ConnectionId,
1129        module_id: String,
1130        negotiated_ver: u8,
1131        concurrency: Concurrency,
1132        sink: FrameSink,
1133        hello_ack: Frame,
1134    ) -> Result<ModuleEndpointId, ForwardingError> {
1135        self.register_candidate_module_connection_inner(
1136            connection_id,
1137            module_id,
1138            negotiated_ver,
1139            concurrency,
1140            sink,
1141            Some(hello_ack),
1142        )
1143    }
1144
1145    fn register_candidate_module_connection_inner(
1146        &self,
1147        connection_id: ConnectionId,
1148        module_id: String,
1149        negotiated_ver: u8,
1150        concurrency: Concurrency,
1151        sink: FrameSink,
1152        hello_ack: Option<Frame>,
1153    ) -> Result<ModuleEndpointId, ForwardingError> {
1154        let mut inner = self.write_inner()?;
1155        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1156            return Err(ForwardingError::ConnectionClosing { connection_id });
1157        }
1158        check_module_connection_role_locked(&inner, connection_id)?;
1159        if inner.candidates_by_id.contains_key(&module_id) {
1160            return Err(ForwardingError::CandidateSlotOccupied { module_id });
1161        }
1162        // As in `register_module_connection_inner`: after every refusal check,
1163        // before any mutation.
1164        enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1165
1166        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1167        let endpoint = ModuleEndpointId {
1168            connection_id,
1169            generation: inner.next_generation,
1170        };
1171        inner.endpoint_by_connection.insert(connection_id, endpoint);
1172        inner
1173            .module_id_by_endpoint
1174            .insert(endpoint, module_id.clone());
1175        inner.next_module_channel.insert(endpoint, 1);
1176        inner.next_control_corr.insert(endpoint, 1);
1177        inner.candidates_by_id.insert(
1178            module_id,
1179            ModuleConnection {
1180                endpoint,
1181                sink,
1182                negotiated_ver,
1183                concurrency,
1184                verified_prefix_owner: None,
1185            },
1186        );
1187        Ok(endpoint)
1188    }
1189
1190    /// Promote `module_id`'s candidate to the active slot, in ONE forwarding
1191    /// write-lock critical section.
1192    ///
1193    /// That single section is the linearization point of a swap. Every relay
1194    /// reservation resolves the active slot under the same lock, so each one
1195    /// lands wholly before cutover (on the incumbent) or wholly after it (on the
1196    /// promoted candidate), never on a mix. A relay reserved on the incumbent
1197    /// before cutover can no longer commit: `commit_route_locked` requires the
1198    /// reservation's endpoint to be the active one, and an ack for it arriving
1199    /// later is answered by the superseded-endpoint arm of
1200    /// `complete_pending_relay` instead of ending the incumbent's connection.
1201    ///
1202    /// Both endpoints keep their identities: nothing is re-keyed, so the
1203    /// incumbent's bound routes and its pending correlation keys stay exactly
1204    /// where they are until it is drained with [`Self::begin_endpoint_drain`],
1205    /// using the incumbent endpoint this returns. Returns `Ok(None)` when there
1206    /// is no candidate for the id.
1207    pub(crate) fn cutover_candidate(
1208        &self,
1209        module_id: &str,
1210    ) -> Result<Option<ForwardingCutover>, ForwardingError> {
1211        let mut inner = self.write_inner()?;
1212        if inner.daemon_draining {
1213            return Err(ForwardingError::ModuleReloading {
1214                module_id: module_id.to_string(),
1215            });
1216        }
1217        let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
1218            return Ok(None);
1219        };
1220        let promoted = candidate.endpoint;
1221        let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
1222        let incumbent = incumbent.map(|incumbent| {
1223            let endpoint = incumbent.endpoint;
1224            inner.superseded_endpoints.insert(endpoint, incumbent);
1225            endpoint
1226        });
1227        drop(inner);
1228
1229        // A different process now answers for this id; see the matching reset
1230        // in `register_module_connection` for why a verdict about the old one
1231        // must not carry over.
1232        if let Some(discarded) = self
1233            .route_bind_breakers
1234            .reset_for_new_module_connection(module_id)
1235        {
1236            info!(
1237                module_id = %module_id,
1238                discarded_consecutive_timeouts = discarded,
1239                "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
1240            );
1241        }
1242        Ok(Some(ForwardingCutover {
1243            promoted,
1244            incumbent,
1245        }))
1246    }
1247
1248    #[allow(clippy::too_many_arguments)]
1249    pub(crate) async fn begin_route_bind_relay_for(
1250        &self,
1251        client_connection_id: ConnectionId,
1252        client_sink: FrameSink,
1253        client_negotiated_ver: u8,
1254        client_corr: u64,
1255        module_id: &str,
1256        principal: Principal,
1257        scope: Option<BoundScope>,
1258        project_root: Option<ProjectRootId>,
1259        harness: String,
1260        build_class: Option<String>,
1261        operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1262        deadline: Instant,
1263        busy_refusals: bool,
1264    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1265        // Reserve egress capacity before taking the forwarding lock. The permit is
1266        // held until the bind reaches one terminal state, so an accepted bind can
1267        // publish its table entry and RouteOpen response in one critical section.
1268        let client_permit =
1269            client_sink
1270                .reserve_owned()
1271                .await
1272                .map_err(|_| ForwardingError::ClientEgressClosed {
1273                    connection_id: client_connection_id,
1274                })?;
1275        self.begin_route_bind_relay_inner(
1276            client_connection_id,
1277            client_sink,
1278            client_negotiated_ver,
1279            client_corr,
1280            module_id,
1281            principal,
1282            scope,
1283            project_root,
1284            harness,
1285            build_class,
1286            operator_app,
1287            deadline,
1288            client_permit,
1289            busy_refusals,
1290        )
1291    }
1292
1293    #[cfg(test)]
1294    pub(crate) fn begin_route_bind_relay_for_test(
1295        &self,
1296        client_connection_id: ConnectionId,
1297        client_sink: FrameSink,
1298        client_corr: u64,
1299        module_id: &str,
1300    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1301        self.begin_route_bind_relay_with_busy_for_test(
1302            client_connection_id,
1303            client_sink,
1304            client_corr,
1305            module_id,
1306            false,
1307        )
1308    }
1309
1310    #[cfg(test)]
1311    pub(crate) fn begin_route_bind_relay_with_busy_for_test(
1312        &self,
1313        client_connection_id: ConnectionId,
1314        client_sink: FrameSink,
1315        client_corr: u64,
1316        module_id: &str,
1317        busy_refusals: bool,
1318    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1319        let permit =
1320            client_sink
1321                .try_reserve_owned()
1322                .map_err(|_| ForwardingError::ClientEgressClosed {
1323                    connection_id: client_connection_id,
1324                })?;
1325        self.begin_route_bind_relay_inner(
1326            client_connection_id,
1327            client_sink,
1328            subc_protocol::PROTOCOL_VERSION,
1329            client_corr,
1330            module_id,
1331            Principal::Direct,
1332            None,
1333            None,
1334            String::new(),
1335            None,
1336            None,
1337            Instant::now() + std::time::Duration::from_secs(60),
1338            permit,
1339            busy_refusals,
1340        )
1341    }
1342
1343    pub(crate) fn begin_module_control_rpc_for(
1344        &self,
1345        module_id: &str,
1346        expected_op: &str,
1347        deadline: Instant,
1348    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1349        self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1350    }
1351
1352    pub(crate) fn begin_health_probe_rpc_for(
1353        &self,
1354        module_id: &str,
1355        expected_op: &str,
1356        probe_started_at: Instant,
1357        deadline: Instant,
1358    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1359        self.begin_module_control_rpc_inner(
1360            module_id,
1361            expected_op,
1362            deadline,
1363            Some(probe_started_at),
1364            false,
1365        )
1366    }
1367
1368    pub(crate) fn begin_drain_health_probe_rpc_for(
1369        &self,
1370        module_id: &str,
1371        expected_op: &str,
1372        probe_started_at: Instant,
1373        deadline: Instant,
1374    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1375        self.begin_module_control_rpc_inner(
1376            module_id,
1377            expected_op,
1378            deadline,
1379            Some(probe_started_at),
1380            true,
1381        )
1382    }
1383
1384    /// A health probe addressed to one endpoint, in whichever slot it is.
1385    ///
1386    /// Every other control RPC resolves the id's ACTIVE endpoint, which is how
1387    /// a swap candidate (not active until cutover) and a superseded incumbent
1388    /// (not active after it) are unreachable by them. A swap needs to probe
1389    /// exactly those two: the candidate before promoting it, and the incumbent
1390    /// for its busy gauges while it drains.
1391    pub(crate) fn begin_endpoint_health_probe_rpc_for(
1392        &self,
1393        endpoint: ModuleEndpointId,
1394        expected_op: &str,
1395        probe_started_at: Instant,
1396        deadline: Instant,
1397    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1398        let inner = self.write_inner()?;
1399        let module = module_connection_for_endpoint_locked(&inner, endpoint)
1400            .cloned()
1401            .ok_or(ForwardingError::NoModuleConnection)?;
1402        let module_id = inner
1403            .module_id_by_endpoint
1404            .get(&endpoint)
1405            .cloned()
1406            .unwrap_or_default();
1407        // A draining incumbent is exactly what this probes for busy gauges, so
1408        // draining is allowed, as it is for the by-id drain probe.
1409        self.begin_control_rpc_locked(
1410            inner,
1411            &module_id,
1412            module,
1413            expected_op,
1414            deadline,
1415            Some(probe_started_at),
1416            true,
1417        )
1418    }
1419
1420    fn begin_module_control_rpc_inner(
1421        &self,
1422        module_id: &str,
1423        expected_op: &str,
1424        deadline: Instant,
1425        health_probe_started_at: Option<Instant>,
1426        allow_draining: bool,
1427    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1428        let inner = self.write_inner()?;
1429        let module = inner
1430            .modules_by_id
1431            .get(module_id)
1432            .cloned()
1433            .ok_or(ForwardingError::NoModuleConnection)?;
1434        self.begin_control_rpc_locked(
1435            inner,
1436            module_id,
1437            module,
1438            expected_op,
1439            deadline,
1440            health_probe_started_at,
1441            allow_draining,
1442        )
1443    }
1444
1445    #[allow(clippy::too_many_arguments)]
1446    fn begin_control_rpc_locked(
1447        &self,
1448        mut inner: RwLockWriteGuard<'_, ForwardingInner>,
1449        module_id: &str,
1450        module: ModuleConnection,
1451        expected_op: &str,
1452        deadline: Instant,
1453        health_probe_started_at: Option<Instant>,
1454        allow_draining: bool,
1455    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1456        if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1457            return Err(ForwardingError::ModuleReloading {
1458                module_id: module_id.to_string(),
1459            });
1460        }
1461        if inner
1462            .closing_connections
1463            .contains(&module.endpoint.connection_id)
1464        {
1465            return Err(ForwardingError::ConnectionClosing {
1466                connection_id: module.endpoint.connection_id,
1467            });
1468        }
1469        if health_probe_started_at.is_some() {
1470            // Recurring health probes are serial per endpoint. Once the next one
1471            // starts, an older answer can no longer improve the current snapshot,
1472            // so retaining more than the newest unanswered probe has no value.
1473            inner
1474                .health_probe_tombstones
1475                .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1476        }
1477        let corr = match inner.allocate_control_corr(module.endpoint) {
1478            Ok(corr) => corr,
1479            Err(err) => {
1480                drop(inner);
1481                self.request_connection_close(
1482                    module.endpoint.connection_id,
1483                    CloseReason::new(
1484                        "control_correlation_exhausted",
1485                        "daemon-originated channel-0 correlation space exhausted",
1486                    ),
1487                );
1488                return Err(err);
1489            }
1490        };
1491        let (sender, receiver) = oneshot::channel();
1492        inner.pending_control_rpcs.insert(
1493            (module.endpoint, corr),
1494            PendingModuleControlRpcEntry {
1495                expected_op: expected_op.to_string(),
1496                deadline,
1497                health_probe_started_at,
1498                sender,
1499            },
1500        );
1501
1502        Ok(PendingModuleControlRpc {
1503            endpoint: module.endpoint,
1504            module_sink: module.sink,
1505            negotiated_ver: module.negotiated_ver,
1506            corr,
1507            receiver,
1508        })
1509    }
1510
1511    #[allow(clippy::too_many_arguments)]
1512    fn begin_route_bind_relay_inner(
1513        &self,
1514        client_connection_id: ConnectionId,
1515        client_sink: FrameSink,
1516        client_negotiated_ver: u8,
1517        client_corr: u64,
1518        expected_module_id: &str,
1519        principal: Principal,
1520        scope: Option<BoundScope>,
1521        project_root: Option<ProjectRootId>,
1522        harness: String,
1523        build_class: Option<String>,
1524        operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1525        deadline: Instant,
1526        client_permit: crate::router::EgressPermit,
1527        busy_refusals: bool,
1528    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1529        let mut inner = self.write_inner()?;
1530        // HELLO and route.open can run on different tasks. Decide the role under
1531        // the same lock that installs endpoints and reserves client channels so
1532        // neither task can turn one connection into both kinds of owner.
1533        if inner
1534            .endpoint_by_connection
1535            .contains_key(&client_connection_id)
1536        {
1537            return Err(ForwardingError::ConnectionRoleConflict {
1538                connection_id: client_connection_id,
1539            });
1540        }
1541        if inner.closing_connections.contains(&client_connection_id) {
1542            return Err(ForwardingError::ConnectionClosing {
1543                connection_id: client_connection_id,
1544            });
1545        }
1546        let module = inner
1547            .modules_by_id
1548            .get(expected_module_id)
1549            .cloned()
1550            .ok_or(ForwardingError::NoModuleConnection)?;
1551        if inner.draining_endpoints.contains_key(&module.endpoint) {
1552            return Err(ForwardingError::ModuleReloading {
1553                module_id: expected_module_id.to_string(),
1554            });
1555        }
1556        if inner
1557            .closing_connections
1558            .contains(&module.endpoint.connection_id)
1559        {
1560            return Err(ForwardingError::ConnectionClosing {
1561                connection_id: module.endpoint.connection_id,
1562            });
1563        }
1564
1565        let corr = match inner.allocate_control_corr(module.endpoint) {
1566            Ok(corr) => corr,
1567            Err(err) => {
1568                drop(inner);
1569                self.request_connection_close(
1570                    module.endpoint.connection_id,
1571                    CloseReason::new(
1572                        "control_correlation_exhausted",
1573                        "daemon-originated channel-0 correlation space exhausted",
1574                    ),
1575                );
1576                return Err(err);
1577            }
1578        };
1579        let (client_channel, client_epoch, module_channel, module_epoch) =
1580            inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1581        let client_key = ClientRouteKey {
1582            connection_id: client_connection_id,
1583            channel: client_channel,
1584        };
1585        let module_key = ModuleRouteKey {
1586            endpoint: module.endpoint,
1587            channel: module_channel,
1588        };
1589        let reservation = RouteReservation {
1590            client_key,
1591            module_key,
1592            client_epoch,
1593            module_epoch,
1594            project_root,
1595        };
1596        let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
1597            request_window: None,
1598            watch_window: None,
1599            route_channel: client_channel,
1600            route_epoch: client_epoch,
1601        })
1602        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1603        let route_open_frame = Frame::build_with_version(
1604            client_negotiated_ver,
1605            FrameType::Response,
1606            Flags::new(false, Priority::Passive, false),
1607            0,
1608            0,
1609            client_corr,
1610            response_body,
1611        )
1612        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1613        let (sender, receiver) = oneshot::channel();
1614        inner.reserved_client.insert(client_key, module_key);
1615        inner.reserved_module.insert(module_key, client_key);
1616        inner.pending_relays.insert(
1617            (module.endpoint, corr),
1618            PendingRouteBindRelayEntry {
1619                reservation,
1620                client_sink,
1621                client_negotiated_ver,
1622                client_permit,
1623                route_open_frame,
1624                busy_refusals,
1625                request_window: window_for(&module.concurrency),
1626                watch_window: (busy_refusals && module.concurrency == Concurrency::ModuleManaged)
1627                    .then_some(DEFAULT_MODULE_MANAGED_WATCH_WINDOW),
1628                principal,
1629                scope,
1630                harness,
1631                build_class,
1632                operator_app,
1633                deadline,
1634                relay_enqueued: false,
1635                sender,
1636            },
1637        );
1638
1639        Ok(PendingRouteBindRelay {
1640            endpoint: module.endpoint,
1641            module_sink: module.sink,
1642            negotiated_ver: module.negotiated_ver,
1643            client_channel,
1644            client_epoch,
1645            module_channel,
1646            module_epoch,
1647            corr,
1648            receiver,
1649        })
1650    }
1651
1652    /// Called by `route.open` right after the route.bind entered the module's
1653    /// egress queue.
1654    ///
1655    /// The module reader runs concurrently with `route.open`, so a fast module
1656    /// can receive the bind, ack it, and have the reader commit the route
1657    /// before this runs. A missing entry therefore does not mean the bind was
1658    /// abandoned: only a removal recorded in `relays_released_before_enqueue`
1659    /// does. Telling the two apart is what stops `route.open` from sending a
1660    /// GOODBYE for a route that was just committed.
1661    pub(crate) fn mark_route_bind_relay_enqueued(
1662        &self,
1663        endpoint: ModuleEndpointId,
1664        corr: u64,
1665    ) -> Result<RouteBindRelayMark, ForwardingError> {
1666        let mut inner = self.write_inner()?;
1667        if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
1668            pending.relay_enqueued = true;
1669            return Ok(RouteBindRelayMark::Marked);
1670        }
1671        if inner
1672            .relays_released_before_enqueue
1673            .remove(&(endpoint, corr))
1674        {
1675            return Ok(RouteBindRelayMark::ReleasedWithoutReply);
1676        }
1677        Ok(RouteBindRelayMark::SettledByReply)
1678    }
1679
1680    pub(crate) fn release_client_route(
1681        &self,
1682        client_connection_id: ConnectionId,
1683        client_channel: u16,
1684        expected_epoch: u32,
1685    ) -> Result<RouteRelease, ForwardingError> {
1686        let mut inner = self.write_inner()?;
1687        let release = release_client_route_locked(
1688            &mut inner,
1689            ClientRouteKey {
1690                connection_id: client_connection_id,
1691                channel: client_channel,
1692            },
1693            expected_epoch,
1694        );
1695        self.record_route_release(&release);
1696        Ok(release)
1697    }
1698
1699    pub(crate) fn release_module_route(
1700        &self,
1701        module_connection_id: ConnectionId,
1702        module_channel: u16,
1703        expected_epoch: u32,
1704    ) -> Result<RouteRelease, ForwardingError> {
1705        let mut inner = self.write_inner()?;
1706        let Some(endpoint) = inner
1707            .endpoint_by_connection
1708            .get(&module_connection_id)
1709            .copied()
1710        else {
1711            return Ok(RouteRelease::Absent);
1712        };
1713        let release = release_module_route_locked(
1714            &mut inner,
1715            ModuleRouteKey {
1716                endpoint,
1717                channel: module_channel,
1718            },
1719            expected_epoch,
1720        );
1721        self.record_route_release(&release);
1722        Ok(release)
1723    }
1724
1725    pub(crate) fn abort_pending_relay(
1726        &self,
1727        endpoint: ModuleEndpointId,
1728        corr: u64,
1729        outcome: RouteBindRelayOutcome,
1730    ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
1731        let mut inner = self.write_inner()?;
1732        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1733            // If another path dropped the relay while `route.open` was still
1734            // sending the bind, and that open is now giving up without marking
1735            // it (its send failed or it was cancelled), forget the record so it
1736            // cannot outlive the open.
1737            inner
1738                .relays_released_before_enqueue
1739                .remove(&(endpoint, corr));
1740            return Ok(None);
1741        };
1742        release_reserved_route_locked(
1743            &mut inner,
1744            pending.reservation.client_key,
1745            pending.reservation.module_key,
1746        );
1747        let target = pending
1748            .relay_enqueued
1749            .then(|| abandoned_route_target(&inner, &pending.reservation));
1750        let _ = pending.sender.send(outcome);
1751        Ok(target.flatten())
1752    }
1753
1754    pub(crate) fn cancel_module_control_rpc(
1755        &self,
1756        endpoint: ModuleEndpointId,
1757        corr: u64,
1758    ) -> Result<(), ForwardingError> {
1759        self.write_inner()?
1760            .pending_control_rpcs
1761            .remove(&(endpoint, corr));
1762        Ok(())
1763    }
1764
1765    pub(crate) fn tombstone_health_probe_rpc(
1766        &self,
1767        endpoint: ModuleEndpointId,
1768        corr: u64,
1769    ) -> Result<bool, ForwardingError> {
1770        let key = (endpoint, corr);
1771        let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
1772        {
1773            let mut inner = self.write_inner()?;
1774            let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
1775                return Ok(false);
1776            };
1777            let Some(probe_started_at) = pending.health_probe_started_at else {
1778                inner.pending_control_rpcs.insert(key, pending);
1779                return Ok(false);
1780            };
1781            let module_id = inner
1782                .module_id_by_endpoint
1783                .get(&endpoint)
1784                .cloned()
1785                .unwrap_or_else(|| "unknown".to_string());
1786            inner.health_probe_tombstones.insert(
1787                key,
1788                HealthProbeTombstone {
1789                    expected_op: pending.expected_op,
1790                    module_id,
1791                    probe_started_at,
1792                    expires_at,
1793                },
1794            );
1795        }
1796        self.schedule_health_probe_tombstone_expiration(key, expires_at);
1797        Ok(true)
1798    }
1799
1800    fn schedule_health_probe_tombstone_expiration(
1801        &self,
1802        key: (ModuleEndpointId, u64),
1803        expires_at: Instant,
1804    ) {
1805        let inner = Arc::downgrade(&self.inner);
1806        tokio::spawn(async move {
1807            tokio::time::sleep_until(expires_at).await;
1808            let Some(inner) = inner.upgrade() else {
1809                return;
1810            };
1811            let Ok(mut inner) = inner.write() else {
1812                return;
1813            };
1814            let expired = inner
1815                .health_probe_tombstones
1816                .get(&key)
1817                .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
1818            if expired {
1819                inner.health_probe_tombstones.remove(&key);
1820            }
1821        });
1822    }
1823
1824    pub(crate) fn complete_pending_relay(
1825        &self,
1826        connection_id: ConnectionId,
1827        corr: u64,
1828        outcome: RouteBindRelayOutcome,
1829    ) -> Result<PendingRelayCompletion, ForwardingError> {
1830        let mut inner = self.write_inner()?;
1831        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1832            return Ok(PendingRelayCompletion {
1833                settled: false,
1834                abandoned: None,
1835            });
1836        };
1837        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1838            return Ok(PendingRelayCompletion {
1839                settled: false,
1840                abandoned: None,
1841            });
1842        };
1843
1844        if Instant::now() >= pending.deadline {
1845            release_reserved_route_locked(
1846                &mut inner,
1847                pending.reservation.client_key,
1848                pending.reservation.module_key,
1849            );
1850            let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
1851                .then(|| abandoned_route_target(&inner, &pending.reservation))
1852                .flatten();
1853            let _ = pending
1854                .sender
1855                .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody {
1856                    code: "module_timeout".to_string(),
1857                    message: "route.bind response arrived after its daemon deadline".to_string(),
1858                    detail: None,
1859                }));
1860            return Ok(PendingRelayCompletion {
1861                settled: true,
1862                abandoned,
1863            });
1864        }
1865
1866        match outcome {
1867            // Two shapes of "the client is not there to receive this route" that
1868            // must resolve identically: its egress is already closed, or it is
1869            // marked closing (its connection loop has been asked to end, but has
1870            // not drained yet, so the sink is still open).
1871            //
1872            // Only the first used to be caught here. The second fell through to
1873            // `commit_route_locked`, which refuses a closing client with
1874            // `ConnectionClosing` -- and this function is called from the MODULE
1875            // connection's frame handler, so that refusal ended the module's
1876            // connection instead of this one client's route. One dying client's
1877            // route.open then took down a connection carrying every other
1878            // client's routes to that module.
1879            //
1880            // The remedy for both is the same, which is why they share an arm:
1881            // give back the reserved handle pair, tell the waiting route.open the
1882            // route is gone, and report the module-side channel so the caller can
1883            // send a channel-scoped GOODBYE for the binding the module just
1884            // created. Nothing here touches the module connection.
1885            RouteBindRelayOutcome::Accepted
1886                if pending.client_sink.is_closed()
1887                    || inner
1888                        .closing_connections
1889                        .contains(&pending.reservation.client_key.connection_id) =>
1890            {
1891                let reason = if pending.client_sink.is_closed() {
1892                    "client egress closed before route publication"
1893                } else {
1894                    "client connection is closing before route publication"
1895                };
1896                release_reserved_route_locked(
1897                    &mut inner,
1898                    pending.reservation.client_key,
1899                    pending.reservation.module_key,
1900                );
1901                // Not gated on `relay_enqueued`: an ack proves the module
1902                // received the bind, and `route.open` sends nothing for a
1903                // relay a reply already settled, so this is the only GOODBYE.
1904                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1905                let _ = pending
1906                    .sender
1907                    .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
1908                return Ok(PendingRelayCompletion {
1909                    settled: true,
1910                    abandoned,
1911                });
1912            }
1913            // The endpoint that acked this bind is no longer accepting new
1914            // routes, for one of two reasons. Either it was the module's active
1915            // endpoint when this relay was reserved and a blue/green swap has
1916            // since promoted a candidate (the replacement process registered
1917            // beside it) into the active slot, superseding it; or a daemon
1918            // drain has gated it, and the ack arrived before the ordered module
1919            // drain settled pending relays. The module did nothing wrong: it
1920            // bound the route it was asked to bind, and the endpoint still
1921            // carries every other client's routes until it is drained. So the
1922            // ack is settled here, while the pending entry still holds the
1923            // client's sender and the reserved handle pair: release the pair,
1924            // refuse the waiting route.open with `module_reloading`, which
1925            // callers retry (after a swap the retry reserves on the promoted
1926            // endpoint), and hand back the module-side channel so the caller
1927            // sends one channel-scoped GOODBYE for the binding the module just
1928            // created. Returning an error instead would end the module
1929            // connection all those other routes share; nothing here touches it.
1930            //
1931            // An endpoint that stopped being active any other way is stale
1932            // (replaced, but not by a swap and not draining). It still falls
1933            // through to `commit_route_locked`, which refuses it with
1934            // `StaleModuleEndpoint` exactly as before swaps existed.
1935            RouteBindRelayOutcome::Accepted
1936                if inner.superseded_endpoints.contains_key(&endpoint)
1937                    || inner.draining_endpoints.contains_key(&endpoint) =>
1938            {
1939                release_reserved_route_locked(
1940                    &mut inner,
1941                    pending.reservation.client_key,
1942                    pending.reservation.module_key,
1943                );
1944                // Not gated on `relay_enqueued`: an ack proves the module
1945                // received the bind, whether or not the enqueue mark was set.
1946                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1947                let module_id = inner
1948                    .module_id_by_endpoint
1949                    .get(&endpoint)
1950                    .cloned()
1951                    .unwrap_or_else(|| "unknown".to_string());
1952                let _ = pending
1953                    .sender
1954                    .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
1955                        "module_reloading",
1956                        format!("module_id '{module_id}' is reloading"),
1957                    )));
1958                return Ok(PendingRelayCompletion {
1959                    settled: true,
1960                    abandoned,
1961                });
1962            }
1963            // The scope the open was admitted under ended or changed after
1964            // admission. The module bound a route under a stamp that is no
1965            // longer true, so it must not become routable. Settled here like
1966            // the superseded arm above, never as an error from
1967            // `commit_route_locked`, which would close the module's whole
1968            // connection and every other client's routes to it: release the
1969            // pair, answer the waiting route.open by name, and hand back the
1970            // module-side channel for one channel-scoped GOODBYE.
1971            RouteBindRelayOutcome::Accepted
1972                if pending
1973                    .scope
1974                    .as_ref()
1975                    .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
1976            {
1977                let refusal = pending
1978                    .scope
1979                    .as_ref()
1980                    .and_then(|scope| scope_refusal_locked(&inner, scope))
1981                    .expect("guard matched a refusal under the same lock");
1982                release_reserved_route_locked(
1983                    &mut inner,
1984                    pending.reservation.client_key,
1985                    pending.reservation.module_key,
1986                );
1987                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1988                let _ = pending
1989                    .sender
1990                    .send(RouteBindRelayOutcome::DaemonRejected(refusal));
1991                return Ok(PendingRelayCompletion {
1992                    settled: true,
1993                    abandoned,
1994                });
1995            }
1996            RouteBindRelayOutcome::Accepted => {
1997                let abandoned = commit_route_locked(&mut inner, pending)?;
1998                return Ok(PendingRelayCompletion {
1999                    settled: true,
2000                    abandoned,
2001                });
2002            }
2003            terminal => {
2004                release_reserved_route_locked(
2005                    &mut inner,
2006                    pending.reservation.client_key,
2007                    pending.reservation.module_key,
2008                );
2009                let _ = pending.sender.send(terminal);
2010            }
2011        }
2012        Ok(PendingRelayCompletion {
2013            settled: true,
2014            abandoned: None,
2015        })
2016    }
2017
2018    pub(crate) fn pending_module_control_op(
2019        &self,
2020        connection_id: ConnectionId,
2021        corr: u64,
2022    ) -> Result<Option<String>, ForwardingError> {
2023        let inner = self.read_inner()?;
2024        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2025            return Ok(None);
2026        };
2027        let key = (endpoint, corr);
2028        Ok(inner
2029            .pending_control_rpcs
2030            .get(&key)
2031            .map(|pending| pending.expected_op.clone())
2032            .or_else(|| {
2033                inner
2034                    .health_probe_tombstones
2035                    .get(&key)
2036                    .filter(|tombstone| tombstone.expires_at > Instant::now())
2037                    .map(|tombstone| tombstone.expected_op.clone())
2038            }))
2039    }
2040
2041    pub(crate) fn complete_module_control_rpc(
2042        &self,
2043        connection_id: ConnectionId,
2044        corr: u64,
2045        actual_op: Option<&str>,
2046        outcome: ModuleControlRpcOutcome,
2047    ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
2048        let now = Instant::now();
2049        let mut inner = self.write_inner()?;
2050        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2051            return Ok(ModuleControlRpcCompletion::Unknown);
2052        };
2053        let key = (endpoint, corr);
2054        if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2055            if now >= pending.deadline {
2056                let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
2057                    ModuleControlRpcCompletion::LateHealthAnswer {
2058                        module_id: inner
2059                            .module_id_by_endpoint
2060                            .get(&endpoint)
2061                            .cloned()
2062                            .unwrap_or_else(|| "unknown".to_string()),
2063                        latency: now.saturating_duration_since(probe_started_at),
2064                    }
2065                });
2066                let _ = pending
2067                    .sender
2068                    .send(ModuleControlRpcOutcome::DeadlineElapsed);
2069                return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
2070            }
2071            let outcome = match actual_op {
2072                Some(actual) if actual != pending.expected_op => {
2073                    ModuleControlRpcOutcome::UnexpectedOp {
2074                        expected: pending.expected_op,
2075                        actual: actual.to_string(),
2076                    }
2077                }
2078                _ => outcome,
2079            };
2080            let _ = pending.sender.send(outcome);
2081            return Ok(ModuleControlRpcCompletion::Settled);
2082        }
2083
2084        let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
2085            return Ok(ModuleControlRpcCompletion::Unknown);
2086        };
2087        if tombstone.expires_at <= now {
2088            return Ok(ModuleControlRpcCompletion::Unknown);
2089        }
2090        Ok(ModuleControlRpcCompletion::LateHealthAnswer {
2091            module_id: tombstone.module_id,
2092            latency: now.saturating_duration_since(tombstone.probe_started_at),
2093        })
2094    }
2095
2096    #[cfg(test)]
2097    pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
2098        Ok(self.read_inner()?.health_probe_tombstones.len())
2099    }
2100
2101    #[cfg(test)]
2102    pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
2103        Ok(self.read_inner()?.closing_connections.len())
2104    }
2105
2106    /// Reserved-but-uncommitted route handles, counted on both index sides, so
2107    /// a test can assert a reservation pair was actually given back.
2108    #[cfg(test)]
2109    pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
2110        let inner = self.read_inner()?;
2111        Ok((inner.reserved_client.len(), inner.reserved_module.len()))
2112    }
2113
2114    pub(crate) fn module_endpoint_for_connection(
2115        &self,
2116        connection_id: ConnectionId,
2117    ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
2118        Ok(self
2119            .read_inner()?
2120            .endpoint_by_connection
2121            .get(&connection_id)
2122            .copied())
2123    }
2124
2125    /// Looks up the module registered on a data-plane connection so route-drop
2126    /// diagnostics name the emitter instead of only reporting a daemon total.
2127    pub(crate) fn module_id_for_connection(
2128        &self,
2129        connection_id: ConnectionId,
2130    ) -> Result<Option<String>, ForwardingError> {
2131        let inner = self.read_inner()?;
2132        Ok(inner
2133            .endpoint_by_connection
2134            .get(&connection_id)
2135            .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
2136            .cloned())
2137    }
2138
2139    /// Whether the daemon ever allocated `(channel, epoch)` on this module
2140    /// connection. Epochs on a module channel are handed out as 1, 2, 3, ...
2141    /// and the last one handed out is remembered until the connection ends, so
2142    /// every epoch from 1 up to that one was a real route at some point. Used
2143    /// only on the drop path, to tell a module still sending on a route the
2144    /// daemon released from one sending on a route that never existed.
2145    pub(crate) fn module_route_epoch_was_allocated(
2146        &self,
2147        connection_id: ConnectionId,
2148        channel: u16,
2149        epoch: u32,
2150    ) -> Result<bool, ForwardingError> {
2151        let inner = self.read_inner()?;
2152        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2153            return Ok(false);
2154        };
2155        Ok(inner
2156            .module_slot_epochs
2157            .get(&ModuleRouteKey { endpoint, channel })
2158            .is_some_and(|last| epoch != 0 && epoch <= *last))
2159    }
2160
2161    pub(crate) fn has_live_module_connection(
2162        &self,
2163        module_id: &str,
2164    ) -> Result<bool, ForwardingError> {
2165        Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
2166    }
2167
2168    /// Admission and route teardown share the forwarding-then-flow lock order.
2169    /// No asynchronous work may split validation from recording the credits.
2170    pub(crate) fn try_admit_request(
2171        &self,
2172        route: &Arc<RouteBinding>,
2173        epoch: u32,
2174        corr: u64,
2175        subscription: bool,
2176    ) -> Result<RequestAdmission, ForwardingError> {
2177        let inner = self.read_inner()?;
2178        let key = ClientRouteKey {
2179            connection_id: route.client_connection_id,
2180            channel: route.client_channel,
2181        };
2182        if epoch != route.client_epoch
2183            || !inner
2184                .client_to_module
2185                .get(&key)
2186                .is_some_and(|current| Arc::ptr_eq(current, route))
2187        {
2188            return Ok(RequestAdmission::Stale);
2189        }
2190        let admission = route.flow.try_acquire_tagged(corr, subscription);
2191        Ok(match admission {
2192            RequestAdmission::Stale
2193                if inner
2194                    .draining_endpoints
2195                    .contains_key(&route.module_endpoint) =>
2196            {
2197                RequestAdmission::ModuleReloading
2198            }
2199            admission => admission,
2200        })
2201    }
2202
2203    pub(crate) fn lookup_data_route(
2204        &self,
2205        connection_id: ConnectionId,
2206        channel: u16,
2207        epoch: u32,
2208    ) -> Result<DataRoute, ForwardingError> {
2209        let inner = self.read_inner()?;
2210        let state = if let Some(endpoint) =
2211            inner.endpoint_by_connection.get(&connection_id).copied()
2212        {
2213            let key = ModuleRouteKey { endpoint, channel };
2214            match inner.module_to_client.get(&key) {
2215                Some(route) if route.module_epoch == epoch => {
2216                    DataRouteState::Bound(Arc::clone(route))
2217                }
2218                Some(_) => DataRouteState::EpochMismatch,
2219                None if inner.reserved_module.contains_key(&key)
2220                    && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
2221                {
2222                    DataRouteState::Reserved
2223                }
2224                None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
2225                None => DataRouteState::Absent,
2226            }
2227        } else {
2228            let key = ClientRouteKey {
2229                connection_id,
2230                channel,
2231            };
2232            match inner.client_to_module.get(&key) {
2233                Some(route) if route.client_epoch == epoch => {
2234                    DataRouteState::Bound(Arc::clone(route))
2235                }
2236                Some(_) => DataRouteState::EpochMismatch,
2237                None if inner.reserved_client.contains_key(&key)
2238                    && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
2239                {
2240                    DataRouteState::Reserved
2241                }
2242                None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
2243                None => DataRouteState::Absent,
2244            }
2245        };
2246        Ok(
2247            if inner.endpoint_by_connection.contains_key(&connection_id) {
2248                DataRoute::Module(state)
2249            } else {
2250                DataRoute::Client(state)
2251            },
2252        )
2253    }
2254
2255    #[cfg(test)]
2256    pub(crate) fn inject_client_slot_epoch(
2257        &self,
2258        connection_id: ConnectionId,
2259        channel: u16,
2260        last_epoch: u32,
2261    ) {
2262        let mut inner = self.write_inner().expect("forwarding lock");
2263        inner.client_slot_epochs.insert(
2264            ClientRouteKey {
2265                connection_id,
2266                channel,
2267            },
2268            last_epoch,
2269        );
2270        inner.next_client_channel.insert(connection_id, channel);
2271    }
2272
2273    #[cfg(test)]
2274    pub(crate) fn inject_module_slot_epoch(
2275        &self,
2276        endpoint: ModuleEndpointId,
2277        channel: u16,
2278        last_epoch: u32,
2279    ) {
2280        let mut inner = self.write_inner().expect("forwarding lock");
2281        inner
2282            .module_slot_epochs
2283            .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
2284        inner.next_module_channel.insert(endpoint, channel);
2285    }
2286
2287    #[cfg(test)]
2288    pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
2289        self.write_inner()
2290            .expect("forwarding lock")
2291            .next_control_corr
2292            .insert(endpoint, next_corr);
2293    }
2294
2295    pub(crate) fn cache_status(
2296        &self,
2297        endpoint: ModuleEndpointId,
2298        module_channel: u16,
2299        module_epoch: u32,
2300        status: String,
2301    ) -> Result<bool, ForwardingError> {
2302        let mut inner = self.write_inner()?;
2303        if !inner.module_id_by_endpoint.contains_key(&endpoint) {
2304            return Err(ForwardingError::StaleModuleEndpoint);
2305        }
2306
2307        let module_key = ModuleRouteKey {
2308            endpoint,
2309            channel: module_channel,
2310        };
2311        let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
2312            (route.module_epoch == module_epoch).then_some((
2313                ClientRouteKey {
2314                    connection_id: route.client_connection_id,
2315                    channel: route.client_channel,
2316                },
2317                route.client_epoch,
2318            ))
2319        } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
2320            (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
2321                client_key,
2322                inner
2323                    .client_slot_epochs
2324                    .get(&client_key)
2325                    .copied()
2326                    .unwrap_or(0),
2327            ))
2328        } else {
2329            None
2330        };
2331
2332        if let Some(handle) = handle {
2333            inner.status.insert(handle, status);
2334            Ok(true)
2335        } else {
2336            debug!(
2337                module_channel,
2338                module_epoch,
2339                generation = endpoint.generation,
2340                connection_id = endpoint.connection_id.get(),
2341                "dropping stale status update for module route handle"
2342            );
2343            Ok(false)
2344        }
2345    }
2346
2347    pub(crate) fn route_poll_snapshot(
2348        &self,
2349        client_connection_id: ConnectionId,
2350        client_channel: u16,
2351        client_epoch: u32,
2352    ) -> Result<RoutePollSnapshot, ForwardingError> {
2353        let inner = self.read_inner()?;
2354        let client_key = ClientRouteKey {
2355            connection_id: client_connection_id,
2356            channel: client_channel,
2357        };
2358        let Some(route) = inner.client_to_module.get(&client_key) else {
2359            return Ok(RoutePollSnapshot::Absent);
2360        };
2361        if route.client_epoch != client_epoch
2362            || !inner
2363                .module_id_by_endpoint
2364                .contains_key(&route.module_endpoint)
2365        {
2366            return Ok(RoutePollSnapshot::Absent);
2367        }
2368        Ok(RoutePollSnapshot::Bound {
2369            module_id: route.module_id.clone(),
2370            status: inner.status.get(&(client_key, client_epoch)).cloned(),
2371        })
2372    }
2373
2374    pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2375        Ok(self.read_inner()?.client_to_module.len())
2376    }
2377
2378    /// How many distinct client connections hold at least one committed route,
2379    /// alongside the largest number of routes any single connection holds.
2380    ///
2381    /// `connected_clients` alone cannot distinguish many clients with a route
2382    /// each from one client accumulating hundreds, and those have opposite
2383    /// causes. Reading it required an out-of-band `lsof` during a live
2384    /// investigation, and the count of connections was mistaken for a count of
2385    /// client processes — which sent two of us after cleanup paths that were
2386    /// working correctly.
2387    pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2388        let inner = self.read_inner()?;
2389        let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2390        for key in inner.client_to_module.keys() {
2391            *per_connection.entry(key.connection_id).or_insert(0) += 1;
2392        }
2393        let max = per_connection.values().copied().max().unwrap_or(0);
2394        Ok((per_connection.len(), max))
2395    }
2396
2397    pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2398        let inner = self.read_inner()?;
2399        Ok(inner
2400            .client_to_module
2401            .keys()
2402            .any(|key| key.channel == route_channel))
2403    }
2404
2405    /// Whether daemon-wide shutdown has begun draining all providers.
2406    /// `None` means normal operation; during daemon shutdown, return the
2407    /// endpoint's measured result, or false if no completed measurement exists.
2408    pub(crate) fn daemon_drain_result_for_connection(
2409        &self,
2410        connection_id: ConnectionId,
2411    ) -> Result<Option<bool>, ForwardingError> {
2412        let inner = self.read_inner()?;
2413        Ok(inner.daemon_draining.then(|| {
2414            inner
2415                .endpoint_by_connection
2416                .get(&connection_id)
2417                .and_then(|endpoint| inner.daemon_drain_results.get(endpoint))
2418                .copied()
2419                .unwrap_or(false)
2420        }))
2421    }
2422
2423    #[cfg(unix)]
2424    pub(crate) fn record_daemon_drain_result(
2425        &self,
2426        endpoint: ModuleEndpointId,
2427        drained: bool,
2428    ) -> Result<(), ForwardingError> {
2429        let mut inner = self.write_inner()?;
2430        // Only update an endpoint captured at the start of daemon shutdown.
2431        // Teardown removes its entry, so a late wait cannot recreate the result
2432        // or attribute it to a replacement connection.
2433        if let Some(result) = inner.daemon_drain_results.get_mut(&endpoint) {
2434            *result = drained;
2435        }
2436        Ok(())
2437    }
2438
2439    /// Gate every provider atomically, including registrations racing shutdown.
2440    /// No supervisor lock is held while taking the forwarding lock.
2441    #[cfg(unix)]
2442    pub(crate) fn begin_daemon_drain(
2443        &self,
2444    ) -> Result<Vec<(String, ModuleEndpointId)>, ForwardingError> {
2445        let mut inner = self.write_inner()?;
2446        if inner.daemon_draining {
2447            return Ok(Vec::new());
2448        }
2449        inner.daemon_draining = true;
2450        let modules = inner
2451            .modules_by_id
2452            .iter()
2453            .map(|(id, module)| (id.clone(), module.endpoint))
2454            .collect::<Vec<_>>();
2455        for (_, endpoint) in &modules {
2456            inner.daemon_drain_results.insert(*endpoint, false);
2457            inner
2458                .draining_endpoints
2459                .insert(*endpoint, RouteCloseReason::Restart);
2460        }
2461        // Swap candidates and superseded incumbents are not routable, but they
2462        // are live endpoints that could still be sent module-control work, so
2463        // the daemon-wide gate covers them too. Only the active endpoints are
2464        // returned to the supervisor for measurement, one per module id.
2465        let off_slot_endpoints = inner
2466            .candidates_by_id
2467            .values()
2468            .map(|module| module.endpoint)
2469            .chain(inner.superseded_endpoints.keys().copied())
2470            .collect::<Vec<_>>();
2471        for endpoint in off_slot_endpoints {
2472            inner
2473                .draining_endpoints
2474                .insert(endpoint, RouteCloseReason::Restart);
2475        }
2476        Ok(modules)
2477    }
2478
2479    /// Begin draining whatever endpoint is ACTIVE for `module_id`.
2480    ///
2481    /// After a swap's cutover the active endpoint is the promoted candidate, so
2482    /// this must not be used to drain the incumbent; use
2483    /// [`Self::begin_endpoint_drain`] with the endpoint `cutover_candidate`
2484    /// returned.
2485    pub(crate) fn begin_module_drain(
2486        &self,
2487        module_id: &str,
2488        reason: RouteCloseReason,
2489    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2490        let mut inner = self.write_inner()?;
2491        let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2492            return Ok(None);
2493        };
2494        Ok(Some(begin_drain_locked(
2495            &mut inner, module_id, module, reason,
2496        )))
2497    }
2498
2499    /// Begin draining one specific endpoint, whichever slot it is in.
2500    ///
2501    /// This is how a swap drains its incumbent after cutover: the incumbent's
2502    /// endpoint is captured by `cutover_candidate`, and resolving it by module id
2503    /// instead would find the promoted candidate and leave neither process
2504    /// routable. Returns `Ok(None)` when the endpoint is no longer registered.
2505    pub(crate) fn begin_endpoint_drain(
2506        &self,
2507        endpoint: ModuleEndpointId,
2508        reason: RouteCloseReason,
2509    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2510        let mut inner = self.write_inner()?;
2511        let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2512            return Ok(None);
2513        };
2514        let module_id = inner
2515            .module_id_by_endpoint
2516            .get(&endpoint)
2517            .cloned()
2518            .expect("an endpoint resolved to a module connection has a module id");
2519        Ok(Some(begin_drain_locked(
2520            &mut inner, &module_id, module, reason,
2521        )))
2522    }
2523}
2524
2525/// Mark `module`'s endpoint draining and settle everything still pending on it.
2526/// Shared by the by-id and by-endpoint drain entry points, which differ only in
2527/// how they find the endpoint.
2528fn begin_drain_locked(
2529    inner: &mut ForwardingInner,
2530    module_id: &str,
2531    module: ModuleConnection,
2532    reason: RouteCloseReason,
2533) -> ModuleDrainTarget {
2534    {
2535        let endpoint = module.endpoint;
2536        inner.draining_endpoints.insert(endpoint, reason);
2537
2538        let flows = inner
2539            .client_to_module
2540            .values()
2541            .filter(|route| route.module_endpoint == endpoint)
2542            .map(|route| Arc::clone(&route.flow))
2543            .collect::<Vec<_>>();
2544        let excluded_subscriptions = flows
2545            .into_iter()
2546            .map(|flow| flow.begin_drain())
2547            .fold(0u32, u32::saturating_add);
2548
2549        let pending_keys = inner
2550            .pending_relays
2551            .keys()
2552            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2553            .copied()
2554            .collect::<Vec<_>>();
2555        let mut abandoned_bindings = Vec::new();
2556        for key in pending_keys {
2557            let Some(pending) = inner.pending_relays.remove(&key) else {
2558                continue;
2559            };
2560            release_reserved_route_locked(
2561                inner,
2562                pending.reservation.client_key,
2563                pending.reservation.module_key,
2564            );
2565            if pending.relay_enqueued {
2566                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2567                    abandoned_bindings.push(target);
2568                }
2569            } else {
2570                // The bind may still be on its way into the module's queue;
2571                // a GOODBYE sent now could overtake it. `route.open` sends
2572                // it once its send completes.
2573                inner.relays_released_before_enqueue.insert(key);
2574            }
2575            let _ = pending
2576                .sender
2577                .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
2578                    "module_reloading",
2579                    format!("module_id '{module_id}' is reloading"),
2580                )));
2581        }
2582
2583        let pending_control_keys = inner
2584            .pending_control_rpcs
2585            .keys()
2586            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2587            .copied()
2588            .collect::<Vec<_>>();
2589        for key in pending_control_keys {
2590            if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2591                let _ = pending
2592                    .sender
2593                    .send(ModuleControlRpcOutcome::ModuleGone(format!(
2594                        "module '{module_id}' began draining during module-control RPC"
2595                    )));
2596            }
2597        }
2598
2599        ModuleDrainTarget {
2600            endpoint,
2601            sink: module.sink,
2602            negotiated_ver: module.negotiated_ver,
2603            abandoned_bindings,
2604            excluded_subscriptions,
2605        }
2606    }
2607}
2608
2609/// What a drain that timed out was still waiting on, for the log line that
2610/// reports the timeout. Per-request ages are not tracked.
2611#[derive(Debug, Default, PartialEq, Eq)]
2612pub(crate) struct DrainHoldouts {
2613    /// Requests still counted against the drain (subscriptions flagged as
2614    /// such are excluded from the drain and not counted here).
2615    pub(crate) requests: usize,
2616    /// Routes holding at least one of those requests.
2617    pub(crate) routes: usize,
2618    /// Every route on the endpoint, for scale.
2619    pub(crate) total_routes: usize,
2620    /// The client connections holding the most requests, largest first, at
2621    /// most three: enough to name the consumer without listing every route.
2622    pub(crate) top_connections: Vec<(u64, usize)>,
2623    /// The held requests themselves, as the module saw them, so its own log
2624    /// can say what each one was: `(module channel, corr)`, ordered by channel
2625    /// then corr, at most [`DRAIN_HELD_REQUESTS_LISTED`]. The daemon never reads
2626    /// request bodies, so it cannot name a request's method; the module can,
2627    /// from the channel and corr. A request is released only when the module
2628    /// sends a terminal frame (Response, Error or StreamEnd) with its corr on
2629    /// its route, so every pair listed here is one the module ended, if at all,
2630    /// without sending that frame.
2631    pub(crate) held: Vec<(u16, u64)>,
2632}
2633
2634/// How many held requests the drain-timeout line lists by channel and corr.
2635pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
2636
2637impl ForwardingTable {
2638    /// Summarise the requests one endpoint's drain is still waiting on.
2639    pub(crate) fn endpoint_drain_holdouts(
2640        &self,
2641        endpoint: ModuleEndpointId,
2642    ) -> Result<DrainHoldouts, ForwardingError> {
2643        let inner = self.read_inner()?;
2644        let mut holdouts = DrainHoldouts::default();
2645        let mut by_connection: HashMap<u64, usize> = HashMap::new();
2646        for (key, route) in &inner.client_to_module {
2647            if route.module_endpoint != endpoint {
2648                continue;
2649            }
2650            holdouts.total_routes += 1;
2651            let held = route.flow.drain_in_flight();
2652            if held == 0 {
2653                continue;
2654            }
2655            holdouts.requests += held;
2656            holdouts.routes += 1;
2657            *by_connection.entry(key.connection_id.get()).or_default() += held;
2658            holdouts.held.extend(
2659                route
2660                    .flow
2661                    .drain_held_corrs()
2662                    .into_iter()
2663                    .map(|corr| (route.module_channel, corr)),
2664            );
2665        }
2666        holdouts.held.sort_unstable();
2667        holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
2668        let mut connections = by_connection.into_iter().collect::<Vec<_>>();
2669        connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
2670        connections.truncate(3);
2671        holdouts.top_connections = connections;
2672        Ok(holdouts)
2673    }
2674
2675    pub(crate) fn endpoint_in_flight_count(
2676        &self,
2677        endpoint: ModuleEndpointId,
2678    ) -> Result<usize, ForwardingError> {
2679        let inner = self.read_inner()?;
2680        Ok(inner
2681            .client_to_module
2682            .values()
2683            .filter(|route| route.module_endpoint == endpoint)
2684            .map(|route| route.flow.drain_in_flight())
2685            .sum())
2686    }
2687
2688    pub(crate) fn endpoint_is_draining(
2689        &self,
2690        endpoint: ModuleEndpointId,
2691    ) -> Result<bool, ForwardingError> {
2692        Ok(self
2693            .read_inner()?
2694            .draining_endpoints
2695            .contains_key(&endpoint))
2696    }
2697
2698    pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
2699        let inner = self.read_inner()?;
2700        Ok(inner
2701            .modules_by_id
2702            .get(module_id)
2703            .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
2704    }
2705
2706    pub(crate) fn release_module_endpoint_routes(
2707        &self,
2708        endpoint: ModuleEndpointId,
2709    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2710        let mut inner = self.write_inner()?;
2711        let routes = inner
2712            .module_to_client
2713            .iter()
2714            .filter(|(module_key, _)| module_key.endpoint == endpoint)
2715            .map(|(module_key, route)| (*module_key, route.module_epoch))
2716            .collect::<Vec<_>>();
2717        let mut released = Vec::with_capacity(routes.len());
2718        for (module_key, epoch) in routes {
2719            if let RouteRelease::Removed(target) =
2720                release_module_route_locked(&mut inner, module_key, epoch)
2721            {
2722                released.push(target);
2723            }
2724        }
2725        Ok(released)
2726    }
2727
2728    /// Enumerate one endpoint's current routes without contacting the module.
2729    ///
2730    /// The read lock makes this safe while the endpoint drains: the returned
2731    /// `draining` marker describes the same table state that owns the route,
2732    /// rather than inferring liveness from a module that may be stopping.
2733    pub(crate) fn endpoint_routes(
2734        &self,
2735        endpoint: ModuleEndpointId,
2736    ) -> Result<Vec<EndpointRoute>, ForwardingError> {
2737        let inner = self.read_inner()?;
2738        Ok(endpoint_routes_locked(&inner, endpoint))
2739    }
2740
2741    /// Snapshot all live endpoint route sets under one forwarding-table read lock.
2742    pub(crate) fn route_census(
2743        &self,
2744        module_id: Option<&str>,
2745    ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
2746        let inner = self.read_inner()?;
2747        let mut endpoints = inner
2748            .modules_by_id
2749            .iter()
2750            .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
2751            .map(|(id, module)| (id.clone(), module.endpoint))
2752            .collect::<Vec<_>>();
2753        endpoints.sort_by(|left, right| left.0.cmp(&right.0));
2754        Ok(endpoints
2755            .into_iter()
2756            .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
2757            .collect())
2758    }
2759
2760    /// Snapshot only the endpoint currently routable under this module id.
2761    pub(crate) fn live_roots(
2762        &self,
2763        module_id: &str,
2764    ) -> Result<ModuleControlResponseToModule, ForwardingError> {
2765        let inner = self.read_inner()?;
2766        let endpoint = inner
2767            .modules_by_id
2768            .get(module_id)
2769            .map(|module| module.endpoint);
2770        let mut roots = BTreeMap::new();
2771        let mut unknown_root_bindings = 0;
2772        let mut total_bindings = 0;
2773        if let Some(endpoint) = endpoint {
2774            for binding in inner
2775                .module_to_client
2776                .values()
2777                .filter(|binding| binding.module_endpoint == endpoint)
2778            {
2779                total_bindings += 1;
2780                let workdir = binding
2781                    .scope
2782                    .as_ref()
2783                    .and_then(|scope| scope.workdir.as_deref())
2784                    .map(std::path::Path::new);
2785                if let Some(root) = &binding.project_root {
2786                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2787                    entry.0 += 1;
2788                } else if workdir.is_none() {
2789                    unknown_root_bindings += 1;
2790                }
2791                if let Some(workdir) = workdir.filter(|workdir| {
2792                    binding.project_root.as_ref().map(ProjectRootId::as_path) != Some(*workdir)
2793                }) {
2794                    roots.entry(workdir.to_path_buf()).or_insert((0, 0)).0 += 1;
2795                }
2796            }
2797            for pending in inner
2798                .pending_relays
2799                .values()
2800                .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
2801            {
2802                total_bindings += 1;
2803                if let Some(root) = &pending.reservation.project_root {
2804                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2805                    entry.1 += 1;
2806                } else {
2807                    unknown_root_bindings += 1;
2808                }
2809            }
2810        }
2811        Ok(ModuleControlResponseToModule::LiveRoots {
2812            roots: roots
2813                .into_iter()
2814                .map(|(project_root, (bound, pending))| LiveRoot {
2815                    project_root,
2816                    bound,
2817                    pending,
2818                })
2819                .collect(),
2820            unknown_root_bindings,
2821            total_bindings,
2822        })
2823    }
2824
2825    /// True if this connection already owns committed or reserved CLIENT routes.
2826    /// A module registers (HELLO) before serving and never opens client routes, so
2827    /// a connection that has client routes must not also become a module endpoint
2828    /// — otherwise one connection holds both client and module state and cleanup
2829    /// only releases one side.
2830    pub(crate) fn connection_has_client_routes(
2831        &self,
2832        connection_id: ConnectionId,
2833    ) -> Result<bool, ForwardingError> {
2834        let inner = self.read_inner()?;
2835        Ok(connection_has_client_routes_locked(&inner, connection_id))
2836    }
2837
2838    pub(crate) fn cleanup_connection(
2839        &self,
2840        connection_id: ConnectionId,
2841    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2842        self.cleanup_connection_counted(connection_id)
2843            .map(|cleanup| cleanup.released)
2844    }
2845
2846    /// [`Self::cleanup_connection`], also reporting how many pending
2847    /// route.bind relays to the closed module were aborted. That count is the
2848    /// `abandoned` figure of the `route.closed` push sent for a lost module
2849    /// connection; it is always zero for a client connection.
2850    pub(crate) fn cleanup_connection_counted(
2851        &self,
2852        connection_id: ConnectionId,
2853    ) -> Result<ConnectionCleanup, ForwardingError> {
2854        self.frame_drop_warnings
2855            .lock()
2856            .unwrap_or_else(std::sync::PoisonError::into_inner)
2857            .remove(&connection_id);
2858        let mut inner = self.write_inner()?;
2859        inner.closing_connections.insert(connection_id);
2860        let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
2861            remove_module_connection_locked(&mut inner, endpoint)
2862        } else {
2863            ConnectionCleanup {
2864                released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
2865                abandoned_relays: 0,
2866            }
2867        };
2868        // The closing mark refuses new work for a connection whose teardown is
2869        // still pending. This is the latest point at which lifting it is safe:
2870        // teardown has just removed every per-connection entry above, under
2871        // this same write lock, so no lookup can still find live state for the
2872        // id; and ids come from a monotonic counter that never reissues one,
2873        // so the id can never name a future connection either. Keeping the
2874        // mark past this point only grew the set by one entry per connection
2875        // for the life of the daemon.
2876        inner.closing_connections.remove(&connection_id);
2877        Ok(cleanup)
2878    }
2879
2880    fn cleanup_client_connection_locked(
2881        inner: &mut ForwardingInner,
2882        connection_id: ConnectionId,
2883    ) -> Vec<GoodbyeTarget> {
2884        let routes = inner
2885            .client_to_module
2886            .iter()
2887            .filter(|(key, _)| key.connection_id == connection_id)
2888            .map(|(key, route)| (*key, route.client_epoch))
2889            .collect::<Vec<_>>();
2890        let mut released = Vec::with_capacity(routes.len());
2891        for (client_key, epoch) in routes {
2892            if let RouteRelease::Removed(target) =
2893                release_client_route_locked(inner, client_key, epoch)
2894            {
2895                released.push(target);
2896            }
2897        }
2898
2899        let pending_keys = inner
2900            .pending_relays
2901            .iter()
2902            .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
2903            .map(|(key, _)| *key)
2904            .collect::<Vec<_>>();
2905        for key in pending_keys {
2906            let Some(pending) = inner.pending_relays.remove(&key) else {
2907                continue;
2908            };
2909            release_reserved_route_locked(
2910                inner,
2911                pending.reservation.client_key,
2912                pending.reservation.module_key,
2913            );
2914            if pending.relay_enqueued {
2915                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2916                    released.push(target);
2917                }
2918            } else {
2919                // The bind may still be on its way into the module's queue;
2920                // a GOODBYE sent now could overtake it. `route.open` sends
2921                // it once its send completes.
2922                inner.relays_released_before_enqueue.insert(key);
2923            }
2924            let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
2925                "client connection closed during route.bind relay".to_string(),
2926            ));
2927        }
2928
2929        let orphaned = inner
2930            .reserved_client
2931            .iter()
2932            .filter(|(key, _)| key.connection_id == connection_id)
2933            .map(|(client, module)| (*client, *module))
2934            .collect::<Vec<_>>();
2935        for (client_key, module_key) in orphaned {
2936            release_reserved_route_locked(inner, client_key, module_key);
2937        }
2938        inner.next_client_channel.remove(&connection_id);
2939        inner
2940            .client_slot_epochs
2941            .retain(|key, _| key.connection_id != connection_id);
2942        inner
2943            .last_published_epoch
2944            .retain(|key, _| key.connection_id != connection_id);
2945        inner
2946            .status
2947            .retain(|(key, _), _| key.connection_id != connection_id);
2948
2949        released
2950    }
2951
2952    /// Close a client connection whose egress queue refused a frame for the
2953    /// route `(connection_id, channel)` at `expected_epoch`. The whole
2954    /// connection closes because its queue is shared by every route on it.
2955    ///
2956    /// The first request that actually closes the connection logs one WARN with
2957    /// the diagnosis: which principals the connection's routes belonged to,
2958    /// which module's frame did not fit and on which client channel, and what
2959    /// the queue held at that moment.
2960    pub(crate) fn escalate_client_delivery_failure(
2961        &self,
2962        connection_id: ConnectionId,
2963        channel: u16,
2964        expected_epoch: u32,
2965        reason: CloseReason,
2966        undelivered: UndeliveredFrame<'_>,
2967    ) -> Result<bool, ForwardingError> {
2968        self.escalate_client_delivery_failure_diagnosed(
2969            connection_id,
2970            channel,
2971            expected_epoch,
2972            reason,
2973            undelivered,
2974            None,
2975        )
2976    }
2977
2978    pub(crate) fn escalate_client_delivery_failure_diagnosed(
2979        &self,
2980        connection_id: ConnectionId,
2981        channel: u16,
2982        expected_epoch: u32,
2983        reason: CloseReason,
2984        undelivered: UndeliveredFrame<'_>,
2985        diagnosis: Option<&crate::router::ClientDeliveryDiagnosis>,
2986    ) -> Result<bool, ForwardingError> {
2987        let principals = {
2988            let mut inner = self.write_inner()?;
2989            let key = ClientRouteKey {
2990                connection_id,
2991                channel,
2992            };
2993            if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
2994                None
2995            } else {
2996                inner.closing_connections.insert(connection_id);
2997                Some(connection_principals_locked(&inner, connection_id))
2998            }
2999        };
3000        let Some(principals) = principals else {
3001            // A concurrent release can remove the epoch before escalation.
3002            // The frame was still lost, even though this route can no longer
3003            // authorize closing the shared client connection.
3004            self.warn_frame_drop(
3005                connection_id,
3006                channel,
3007                expected_epoch,
3008                "module_to_client",
3009                diagnosis
3010                    .map(|failure| failure.cause)
3011                    .unwrap_or("delivery_failed"),
3012            );
3013            return Ok(false);
3014        };
3015        let backlog = undelivered.sink.backlog();
3016        let close_reason = reason.to_string();
3017        if self.request_connection_close(connection_id, reason) {
3018            warn!(
3019                connection_id = connection_id.get(),
3020                principals = %principals,
3021                module_id = undelivered.module_id.unwrap_or("unknown"),
3022                client_channel = channel,
3023                queued_bytes = backlog.queued_bytes,
3024                queued_frames = backlog.queued_frames,
3025                frame_type = ?diagnosis.map(|failure| failure.frame_type),
3026                frame_flags = ?diagnosis.map(|failure| failure.flags.0),
3027                close_cause = diagnosis.map(|failure| failure.cause).unwrap_or("delivery_failed"),
3028                oldest_queued_ms = backlog
3029                    .oldest_age
3030                    .map(|age| age.as_millis() as u64)
3031                    .unwrap_or(0),
3032                close_reason = %close_reason,
3033                "closing client connection: its egress queue could not take a frame"
3034            );
3035        }
3036        Ok(true)
3037    }
3038
3039    /// Publish the tags a scope sync changed and close the live routes its
3040    /// drain table selects, in one critical section.
3041    ///
3042    /// The caller holds the scope table's write lock (scope table, then this
3043    /// table, always). Publishing and selecting under one lock is what closes
3044    /// the race with a bind commit: a commit before this sees the old tag and
3045    /// its route is selected here like any other; a commit after it sees the
3046    /// new tag and is refused.
3047    ///
3048    /// Every client route index entry is scanned, which includes routes on a
3049    /// swap's superseded endpoint (the old process of a module being replaced
3050    /// blue/green, which keeps its existing routes until they drain): those
3051    /// stay indexed until drained, so ending a scope reaches them too.
3052    pub(crate) fn publish_scope_changes(
3053        &self,
3054        changes: &[ScopeTagChange],
3055    ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
3056        let mut inner = self.write_inner()?;
3057        let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
3058        for change in changes {
3059            let key = (change.owner.clone(), change.scope_ref.clone());
3060            match change.after {
3061                Some(tag) => {
3062                    inner.scope_tags.insert(key, tag);
3063                }
3064                None => {
3065                    inner.scope_tags.remove(&key);
3066                }
3067            }
3068            by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
3069        }
3070        let mut selected = Vec::new();
3071        for (client_key, route) in &inner.client_to_module {
3072            let Some(scope) = &route.scope else {
3073                continue;
3074            };
3075            let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
3076            else {
3077                continue;
3078            };
3079            if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
3080                continue;
3081            }
3082            let reason = match &change.drain {
3083                ScopeDrain::Nothing => continue,
3084                ScopeDrain::All(reason) => *reason,
3085                ScopeDrain::Carriers(narrowed) => {
3086                    let owner = Principal::Reserved {
3087                        module_id: scope.owner.clone(),
3088                    };
3089                    let hit = route.principal != owner
3090                        && narrowed.iter().any(|(principal, allowed)| {
3091                            *principal == route.principal
3092                                && allowed
3093                                    .as_ref()
3094                                    .is_none_or(|targets| !targets.contains(&route.module_id))
3095                        });
3096                    if !hit {
3097                        continue;
3098                    }
3099                    RouteCloseReason::ScopeCarrierRemoved
3100                }
3101            };
3102            selected.push((*client_key, route.client_epoch, reason));
3103        }
3104        let mut drained = Vec::new();
3105        for (client_key, client_epoch, reason) in selected {
3106            let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
3107                continue;
3108            };
3109            let release = release_client_route_locked(&mut inner, client_key, client_epoch);
3110            self.record_route_release(&release);
3111            if let RouteRelease::Removed(module) = release {
3112                drained.push(ScopeDrainedRoute {
3113                    reason,
3114                    scope: route
3115                        .scope
3116                        .clone()
3117                        .expect("selected scope drain has a bound scope"),
3118                    module_id: route.module_id.clone(),
3119                    client: GoodbyeTarget {
3120                        connection_id: route.client_connection_id,
3121                        sink: route.client_sink.clone(),
3122                        negotiated_ver: route.client_negotiated_ver,
3123                        channel: route.client_channel,
3124                        epoch: route.client_epoch,
3125                        kind: GoodbyeTargetKind::Client,
3126                        module_id: Some(route.module_id.clone()),
3127                    },
3128                    module,
3129                });
3130            }
3131        }
3132        Ok(drained)
3133    }
3134
3135    /// The published tag of one scope, for tests of the commit re-check.
3136    #[cfg(test)]
3137    pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
3138        self.read_inner()
3139            .ok()?
3140            .scope_tags
3141            .get(&(owner.to_string(), scope_ref.to_string()))
3142            .copied()
3143    }
3144
3145    fn record_route_release(&self, release: &RouteRelease) {
3146        match release {
3147            RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
3148            RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
3149            RouteRelease::Absent => {}
3150        }
3151    }
3152
3153    fn read_inner(&self) -> Result<RwLockReadGuard<'_, ForwardingInner>, ForwardingError> {
3154        self.inner.read().map_err(|_| ForwardingError::Poisoned)
3155    }
3156
3157    fn write_inner(&self) -> Result<RwLockWriteGuard<'_, ForwardingInner>, ForwardingError> {
3158        self.inner.write().map_err(|_| ForwardingError::Poisoned)
3159    }
3160
3161    fn lock_close_registry(
3162        &self,
3163    ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
3164        self.close_registry
3165            .lock()
3166            .unwrap_or_else(|poisoned| poisoned.into_inner())
3167    }
3168}
3169
3170impl ForwardingInner {
3171    fn allocate_route_slots(
3172        &mut self,
3173        connection_id: ConnectionId,
3174        endpoint: ModuleEndpointId,
3175    ) -> Result<(u16, u32, u16, u32), ForwardingError> {
3176        let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
3177        let mut client_channel = client_start;
3178        let client_channel = loop {
3179            let key = ClientRouteKey {
3180                connection_id,
3181                channel: client_channel,
3182            };
3183            let eligible = !self.client_to_module.contains_key(&key)
3184                && !self.reserved_client.contains_key(&key)
3185                && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3186            if eligible {
3187                break client_channel;
3188            }
3189            client_channel = next_channel(client_channel);
3190            if client_channel == client_start {
3191                return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
3192            }
3193        };
3194
3195        let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
3196        let mut module_channel = module_start;
3197        let module_channel = loop {
3198            let key = ModuleRouteKey {
3199                endpoint,
3200                channel: module_channel,
3201            };
3202            let eligible = !self.module_to_client.contains_key(&key)
3203                && !self.reserved_module.contains_key(&key)
3204                && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3205            if eligible {
3206                break module_channel;
3207            }
3208            module_channel = next_channel(module_channel);
3209            if module_channel == module_start {
3210                return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
3211            }
3212        };
3213
3214        let client_key = ClientRouteKey {
3215            connection_id,
3216            channel: client_channel,
3217        };
3218        let module_key = ModuleRouteKey {
3219            endpoint,
3220            channel: module_channel,
3221        };
3222        let client_epoch = self
3223            .client_slot_epochs
3224            .get(&client_key)
3225            .copied()
3226            .unwrap_or(0)
3227            + 1;
3228        let module_epoch = self
3229            .module_slot_epochs
3230            .get(&module_key)
3231            .copied()
3232            .unwrap_or(0)
3233            + 1;
3234        self.client_slot_epochs.insert(client_key, client_epoch);
3235        self.module_slot_epochs.insert(module_key, module_epoch);
3236        self.next_client_channel
3237            .insert(connection_id, next_channel(client_channel));
3238        self.next_module_channel
3239            .insert(endpoint, next_channel(module_channel));
3240        Ok((client_channel, client_epoch, module_channel, module_epoch))
3241    }
3242
3243    fn allocate_control_corr(
3244        &mut self,
3245        endpoint: ModuleEndpointId,
3246    ) -> Result<u64, ForwardingError> {
3247        let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
3248        if candidate == 0 {
3249            self.closing_connections.insert(endpoint.connection_id);
3250            return Err(ForwardingError::RelayCorrelationExhausted);
3251        }
3252        self.next_control_corr.insert(
3253            endpoint,
3254            if candidate == u64::MAX {
3255                0
3256            } else {
3257                candidate + 1
3258            },
3259        );
3260        Ok(candidate)
3261    }
3262}
3263
3264fn connection_has_client_routes_locked(
3265    inner: &ForwardingInner,
3266    connection_id: ConnectionId,
3267) -> bool {
3268    inner
3269        .client_to_module
3270        .keys()
3271        .any(|key| key.connection_id == connection_id)
3272        || inner
3273            .reserved_client
3274            .keys()
3275            .any(|key| key.connection_id == connection_id)
3276}
3277
3278fn check_module_connection_role_locked(
3279    inner: &ForwardingInner,
3280    connection_id: ConnectionId,
3281) -> Result<(), ForwardingError> {
3282    if inner.endpoint_by_connection.contains_key(&connection_id)
3283        || connection_has_client_routes_locked(inner, connection_id)
3284    {
3285        return Err(ForwardingError::ConnectionRoleConflict { connection_id });
3286    }
3287    Ok(())
3288}
3289
3290fn next_channel(channel: u16) -> u16 {
3291    let next = channel.wrapping_add(1);
3292    if next == 0 {
3293        1
3294    } else {
3295        next
3296    }
3297}
3298
3299fn endpoint_routes_locked(
3300    inner: &ForwardingInner,
3301    endpoint: ModuleEndpointId,
3302) -> Vec<EndpointRoute> {
3303    let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
3304    let draining = drain_reason.is_some();
3305    let mut routes = inner
3306        .module_to_client
3307        .iter()
3308        .filter(|(module_key, _)| module_key.endpoint == endpoint)
3309        .map(|(_, route)| EndpointRoute {
3310            goodbye_target: GoodbyeTarget {
3311                connection_id: route.client_connection_id,
3312                sink: route.client_sink.clone(),
3313                negotiated_ver: route.client_negotiated_ver,
3314                channel: route.client_channel,
3315                epoch: route.client_epoch,
3316                kind: GoodbyeTargetKind::Client,
3317                module_id: Some(route.module_id.clone()),
3318            },
3319            principal: route.principal.clone(),
3320            bound_at: route.bound_at,
3321            draining,
3322            drain_reason,
3323        })
3324        .collect::<Vec<_>>();
3325    routes.sort_by_key(|route| {
3326        (
3327            route.goodbye_target.connection_id.get(),
3328            route.goodbye_target.channel,
3329            route.goodbye_target.epoch,
3330        )
3331    });
3332    routes
3333}
3334
3335/// Why a bind admitted under `scope` may no longer commit, read from the
3336/// published tags: `scope_ended` when the scope is gone or at another epoch,
3337/// `scope_changed` (retryable) when only its content moved.
3338fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
3339    let key = (scope.owner.clone(), scope.scope_ref.clone());
3340    match inner.scope_tags.get(&key) {
3341        Some(current) if *current == scope.tag => None,
3342        Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
3343            subc_protocol::error_codes::SCOPE_CHANGED,
3344            format!(
3345                "scope '{}' of {} changed while the route was being bound; re-open it",
3346                scope.scope_ref, scope.owner
3347            ),
3348        )),
3349        _ => Some(ErrorBody::new(
3350            subc_protocol::error_codes::SCOPE_ENDED,
3351            format!(
3352                "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
3353                scope.scope_ref, scope.owner, scope.tag.scope_epoch
3354            ),
3355        )),
3356    }
3357}
3358
3359fn release_reserved_route_locked(
3360    inner: &mut ForwardingInner,
3361    client_key: ClientRouteKey,
3362    module_key: ModuleRouteKey,
3363) {
3364    if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
3365        inner.reserved_client.remove(&client_key);
3366    }
3367    if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
3368        inner.reserved_module.remove(&module_key);
3369    }
3370    inner.status.retain(|(key, _), _| *key != client_key);
3371}
3372
3373fn release_client_route_locked(
3374    inner: &mut ForwardingInner,
3375    client_key: ClientRouteKey,
3376    expected_epoch: u32,
3377) -> RouteRelease {
3378    let Some(route) = inner.client_to_module.get(&client_key) else {
3379        return RouteRelease::Absent;
3380    };
3381    if route.client_epoch != expected_epoch {
3382        return RouteRelease::Stale;
3383    }
3384    let route = inner
3385        .client_to_module
3386        .remove(&client_key)
3387        .expect("route checked under the same forwarding lock");
3388    route.flow.close();
3389    if let Some(scope) = &route.scope {
3390        scope.release_bind();
3391    }
3392    inner.module_to_client.remove(&ModuleRouteKey {
3393        endpoint: route.module_endpoint,
3394        channel: route.module_channel,
3395    });
3396    inner.operator_confirms.route_closed(ModuleRouteKey {
3397        endpoint: route.module_endpoint,
3398        channel: route.module_channel,
3399    });
3400    inner.status.remove(&(client_key, expected_epoch));
3401    RouteRelease::Removed(GoodbyeTarget {
3402        connection_id: route.module_endpoint.connection_id,
3403        sink: route.module_sink.clone(),
3404        negotiated_ver: route.module_negotiated_ver,
3405        channel: route.module_channel,
3406        epoch: route.module_epoch,
3407        kind: GoodbyeTargetKind::Module,
3408        module_id: Some(route.module_id.clone()),
3409    })
3410}
3411
3412fn release_module_route_locked(
3413    inner: &mut ForwardingInner,
3414    module_key: ModuleRouteKey,
3415    expected_epoch: u32,
3416) -> RouteRelease {
3417    let Some(route) = inner.module_to_client.get(&module_key) else {
3418        return RouteRelease::Absent;
3419    };
3420    if route.module_epoch != expected_epoch {
3421        return RouteRelease::Stale;
3422    }
3423    let route = inner
3424        .module_to_client
3425        .remove(&module_key)
3426        .expect("route checked under the same forwarding lock");
3427    inner.operator_confirms.route_closed(module_key);
3428    route.flow.close();
3429    if let Some(scope) = &route.scope {
3430        scope.release_bind();
3431    }
3432    let client_key = ClientRouteKey {
3433        connection_id: route.client_connection_id,
3434        channel: route.client_channel,
3435    };
3436    inner.client_to_module.remove(&client_key);
3437    inner.status.remove(&(client_key, route.client_epoch));
3438    RouteRelease::Removed(GoodbyeTarget {
3439        connection_id: route.client_connection_id,
3440        sink: route.client_sink.clone(),
3441        negotiated_ver: route.client_negotiated_ver,
3442        channel: route.client_channel,
3443        epoch: route.client_epoch,
3444        kind: GoodbyeTargetKind::Client,
3445        module_id: Some(route.module_id.clone()),
3446    })
3447}
3448
3449fn commit_route_locked(
3450    inner: &mut ForwardingInner,
3451    mut pending: PendingRouteBindRelayEntry,
3452) -> Result<Option<GoodbyeTarget>, ForwardingError> {
3453    let reservation = pending.reservation;
3454    if inner
3455        .closing_connections
3456        .contains(&reservation.client_key.connection_id)
3457    {
3458        return Err(ForwardingError::ConnectionClosing {
3459            connection_id: reservation.client_key.connection_id,
3460        });
3461    }
3462    let module_id = inner
3463        .module_id_by_endpoint
3464        .get(&reservation.module_key.endpoint)
3465        .cloned()
3466        .ok_or(ForwardingError::StaleModuleEndpoint)?;
3467    if inner
3468        .draining_endpoints
3469        .contains_key(&reservation.module_key.endpoint)
3470    {
3471        return Err(ForwardingError::ModuleReloading { module_id });
3472    }
3473    if pending.busy_refusals {
3474        let body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
3475            route_channel: reservation.client_key.channel,
3476            route_epoch: reservation.client_epoch,
3477            request_window: Some(pending.request_window),
3478            watch_window: pending.watch_window,
3479        })
3480        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
3481        pending.route_open_frame = Frame::build_with_version(
3482            pending.client_negotiated_ver,
3483            FrameType::Response,
3484            pending.route_open_frame.header.flags,
3485            0,
3486            0,
3487            pending.route_open_frame.header.corr,
3488            body,
3489        )
3490        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
3491    }
3492    if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
3493        || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
3494    {
3495        return Err(ForwardingError::UnknownReservation {
3496            client_channel: reservation.client_key.channel,
3497            module_channel: reservation.module_key.channel,
3498        });
3499    }
3500    let module = inner
3501        .modules_by_id
3502        .get(&module_id)
3503        .filter(|module| module.endpoint == reservation.module_key.endpoint)
3504        .cloned()
3505        .ok_or(ForwardingError::StaleModuleEndpoint)?;
3506    let binding = Arc::new(RouteBinding {
3507        client_connection_id: reservation.client_key.connection_id,
3508        client_sink: pending.client_sink,
3509        client_negotiated_ver: pending.client_negotiated_ver,
3510        client_channel: reservation.client_key.channel,
3511        client_epoch: reservation.client_epoch,
3512        module_id,
3513        module_endpoint: reservation.module_key.endpoint,
3514        module_sink: module.sink,
3515        module_negotiated_ver: module.negotiated_ver,
3516        module_channel: reservation.module_key.channel,
3517        module_epoch: reservation.module_epoch,
3518        principal: pending.principal,
3519        harness: pending.harness,
3520        build_class: pending.build_class,
3521        operator_app: pending.operator_app,
3522        project_root: reservation.project_root.clone(),
3523        bound_at: Instant::now(),
3524        flow: Arc::new(if pending.busy_refusals {
3525            ChannelFlow::with_admission(pending.request_window, true, pending.watch_window)
3526        } else {
3527            ChannelFlow::new(pending.request_window)
3528        }),
3529        scope: pending.scope,
3530    });
3531    inner
3532        .client_to_module
3533        .insert(reservation.client_key, Arc::clone(&binding));
3534    inner
3535        .module_to_client
3536        .insert(reservation.module_key, Arc::clone(&binding));
3537    let previous_published = inner
3538        .last_published_epoch
3539        .insert(reservation.client_key, reservation.client_epoch);
3540
3541    // This point is reachable only through an accepted route.bind acknowledgement.
3542    // Publish history before notifying the opener, so a subsequent
3543    // `scope.served_by` query sees the acknowledged bind.
3544    if let Some(scope) = &binding.scope {
3545        scope.record_served(&binding.module_id, module.verified_prefix_owner.as_deref());
3546    }
3547
3548    // Sending through the reserved slot cannot fail, but it reports a receiver
3549    // that closed after reservation and before this locked publication point.
3550    // The frame is stamped and charged to the queued-byte count here, when it
3551    // actually enters the queue, not when the slot was reserved.
3552    let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
3553    if client_writer_closed {
3554        // Not gated on `relay_enqueued`: only a module ack reaches this
3555        // commit, which proves the module holds the binding, and `route.open`
3556        // sends nothing for a relay a reply already settled.
3557        let abandoned = abandoned_route_target(inner, &reservation);
3558        if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
3559            route.flow.close();
3560        }
3561        inner.module_to_client.remove(&reservation.module_key);
3562        inner
3563            .status
3564            .remove(&(reservation.client_key, reservation.client_epoch));
3565        match previous_published {
3566            Some(epoch) => {
3567                inner
3568                    .last_published_epoch
3569                    .insert(reservation.client_key, epoch);
3570            }
3571            None => {
3572                inner.last_published_epoch.remove(&reservation.client_key);
3573            }
3574        }
3575        let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3576            "client egress closed during route publication".to_string(),
3577        ));
3578        return Ok(abandoned);
3579    }
3580
3581    if let Some(scope) = &binding.scope {
3582        scope.commit_presence();
3583    }
3584    let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
3585    Ok(None)
3586}
3587
3588/// The live module connection registered under exactly `endpoint`, whichever
3589/// slot holds it: active, swap candidate, or superseded incumbent.
3590///
3591/// Resolving by endpoint rather than by module id is what keeps an incumbent
3592/// addressable after a swap promoted a candidate over its id. An endpoint that
3593/// is in none of the three (a stale one, replaced without a promotion) resolves
3594/// to nothing, as it always has.
3595fn module_connection_for_endpoint_locked(
3596    inner: &ForwardingInner,
3597    endpoint: ModuleEndpointId,
3598) -> Option<&ModuleConnection> {
3599    let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
3600    inner
3601        .modules_by_id
3602        .get(module_id)
3603        .filter(|module| module.endpoint == endpoint)
3604        .or_else(|| {
3605            inner
3606                .candidates_by_id
3607                .get(module_id)
3608                .filter(|module| module.endpoint == endpoint)
3609        })
3610        .or_else(|| inner.superseded_endpoints.get(&endpoint))
3611}
3612
3613fn abandoned_route_target(
3614    inner: &ForwardingInner,
3615    reservation: &RouteReservation,
3616) -> Option<GoodbyeTarget> {
3617    let module_id = inner
3618        .module_id_by_endpoint
3619        .get(&reservation.module_key.endpoint)?;
3620    let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
3621    (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
3622        connection_id: module.endpoint.connection_id,
3623        sink: module.sink.clone(),
3624        negotiated_ver: module.negotiated_ver,
3625        channel: reservation.module_key.channel,
3626        epoch: reservation.module_epoch,
3627        kind: GoodbyeTargetKind::Module,
3628        module_id: Some(module_id.clone()),
3629    })
3630}
3631
3632/// Queue a registering module's HELLO_ACK on its sink. Called with the
3633/// forwarding write lock held, before the endpoint is inserted, which is what
3634/// puts the ack ahead of any `route.bind` or control RPC routed to the module.
3635/// `try_send` never waits, so holding the lock across it is safe.
3636fn enqueue_hello_ack_locked(
3637    sink: &FrameSink,
3638    connection_id: ConnectionId,
3639    hello_ack: Option<Frame>,
3640) -> Result<(), ForwardingError> {
3641    let Some(hello_ack) = hello_ack else {
3642        return Ok(());
3643    };
3644    sink.try_send(hello_ack)
3645        .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
3646}
3647
3648fn remove_module_connection_locked(
3649    inner: &mut ForwardingInner,
3650    endpoint: ModuleEndpointId,
3651) -> ConnectionCleanup {
3652    // Commit the disconnect before its routes are released, so their teardown
3653    // cannot replace module_closed with route_closed.
3654    inner.operator_confirms.module_closed(endpoint);
3655    inner.draining_endpoints.remove(&endpoint);
3656    inner.daemon_drain_results.remove(&endpoint);
3657    let module_id = inner.module_id_by_endpoint.remove(&endpoint);
3658    if let Some(module_id) = module_id.as_ref() {
3659        if inner
3660            .modules_by_id
3661            .get(module_id)
3662            .is_some_and(|module| module.endpoint == endpoint)
3663        {
3664            inner.modules_by_id.remove(module_id);
3665        }
3666        if inner
3667            .candidates_by_id
3668            .get(module_id)
3669            .is_some_and(|module| module.endpoint == endpoint)
3670        {
3671            inner.candidates_by_id.remove(module_id);
3672        }
3673    }
3674    inner.superseded_endpoints.remove(&endpoint);
3675    inner.endpoint_by_connection.remove(&endpoint.connection_id);
3676    inner.next_module_channel.remove(&endpoint);
3677    inner.next_control_corr.remove(&endpoint);
3678    inner
3679        .health_probe_tombstones
3680        .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
3681    inner
3682        .module_slot_epochs
3683        .retain(|key, _| key.endpoint != endpoint);
3684    let reserved_module_keys: Vec<ModuleRouteKey> = inner
3685        .reserved_module
3686        .keys()
3687        .filter(|module_key| module_key.endpoint == endpoint)
3688        .copied()
3689        .collect();
3690    for module_key in reserved_module_keys {
3691        if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
3692            release_reserved_route_locked(inner, client_key, module_key);
3693        }
3694    }
3695
3696    let pending_keys: Vec<_> = inner
3697        .pending_relays
3698        .keys()
3699        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3700        .copied()
3701        .collect();
3702    let pending: Vec<_> = pending_keys
3703        .into_iter()
3704        .filter_map(|key| {
3705            inner
3706                .pending_relays
3707                .remove(&key)
3708                .map(|pending| (key, pending))
3709        })
3710        .collect();
3711    let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
3712    for (key, pending) in pending {
3713        // Recorded like every other removal without a reply, so a `route.open`
3714        // still sending the bind learns the relay was released rather than
3715        // settled by the module.
3716        if !pending.relay_enqueued {
3717            inner.relays_released_before_enqueue.insert(key);
3718        }
3719        let module_label = module_id.as_deref().unwrap_or("unknown");
3720        let _ = pending
3721            .sender
3722            .send(RouteBindRelayOutcome::ModuleGone(format!(
3723                "module '{module_label}' connection closed during route.bind relay"
3724            )));
3725    }
3726
3727    let pending_control_keys: Vec<_> = inner
3728        .pending_control_rpcs
3729        .keys()
3730        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3731        .copied()
3732        .collect();
3733    let pending_control: Vec<_> = pending_control_keys
3734        .into_iter()
3735        .filter_map(|key| inner.pending_control_rpcs.remove(&key))
3736        .collect();
3737    for pending in pending_control {
3738        let module_label = module_id.as_deref().unwrap_or("unknown");
3739        let _ = pending
3740            .sender
3741            .send(ModuleControlRpcOutcome::ModuleGone(format!(
3742                "module '{module_label}' connection closed during module-control RPC"
3743            )));
3744    }
3745
3746    let module_routes = inner
3747        .module_to_client
3748        .iter()
3749        .filter(|(module_key, _)| module_key.endpoint == endpoint)
3750        .map(|(module_key, route)| (*module_key, route.module_epoch))
3751        .collect::<Vec<_>>();
3752    let mut released = Vec::with_capacity(module_routes.len());
3753    for (module_key, epoch) in module_routes {
3754        if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
3755        {
3756            released.push(target);
3757        }
3758    }
3759    ConnectionCleanup {
3760        released,
3761        abandoned_relays,
3762    }
3763}
3764
3765#[derive(Debug, Clone, Copy)]
3766struct RequestCredit {
3767    subscription: bool,
3768    watch_credit: bool,
3769    excluded_from_drain: bool,
3770}
3771
3772#[derive(Debug, Default)]
3773struct CreditLedger {
3774    by_corr: HashMap<u64, Vec<RequestCredit>>,
3775}
3776
3777impl CreditLedger {
3778    fn acquire(&mut self, corr: u64, subscription: bool, watch_credit: bool) {
3779        self.by_corr.entry(corr).or_default().push(RequestCredit {
3780            subscription,
3781            watch_credit,
3782            excluded_from_drain: false,
3783        });
3784    }
3785
3786    fn release(&mut self, corr: u64) -> Option<RequestCredit> {
3787        let credits = self.by_corr.get_mut(&corr)?;
3788        let released = credits.pop();
3789        if credits.is_empty() {
3790            self.by_corr.remove(&corr);
3791        }
3792        released
3793    }
3794
3795    fn capture_subscription_exclusions(&mut self) -> u32 {
3796        let mut excluded = 0u32;
3797        for credit in self.by_corr.values_mut().flatten() {
3798            if credit.subscription && !credit.excluded_from_drain {
3799                credit.excluded_from_drain = true;
3800                excluded = excluded.saturating_add(1);
3801            }
3802        }
3803        excluded
3804    }
3805
3806    fn in_flight(&self) -> usize {
3807        self.by_corr.values().map(Vec::len).sum()
3808    }
3809
3810    fn drain_in_flight(&self) -> usize {
3811        self.by_corr
3812            .values()
3813            .flatten()
3814            .filter(|credit| !credit.excluded_from_drain)
3815            .count()
3816    }
3817
3818    /// The correlation ids of the requests a drain still waits on, one entry
3819    /// per held credit, in ascending order.
3820    fn drain_held_corrs(&self) -> Vec<u64> {
3821        let mut corrs = self
3822            .by_corr
3823            .iter()
3824            .flat_map(|(corr, credits)| {
3825                credits
3826                    .iter()
3827                    .filter(|credit| !credit.excluded_from_drain)
3828                    .map(move |_| *corr)
3829            })
3830            .collect::<Vec<_>>();
3831        corrs.sort_unstable();
3832        corrs
3833    }
3834}
3835
3836#[derive(Debug, Default)]
3837struct ChannelFlowState {
3838    closed: bool,
3839    credits: CreditLedger,
3840    last_busy_warn: Option<tokio::time::Instant>,
3841}
3842
3843/// Per-channel request-credit accounting shared by the client and module route halves.
3844#[derive(Debug)]
3845pub(crate) struct ChannelFlow {
3846    sem: Semaphore,
3847    window: usize,
3848    busy_refusals: bool,
3849    watch_sem: Option<Semaphore>,
3850    watch_window: Option<usize>,
3851    state: Mutex<ChannelFlowState>,
3852}
3853
3854impl ChannelFlow {
3855    pub(crate) fn new(window: usize) -> Self {
3856        Self::with_admission(window, false, None)
3857    }
3858
3859    fn with_admission(window: usize, busy_refusals: bool, watch_window: Option<usize>) -> Self {
3860        debug_assert!(window > 0, "flow-control window must be non-zero");
3861        Self {
3862            sem: Semaphore::new(window),
3863            window,
3864            busy_refusals,
3865            watch_sem: watch_window.map(Semaphore::new),
3866            watch_window,
3867            state: Mutex::new(ChannelFlowState::default()),
3868        }
3869    }
3870
3871    pub(crate) fn busy_refusals(&self) -> bool {
3872        self.busy_refusals
3873    }
3874
3875    fn try_acquire_tagged(&self, corr: u64, subscription: bool) -> RequestAdmission {
3876        let mut state = self
3877            .state
3878            .lock()
3879            .unwrap_or_else(|poisoned| poisoned.into_inner());
3880        if state.closed {
3881            return RequestAdmission::Stale;
3882        }
3883        let busy = |state: &ChannelFlowState| {
3884            RequestAdmission::Busy(subc_protocol::RouteBusyDetail {
3885                reason: "no_credit".into(),
3886                request_in_flight: state.credits.in_flight(),
3887                watch_in_flight: state
3888                    .credits
3889                    .by_corr
3890                    .values()
3891                    .flatten()
3892                    .filter(|credit| credit.subscription)
3893                    .count(),
3894                request_window: self.window,
3895                watch_window: self.watch_window,
3896            })
3897        };
3898        let Ok(total) = self.sem.try_acquire() else {
3899            return busy(&state);
3900        };
3901        // A watch takes the total credit first. Dropping it on watch-lane
3902        // exhaustion rolls back before any ledger entry becomes visible.
3903        let watch = if subscription {
3904            if let Some(sem) = &self.watch_sem {
3905                match sem.try_acquire() {
3906                    Ok(permit) => Some(permit),
3907                    Err(_) => return busy(&state),
3908                }
3909            } else {
3910                None
3911            }
3912        } else {
3913            None
3914        };
3915        state.credits.acquire(corr, subscription, watch.is_some());
3916        total.forget();
3917        if let Some(watch) = watch {
3918            watch.forget();
3919        }
3920        RequestAdmission::Admitted
3921    }
3922
3923    pub(crate) fn should_warn_busy(&self) -> bool {
3924        let mut state = self
3925            .state
3926            .lock()
3927            .unwrap_or_else(|poisoned| poisoned.into_inner());
3928        let now = tokio::time::Instant::now();
3929        if state
3930            .last_busy_warn
3931            .is_some_and(|last| now.duration_since(last) < std::time::Duration::from_secs(60))
3932        {
3933            return false;
3934        }
3935        state.last_busy_warn = Some(now);
3936        true
3937    }
3938
3939    #[cfg(test)]
3940    pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
3941        self.acquire_tagged(0, false).await
3942    }
3943
3944    pub(crate) async fn acquire_tagged(
3945        &self,
3946        corr: u64,
3947        subscription: bool,
3948    ) -> Result<(), ChannelFlowClosed> {
3949        let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
3950        let mut state = self
3951            .state
3952            .lock()
3953            .unwrap_or_else(|poisoned| poisoned.into_inner());
3954        if state.closed {
3955            return Err(ChannelFlowClosed);
3956        }
3957        state.credits.acquire(corr, subscription, false);
3958        permit.forget();
3959        Ok(())
3960    }
3961
3962    #[cfg(test)]
3963    pub(crate) fn release(&self) {
3964        self.release_corr(0);
3965    }
3966
3967    pub(crate) fn release_corr(&self, corr: u64) {
3968        let mut state = self
3969            .state
3970            .lock()
3971            .unwrap_or_else(|poisoned| poisoned.into_inner());
3972        let Some(released) = state.credits.release(corr) else {
3973            // Protocol-conforming modules emit exactly one terminal per request.
3974            // This guard is a best-effort safety net against window growth, not a
3975            // security boundary against malicious peers.
3976            warn!(
3977                window = self.window,
3978                available = self.sem.available_permits(),
3979                "flow-control over-release ignored"
3980            );
3981            return;
3982        };
3983        if !self.sem.is_closed() {
3984            self.sem.add_permits(1);
3985        }
3986        if released.watch_credit {
3987            if let Some(watch_sem) = self.watch_sem.as_ref().filter(|sem| !sem.is_closed()) {
3988                watch_sem.add_permits(1);
3989            }
3990        }
3991    }
3992
3993    #[cfg(test)]
3994    pub(crate) fn in_flight(&self) -> usize {
3995        self.state
3996            .lock()
3997            .unwrap_or_else(|poisoned| poisoned.into_inner())
3998            .credits
3999            .in_flight()
4000    }
4001
4002    pub(crate) fn drain_in_flight(&self) -> usize {
4003        self.state
4004            .lock()
4005            .unwrap_or_else(|poisoned| poisoned.into_inner())
4006            .credits
4007            .drain_in_flight()
4008    }
4009
4010    pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
4011        self.state
4012            .lock()
4013            .unwrap_or_else(|poisoned| poisoned.into_inner())
4014            .credits
4015            .drain_held_corrs()
4016    }
4017
4018    #[cfg(test)]
4019    pub(crate) fn available_permits(&self) -> usize {
4020        self.sem.available_permits()
4021    }
4022
4023    pub(crate) fn begin_drain(&self) -> u32 {
4024        let mut state = self
4025            .state
4026            .lock()
4027            .unwrap_or_else(|poisoned| poisoned.into_inner());
4028        state.closed = true;
4029        self.sem.close();
4030        if let Some(sem) = &self.watch_sem {
4031            sem.close();
4032        }
4033        state.credits.capture_subscription_exclusions()
4034    }
4035
4036    pub(crate) fn close(&self) {
4037        let mut state = self
4038            .state
4039            .lock()
4040            .unwrap_or_else(|poisoned| poisoned.into_inner());
4041        state.closed = true;
4042        self.sem.close();
4043        if let Some(sem) = &self.watch_sem {
4044            sem.close();
4045        }
4046    }
4047}
4048
4049#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4050pub(crate) struct ChannelFlowClosed;
4051
4052#[derive(Debug, Clone, PartialEq, Eq)]
4053pub(crate) enum RequestAdmission {
4054    Admitted,
4055    Busy(subc_protocol::RouteBusyDetail),
4056    ModuleReloading,
4057    Stale,
4058}
4059
4060impl fmt::Display for ChannelFlowClosed {
4061    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4062        write!(f, "flow-control window closed")
4063    }
4064}
4065
4066impl Error for ChannelFlowClosed {}
4067
4068#[cfg(test)]
4069#[path = "route_busy_flow_tests.rs"]
4070mod route_busy_flow_tests;
4071
4072fn window_for(concurrency: &Concurrency) -> usize {
4073    match concurrency {
4074        Concurrency::Serial => 1,
4075        Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
4076        Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
4077    }
4078}
4079
4080#[derive(Debug, Clone, PartialEq, Eq)]
4081pub enum ForwardingError {
4082    /// A connection cannot hold both client routes and a module endpoint, or
4083    /// replace a module identity while that endpoint is still registered.
4084    ConnectionRoleConflict {
4085        connection_id: ConnectionId,
4086    },
4087    NoModuleConnection,
4088    ModuleReloading {
4089        module_id: String,
4090    },
4091    StaleModuleEndpoint,
4092    UnknownReservation {
4093        client_channel: u16,
4094        module_channel: u16,
4095    },
4096    ClientRouteChannelExhausted {
4097        connection_id: ConnectionId,
4098    },
4099    ModuleRouteChannelExhausted {
4100        endpoint: ModuleEndpointId,
4101    },
4102    RelayCorrelationExhausted,
4103    ConnectionClosing {
4104        connection_id: ConnectionId,
4105    },
4106    ClientEgressClosed {
4107        connection_id: ConnectionId,
4108    },
4109    RouteOpenBuild(String),
4110    /// A swap candidate is already registered for this module id.
4111    CandidateSlotOccupied {
4112        module_id: String,
4113    },
4114    /// A registering module's HELLO_ACK could not be queued because its
4115    /// outbound queue is closed or full.
4116    ModuleEgressUnavailable {
4117        connection_id: ConnectionId,
4118    },
4119    Poisoned,
4120}
4121
4122impl fmt::Display for ForwardingError {
4123    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4124        match self {
4125            Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
4126            Self::NoModuleConnection => write!(f, "no module connection is registered"),
4127            Self::ModuleReloading { module_id } => {
4128                write!(f, "module_id '{module_id}' is reloading")
4129            }
4130            Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
4131            Self::UnknownReservation {
4132                client_channel,
4133                module_channel,
4134            } => write!(
4135                f,
4136                "route reservation client channel {client_channel} / module channel {module_channel} was not found"
4137            ),
4138            Self::ClientRouteChannelExhausted { connection_id } => write!(
4139                f,
4140                "no client route channels are available for connection {}",
4141                connection_id.get()
4142            ),
4143            Self::ModuleRouteChannelExhausted { endpoint } => write!(
4144                f,
4145                "no module route channels are available for endpoint generation {} on connection {}",
4146                endpoint.generation,
4147                endpoint.connection_id.get()
4148            ),
4149            Self::RelayCorrelationExhausted => {
4150                write!(f, "module control correlation ids are exhausted")
4151            }
4152            Self::ConnectionClosing { connection_id } => write!(
4153                f,
4154                "connection {} is closing and cannot accept route allocation",
4155                connection_id.get()
4156            ),
4157            Self::ClientEgressClosed { connection_id } => write!(
4158                f,
4159                "client connection {} egress is closed",
4160                connection_id.get()
4161            ),
4162            Self::RouteOpenBuild(message) => {
4163                write!(f, "failed to prebuild route.open response: {message}")
4164            }
4165            Self::CandidateSlotOccupied { module_id } => write!(
4166                f,
4167                "module_id '{module_id}' already has a swap candidate registered"
4168            ),
4169            Self::ModuleEgressUnavailable { connection_id } => write!(
4170                f,
4171                "module connection {} egress is unavailable; HELLO_ACK could not be queued",
4172                connection_id.get()
4173            ),
4174            Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
4175        }
4176    }
4177}
4178
4179impl Error for ForwardingError {}
4180
4181#[cfg(test)]
4182mod tests {
4183    use std::time::Duration;
4184
4185    use super::*;
4186    use tokio::sync::mpsc;
4187
4188    #[test]
4189    fn ordinary_long_running_request_is_not_excluded_from_drain() {
4190        let mut ledger = CreditLedger::default();
4191        ledger.acquire(1, false, false);
4192
4193        assert_eq!(ledger.capture_subscription_exclusions(), 0);
4194        assert_eq!(ledger.drain_in_flight(), 1);
4195    }
4196
4197    #[test]
4198    fn bit_set_subscription_is_excluded_and_counted() {
4199        let mut ledger = CreditLedger::default();
4200        ledger.acquire(1, true, false);
4201
4202        assert_eq!(ledger.capture_subscription_exclusions(), 1);
4203        assert_eq!(ledger.drain_in_flight(), 0);
4204    }
4205
4206    #[test]
4207    fn subscription_opened_after_drain_snapshot_is_not_excluded() {
4208        let mut ledger = CreditLedger::default();
4209        ledger.acquire(1, true, false);
4210        assert_eq!(ledger.capture_subscription_exclusions(), 1);
4211
4212        ledger.acquire(2, true, false);
4213
4214        assert_eq!(ledger.drain_in_flight(), 1);
4215    }
4216
4217    #[test]
4218    fn drain_with_no_subscriptions_reports_zero_excluded() {
4219        let mut ledger = CreditLedger::default();
4220        assert_eq!(ledger.capture_subscription_exclusions(), 0);
4221    }
4222
4223    fn test_hello_ack(corr: u64) -> Frame {
4224        Frame::build(
4225            FrameType::HelloAck,
4226            Flags::new(false, Priority::Passive, false),
4227            0,
4228            0,
4229            corr,
4230            Vec::new(),
4231        )
4232        .unwrap()
4233    }
4234
4235    /// An acked registration whose HELLO_ACK cannot be queued must leave no
4236    /// endpoint behind: a routable module that never got its ack would read a
4237    /// route.bind first and exit. A closed queue and a full one both refuse.
4238    #[test]
4239    fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
4240        let forwarding = ForwardingTable::default();
4241
4242        let (closed_tx, closed_rx) = mpsc::channel(8);
4243        drop(closed_rx);
4244        let closed = ConnectionId::new(1);
4245        assert_eq!(
4246            forwarding.register_module_connection_acked(
4247                closed,
4248                "closed".to_string(),
4249                2,
4250                Concurrency::ModuleManaged,
4251                FrameSink::new(closed_tx),
4252                ModuleHelloAck {
4253                    frame: test_hello_ack(1),
4254                    verified_prefix_owner: None
4255                },
4256            ),
4257            Err(ForwardingError::ModuleEgressUnavailable {
4258                connection_id: closed
4259            })
4260        );
4261
4262        let (full_tx, _full_rx) = mpsc::channel(1);
4263        let full_sink = FrameSink::new(full_tx);
4264        full_sink.try_send(test_hello_ack(99)).unwrap();
4265        let full = ConnectionId::new(2);
4266        assert_eq!(
4267            forwarding.register_module_connection_acked(
4268                full,
4269                "full".to_string(),
4270                2,
4271                Concurrency::ModuleManaged,
4272                full_sink.clone(),
4273                ModuleHelloAck {
4274                    frame: test_hello_ack(2),
4275                    verified_prefix_owner: None
4276                },
4277            ),
4278            Err(ForwardingError::ModuleEgressUnavailable {
4279                connection_id: full
4280            })
4281        );
4282        assert_eq!(
4283            forwarding.register_candidate_module_connection_acked(
4284                full,
4285                "full".to_string(),
4286                2,
4287                Concurrency::ModuleManaged,
4288                full_sink,
4289                test_hello_ack(3),
4290            ),
4291            Err(ForwardingError::ModuleEgressUnavailable {
4292                connection_id: full
4293            })
4294        );
4295
4296        for (connection, module_id) in [(closed, "closed"), (full, "full")] {
4297            assert_eq!(
4298                forwarding
4299                    .module_endpoint_for_connection(connection)
4300                    .unwrap(),
4301                None
4302            );
4303            let (client_tx, _client_rx) = mpsc::channel(8);
4304            assert_eq!(
4305                forwarding
4306                    .begin_route_bind_relay_for_test(
4307                        ConnectionId::new(50),
4308                        FrameSink::new(client_tx),
4309                        1,
4310                        module_id,
4311                    )
4312                    .err(),
4313                Some(ForwardingError::NoModuleConnection)
4314            );
4315        }
4316        assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
4317    }
4318
4319    /// Both acked registration forms put the HELLO_ACK on the module's queue
4320    /// by the time the endpoint can be resolved.
4321    #[test]
4322    fn acked_registration_queues_the_hello_ack_first() {
4323        let forwarding = ForwardingTable::default();
4324        let (active_tx, mut active_rx) = mpsc::channel(8);
4325        forwarding
4326            .register_module_connection_acked(
4327                ConnectionId::new(1),
4328                "acked".to_string(),
4329                2,
4330                Concurrency::ModuleManaged,
4331                FrameSink::new(active_tx),
4332                ModuleHelloAck {
4333                    frame: test_hello_ack(11),
4334                    verified_prefix_owner: None,
4335                },
4336            )
4337            .unwrap();
4338        let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
4339        forwarding
4340            .register_candidate_module_connection_acked(
4341                ConnectionId::new(2),
4342                "acked".to_string(),
4343                2,
4344                Concurrency::ModuleManaged,
4345                FrameSink::new(candidate_tx),
4346                test_hello_ack(12),
4347            )
4348            .unwrap();
4349
4350        let active_first = active_rx.try_recv().unwrap().frame;
4351        assert_eq!(active_first.header.ty, FrameType::HelloAck);
4352        assert_eq!(active_first.header.corr, 11);
4353        let candidate_first = candidate_rx.try_recv().unwrap().frame;
4354        assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
4355        assert_eq!(candidate_first.header.corr, 12);
4356    }
4357
4358    #[test]
4359    fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
4360        let forwarding = ForwardingTable::default();
4361        let module_connection = ConnectionId::new(10);
4362        let exhausted_client = ConnectionId::new(20);
4363        let second_client = ConnectionId::new(30);
4364        let (module_tx, _module_rx) = mpsc::channel(1);
4365        let endpoint = forwarding
4366            .register_module_connection(
4367                module_connection,
4368                "route-limit-provider".to_string(),
4369                1,
4370                Concurrency::ModuleManaged,
4371                FrameSink::new(module_tx),
4372            )
4373            .unwrap();
4374
4375        {
4376            let mut inner = forwarding.inner.write().unwrap();
4377            for channel in 1..=u16::MAX {
4378                inner.reserved_client.insert(
4379                    ClientRouteKey {
4380                        connection_id: exhausted_client,
4381                        channel,
4382                    },
4383                    ModuleRouteKey {
4384                        endpoint,
4385                        channel: 1,
4386                    },
4387                );
4388            }
4389        }
4390
4391        let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
4392        let err = forwarding
4393            .begin_route_bind_relay_for_test(
4394                exhausted_client,
4395                FrameSink::new(exhausted_tx),
4396                1,
4397                "route-limit-provider",
4398            )
4399            .unwrap_err();
4400        assert!(matches!(
4401            err,
4402            ForwardingError::ClientRouteChannelExhausted { connection_id }
4403                if connection_id == exhausted_client
4404        ));
4405
4406        let (second_tx, _second_rx) = mpsc::channel(1);
4407        let pending = forwarding
4408            .begin_route_bind_relay_for_test(
4409                second_client,
4410                FrameSink::new(second_tx),
4411                2,
4412                "route-limit-provider",
4413            )
4414            .unwrap();
4415        assert_eq!(pending.client_channel, 1);
4416    }
4417
4418    #[test]
4419    fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
4420        let forwarding = ForwardingTable::default();
4421        let module_connection = ConnectionId::new(40);
4422        let client = ConnectionId::new(50);
4423        let (module_tx, _module_rx) = mpsc::channel(1);
4424        forwarding
4425            .register_module_connection(
4426                module_connection,
4427                "slot-reuse-provider".to_string(),
4428                1,
4429                Concurrency::ModuleManaged,
4430                FrameSink::new(module_tx),
4431            )
4432            .unwrap();
4433
4434        let (client_tx, _client_rx) = mpsc::channel(1);
4435        let client_sink = FrameSink::new(client_tx);
4436        let mut wrapped_channel = None;
4437        for index in 0..=usize::from(u16::MAX) {
4438            let pending = forwarding
4439                .begin_route_bind_relay_for_test(
4440                    client,
4441                    client_sink.clone(),
4442                    index as u64 + 1,
4443                    "slot-reuse-provider",
4444                )
4445                .unwrap();
4446            if index == usize::from(u16::MAX) {
4447                wrapped_channel = Some(pending.module_channel);
4448            }
4449            forwarding
4450                .abort_pending_relay(
4451                    pending.endpoint,
4452                    pending.corr,
4453                    RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
4454                )
4455                .unwrap();
4456        }
4457
4458        assert_eq!(wrapped_channel, Some(1));
4459    }
4460
4461    #[test]
4462    fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
4463        let forwarding = ForwardingTable::default();
4464        let client = ConnectionId::new(60);
4465        forwarding
4466            .inner
4467            .write()
4468            .unwrap()
4469            .next_client_channel
4470            .insert(client, 41);
4471
4472        let released = forwarding.cleanup_connection(client).unwrap();
4473
4474        assert!(released.is_empty());
4475        assert!(!forwarding
4476            .inner
4477            .read()
4478            .unwrap()
4479            .next_client_channel
4480            .contains_key(&client));
4481    }
4482
4483    #[test]
4484    fn stale_module_cleanup_preserves_fast_reconnect_successor() {
4485        let forwarding = ForwardingTable::default();
4486        let module_id = "fast-reconnect-provider";
4487        let first_connection = ConnectionId::new(70);
4488        let second_connection = ConnectionId::new(80);
4489        let (first_tx, _first_rx) = mpsc::channel(1);
4490        let first_endpoint = forwarding
4491            .register_module_connection(
4492                first_connection,
4493                module_id.to_string(),
4494                1,
4495                Concurrency::ModuleManaged,
4496                FrameSink::new(first_tx),
4497            )
4498            .unwrap();
4499        let (second_tx, _second_rx) = mpsc::channel(1);
4500        let second_endpoint = forwarding
4501            .register_module_connection(
4502                second_connection,
4503                module_id.to_string(),
4504                1,
4505                Concurrency::ModuleManaged,
4506                FrameSink::new(second_tx),
4507            )
4508            .unwrap();
4509        assert_ne!(first_endpoint, second_endpoint);
4510
4511        let released = forwarding.cleanup_connection(first_connection).unwrap();
4512
4513        assert!(released.is_empty());
4514        assert_eq!(
4515            forwarding
4516                .inner
4517                .read()
4518                .unwrap()
4519                .modules_by_id
4520                .get(module_id)
4521                .map(|module| module.endpoint),
4522            Some(second_endpoint)
4523        );
4524        assert!(forwarding.has_live_module_connection(module_id).unwrap());
4525        let control_rpc = forwarding
4526            .begin_module_control_rpc_for(
4527                module_id,
4528                "health.check",
4529                Instant::now() + Duration::from_secs(1),
4530            )
4531            .unwrap();
4532        assert_eq!(control_rpc.endpoint, second_endpoint);
4533    }
4534
4535    fn route_fixture(
4536        module_id: &str,
4537    ) -> (
4538        ForwardingTable,
4539        ConnectionId,
4540        ModuleEndpointId,
4541        ConnectionId,
4542        FrameSink,
4543        mpsc::Receiver<crate::router::OutboundFrame>,
4544    ) {
4545        let forwarding = ForwardingTable::default();
4546        let module_connection = ConnectionId::new(100);
4547        let client_connection = ConnectionId::new(200);
4548        let (module_tx, _module_rx) = mpsc::channel(8);
4549        let endpoint = forwarding
4550            .register_module_connection(
4551                module_connection,
4552                module_id.to_string(),
4553                2,
4554                Concurrency::ModuleManaged,
4555                FrameSink::new(module_tx),
4556            )
4557            .unwrap();
4558        let (client_tx, client_rx) = mpsc::channel(8);
4559        (
4560            forwarding,
4561            module_connection,
4562            endpoint,
4563            client_connection,
4564            FrameSink::new(client_tx),
4565            client_rx,
4566        )
4567    }
4568
4569    #[test]
4570    #[cfg(unix)]
4571    fn daemon_drain_gates_current_and_racing_provider_registrations() {
4572        let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
4573        assert_eq!(
4574            forwarding.begin_daemon_drain().unwrap(),
4575            [("provider".to_string(), endpoint)]
4576        );
4577        assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
4578        assert!(matches!(
4579            forwarding.register_module_connection(
4580                ConnectionId::new(300),
4581                "late-provider".into(),
4582                2,
4583                Concurrency::ModuleManaged,
4584                sink,
4585            ),
4586            Err(ForwardingError::ConnectionClosing { .. })
4587        ));
4588    }
4589
4590    #[test]
4591    #[cfg(unix)]
4592    fn daemon_drain_results_are_endpoint_fenced_and_removed_on_cleanup() {
4593        let (forwarding, connection, endpoint, _, _, _) = route_fixture("provider");
4594        assert_eq!(
4595            forwarding
4596                .daemon_drain_result_for_connection(connection)
4597                .unwrap(),
4598            None
4599        );
4600        forwarding
4601            .record_daemon_drain_result(endpoint, true)
4602            .unwrap();
4603        forwarding.begin_daemon_drain().unwrap();
4604        assert_eq!(
4605            forwarding
4606                .daemon_drain_result_for_connection(connection)
4607                .unwrap(),
4608            Some(false)
4609        );
4610        let replacement = ModuleEndpointId {
4611            generation: endpoint.generation + 1,
4612            ..endpoint
4613        };
4614        forwarding
4615            .record_daemon_drain_result(replacement, true)
4616            .unwrap();
4617        assert_eq!(
4618            forwarding
4619                .daemon_drain_result_for_connection(connection)
4620                .unwrap(),
4621            Some(false)
4622        );
4623        forwarding
4624            .record_daemon_drain_result(endpoint, true)
4625            .unwrap();
4626        assert_eq!(
4627            forwarding
4628                .daemon_drain_result_for_connection(connection)
4629                .unwrap(),
4630            Some(true)
4631        );
4632        assert_eq!(
4633            forwarding
4634                .daemon_drain_result_for_connection(ConnectionId::new(300))
4635                .unwrap(),
4636            Some(false)
4637        );
4638        forwarding.cleanup_connection(connection).unwrap();
4639        forwarding
4640            .record_daemon_drain_result(endpoint, true)
4641            .unwrap();
4642        assert_eq!(
4643            forwarding
4644                .daemon_drain_result_for_connection(connection)
4645                .unwrap(),
4646            Some(false)
4647        );
4648        assert!(forwarding
4649            .read_inner()
4650            .unwrap()
4651            .daemon_drain_results
4652            .is_empty());
4653    }
4654
4655    #[test]
4656    #[cfg(unix)]
4657    fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
4658        let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
4659            route_fixture("provider");
4660        let mut pending =
4661            begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
4662        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
4663        forwarding.begin_daemon_drain().unwrap();
4664        let completion = forwarding
4665            .complete_pending_relay(
4666                module_connection,
4667                pending.corr,
4668                RouteBindRelayOutcome::Accepted,
4669            )
4670            .expect("draining admission is not a fatal module error");
4671        assert!(completion.settled);
4672        let abandoned = completion
4673            .abandoned
4674            .expect("late accepted bind needs a route GOODBYE");
4675        assert_eq!(abandoned.channel, pending.module_channel);
4676        assert!(matches!(pending.receiver.try_recv().unwrap(),
4677            RouteBindRelayOutcome::DaemonRejected(error) if error.code == "module_reloading"));
4678        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4679        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4680        assert!(
4681            rx.try_recv().is_err(),
4682            "no route.open may be published during drain"
4683        );
4684        assert_eq!(
4685            forwarding
4686                .module_endpoint_for_connection(module_connection)
4687                .unwrap(),
4688            Some(endpoint)
4689        );
4690        assert!(
4691            !forwarding
4692                .complete_pending_relay(
4693                    module_connection,
4694                    pending.corr,
4695                    RouteBindRelayOutcome::Accepted,
4696                )
4697                .unwrap()
4698                .settled
4699        );
4700    }
4701
4702    fn test_ping(corr: u64) -> Frame {
4703        Frame::build(
4704            FrameType::Ping,
4705            Flags::new(false, Priority::Passive, false),
4706            0,
4707            0,
4708            corr,
4709            Vec::new(),
4710        )
4711        .unwrap()
4712    }
4713
4714    fn begin_test_route(
4715        forwarding: &ForwardingTable,
4716        client_connection: ConnectionId,
4717        client_sink: FrameSink,
4718        corr: u64,
4719        module_id: &str,
4720    ) -> PendingRouteBindRelay {
4721        forwarding
4722            .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
4723            .unwrap()
4724    }
4725
4726    #[test]
4727    fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
4728        for candidate in [false, true] {
4729            for register_first in [false, true] {
4730                let (forwarding, module_connection, _, client_connection, sink, _rx) =
4731                    route_fixture("target");
4732                let register = || {
4733                    if candidate {
4734                        forwarding.register_candidate_module_connection(
4735                            client_connection,
4736                            "source".into(),
4737                            2,
4738                            Concurrency::ModuleManaged,
4739                            sink.clone(),
4740                        )
4741                    } else {
4742                        forwarding.register_module_connection(
4743                            client_connection,
4744                            "source".into(),
4745                            2,
4746                            Concurrency::ModuleManaged,
4747                            sink.clone(),
4748                        )
4749                    }
4750                };
4751                if register_first {
4752                    register().unwrap();
4753                    assert!(
4754                        forwarding
4755                            .begin_route_bind_relay_for_test(
4756                                client_connection,
4757                                sink.clone(),
4758                                1,
4759                                "target",
4760                            )
4761                            .is_err(),
4762                        "a deferred route.open cannot reserve after HELLO"
4763                    );
4764                    assert!(forwarding
4765                        .cleanup_connection(client_connection)
4766                        .unwrap()
4767                        .is_empty());
4768                } else {
4769                    let pending =
4770                        begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
4771                    assert!(
4772                        register().is_err(),
4773                        "HELLO cannot register after a route reservation"
4774                    );
4775                    forwarding
4776                        .complete_pending_relay(
4777                            module_connection,
4778                            pending.corr,
4779                            RouteBindRelayOutcome::Accepted,
4780                        )
4781                        .unwrap();
4782                    assert_eq!(
4783                        forwarding
4784                            .cleanup_connection(client_connection)
4785                            .unwrap()
4786                            .len(),
4787                        1
4788                    );
4789                }
4790                assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4791                assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4792            }
4793        }
4794    }
4795
4796    /// A pending route.open reserves its queue slot with `reserve_owned` and
4797    /// sends its prebuilt response through that slot when the module accepts.
4798    /// A queue already holding many data frames (far more than the old 64-frame
4799    /// queue) must not stop the open from reserving or completing, and the
4800    /// response must be charged to the queue's byte count when it is sent.
4801    #[tokio::test]
4802    async fn pending_route_open_completes_behind_queued_data_frames() {
4803        assert_eq!(
4804            crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
4805            8,
4806            "the per-connection pending route.open limit is its own constant"
4807        );
4808        let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
4809            route_fixture("open-behind-data");
4810        let (sink, mut client_rx) = crate::server::connection_egress();
4811        const DATA_FRAMES: usize = 1_000;
4812        let data = |corr: u64| {
4813            Frame::build(
4814                FrameType::StreamData,
4815                Flags::new(false, Priority::Interactive, false),
4816                9,
4817                1,
4818                corr,
4819                vec![b'x'; 200],
4820            )
4821            .unwrap()
4822        };
4823        for corr in 0..DATA_FRAMES as u64 {
4824            sink.try_send(data(corr)).unwrap();
4825        }
4826        let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
4827        assert_eq!(sink.backlog().queued_bytes, data_bytes);
4828
4829        let pending = tokio::time::timeout(
4830            Duration::from_secs(5),
4831            forwarding.begin_route_bind_relay_for(
4832                client,
4833                sink.clone(),
4834                subc_protocol::PROTOCOL_VERSION,
4835                4_242,
4836                "open-behind-data",
4837                Principal::Direct,
4838                None,
4839                None,
4840                "fixture".into(),
4841                None,
4842                None,
4843                Instant::now() + Duration::from_secs(60),
4844                false,
4845            ),
4846        )
4847        .await
4848        .expect("reserving the route.open slot must not wait behind data frames")
4849        .unwrap();
4850        forwarding
4851            .complete_pending_relay(
4852                module_connection,
4853                pending.corr,
4854                RouteBindRelayOutcome::Accepted,
4855            )
4856            .unwrap();
4857
4858        let backlog = sink.backlog();
4859        assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
4860        assert!(
4861            backlog.queued_bytes > data_bytes,
4862            "the route.open response must be counted in queued bytes: {backlog:?}"
4863        );
4864        for corr in 0..DATA_FRAMES as u64 {
4865            assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
4866        }
4867        let open = client_rx.recv().await.unwrap();
4868        assert_eq!(open.header.corr, 4_242);
4869        assert_eq!(open.header.ty, FrameType::Response);
4870        drop(open);
4871        assert_eq!(sink.backlog().queued_bytes, 0);
4872        assert_eq!(sink.backlog().queued_frames, 0);
4873    }
4874
4875    /// The drain-timeout line reports these numbers, so they must count the
4876    /// requests the drain is waiting on and no others: a route with nothing in
4877    /// flight is not a holdout, and a flagged subscription is excluded from the
4878    /// drain and so from the count.
4879    #[tokio::test]
4880    async fn drain_holdouts_count_held_requests_and_name_the_connection() {
4881        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4882            route_fixture("holdouts");
4883        let mut bound = |corr| {
4884            let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
4885            forwarding
4886                .complete_pending_relay(
4887                    module_connection,
4888                    route.corr,
4889                    RouteBindRelayOutcome::Accepted,
4890                )
4891                .unwrap();
4892            client_rx.try_recv().unwrap();
4893            match forwarding
4894                .lookup_data_route(client, route.client_channel, route.client_epoch)
4895                .unwrap()
4896            {
4897                DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4898                other => panic!("expected live route, got {other:?}"),
4899            }
4900        };
4901        let holding = bound(61);
4902        let _idle = bound(62);
4903        holding.flow.acquire_tagged(7, false).await.unwrap();
4904        holding.flow.acquire_tagged(2, false).await.unwrap();
4905        holding.flow.acquire_tagged(3, true).await.unwrap();
4906        forwarding
4907            .begin_module_drain("holdouts", RouteCloseReason::Restart)
4908            .unwrap();
4909
4910        let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
4911        assert_eq!(
4912            holdouts,
4913            DrainHoldouts {
4914                requests: 2,
4915                routes: 1,
4916                total_routes: 2,
4917                top_connections: vec![(client.get(), 2)],
4918                // The held requests by the module's channel and corr, ascending,
4919                // without the excluded subscription (corr 3).
4920                held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
4921            }
4922        );
4923    }
4924
4925    #[test]
4926    fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
4927        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4928            route_fixture("census");
4929        let pending = begin_test_route(&forwarding, client, sink, 1, "census");
4930        forwarding
4931            .complete_pending_relay(
4932                module_connection,
4933                pending.corr,
4934                RouteBindRelayOutcome::Accepted,
4935            )
4936            .unwrap();
4937
4938        let routes = forwarding.endpoint_routes(endpoint).unwrap();
4939        assert_eq!(routes.len(), 1);
4940        assert!(matches!(routes[0].principal, Principal::Direct));
4941        assert_eq!(routes[0].goodbye_target.connection_id, client);
4942        assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
4943        assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
4944        assert!(!routes[0].draining);
4945
4946        forwarding
4947            .begin_module_drain("census", RouteCloseReason::Restart)
4948            .unwrap();
4949        let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
4950        assert_eq!(draining_routes.len(), 1);
4951        assert!(draining_routes[0].draining);
4952    }
4953
4954    #[test]
4955    fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
4956        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
4957        let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
4958        assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
4959        forwarding
4960            .abort_pending_relay(
4961                first.endpoint,
4962                first.corr,
4963                RouteBindRelayOutcome::ModuleGone("abort".into()),
4964            )
4965            .unwrap();
4966        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4967        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4968
4969        let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
4970        assert_eq!(second.client_channel, first.client_channel);
4971        assert_eq!(second.module_channel, first.module_channel);
4972        assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
4973    }
4974
4975    #[test]
4976    fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
4977        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4978            route_fixture("epoch-release");
4979        let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
4980        forwarding
4981            .complete_pending_relay(
4982                module_connection,
4983                first.corr,
4984                RouteBindRelayOutcome::Accepted,
4985            )
4986            .unwrap();
4987        assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
4988        assert!(matches!(
4989            forwarding
4990                .release_client_route(client, first.client_channel, first.client_epoch)
4991                .unwrap(),
4992            RouteRelease::Removed(_)
4993        ));
4994        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4995        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4996
4997        let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
4998        forwarding
4999            .complete_pending_relay(
5000                module_connection,
5001                second.corr,
5002                RouteBindRelayOutcome::Accepted,
5003            )
5004            .unwrap();
5005        assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
5006        assert!(matches!(
5007            forwarding
5008                .release_client_route(client, second.client_channel, first.client_epoch)
5009                .unwrap(),
5010            RouteRelease::Stale
5011        ));
5012        assert!(!forwarding
5013            .cache_status(
5014                endpoint,
5015                second.module_channel,
5016                first.module_epoch,
5017                "stale".into(),
5018            )
5019            .unwrap());
5020        assert!(forwarding
5021            .cache_status(
5022                endpoint,
5023                second.module_channel,
5024                second.module_epoch,
5025                "current".into(),
5026            )
5027            .unwrap());
5028        match forwarding
5029            .route_poll_snapshot(client, second.client_channel, second.client_epoch)
5030            .unwrap()
5031        {
5032            RoutePollSnapshot::Bound { status, .. } => {
5033                assert_eq!(status.as_deref(), Some("current"));
5034            }
5035            RoutePollSnapshot::Absent => panic!("successor binding was removed"),
5036        }
5037        let counters = forwarding.counters().snapshot();
5038        assert_eq!(counters["route_released_epoch_fenced"], 1);
5039        assert_eq!(counters["route_release_stale_skipped"], 1);
5040    }
5041
5042    #[test]
5043    fn max_epoch_reservation_retires_only_that_slot() {
5044        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
5045        forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
5046        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
5047        let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
5048        assert_eq!(
5049            (final_use.client_channel, final_use.client_epoch),
5050            (7, u32::MAX)
5051        );
5052        assert_eq!(
5053            (final_use.module_channel, final_use.module_epoch),
5054            (9, u32::MAX)
5055        );
5056        forwarding
5057            .abort_pending_relay(
5058                endpoint,
5059                final_use.corr,
5060                RouteBindRelayOutcome::ModuleGone("abort".into()),
5061            )
5062            .unwrap();
5063        forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
5064        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
5065        let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
5066        assert_ne!(next.client_channel, 7);
5067        assert_ne!(next.module_channel, 9);
5068        assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
5069    }
5070
5071    #[test]
5072    fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
5073        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
5074            route_fixture("corr-shared");
5075        let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
5076        assert_eq!(bind.corr, 1);
5077        forwarding
5078            .abort_pending_relay(
5079                endpoint,
5080                bind.corr,
5081                RouteBindRelayOutcome::ModuleGone("abort".into()),
5082            )
5083            .unwrap();
5084        let rpc = forwarding
5085            .begin_module_control_rpc_for(
5086                "corr-shared",
5087                "health.check",
5088                Instant::now() - Duration::from_millis(1),
5089            )
5090            .unwrap();
5091        assert_eq!(rpc.corr, 2);
5092        assert_eq!(
5093            forwarding
5094                .complete_module_control_rpc(
5095                    module_connection,
5096                    rpc.corr,
5097                    Some("health.check"),
5098                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
5099                        status: subc_protocol::session::HealthStatus::Ok,
5100                        detail: None,
5101                        metrics: None,
5102                    }),
5103                )
5104                .unwrap(),
5105            ModuleControlRpcCompletion::Settled
5106        );
5107        assert!(matches!(
5108            rpc.receiver.blocking_recv().unwrap(),
5109            ModuleControlRpcOutcome::DeadlineElapsed
5110        ));
5111    }
5112
5113    #[tokio::test(start_paused = true)]
5114    async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
5115        let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
5116        let probe_started_at = Instant::now();
5117        let rpc = forwarding
5118            .begin_health_probe_rpc_for(
5119                "tombstone-ttl",
5120                "health.check",
5121                probe_started_at,
5122                probe_started_at + Duration::from_secs(5),
5123            )
5124            .unwrap();
5125        assert!(forwarding
5126            .tombstone_health_probe_rpc(endpoint, rpc.corr)
5127            .unwrap());
5128        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
5129
5130        tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
5131        tokio::task::yield_now().await;
5132
5133        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
5134    }
5135
5136    #[test]
5137    fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
5138        let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
5139        let mut close = forwarding.register_connection_close(endpoint.connection_id);
5140        forwarding.inject_control_corr(endpoint, u64::MAX);
5141        let final_rpc = forwarding
5142            .begin_module_control_rpc_for(
5143                "corr-max",
5144                "health.check",
5145                Instant::now() + Duration::from_secs(1),
5146            )
5147            .unwrap();
5148        assert_eq!(final_rpc.corr, u64::MAX);
5149        forwarding
5150            .cancel_module_control_rpc(endpoint, final_rpc.corr)
5151            .unwrap();
5152        assert!(matches!(
5153            forwarding.begin_module_control_rpc_for(
5154                "corr-max",
5155                "health.check",
5156                Instant::now() + Duration::from_secs(1),
5157            ),
5158            Err(ForwardingError::RelayCorrelationExhausted)
5159        ));
5160        assert!(close.try_recv().is_ok());
5161    }
5162
5163    #[test]
5164    fn publication_epoch_controls_delivery_failure_escalation() {
5165        fn setup_successor(
5166            commit_successor: Option<bool>,
5167        ) -> (ForwardingTable, ConnectionId, u16, u32) {
5168            let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5169                route_fixture("escalation");
5170            let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
5171            forwarding
5172                .complete_pending_relay(
5173                    module_connection,
5174                    first.corr,
5175                    RouteBindRelayOutcome::Accepted,
5176                )
5177                .unwrap();
5178            client_rx.try_recv().unwrap();
5179            assert!(matches!(
5180                forwarding
5181                    .release_client_route(client, first.client_channel, first.client_epoch)
5182                    .unwrap(),
5183                RouteRelease::Removed(_)
5184            ));
5185            if let Some(commit_successor) = commit_successor {
5186                forwarding.inject_client_slot_epoch(
5187                    client,
5188                    first.client_channel,
5189                    first.client_epoch,
5190                );
5191                forwarding.inject_module_slot_epoch(
5192                    endpoint,
5193                    first.module_channel,
5194                    first.module_epoch,
5195                );
5196                let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
5197                if commit_successor {
5198                    forwarding
5199                        .complete_pending_relay(
5200                            module_connection,
5201                            successor.corr,
5202                            RouteBindRelayOutcome::Accepted,
5203                        )
5204                        .unwrap();
5205                    client_rx.try_recv().unwrap();
5206                } else {
5207                    forwarding
5208                        .abort_pending_relay(
5209                            endpoint,
5210                            successor.corr,
5211                            RouteBindRelayOutcome::ModuleGone("abort".into()),
5212                        )
5213                        .unwrap();
5214                }
5215            }
5216            (forwarding, client, first.client_channel, first.client_epoch)
5217        }
5218
5219        let probe_sink = FrameSink::new(mpsc::channel(1).0);
5220        let (no_successor, client, channel, epoch) = setup_successor(None);
5221        let mut close = no_successor.register_connection_close(client);
5222        assert!(no_successor
5223            .escalate_client_delivery_failure(
5224                client,
5225                channel,
5226                epoch,
5227                CloseReason::new("delivery", "failed"),
5228                UndeliveredFrame {
5229                    module_id: None,
5230                    sink: &probe_sink,
5231                },
5232            )
5233            .unwrap());
5234        assert!(close.try_recv().is_ok());
5235
5236        let (aborted, client, channel, epoch) = setup_successor(Some(false));
5237        let mut close = aborted.register_connection_close(client);
5238        assert!(aborted
5239            .escalate_client_delivery_failure(
5240                client,
5241                channel,
5242                epoch,
5243                CloseReason::new("delivery", "failed"),
5244                UndeliveredFrame {
5245                    module_id: None,
5246                    sink: &probe_sink,
5247                },
5248            )
5249            .unwrap());
5250        assert!(close.try_recv().is_ok());
5251
5252        let (published, client, channel, epoch) = setup_successor(Some(true));
5253        let mut close = published.register_connection_close(client);
5254        assert!(!published
5255            .escalate_client_delivery_failure(
5256                client,
5257                channel,
5258                epoch,
5259                CloseReason::new("delivery", "stale failure"),
5260                UndeliveredFrame {
5261                    module_id: None,
5262                    sink: &probe_sink,
5263                },
5264            )
5265            .unwrap());
5266        assert!(close.try_recv().is_err());
5267    }
5268
5269    #[test]
5270    fn route_concentration_separates_client_count_from_routes_per_client() {
5271        // The distinction this asserts is the one a bare connection count cannot
5272        // make: two connections holding one route each and one connection
5273        // holding two are the same total, and have opposite causes.
5274        let (forwarding, module_connection, _, client, sink, _client_rx) =
5275            route_fixture("concentration");
5276        assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
5277
5278        for corr in [70_u64, 71] {
5279            let pending =
5280                begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
5281            forwarding
5282                .complete_pending_relay(
5283                    module_connection,
5284                    pending.corr,
5285                    RouteBindRelayOutcome::Accepted,
5286                )
5287                .unwrap();
5288        }
5289
5290        // One connection, two routes — not two connections with a route each.
5291        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5292        assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
5293    }
5294
5295    #[test]
5296    fn cleanup_and_accepted_resolution_have_one_lock_winner() {
5297        let (forwarding, module_connection, _, client, sink, mut client_rx) =
5298            route_fixture("cleanup-race");
5299        let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
5300        forwarding
5301            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5302            .unwrap();
5303        let released = forwarding.cleanup_connection(client).unwrap();
5304        assert_eq!(released.len(), 1);
5305        let completion = forwarding
5306            .complete_pending_relay(
5307                module_connection,
5308                pending.corr,
5309                RouteBindRelayOutcome::Accepted,
5310            )
5311            .unwrap();
5312        assert!(!completion.settled);
5313        assert!(client_rx.try_recv().is_err());
5314        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5315
5316        let (forwarding, module_connection, _, client, sink, mut client_rx) =
5317            route_fixture("accepted-race");
5318        let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
5319        forwarding
5320            .complete_pending_relay(
5321                module_connection,
5322                pending.corr,
5323                RouteBindRelayOutcome::Accepted,
5324            )
5325            .unwrap();
5326        assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
5327        let released = forwarding.cleanup_connection(client).unwrap();
5328        assert_eq!(released.len(), 1);
5329        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5330    }
5331
5332    #[test]
5333    fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
5334        let (forwarding, module_connection, _, client, sink, mut client_rx) =
5335            route_fixture("drain-gap");
5336        let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
5337        forwarding
5338            .complete_pending_relay(
5339                module_connection,
5340                live.corr,
5341                RouteBindRelayOutcome::Accepted,
5342            )
5343            .unwrap();
5344        client_rx.try_recv().unwrap();
5345        let binding = match forwarding
5346            .lookup_data_route(client, live.client_channel, live.client_epoch)
5347            .unwrap()
5348        {
5349            DataRoute::Client(DataRouteState::Bound(binding)) => binding,
5350            other => panic!("expected live route, got {other:?}"),
5351        };
5352
5353        let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
5354        forwarding
5355            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5356            .unwrap();
5357        let control_rpc = forwarding
5358            .begin_module_control_rpc_for(
5359                "drain-gap",
5360                "health.check",
5361                Instant::now() + Duration::from_secs(1),
5362            )
5363            .unwrap();
5364        let target = forwarding
5365            .begin_module_drain("drain-gap", RouteCloseReason::Reload)
5366            .unwrap()
5367            .unwrap();
5368        assert!(matches!(
5369            control_rpc.receiver.blocking_recv().unwrap(),
5370            ModuleControlRpcOutcome::ModuleGone(_)
5371        ));
5372        assert_eq!(target.abandoned_bindings.len(), 1);
5373        assert!(binding.flow.sem.is_closed());
5374        assert!(
5375            !forwarding
5376                .complete_pending_relay(
5377                    module_connection,
5378                    pending.corr,
5379                    RouteBindRelayOutcome::Accepted,
5380                )
5381                .unwrap()
5382                .settled
5383        );
5384        assert!(matches!(
5385            forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
5386            Err(ForwardingError::ModuleReloading { .. })
5387        ));
5388        let released = forwarding
5389            .release_module_endpoint_routes(target.endpoint)
5390            .unwrap();
5391        assert_eq!(released.len(), 1);
5392        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5393    }
5394
5395    /// A client can be marked closing while its egress is still open: the daemon
5396    /// asks a connection to close (here through the production path, a module
5397    /// frame that its egress refused) and the connection loop tears down a moment
5398    /// later. A route.bind ack that lands inside that window is answered on the
5399    /// MODULE connection's frame handler, so resolving it must not produce an
5400    /// error -- an error there ends the module connection, and that connection
5401    /// carries every other client's routes to the module.
5402    #[test]
5403    fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
5404        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5405            route_fixture("closing-client");
5406
5407        // A published route on this client: escalate_client_delivery_failure only
5408        // marks a connection closing for a route it has already published.
5409        let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
5410        forwarding
5411            .complete_pending_relay(
5412                module_connection,
5413                live.corr,
5414                RouteBindRelayOutcome::Accepted,
5415            )
5416            .unwrap();
5417        client_rx.try_recv().unwrap();
5418
5419        // A second route.open from the same client, relayed and awaiting its ack.
5420        let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
5421        forwarding
5422            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5423            .unwrap();
5424
5425        // The window: closing, but the sink is still open.
5426        assert!(forwarding
5427            .escalate_client_delivery_failure(
5428                client,
5429                live.client_channel,
5430                live.client_epoch,
5431                CloseReason::new(
5432                    "module_to_client_delivery_failed",
5433                    "client egress refused a module frame",
5434                ),
5435                UndeliveredFrame {
5436                    module_id: None,
5437                    sink: &sink,
5438                },
5439            )
5440            .unwrap());
5441        assert!(!sink.is_closed());
5442
5443        let completion = forwarding
5444            .complete_pending_relay(
5445                module_connection,
5446                pending.corr,
5447                RouteBindRelayOutcome::Accepted,
5448            )
5449            .expect("a closing client must not turn a module's ack into an error");
5450
5451        assert!(completion.settled);
5452        let abandoned = completion
5453            .abandoned
5454            .expect("the module must be told to drop the binding it just created");
5455        assert_eq!(abandoned.connection_id, module_connection);
5456        assert_eq!(abandoned.channel, pending.module_channel);
5457        assert_eq!(abandoned.epoch, pending.module_epoch);
5458        assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
5459        assert!(matches!(
5460            pending.receiver.blocking_recv().unwrap(),
5461            RouteBindRelayOutcome::ModuleGone(_)
5462        ));
5463        // No route was published to a client that is on its way out, and the
5464        // reserved handle pair went back.
5465        assert!(client_rx.try_recv().is_err());
5466        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
5467
5468        // The module endpoint is untouched: still live, and still able to take a
5469        // route from another client.
5470        assert!(forwarding
5471            .has_live_module_connection("closing-client")
5472            .unwrap());
5473        let cotenant = ConnectionId::new(201);
5474        let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
5475        let cotenant_route = begin_test_route(
5476            &forwarding,
5477            cotenant,
5478            FrameSink::new(cotenant_tx),
5479            62,
5480            "closing-client",
5481        );
5482        assert_eq!(cotenant_route.endpoint, endpoint);
5483        forwarding
5484            .complete_pending_relay(
5485                module_connection,
5486                cotenant_route.corr,
5487                RouteBindRelayOutcome::Accepted,
5488            )
5489            .unwrap();
5490        assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
5491        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5492    }
5493
5494    #[test]
5495    fn pending_route_permit_is_released_on_rejection_and_abort() {
5496        let forwarding = ForwardingTable::default();
5497        let module_connection = ConnectionId::new(300);
5498        let client = ConnectionId::new(301);
5499        let (module_tx, _module_rx) = mpsc::channel(1);
5500        let endpoint = forwarding
5501            .register_module_connection(
5502                module_connection,
5503                "permit".into(),
5504                2,
5505                Concurrency::ModuleManaged,
5506                FrameSink::new(module_tx),
5507            )
5508            .unwrap();
5509        let (client_tx, mut client_rx) = mpsc::channel(1);
5510        let sink = FrameSink::new(client_tx);
5511        let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
5512        assert!(sink.try_send(test_ping(999)).is_err());
5513        forwarding
5514            .complete_pending_relay(
5515                module_connection,
5516                rejected.corr,
5517                RouteBindRelayOutcome::Rejected(ErrorBody {
5518                    code: "no".into(),
5519                    message: "rejected".into(),
5520                    detail: None,
5521                }),
5522            )
5523            .unwrap();
5524        sink.try_send(test_ping(1000)).unwrap();
5525        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
5526
5527        let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
5528        assert!(sink.try_send(test_ping(1001)).is_err());
5529        forwarding
5530            .abort_pending_relay(
5531                endpoint,
5532                aborted.corr,
5533                RouteBindRelayOutcome::ModuleGone("abort".into()),
5534            )
5535            .unwrap();
5536        sink.try_send(test_ping(1002)).unwrap();
5537        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
5538
5539        let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
5540        forwarding
5541            .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
5542            .unwrap();
5543        drop(client_rx);
5544        let completion = forwarding
5545            .complete_pending_relay(
5546                module_connection,
5547                receiver_closed.corr,
5548                RouteBindRelayOutcome::Accepted,
5549            )
5550            .unwrap();
5551        assert!(completion.abandoned.is_some());
5552        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5553    }
5554
5555    /// Every connection teardown passes through `cleanup_connection`, and
5556    /// connection ids come from a monotonic counter that never hands an id out
5557    /// twice. If the closing mark survives teardown, the set grows by one entry
5558    /// per connection for the life of the daemon -- the self-watchdog alone
5559    /// reconnects once a minute.
5560    #[test]
5561    fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
5562        let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
5563            route_fixture("closing-set-leak");
5564
5565        const CONNECTIONS: u64 = 32;
5566        for index in 0..CONNECTIONS {
5567            let client = ConnectionId::new(1000 + index);
5568            let (client_tx, _client_rx) = mpsc::channel(8);
5569            let route = begin_test_route(
5570                &forwarding,
5571                client,
5572                FrameSink::new(client_tx),
5573                index + 1,
5574                "closing-set-leak",
5575            );
5576            forwarding
5577                .complete_pending_relay(
5578                    module_connection,
5579                    route.corr,
5580                    RouteBindRelayOutcome::Accepted,
5581                )
5582                .unwrap();
5583            forwarding.cleanup_connection(client).unwrap();
5584        }
5585        forwarding.cleanup_connection(module_connection).unwrap();
5586
5587        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5588    }
5589
5590    /// The closing mark exists to refuse new work for a connection that is on
5591    /// its way out but whose teardown has not run yet: the daemon asks the
5592    /// connection loop to end, and only when the loop reacts does
5593    /// `cleanup_connection` strip the connection's state. Inside that window an
5594    /// operation for the dying connection must still be refused; only after
5595    /// cleanup completes may the mark go.
5596    #[test]
5597    fn closing_connection_is_refused_new_work_until_cleanup_completes() {
5598        let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
5599            route_fixture("closing-gate");
5600
5601        // A published route: escalate_client_delivery_failure only marks a
5602        // connection closing for a route it has already published.
5603        let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
5604        forwarding
5605            .complete_pending_relay(
5606                module_connection,
5607                live.corr,
5608                RouteBindRelayOutcome::Accepted,
5609            )
5610            .unwrap();
5611        client_rx.try_recv().unwrap();
5612
5613        // Mark the connection closing through the production path without
5614        // running teardown, pinning the window open.
5615        assert!(forwarding
5616            .escalate_client_delivery_failure(
5617                client,
5618                live.client_channel,
5619                live.client_epoch,
5620                CloseReason::new(
5621                    "module_to_client_delivery_failed",
5622                    "client egress refused a module frame",
5623                ),
5624                UndeliveredFrame {
5625                    module_id: None,
5626                    sink: &sink,
5627                },
5628            )
5629            .unwrap());
5630        assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
5631
5632        // A late route.open for the closing client is refused ...
5633        assert!(matches!(
5634            forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
5635            Err(ForwardingError::ConnectionClosing { connection_id })
5636                if connection_id == client
5637        ));
5638        // ... and so is a late attempt to register the connection as a module.
5639        let (late_tx, _late_rx) = mpsc::channel(1);
5640        assert!(matches!(
5641            forwarding.register_module_connection(
5642                client,
5643                "late-module".into(),
5644                2,
5645                Concurrency::ModuleManaged,
5646                FrameSink::new(late_tx),
5647            ),
5648            Err(ForwardingError::ConnectionClosing { connection_id })
5649                if connection_id == client
5650        ));
5651
5652        // Teardown is the point that lifts the mark: it has just removed every
5653        // per-connection entry under the same lock, so the gate has nothing
5654        // left to protect for this id.
5655        forwarding.cleanup_connection(client).unwrap();
5656        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5657    }
5658}
5659
5660/// Blue/green swap slots: a candidate registered beside the active endpoint,
5661/// promoted by `cutover_candidate`, with the old incumbent drained by endpoint.
5662#[cfg(test)]
5663mod swap_slot_tests {
5664    use std::time::Duration;
5665
5666    use super::*;
5667    use tokio::sync::mpsc;
5668
5669    const MODULE_ID: &str = "swapped";
5670
5671    struct SwapFixture {
5672        forwarding: ForwardingTable,
5673        incumbent_connection: ConnectionId,
5674        incumbent: ModuleEndpointId,
5675        candidate_connection: ConnectionId,
5676        candidate: ModuleEndpointId,
5677        _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
5678    }
5679
5680    fn swap_fixture() -> SwapFixture {
5681        let forwarding = ForwardingTable::default();
5682        let incumbent_connection = ConnectionId::new(100);
5683        let candidate_connection = ConnectionId::new(110);
5684        let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
5685        let incumbent = forwarding
5686            .register_module_connection(
5687                incumbent_connection,
5688                MODULE_ID.to_string(),
5689                2,
5690                Concurrency::ModuleManaged,
5691                FrameSink::new(incumbent_tx),
5692            )
5693            .unwrap();
5694        let (candidate_tx, candidate_rx) = mpsc::channel(8);
5695        let candidate = forwarding
5696            .register_candidate_module_connection(
5697                candidate_connection,
5698                MODULE_ID.to_string(),
5699                2,
5700                Concurrency::ModuleManaged,
5701                FrameSink::new(candidate_tx),
5702            )
5703            .unwrap();
5704        SwapFixture {
5705            forwarding,
5706            incumbent_connection,
5707            incumbent,
5708            candidate_connection,
5709            candidate,
5710            _module_rxs: vec![incumbent_rx, candidate_rx],
5711        }
5712    }
5713
5714    fn client(
5715        raw: u64,
5716    ) -> (
5717        ConnectionId,
5718        FrameSink,
5719        mpsc::Receiver<crate::router::OutboundFrame>,
5720    ) {
5721        let (tx, rx) = mpsc::channel(8);
5722        (ConnectionId::new(raw), FrameSink::new(tx), rx)
5723    }
5724
5725    fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
5726        forwarding
5727            .read_inner()
5728            .unwrap()
5729            .client_to_module
5730            .values()
5731            .map(|route| route.module_endpoint)
5732            .collect()
5733    }
5734
5735    #[test]
5736    fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
5737        let fixture = swap_fixture();
5738        let forwarding = &fixture.forwarding;
5739        assert_ne!(fixture.incumbent, fixture.candidate);
5740
5741        // Every by-id consumer still resolves the incumbent.
5742        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5743        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5744        let (client_connection, client_sink, _client_rx) = client(200);
5745        let pending = forwarding
5746            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5747            .unwrap();
5748        assert_eq!(pending.endpoint, fixture.incumbent);
5749        let rpc = forwarding
5750            .begin_module_control_rpc_for(
5751                MODULE_ID,
5752                "health.check",
5753                Instant::now() + Duration::from_secs(1),
5754            )
5755            .unwrap();
5756        assert_eq!(rpc.endpoint, fixture.incumbent);
5757        let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
5758        assert_eq!(census.len(), 1, "the census lists one endpoint per id");
5759
5760        // Connection-keyed lookups see the candidate, so its own frames resolve.
5761        assert_eq!(
5762            forwarding
5763                .module_endpoint_for_connection(fixture.candidate_connection)
5764                .unwrap(),
5765            Some(fixture.candidate)
5766        );
5767        assert_eq!(
5768            forwarding
5769                .module_id_for_connection(fixture.candidate_connection)
5770                .unwrap()
5771                .as_deref(),
5772            Some(MODULE_ID)
5773        );
5774
5775        // One candidate per id.
5776        let (other_tx, _other_rx) = mpsc::channel(1);
5777        assert_eq!(
5778            forwarding.register_candidate_module_connection(
5779                ConnectionId::new(120),
5780                MODULE_ID.to_string(),
5781                2,
5782                Concurrency::ModuleManaged,
5783                FrameSink::new(other_tx),
5784            ),
5785            Err(ForwardingError::CandidateSlotOccupied {
5786                module_id: MODULE_ID.to_string()
5787            })
5788        );
5789    }
5790
5791    /// The cutover linearization point. A relay reserved on the incumbent before
5792    /// cutover must never become a route on the incumbent, and every relay
5793    /// reserved after it must land on the promoted candidate.
5794    #[test]
5795    fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
5796        let fixture = swap_fixture();
5797        let forwarding = &fixture.forwarding;
5798        let (early_client, early_sink, _early_rx) = client(200);
5799        let mut early = forwarding
5800            .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
5801            .unwrap();
5802        assert_eq!(early.endpoint, fixture.incumbent);
5803        assert_eq!(
5804            forwarding
5805                .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
5806                .unwrap(),
5807            RouteBindRelayMark::Marked
5808        );
5809
5810        let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5811        assert_eq!(
5812            cutover,
5813            ForwardingCutover {
5814                promoted: fixture.candidate,
5815                incumbent: Some(fixture.incumbent),
5816            }
5817        );
5818
5819        // A route.open reserved after cutover goes to the promoted candidate.
5820        let (late_client, late_sink, _late_rx) = client(201);
5821        let late = forwarding
5822            .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
5823            .unwrap();
5824        assert_eq!(
5825            late.endpoint, fixture.candidate,
5826            "a route.open after cutover was reserved on the incumbent"
5827        );
5828
5829        // The incumbent acks the early relay after cutover.
5830        let completion = forwarding
5831            .complete_pending_relay(
5832                fixture.incumbent_connection,
5833                early.corr,
5834                RouteBindRelayOutcome::Accepted,
5835            )
5836            .expect("a superseded endpoint's ack is not an error on its connection");
5837        assert!(completion.settled);
5838        assert!(
5839            !committed_endpoints(forwarding).contains(&fixture.incumbent),
5840            "a relay reserved before cutover committed a route on the incumbent"
5841        );
5842        let goodbye = completion
5843            .abandoned
5844            .expect("the incumbent is told to drop the binding it just created");
5845        assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
5846        assert_eq!(goodbye.channel, early.module_channel);
5847        assert_eq!(goodbye.epoch, early.module_epoch);
5848        assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
5849        match early.receiver.try_recv() {
5850            Ok(RouteBindRelayOutcome::DaemonRejected(body)) => {
5851                assert_eq!(body.code, "module_reloading")
5852            }
5853            other => panic!("expected a retryable module_reloading answer, got {other:?}"),
5854        }
5855        assert!(matches!(
5856            forwarding
5857                .lookup_data_route(early_client, early.client_channel, early.client_epoch)
5858                .unwrap(),
5859            DataRoute::Client(DataRouteState::Absent)
5860        ));
5861
5862        // The late relay commits on the candidate; only its pair stays reserved
5863        // until then.
5864        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
5865        forwarding
5866            .complete_pending_relay(
5867                fixture.candidate_connection,
5868                late.corr,
5869                RouteBindRelayOutcome::Accepted,
5870            )
5871            .unwrap();
5872        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5873        assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
5874    }
5875
5876    #[test]
5877    fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
5878        let fixture = swap_fixture();
5879        let forwarding = &fixture.forwarding;
5880        // One bound route and one in-flight relay on the incumbent.
5881        let (bound_client, bound_sink, _bound_rx) = client(200);
5882        let bound = forwarding
5883            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5884            .unwrap();
5885        forwarding
5886            .complete_pending_relay(
5887                fixture.incumbent_connection,
5888                bound.corr,
5889                RouteBindRelayOutcome::Accepted,
5890            )
5891            .unwrap();
5892        let (pending_client, pending_sink, _pending_rx) = client(201);
5893        let mut in_flight = forwarding
5894            .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
5895            .unwrap();
5896        forwarding
5897            .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
5898            .unwrap();
5899
5900        let incumbent = forwarding
5901            .cutover_candidate(MODULE_ID)
5902            .unwrap()
5903            .unwrap()
5904            .incumbent
5905            .unwrap();
5906        let target = forwarding
5907            .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
5908            .unwrap()
5909            .expect("the superseded incumbent is still registered");
5910
5911        assert_eq!(target.endpoint, fixture.incumbent);
5912        assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
5913        assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
5914        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5915        assert_eq!(target.abandoned_bindings.len(), 1);
5916        assert_eq!(
5917            target.abandoned_bindings[0].channel,
5918            in_flight.module_channel
5919        );
5920        assert!(matches!(
5921            in_flight.receiver.try_recv(),
5922            Ok(RouteBindRelayOutcome::DaemonRejected(body)) if body.code == "module_reloading"
5923        ));
5924        assert_eq!(
5925            forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
5926            1,
5927            "the incumbent's bound route stays until its drain finishes"
5928        );
5929
5930        let (next_client, next_sink, _next_rx) = client(202);
5931        let next = forwarding
5932            .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
5933            .expect("the promoted candidate keeps accepting routes");
5934        assert_eq!(next.endpoint, fixture.candidate);
5935    }
5936
5937    /// An endpoint replaced WITHOUT a promotion (a successor registered over it
5938    /// as an ordinary active HELLO) is stale, not superseded, and its ack still
5939    /// fails the acking connection exactly as it did before swap slots existed:
5940    /// `StaleModuleEndpoint`, the reservation indexes already stripped, and the
5941    /// waiting client's sender dropped unanswered.
5942    #[test]
5943    fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
5944        let forwarding = ForwardingTable::default();
5945        let first_connection = ConnectionId::new(70);
5946        let (first_tx, _first_rx) = mpsc::channel(8);
5947        forwarding
5948            .register_module_connection(
5949                first_connection,
5950                MODULE_ID.to_string(),
5951                2,
5952                Concurrency::ModuleManaged,
5953                FrameSink::new(first_tx),
5954            )
5955            .unwrap();
5956        let (client_connection, client_sink, _client_rx) = client(200);
5957        let mut pending = forwarding
5958            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5959            .unwrap();
5960        let (second_tx, _second_rx) = mpsc::channel(8);
5961        forwarding
5962            .register_module_connection(
5963                ConnectionId::new(80),
5964                MODULE_ID.to_string(),
5965                2,
5966                Concurrency::ModuleManaged,
5967                FrameSink::new(second_tx),
5968            )
5969            .unwrap();
5970
5971        assert_eq!(
5972            forwarding
5973                .complete_pending_relay(
5974                    first_connection,
5975                    pending.corr,
5976                    RouteBindRelayOutcome::Accepted
5977                )
5978                .unwrap_err(),
5979            ForwardingError::StaleModuleEndpoint
5980        );
5981        assert!(committed_endpoints(&forwarding).is_empty());
5982        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5983        assert!(matches!(
5984            pending.receiver.try_recv(),
5985            Err(oneshot::error::TryRecvError::Closed)
5986        ));
5987    }
5988
5989    #[test]
5990    fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
5991        // A candidate whose connection drops leaves the incumbent routable.
5992        let fixture = swap_fixture();
5993        let forwarding = &fixture.forwarding;
5994        assert!(forwarding
5995            .cleanup_connection(fixture.candidate_connection)
5996            .unwrap()
5997            .is_empty());
5998        assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
5999        let (client_connection, client_sink, _client_rx) = client(200);
6000        assert_eq!(
6001            forwarding
6002                .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
6003                .unwrap()
6004                .endpoint,
6005            fixture.incumbent
6006        );
6007
6008        // After a cutover, the incumbent's teardown releases its own routes and
6009        // leaves the promoted candidate in place.
6010        let fixture = swap_fixture();
6011        let forwarding = &fixture.forwarding;
6012        let (bound_client, bound_sink, _bound_rx) = client(200);
6013        let bound = forwarding
6014            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
6015            .unwrap();
6016        forwarding
6017            .complete_pending_relay(
6018                fixture.incumbent_connection,
6019                bound.corr,
6020                RouteBindRelayOutcome::Accepted,
6021            )
6022            .unwrap();
6023        forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
6024        let released = forwarding
6025            .cleanup_connection(fixture.incumbent_connection)
6026            .unwrap();
6027        assert_eq!(released.len(), 1);
6028        assert_eq!(released[0].connection_id, bound_client);
6029        assert!(forwarding
6030            .read_inner()
6031            .unwrap()
6032            .superseded_endpoints
6033            .is_empty());
6034        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
6035        let (next_client, next_sink, _next_rx) = client(201);
6036        assert_eq!(
6037            forwarding
6038                .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
6039                .unwrap()
6040                .endpoint,
6041            fixture.candidate
6042        );
6043    }
6044}