Skip to main content

subc_daemon/
forwarding.rs

1//! Forwarding snapshots and route publication. Spawn/claim publication takes
2//! the spawn transition, claim state/index, then forwarding, in that order.
3//! Bind commits read only the launch watch, never the spawn transition lock.
4//! No forwarding guard is held while awaiting a module.
5//! HELLO publication holds Registry before ForwardingTable::inner so an ACK
6//! cannot be observed before registry publication. Code holding inner must not
7//! take the registry lock, including through a callback.
8
9use std::{
10    collections::{BTreeMap, HashMap, HashSet},
11    error::Error,
12    fmt,
13    sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
14    time::Duration,
15};
16
17use subc_control::{ClientControlResponse, RouteCloseReason};
18use subc_protocol::{
19    manifest::Concurrency,
20    session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
21    ErrorBody, Flags, FrameType, Principal, Priority,
22};
23use tokio::sync::{oneshot, Semaphore};
24use tokio::time::Instant;
25use tracing::{debug, info, warn};
26
27use crate::{
28    control::{RouteBindBreakers, RouteBindConcurrency},
29    observability::DaemonCounters,
30    registry::ConnectionId,
31    router::FrameSink,
32    scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
33    Frame, ProjectRootId,
34};
35
36/// Default per-channel request-credit window for modules that schedule internally.
37const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
38
39/// Watches leave eight request credits available for ordinary calls.
40const DEFAULT_MODULE_MANAGED_WATCH_WINDOW: usize = 24;
41
42/// High per-channel cap for stateless modules; this is an OOM guard, not scheduling policy.
43const STATELESS_PARALLEL_WINDOW: usize = 1024;
44
45/// A stopped probe cycle cannot retain its last unanswered correlation forever.
46/// Active endpoints replace the tombstone on their next serial health probe;
47/// this backstop covers endpoints that stop probing altogether.
48const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
49
50/// Module connection identity used in forwarding keys.
51///
52/// The generation is bumped every time a module connection is registered so a future restart cannot
53/// accidentally reuse a stale `(connection_id, route_channel)` binding.
54#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
55pub struct ModuleEndpointId {
56    pub connection_id: ConnectionId,
57    pub generation: u64,
58}
59
60/// Client-local route key. A route channel is unique only within one client connection.
61#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
62pub(crate) struct ClientRouteKey {
63    pub connection_id: ConnectionId,
64    pub channel: u16,
65}
66
67/// Module-local route key. A route channel is unique only within one live module endpoint.
68#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
69pub(crate) struct ModuleRouteKey {
70    pub endpoint: ModuleEndpointId,
71    pub channel: u16,
72}
73
74#[derive(Debug)]
75pub(crate) struct RouteBinding {
76    pub client_connection_id: ConnectionId,
77    pub client_sink: FrameSink,
78    pub client_negotiated_ver: u8,
79    pub client_channel: u16,
80    pub client_epoch: u32,
81    pub module_id: String,
82    pub module_endpoint: ModuleEndpointId,
83    pub module_sink: FrameSink,
84    pub module_negotiated_ver: u8,
85    pub module_channel: u16,
86    pub module_epoch: u32,
87    pub principal: Principal,
88    /// Opener attributes captured at admission for daemon-side prompt policy.
89    #[allow(dead_code)]
90    pub harness: String,
91    /// Only Callosum's exact production claim is retained; never relayed.
92    #[allow(dead_code)]
93    pub build_class: Option<String>,
94    /// Captured at connect, carried through admission; not standing authority.
95    #[allow(dead_code)]
96    pub operator_app: Option<crate::operator_apps::OperatorAppStamp>,
97    pub project_root: Option<ProjectRootId>,
98    pub bound_at: Instant,
99    pub flow: Arc<ChannelFlow>,
100    /// The scope the route was admitted under, with the tag its bind was
101    /// stamped at. A scope change finds the routes to close by it.
102    pub scope: Option<BoundScope>,
103}
104
105#[derive(Debug, Clone)]
106pub(crate) enum DataRoute {
107    Client(DataRouteState),
108    Module(DataRouteState),
109}
110
111#[derive(Debug, Clone)]
112pub(crate) enum DataRouteState {
113    Bound(Arc<RouteBinding>),
114    Reserved,
115    EpochMismatch,
116    Absent,
117}
118
119/// Which kind of peer a route GOODBYE is being delivered to. This decides what
120/// happens when the GOODBYE cannot be enqueued (egress full/closed):
121/// - `Client`: escalate to closing that client connection (a socket close is a
122///   stronger teardown signal, and a full client egress means it is the slow
123///   client we would drop anyway).
124/// - `Module`: never close; deliver late instead (see
125///   [`send_module_route_goodbye`]), and drop only if the module still has no
126///   room after [`LATE_MODULE_GOODBYE_DEADLINE`]. A client-disconnect notifies the
127///   SHARED module that one client's route is gone; closing the module on its
128///   egress backpressure would tear down every co-tenant client (the exact
129///   cross-tenant blast radius this never-close rule exists to prevent — observed when a
130///   flooding dead client filled BOTH its own and the module's egress, so its
131///   route-gone GOODBYE to the module failed and closed the shared connection).
132///   subc has already removed the route from its forwarding state and drops
133///   stale module frames for the released channel (see router.rs), so subc's
134///   own routing is correct. The residual: under SUSTAINED module-egress
135///   backpressure a module-targeted route-gone notification can be lost, which a
136///   module using it for client-refcounting (e.g. AFT's session accounting)
137///   would miss. This is INTENTIONALLY ACCEPTED, not a gap. A consuming module
138///   must bound stale bindings with its own idle-activity reaper (last-touched
139///   TTL) independent of route-gone signals — AFT does exactly this, so a lost
140///   GOODBYE degrades to "the
141///   binding stays warm until its idle TTL" (bounded wasted resources), never an
142///   unbounded leak; disk-durable replay is unaffected. A dedicated reliable
143///   module control lane was evaluated and deliberately NOT
144///   built: it would add starvation-avoidance machinery to the thin core for a
145///   bounded warm-resource window that is not a correctness issue. never-close
146///   is the invariant that matters here.
147#[derive(Debug, Clone, Copy, PartialEq, Eq)]
148pub(crate) enum GoodbyeTargetKind {
149    Client,
150    Module,
151}
152
153#[derive(Debug, Clone)]
154pub(crate) struct GoodbyeTarget {
155    pub connection_id: ConnectionId,
156    pub sink: FrameSink,
157    pub negotiated_ver: u8,
158    pub channel: u16,
159    pub epoch: u32,
160    pub kind: GoodbyeTargetKind,
161    /// The module on the other end of the route: for a module-targeted relay
162    /// the receiving module (attributing a dropped relay), for a client target
163    /// the module whose route is going away (naming it when an undeliverable
164    /// relay closes the client).
165    pub module_id: Option<String>,
166}
167
168/// The frame a client connection's egress queue refused, for the diagnosis
169/// logged when that refusal closes the connection.
170#[derive(Debug, Clone, Copy)]
171pub(crate) struct UndeliveredFrame<'a> {
172    /// The module on the other end of the route the frame belonged to, when known.
173    pub module_id: Option<&'a str>,
174    /// The refusing connection's sink, read for what its queue held.
175    pub sink: &'a FrameSink,
176}
177
178/// How a route's principal appears in logs: `direct`, or `reserved:<module>`.
179pub(crate) fn principal_label(principal: &Principal) -> String {
180    match principal {
181        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
182        Principal::Direct => "direct".to_string(),
183        other => format!("{other:?}"),
184    }
185}
186
187/// The distinct principals of the routes bound on one client connection,
188/// sorted and comma-joined, or `none` when it has no bound route.
189fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
190    let labels = inner
191        .client_to_module
192        .iter()
193        .filter(|(key, _)| key.connection_id == connection_id)
194        .map(|(_, route)| principal_label(&route.principal))
195        .collect::<std::collections::BTreeSet<_>>();
196    if labels.is_empty() {
197        "none".to_string()
198    } else {
199        labels.into_iter().collect::<Vec<_>>().join(",")
200    }
201}
202
203impl GoodbyeTarget {
204    /// True only when an undeliverable GOODBYE should escalate to closing the
205    /// target connection. Never escalate for module recipients.
206    pub(crate) fn close_on_delivery_failure(&self) -> bool {
207        matches!(self.kind, GoodbyeTargetKind::Client)
208    }
209}
210
211/// How long a route GOODBYE that a module's egress queue refused keeps waiting
212/// for room before it is given up and counted as dropped.
213///
214/// A module that stops reading for a moment (a GC pause, a long synchronous
215/// handler, a reconnect herd it is still working through) should still learn
216/// that the route is gone, because a module that never hears the GOODBYE keeps
217/// the route and keeps sending on it for as long as its connection lives. The
218/// value is the default module drain timeout: the daemon already treats that as
219/// the longest it is reasonable to wait on a module that is busy but alive, and
220/// a module that cannot free 21 bytes of egress in that time is wedged rather
221/// than slow. Resolving each module's own configured drain timeout here would
222/// need a registry lookup on a path that holds only the module's sink.
223pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
224
225/// Send a route GOODBYE to a module without ever closing the module's shared
226/// connection, which would sever every other route it serves.
227///
228/// The frame is enqueued at once when the module's egress queue has room. When
229/// it does not, a detached task waits for room (bounded by
230/// [`LATE_MODULE_GOODBYE_DEADLINE`]) and sends it then. Arriving late is safe:
231/// the daemon has already released the route, and module SDKs tear a route
232/// down only when both the channel AND the epoch match the route installed on
233/// that channel, so a late GOODBYE for a channel since reused at a newer epoch
234/// is ignored. The GOODBYE is counted in `goodbye_relay_module_dropped` only
235/// when the connection is closed or the deadline passes.
236pub(crate) fn send_module_route_goodbye(
237    counters: &DaemonCounters,
238    sink: &FrameSink,
239    frame: Frame,
240    module_id: Option<&str>,
241    context: &'static str,
242) {
243    let channel = frame.header.channel;
244    let epoch = frame.header.epoch;
245    let Err(err) = sink.try_send(frame.clone()) else {
246        return;
247    };
248    // Waiting is pointless once the module's writer is gone, and impossible
249    // outside a Tokio runtime (cleanup that runs from a destructor at shutdown).
250    let runtime = match tokio::runtime::Handle::try_current() {
251        Ok(runtime) if !sink.is_closed() => runtime,
252        _ => {
253            counters.increment_goodbye_relay_module_dropped(module_id);
254            warn!(
255            module_id = module_id.unwrap_or("unknown"),
256            route_channel = channel,
257            route_epoch = epoch,
258            error = %err,
259            context,
260            "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
261            );
262            return;
263        }
264    };
265    debug!(
266        module_id = module_id.unwrap_or("unknown"),
267        route_channel = channel,
268        route_epoch = epoch,
269        error = %err,
270        context,
271        "module egress queue refused route GOODBYE; delivering it once the module frees room"
272    );
273    let counters = counters.clone();
274    let sink = sink.clone();
275    let module_id = module_id.map(str::to_string);
276    runtime.spawn(async move {
277        let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
278        let why = match outcome {
279            Ok(Ok(())) => {
280                debug!(
281                    module_id = module_id.as_deref().unwrap_or("unknown"),
282                    route_channel = channel,
283                    route_epoch = epoch,
284                    context,
285                    "late route GOODBYE delivered to module"
286                );
287                return;
288            }
289            Ok(Err(err)) => err.to_string(),
290            Err(_) => format!(
291                "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
292            ),
293        };
294        counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
295        warn!(
296            module_id = module_id.as_deref().unwrap_or("unknown"),
297            route_channel = channel,
298            route_epoch = epoch,
299            error = %why,
300            context,
301            "route GOODBYE to module dropped under backpressure; not closing shared module connection"
302        );
303    });
304}
305
306/// One route currently served by a module endpoint.
307///
308/// `goodbye_target` is deliberately retained alongside the census projection so
309/// a draining caller can address exactly the same route set that this read
310/// reports, without a second forwarding-table pass.
311#[derive(Debug, Clone)]
312pub(crate) struct EndpointRoute {
313    pub goodbye_target: GoodbyeTarget,
314    pub principal: Principal,
315    pub bound_at: Instant,
316    pub draining: bool,
317    /// WHY the endpoint is draining, when it is. Carried per-route so the
318    /// census can answer "closing because of what" without a second lookup;
319    /// `None` exactly when `draining` is false (one source: the drain map).
320    pub drain_reason: Option<RouteCloseReason>,
321}
322
323/// One live route a scope change closed: both ends to send GOODBYE to, and the
324/// reason for the client's `route.closed`.
325#[derive(Debug, Clone)]
326pub(crate) struct ScopeDrainedRoute {
327    pub reason: RouteCloseReason,
328    pub scope: BoundScope,
329    pub module_id: String,
330    pub client: GoodbyeTarget,
331    pub module: GoodbyeTarget,
332}
333
334#[derive(Debug)]
335pub(crate) struct PendingRouteBindRelay {
336    pub endpoint: ModuleEndpointId,
337    pub module_sink: FrameSink,
338    pub negotiated_ver: u8,
339    pub client_channel: u16,
340    pub client_epoch: u32,
341    pub module_channel: u16,
342    pub module_epoch: u32,
343    pub corr: u64,
344    pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
345}
346
347#[derive(Debug, Clone)]
348pub(crate) struct ModuleDrainTarget {
349    pub endpoint: ModuleEndpointId,
350    pub sink: FrameSink,
351    pub negotiated_ver: u8,
352    pub abandoned_bindings: Vec<GoodbyeTarget>,
353    pub excluded_subscriptions: u32,
354}
355
356/// One registered module connection, with its endpoint-bound output sink.
357#[cfg(unix)]
358#[derive(Debug, Clone)]
359pub(crate) struct ModuleConnectionTarget {
360    pub module_id: String,
361    pub endpoint: ModuleEndpointId,
362    pub sink: FrameSink,
363    pub negotiated_ver: u8,
364}
365
366#[derive(Debug, Clone)]
367pub(crate) enum RouteBindRelayOutcome {
368    Accepted,
369    /// The module refused the bind with its own body. Relayed to the client
370    /// without the daemon-origin bit: the code and message are module-chosen,
371    /// so the frame must not claim daemon authorship, even when the module
372    /// picks a daemon code like `module_reloading`.
373    Rejected(ErrorBody),
374    /// The daemon refused the bind itself — a drain or superseded endpoint
375    /// (`module_reloading`), a reply that arrived after the deadline
376    /// (`module_timeout`), or a scope that ended mid-bind. Carries the
377    /// daemon-origin bit because the daemon wrote it. The module may still
378    /// have received the bind (a late reply, or a drain that began after it
379    /// accepted); the daemon then ends that binding with a GOODBYE, so no
380    /// call runs on it.
381    DaemonRejected(ErrorBody),
382    ModuleGone(String),
383}
384
385/// What [`ForwardingTable::cutover_candidate`] did.
386#[derive(Debug, Clone, Copy, PartialEq, Eq)]
387pub(crate) struct ForwardingCutover {
388    /// The endpoint now in the active slot (the former candidate).
389    pub promoted: ModuleEndpointId,
390    /// The endpoint demoted out of the active slot, to be drained by endpoint.
391    /// `None` when the incumbent's connection was already gone.
392    pub incumbent: Option<ModuleEndpointId>,
393}
394
395/// What tearing down one connection released.
396#[derive(Debug)]
397pub(crate) struct ConnectionCleanup {
398    /// Routes whose other end must be sent GOODBYE.
399    pub released: Vec<GoodbyeTarget>,
400    /// Pending route.bind relays to the closed module that were aborted: opens
401    /// that had not been bound yet. Zero when a client connection closed.
402    pub abandoned_relays: u32,
403}
404
405#[derive(Debug, Clone)]
406pub(crate) struct PendingRelayCompletion {
407    pub settled: bool,
408    pub abandoned: Option<GoodbyeTarget>,
409}
410
411#[derive(Debug)]
412pub(crate) struct PendingModuleControlRpc {
413    pub endpoint: ModuleEndpointId,
414    pub module_sink: FrameSink,
415    pub negotiated_ver: u8,
416    pub corr: u64,
417    pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
418}
419
420#[derive(Debug, Clone)]
421pub(crate) enum ModuleControlRpcOutcome {
422    RawResponse(Vec<u8>),
423    Response(ModuleControlResponse),
424    Rejected(ErrorBody),
425    ModuleGone(String),
426    MalformedResponse(String),
427    UnexpectedOp { expected: String, actual: String },
428    DeadlineElapsed,
429}
430
431#[derive(Debug, Clone, PartialEq, Eq)]
432pub(crate) enum ModuleControlRpcCompletion {
433    Unknown,
434    Settled,
435    LateHealthAnswer {
436        module_id: String,
437        latency: Duration,
438    },
439}
440
441#[derive(Debug)]
442struct PendingModuleControlRpcEntry {
443    expected_op: String,
444    deadline: Instant,
445    health_probe_started_at: Option<Instant>,
446    sender: oneshot::Sender<ModuleControlRpcOutcome>,
447}
448
449#[derive(Debug)]
450struct HealthProbeTombstone {
451    expected_op: String,
452    module_id: String,
453    probe_started_at: Instant,
454    expires_at: Instant,
455}
456
457#[derive(Debug, Clone)]
458struct RouteReservation {
459    client_key: ClientRouteKey,
460    module_key: ModuleRouteKey,
461    client_epoch: u32,
462    module_epoch: u32,
463    project_root: Option<ProjectRootId>,
464}
465
466#[derive(Debug)]
467struct PendingRouteBindRelayEntry {
468    reservation: RouteReservation,
469    client_sink: FrameSink,
470    client_negotiated_ver: u8,
471    client_permit: crate::router::EgressPermit,
472    route_open_frame: Frame,
473    busy_refusals: bool,
474    request_window: usize,
475    watch_window: Option<usize>,
476    principal: Principal,
477    /// The scope admitted at `route.open`, with the tag captured then. Commit
478    /// compares it with the published tag.
479    scope: Option<BoundScope>,
480    harness: String,
481    build_class: Option<String>,
482    operator_app: Option<crate::operator_apps::OperatorAppStamp>,
483    deadline: Instant,
484    /// Set once `route.open` has finished putting the route.bind on the
485    /// module's egress queue. A path that drops this entry WITHOUT a module
486    /// reply (client teardown, module drain, the open's own abort) sends the
487    /// module a GOODBYE for the bind only when this is set, because only then
488    /// is the GOODBYE guaranteed to queue behind the bind. When it is not set,
489    /// the removal is recorded in `relays_released_before_enqueue` instead and
490    /// `route.open` sends that GOODBYE itself once its send has completed.
491    ///
492    /// A module reply does not consult it: an ack proves the module received
493    /// the bind, so the reply path decides about the GOODBYE on its own.
494    relay_enqueued: bool,
495    sender: oneshot::Sender<RouteBindRelayOutcome>,
496}
497
498/// What `mark_route_bind_relay_enqueued` found once `route.open` had put the
499/// route.bind on the module's queue.
500#[derive(Debug, Clone, Copy, PartialEq, Eq)]
501pub(crate) enum RouteBindRelayMark {
502    /// The relay is still pending; later removals handle their own GOODBYE.
503    Marked,
504    /// The module already answered and the reply path settled the relay:
505    /// it committed the route, or it refused it and sent any GOODBYE the
506    /// module needed. `route.open` must send nothing.
507    SettledByReply,
508    /// The relay was dropped without a module reply while the bind was still
509    /// being queued, so nothing has told the module to drop the binding it
510    /// is about to create. `route.open` must send that GOODBYE.
511    ReleasedWithoutReply,
512}
513
514#[derive(Debug, Clone)]
515pub(crate) enum RouteRelease {
516    Removed(GoodbyeTarget),
517    Stale,
518    Absent,
519}
520
521#[derive(Debug, Clone)]
522pub(crate) enum RoutePollSnapshot {
523    Bound {
524        module_id: String,
525        status: Option<String>,
526    },
527    Absent,
528}
529
530#[derive(Debug, Clone)]
531struct ModuleConnection {
532    endpoint: ModuleEndpointId,
533    sink: FrameSink,
534    negotiated_ver: u8,
535    concurrency: Concurrency,
536    // Captured at HELLO, not inferred from the id or a later prefix policy.
537    verified_prefix_owner: Option<String>,
538}
539
540pub(crate) struct ModuleHelloAck {
541    pub(crate) frame: Frame,
542    pub(crate) verified_prefix_owner: Option<String>,
543}
544
545#[derive(Debug, Default)]
546struct ForwardingInner {
547    operator_confirms: Arc<crate::operator_confirm::OperatorConfirms>,
548    daemon_draining: bool,
549    /// Whether forwarding reached quiescence during daemon shutdown, for each
550    /// active endpoint present when shutdown began. A pending wait is false
551    /// because it has not proved quiescence. Endpoint generations prevent a
552    /// result from describing a replacement connection with the same module id.
553    daemon_drain_results: HashMap<ModuleEndpointId, bool>,
554    /// The ACTIVE slot: the one endpoint per module id that routing resolves.
555    /// Every by-id lookup (relay reservation, drain-by-id, liveness, census,
556    /// live roots, module-control RPCs) reads this map and nothing else.
557    modules_by_id: HashMap<String, ModuleConnection>,
558    /// The CANDIDATE slot of a blue/green swap: a second process registered
559    /// under an id that already has an active endpoint. It has a full endpoint
560    /// identity (so its own connection can be looked up and torn down) but no
561    /// by-id lookup sees it, so nothing is routed to it until `cutover_candidate`
562    /// promotes it.
563    candidates_by_id: HashMap<String, ModuleConnection>,
564    /// Former active endpoints demoted by `cutover_candidate`, until their
565    /// connection is removed. Membership is what distinguishes an endpoint that
566    /// was SUPERSEDED by a promotion (still alive, still carrying bound routes,
567    /// about to be drained) from one that is merely STALE (replaced some other
568    /// way, which is treated as a fault on the acking connection).
569    superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
570    endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
571    module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
572    /// Endpoints mid-drain, keyed to the reason the drain was begun with. The
573    /// value serves the census ("draining because restart"); membership alone
574    /// still answers every admission-gate check.
575    draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
576    closing_connections: HashSet<ConnectionId>,
577    /// Pending binds check spawn liveness without keeping the spawn alive.
578    spawn_connections: HashMap<ConnectionId, std::sync::Weak<crate::supervise::RegistrationSpawn>>,
579    next_generation: u64,
580    reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
581    reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
582    next_client_channel: HashMap<ConnectionId, u16>,
583    next_module_channel: HashMap<ModuleEndpointId, u16>,
584    client_slot_epochs: HashMap<ClientRouteKey, u32>,
585    module_slot_epochs: HashMap<ModuleRouteKey, u32>,
586    last_published_epoch: HashMap<ClientRouteKey, u32>,
587    client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
588    module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
589    status: HashMap<(ClientRouteKey, u32), String>,
590    pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
591    /// Pending relays removed without a module reply before `route.open`
592    /// marked them enqueued. `route.open` consumes its key when it marks the
593    /// relay (and sends the GOODBYE the removal could not send safely) or when
594    /// it aborts, so an entry lives only while that open is mid-send.
595    relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
596    /// Each scope's current `(scope_epoch, version)`, keyed `(owner, ref)`,
597    /// published by a scope sync in the same step that changes the record. A
598    /// bind commit reads it here so it never takes the scope table's lock
599    /// while holding this one. Absent means the scope is not live.
600    scope_tags: HashMap<(String, String), ScopeTag>,
601    next_control_corr: HashMap<ModuleEndpointId, u64>,
602    pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
603    health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
604}
605
606#[derive(Debug, Clone)]
607pub(crate) struct CloseReason {
608    code: &'static str,
609    message: String,
610}
611
612impl CloseReason {
613    pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
614        Self {
615            code,
616            message: message.into(),
617        }
618    }
619}
620
621impl fmt::Display for CloseReason {
622    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
623        write!(f, "{}: {}", self.code, self.message)
624    }
625}
626
627pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
628
629/// Dynamic forwarding state shared by the control plane and data-plane router.
630#[derive(Debug, Default)]
631pub struct ForwardingTable {
632    #[cfg(all(test, target_os = "linux"))]
633    route_commit_pause: Mutex<Option<RouteCommitPause>>,
634    #[cfg(all(test, target_os = "linux"))]
635    cleanup_tie_observer: Mutex<Option<oneshot::Sender<(bool, usize)>>>,
636    #[cfg(all(test, target_os = "linux"))]
637    cleanup_pause: Mutex<Option<RouteCommitPause>>,
638    #[cfg(test)]
639    policy_rebind_before_write: Mutex<Option<Arc<std::sync::Barrier>>>,
640    inner: Arc<RwLock<ForwardingInner>>,
641    frame_drop_warnings: Mutex<HashMap<ConnectionId, ChannelDropWarnings>>,
642    #[cfg(feature = "test-support")]
643    route_closing_hold: Mutex<Option<oneshot::Receiver<()>>>,
644    close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
645    counters: DaemonCounters,
646    /// Per-target-module bind-relay breaker state. It lives here, beside the
647    /// module connections it describes, because this is where a module
648    /// connection's identity is established and therefore where a stale
649    /// verdict has to be discarded.
650    route_bind_breakers: RouteBindBreakers,
651    /// Current route.bind relays keyed by target module. Admission is shared
652    /// across every client connection that points at the same endpoint.
653    route_bind_concurrency: RouteBindConcurrency,
654    /// Start and end of each module's `route.open` outage. Held here for the
655    /// same reason as the breakers: every handler built over this table must
656    /// see one outage per module, or each would log its own opening line.
657    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
658}
659
660#[cfg(all(test, target_os = "linux"))]
661#[derive(Debug)]
662struct RouteCommitPause {
663    reached: oneshot::Sender<()>,
664    release: std::sync::mpsc::Receiver<()>,
665}
666
667pub(crate) type PolicyRevocation = (Vec<EndpointRoute>, Vec<GoodbyeTarget>);
668
669#[derive(Debug)]
670struct FrameDropWarning {
671    last_warned: tokio::time::Instant,
672    suppressed: u64,
673}
674
675type ChannelDropWarnings = HashMap<(u16, &'static str), FrameDropWarning>;
676
677impl ForwardingTable {
678    #[cfg(all(test, target_os = "linux"))]
679    pub(crate) fn pause_cleanup_for_test(
680        &self,
681    ) -> (oneshot::Receiver<()>, std::sync::mpsc::Sender<()>) {
682        let (reached, observed) = oneshot::channel();
683        let (release, paused) = std::sync::mpsc::channel();
684        *self.cleanup_pause.lock().unwrap() = Some(RouteCommitPause {
685            reached,
686            release: paused,
687        });
688        (observed, release)
689    }
690    #[cfg(all(test, target_os = "linux"))]
691    pub(crate) fn observe_cleanup_tie_for_test(&self) -> oneshot::Receiver<(bool, usize)> {
692        let (send, receive) = oneshot::channel();
693        *self.cleanup_tie_observer.lock().unwrap() = Some(send);
694        receive
695    }
696    #[cfg(all(test, target_os = "linux"))]
697    pub(crate) fn spawn_connection_recorded_for_test(&self, connection: ConnectionId) -> bool {
698        self.read_inner()
699            .unwrap()
700            .spawn_connections
701            .contains_key(&connection)
702    }
703    #[cfg(all(test, target_os = "linux"))]
704    pub(crate) fn pause_route_commit_for_test(
705        &self,
706    ) -> (oneshot::Receiver<()>, std::sync::mpsc::Sender<()>) {
707        let (reached, observed) = oneshot::channel();
708        let (release, paused) = std::sync::mpsc::channel();
709        *self.route_commit_pause.lock().unwrap() = Some(RouteCommitPause {
710            reached,
711            release: paused,
712        });
713        (observed, release)
714    }
715    pub(crate) fn bind_spawn(
716        &self,
717        connection: ConnectionId,
718        spawn: &Arc<crate::supervise::RegistrationSpawn>,
719    ) -> Result<(), ForwardingError> {
720        self.write_inner()?
721            .spawn_connections
722            .insert(connection, Arc::downgrade(spawn));
723        Ok(())
724    }
725    pub(crate) fn active_module_endpoint(
726        &self,
727        module_id: &str,
728    ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
729        Ok(self
730            .read_inner()?
731            .modules_by_id
732            .get(module_id)
733            .map(|module| module.endpoint))
734    }
735
736    pub(crate) fn config_delivery_sink(
737        &self,
738        module: &str,
739        connection: ConnectionId,
740    ) -> Result<Option<FrameSink>, ForwardingError> {
741        let inner = self.read_inner()?;
742        Ok(inner
743            .modules_by_id
744            .get(module)
745            .filter(|target| {
746                target.endpoint.connection_id == connection
747                    && !inner.closing_connections.contains(&connection)
748            })
749            .map(|target| target.sink.clone()))
750    }
751    /// Keep a silent route diagnosable without letting a broken peer flood logs.
752    /// Epoch and direction describe the dropped frame, but do not split its limit.
753    pub(crate) fn warn_frame_drop(
754        &self,
755        connection_id: ConnectionId,
756        channel: u16,
757        epoch: u32,
758        direction: &'static str,
759        reason: &'static str,
760    ) {
761        let now = tokio::time::Instant::now();
762        let mut warnings = self
763            .frame_drop_warnings
764            .lock()
765            .unwrap_or_else(std::sync::PoisonError::into_inner);
766        let entry = warnings
767            .entry(connection_id)
768            .or_default()
769            .entry((channel, reason));
770        let suppressed = match entry {
771            std::collections::hash_map::Entry::Occupied(mut entry) => {
772                let state = entry.get_mut();
773                if now.saturating_duration_since(state.last_warned) < Duration::from_secs(60) {
774                    state.suppressed = state.suppressed.saturating_add(1);
775                    return;
776                }
777                state.last_warned = now;
778                std::mem::take(&mut state.suppressed)
779            }
780            std::collections::hash_map::Entry::Vacant(entry) => {
781                entry.insert(FrameDropWarning {
782                    last_warned: now,
783                    suppressed: 0,
784                });
785                0
786            }
787        };
788        warn!(
789            connection_id = connection_id.get(),
790            channel, epoch, direction, reason, suppressed, "dropping data-plane frame"
791        );
792    }
793
794    /// Install scripted operator confirmation for an in-process integration harness,
795    /// without bootstrapping a daemon process. Only available with `test-support`.
796    ///
797    /// The script has one response per line: `approve`, `decline`, `unavailable`,
798    /// `hang` (until withdrawn), or `approve when <path>` (wait for the path to
799    /// exist). Unreadable or exhausted scripts fail closed. If supplied, `events`
800    /// receives the provider's JSON-lines event log. `limits: None` uses
801    /// [`OperatorLimits::default`](crate::operator_confirm::OperatorLimits); use
802    /// `Some(OperatorLimits { queue_wait, prompt_timeout, stuck_grace })` to
803    /// override the queue wait, prompt timeout and stuck-provider grace periods.
804    ///
805    /// The current executable's file name **must start with `ckdev-`**. Otherwise
806    /// this logs a warning and keeps the OS provider, just like bootstrap.
807    /// `cortexkit-test-support`'s `ckdev_binary` copies a harness executable under
808    /// a qualifying name; run that copy to enable scripted confirmation. See
809    /// `crates/subc-client-rs/tests/real_daemon/operator_confirm.rs` for the
810    /// reference end-to-end harness.
811    ///
812    /// Configure the table shared by the harness's `ControlHandler` and `Router`
813    /// before serving any requests. This also configures privileged confirmations
814    /// that use the same table.
815    ///
816    /// # Panics
817    ///
818    /// Panics if operator-confirm processing has already started.
819    ///
820    /// # Example
821    ///
822    /// ```no_run
823    /// use std::{path::Path, sync::Arc};
824    /// use subc_daemon::{ControlHandler, ForwardingTable, Registry};
825    ///
826    /// let registry = Arc::new(Registry::default());
827    /// let forwarding = Arc::new(ForwardingTable::default());
828    /// forwarding.configure_operator_script(
829    ///     Path::new("operator.script"),
830    ///     Some(Path::new("operator.events")),
831    ///     None,
832    /// );
833    /// let handler = ControlHandler::with_forwarding(registry, forwarding);
834    /// ```
835    #[cfg(feature = "test-support")]
836    pub fn configure_operator_script(
837        &self,
838        script: &std::path::Path,
839        events: Option<&std::path::Path>,
840        limits: Option<crate::operator_confirm::OperatorLimits>,
841    ) {
842        self.operator_confirms()
843            .configure_script(script, events, limits);
844    }
845
846    /// Hold the next supervisor drain between enqueueing `module.draining` and
847    /// `route.closing`, so a test can forward a response in that window. Dropping
848    /// the returned sender also releases the hold, including on a test panic.
849    #[cfg(feature = "test-support")]
850    pub fn hold_route_closing_for_test(&self) -> oneshot::Sender<()> {
851        let (release, hold) = oneshot::channel();
852        *self.route_closing_hold.lock().unwrap() = Some(hold);
853        release
854    }
855
856    #[cfg(feature = "test-support")]
857    pub(crate) async fn wait_before_route_closing_for_test(&self) {
858        let hold = self.route_closing_hold.lock().unwrap().take();
859        if let Some(hold) = hold {
860            let _ = hold.await;
861        }
862    }
863
864    pub(crate) fn operator_confirms(&self) -> Arc<crate::operator_confirm::OperatorConfirms> {
865        Arc::clone(
866            &self
867                .inner
868                .read()
869                .unwrap_or_else(|p| p.into_inner())
870                .operator_confirms,
871        )
872    }
873
874    #[cfg(test)]
875    pub(crate) fn inject_operator_principal(&self, key: ModuleRouteKey, principal: Principal) {
876        self.alter_operator_binding_for_test(key, |binding| binding.principal = principal);
877    }
878
879    #[cfg(test)]
880    pub(crate) fn alter_operator_binding_for_test(
881        &self,
882        key: ModuleRouteKey,
883        alter: impl FnOnce(&mut RouteBinding),
884    ) {
885        let mut inner = self.write_inner().unwrap();
886        let old = inner.module_to_client.remove(&key).unwrap();
887        let client = ClientRouteKey {
888            connection_id: old.client_connection_id,
889            channel: old.client_channel,
890        };
891        inner.client_to_module.remove(&client);
892        let mut binding = Arc::try_unwrap(old).expect("test binding has no outstanding readers");
893        alter(&mut binding);
894        let binding = Arc::new(binding);
895        inner.client_to_module.insert(client, Arc::clone(&binding));
896        inner.module_to_client.insert(key, binding);
897    }
898
899    /// Mutate the route without delivering its withdrawal notice, so tests can
900    /// exercise the commit-point route check rather than the withdrawal path.
901    #[cfg(test)]
902    pub(crate) fn invalidate_operator_route_for_test(&self, key: ModuleRouteKey, stale: bool) {
903        let mut inner = self.write_inner().unwrap();
904        let old = inner.module_to_client.remove(&key).unwrap();
905        let client = ClientRouteKey {
906            connection_id: old.client_connection_id,
907            channel: old.client_channel,
908        };
909        inner.client_to_module.remove(&client);
910        if stale {
911            let mut binding = Arc::try_unwrap(old).expect("no binding readers at the test pause");
912            binding.module_epoch += 1;
913            let binding = Arc::new(binding);
914            inner.client_to_module.insert(client, binding.clone());
915            inner.module_to_client.insert(key, binding);
916        }
917    }
918
919    /// Keep the binding protected through confirmation admission. Route release
920    /// takes the write lock and then the same confirm-state lock, so it cannot
921    /// miss an admitted request or leave it waiting on a vanished route.
922    pub(crate) fn with_operator_route<T>(
923        &self,
924        connection_id: ConnectionId,
925        channel: u16,
926        epoch: u32,
927        admit: impl FnOnce(Option<&RouteBinding>) -> T,
928    ) -> Result<T, ForwardingError> {
929        self.with_operator_route_epoch(connection_id, channel, |binding| {
930            admit(binding.filter(|binding| binding.module_epoch == epoch))
931        })
932    }
933
934    /// Like admission, but retains a mismatched epoch so a shortcut can audit
935    /// a stale channel separately from a closed one, under the same read lock.
936    pub(crate) fn with_operator_route_epoch<T>(
937        &self,
938        connection_id: ConnectionId,
939        channel: u16,
940        decide: impl FnOnce(Option<&RouteBinding>) -> T,
941    ) -> Result<T, ForwardingError> {
942        let inner = self.read_inner()?;
943        let binding = inner
944            .endpoint_by_connection
945            .get(&connection_id)
946            .and_then(|endpoint| {
947                inner.module_to_client.get(&ModuleRouteKey {
948                    endpoint: *endpoint,
949                    channel,
950                })
951            });
952        Ok(decide(binding.map(Arc::as_ref)))
953    }
954
955    pub(crate) fn counters(&self) -> DaemonCounters {
956        self.counters.clone()
957    }
958
959    pub(crate) fn counters_ref(&self) -> &DaemonCounters {
960        &self.counters
961    }
962
963    pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
964        self.route_bind_breakers.clone()
965    }
966
967    pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
968        self.route_bind_concurrency.clone()
969    }
970
971    pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
972        Arc::clone(&self.route_outages)
973    }
974
975    pub(crate) fn register_connection_close(
976        &self,
977        connection_id: ConnectionId,
978    ) -> ConnectionCloseReceiver {
979        let (sender, receiver) = oneshot::channel();
980        let replaced = self
981            .lock_close_registry()
982            .insert(connection_id, sender)
983            .is_some();
984        if replaced {
985            warn!(
986                connection_id = connection_id.get(),
987                "replaced existing connection close registration"
988            );
989        }
990        receiver
991    }
992
993    pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
994        self.lock_close_registry().remove(&connection_id);
995        // The read loop and its tasks have ended and connection cleanup has
996        // removed its routes and pending relays. The writer may still drain.
997        // A channel-0 GOODBYE preserves the connection-to-launch association
998        // until that cleanup.
999        if let Ok(mut inner) = self.write_inner() {
1000            inner.spawn_connections.remove(&connection_id);
1001        }
1002    }
1003
1004    /// Close every established connection, modules and clients alike, as the
1005    /// last step of an announced daemon shutdown. By the time this runs each
1006    /// registered module has already been sent a module GOODBYE (see
1007    /// `Supervisor::end_children_for_daemon_shutdown`), so the EOF that follows
1008    /// is a planned stop; EOF with no GOODBYE before it stays reserved for a
1009    /// daemon that went away unannounced. Closing while the daemon is still
1010    /// alive lets it wait for the modules' own teardowns before it ends
1011    /// whatever is left.
1012    #[cfg(unix)]
1013    pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
1014        let senders: Vec<_> = self.lock_close_registry().drain().collect();
1015        let count = senders.len();
1016        for (_, sender) in senders {
1017            let _ = sender.send(reason.clone());
1018        }
1019        count
1020    }
1021
1022    /// Every registered module connection, whichever slot it occupies (active,
1023    /// swap candidate, or superseded incumbent), once each. Daemon shutdown
1024    /// uses this to tell each of them it is a planned stop before closing it.
1025    #[cfg(unix)]
1026    pub(crate) fn module_connections(
1027        &self,
1028    ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
1029        let inner = self.read_inner()?;
1030        let mut seen = HashSet::new();
1031        Ok(inner
1032            .modules_by_id
1033            .values()
1034            .chain(inner.candidates_by_id.values())
1035            .chain(inner.superseded_endpoints.values())
1036            .filter(|module| seen.insert(module.endpoint))
1037            .map(|module| ModuleConnectionTarget {
1038                module_id: inner
1039                    .module_id_by_endpoint
1040                    .get(&module.endpoint)
1041                    .cloned()
1042                    .unwrap_or_default(),
1043                endpoint: module.endpoint,
1044                sink: module.sink.clone(),
1045                negotiated_ver: module.negotiated_ver,
1046            })
1047            .collect())
1048    }
1049
1050    /// Ask a registered connection to close. Returns true only for the request
1051    /// that actually reached it; later requests for the same connection, and
1052    /// requests for a connection that is not registered, return false.
1053    pub(crate) fn request_connection_close(
1054        &self,
1055        connection_id: ConnectionId,
1056        reason: CloseReason,
1057    ) -> bool {
1058        let sender = self.lock_close_registry().remove(&connection_id);
1059        if let Some(sender) = sender {
1060            debug!(
1061                connection_id = connection_id.get(),
1062                close_reason = %reason,
1063                "requesting connection close"
1064            );
1065            let _ = sender.send(reason);
1066            true
1067        } else {
1068            debug!(
1069                connection_id = connection_id.get(),
1070                close_reason = %reason,
1071                "connection close request ignored for inactive connection"
1072            );
1073            false
1074        }
1075    }
1076
1077    pub fn register_module_connection(
1078        &self,
1079        connection_id: ConnectionId,
1080        module_id: String,
1081        negotiated_ver: u8,
1082        concurrency: Concurrency,
1083        sink: FrameSink,
1084    ) -> Result<ModuleEndpointId, ForwardingError> {
1085        self.register_module_connection_inner(
1086            connection_id,
1087            module_id,
1088            negotiated_ver,
1089            concurrency,
1090            sink,
1091            None,
1092        )
1093    }
1094
1095    /// Register a module connection and queue its HELLO_ACK as the first frame
1096    /// on its sink, in the same write-lock critical section that makes the
1097    /// endpoint visible.
1098    ///
1099    /// A module reads HELLO_ACK as the first frame after its HELLO and exits on
1100    /// anything else. Once the endpoint is in `modules_by_id`, a `route.open`
1101    /// on any other connection can queue a `route.bind` request onto this sink,
1102    /// so the ack must already be queued by then. Every lookup that can queue a
1103    /// frame for the module takes this lock, so nothing can get in ahead of it.
1104    ///
1105    /// If the ack cannot be queued (the sink is closed or full), the
1106    /// registration fails with nothing inserted.
1107    pub(crate) fn register_module_connection_acked(
1108        &self,
1109        connection_id: ConnectionId,
1110        module_id: String,
1111        negotiated_ver: u8,
1112        concurrency: Concurrency,
1113        sink: FrameSink,
1114        hello_ack: ModuleHelloAck,
1115    ) -> Result<ModuleEndpointId, ForwardingError> {
1116        self.register_module_connection_inner(
1117            connection_id,
1118            module_id,
1119            negotiated_ver,
1120            concurrency,
1121            sink,
1122            Some(hello_ack),
1123        )
1124    }
1125
1126    fn register_module_connection_inner(
1127        &self,
1128        connection_id: ConnectionId,
1129        module_id: String,
1130        negotiated_ver: u8,
1131        concurrency: Concurrency,
1132        sink: FrameSink,
1133        hello_ack: Option<ModuleHelloAck>,
1134    ) -> Result<ModuleEndpointId, ForwardingError> {
1135        let mut inner = self.write_inner()?;
1136        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1137            return Err(ForwardingError::ConnectionClosing { connection_id });
1138        }
1139        check_module_connection_role_locked(&inner, connection_id)?;
1140        // Every refusal check has passed and nothing has been mutated yet, so
1141        // a failed enqueue leaves the table exactly as it was.
1142        let verified_prefix_owner = hello_ack
1143            .as_ref()
1144            .and_then(|hello| hello.verified_prefix_owner.clone());
1145        enqueue_hello_ack_locked(&sink, connection_id, hello_ack.map(|hello| hello.frame))?;
1146
1147        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1148        let endpoint = ModuleEndpointId {
1149            connection_id,
1150            generation: inner.next_generation,
1151        };
1152        inner.endpoint_by_connection.insert(connection_id, endpoint);
1153        inner
1154            .module_id_by_endpoint
1155            .insert(endpoint, module_id.clone());
1156        inner.next_module_channel.insert(endpoint, 1);
1157        inner.next_control_corr.insert(endpoint, 1);
1158        inner.modules_by_id.insert(
1159            module_id.clone(),
1160            ModuleConnection {
1161                endpoint,
1162                sink,
1163                negotiated_ver,
1164                concurrency,
1165                verified_prefix_owner,
1166            },
1167        );
1168        drop(inner);
1169
1170        // A new module connection has arrived under this id, so anything the
1171        // bind-relay breaker learned was learned about a process that is no
1172        // longer the one behind this name. See
1173        // `RouteBindBreakers::reset_for_new_module_connection`.
1174        //
1175        // Keyed on ARRIVAL rather than on teardown deliberately: a connection
1176        // going away is not evidence about anything, and a module whose
1177        // connection drops without coming back should keep its verdict until
1178        // something actually registers in its place. This also covers the
1179        // unclean replacements -- a module killed mid-bind, or one whose
1180        // connection was closing -- because registration is the single path by
1181        // which any module connection becomes usable.
1182        if let Some(discarded) = self
1183            .route_bind_breakers
1184            .reset_for_new_module_connection(&module_id)
1185        {
1186            info!(
1187                module_id = %module_id,
1188                discarded_consecutive_timeouts = discarded,
1189                "route.bind breaker state discarded: a new module connection replaced the process it described"
1190            );
1191        }
1192        Ok(endpoint)
1193    }
1194
1195    /// Register a blue/green swap candidate for `module_id` into the candidate
1196    /// slot, alongside whatever endpoint is active for the id.
1197    ///
1198    /// Unlike [`Self::register_module_connection`], this never touches the
1199    /// active slot and never resets the bind-relay breaker: the incumbent is
1200    /// still the process serving the id, so what the breaker learned about it
1201    /// is still true. The breaker is reset at cutover instead, when the process
1202    /// behind the name actually changes. A second candidate for the same id is
1203    /// refused.
1204    ///
1205    /// Production registers candidates through
1206    /// [`Self::register_candidate_module_connection_acked`]; this ack-less form
1207    /// is for tests that build forwarding state directly.
1208    #[cfg(test)]
1209    pub(crate) fn register_candidate_module_connection(
1210        &self,
1211        connection_id: ConnectionId,
1212        module_id: String,
1213        negotiated_ver: u8,
1214        concurrency: Concurrency,
1215        sink: FrameSink,
1216    ) -> Result<ModuleEndpointId, ForwardingError> {
1217        self.register_candidate_module_connection_inner(
1218            connection_id,
1219            module_id,
1220            negotiated_ver,
1221            concurrency,
1222            sink,
1223            None,
1224        )
1225    }
1226
1227    /// The swap-candidate counterpart of
1228    /// [`Self::register_module_connection_acked`]: the HELLO_ACK is queued on
1229    /// the candidate's sink before its endpoint is inserted. No by-id lookup
1230    /// sees a candidate, but its own connection's endpoint does become
1231    /// resolvable here, and at cutover it becomes routable; the ack has to be
1232    /// ahead of anything either can queue.
1233    pub(crate) fn register_candidate_module_connection_acked(
1234        &self,
1235        connection_id: ConnectionId,
1236        module_id: String,
1237        negotiated_ver: u8,
1238        concurrency: Concurrency,
1239        sink: FrameSink,
1240        hello_ack: Frame,
1241    ) -> Result<ModuleEndpointId, ForwardingError> {
1242        self.register_candidate_module_connection_inner(
1243            connection_id,
1244            module_id,
1245            negotiated_ver,
1246            concurrency,
1247            sink,
1248            Some(hello_ack),
1249        )
1250    }
1251
1252    fn register_candidate_module_connection_inner(
1253        &self,
1254        connection_id: ConnectionId,
1255        module_id: String,
1256        negotiated_ver: u8,
1257        concurrency: Concurrency,
1258        sink: FrameSink,
1259        hello_ack: Option<Frame>,
1260    ) -> Result<ModuleEndpointId, ForwardingError> {
1261        let mut inner = self.write_inner()?;
1262        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1263            return Err(ForwardingError::ConnectionClosing { connection_id });
1264        }
1265        check_module_connection_role_locked(&inner, connection_id)?;
1266        if inner.candidates_by_id.contains_key(&module_id) {
1267            return Err(ForwardingError::CandidateSlotOccupied { module_id });
1268        }
1269        // As in `register_module_connection_inner`: after every refusal check,
1270        // before any mutation.
1271        enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1272
1273        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1274        let endpoint = ModuleEndpointId {
1275            connection_id,
1276            generation: inner.next_generation,
1277        };
1278        inner.endpoint_by_connection.insert(connection_id, endpoint);
1279        inner
1280            .module_id_by_endpoint
1281            .insert(endpoint, module_id.clone());
1282        inner.next_module_channel.insert(endpoint, 1);
1283        inner.next_control_corr.insert(endpoint, 1);
1284        inner.candidates_by_id.insert(
1285            module_id,
1286            ModuleConnection {
1287                endpoint,
1288                sink,
1289                negotiated_ver,
1290                concurrency,
1291                verified_prefix_owner: None,
1292            },
1293        );
1294        Ok(endpoint)
1295    }
1296
1297    /// Promote `module_id`'s candidate to the active slot, in ONE forwarding
1298    /// write-lock critical section.
1299    ///
1300    /// That single section is the linearization point of a swap. Every relay
1301    /// reservation resolves the active slot under the same lock, so each one
1302    /// lands wholly before cutover (on the incumbent) or wholly after it (on the
1303    /// promoted candidate), never on a mix. A relay reserved on the incumbent
1304    /// before cutover can no longer commit: `commit_route_locked` requires the
1305    /// reservation's endpoint to be the active one, and an ack for it arriving
1306    /// later is answered by the superseded-endpoint arm of
1307    /// `complete_pending_relay` instead of ending the incumbent's connection.
1308    ///
1309    /// Both endpoints keep their identities: nothing is re-keyed, so the
1310    /// incumbent's bound routes and its pending correlation keys stay exactly
1311    /// where they are until it is drained with [`Self::begin_endpoint_drain`],
1312    /// using the incumbent endpoint this returns. Returns `Ok(None)` when there
1313    /// is no candidate for the id.
1314    pub(crate) fn cutover_candidate(
1315        &self,
1316        module_id: &str,
1317    ) -> Result<Option<ForwardingCutover>, ForwardingError> {
1318        let mut inner = self.write_inner()?;
1319        if inner.daemon_draining {
1320            return Err(ForwardingError::ModuleReloading {
1321                module_id: module_id.to_string(),
1322            });
1323        }
1324        let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
1325            return Ok(None);
1326        };
1327        let promoted = candidate.endpoint;
1328        let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
1329        let incumbent = incumbent.map(|incumbent| {
1330            let endpoint = incumbent.endpoint;
1331            inner.superseded_endpoints.insert(endpoint, incumbent);
1332            endpoint
1333        });
1334        drop(inner);
1335
1336        // A different process now answers for this id; see the matching reset
1337        // in `register_module_connection` for why a verdict about the old one
1338        // must not carry over.
1339        if let Some(discarded) = self
1340            .route_bind_breakers
1341            .reset_for_new_module_connection(module_id)
1342        {
1343            info!(
1344                module_id = %module_id,
1345                discarded_consecutive_timeouts = discarded,
1346                "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
1347            );
1348        }
1349        Ok(Some(ForwardingCutover {
1350            promoted,
1351            incumbent,
1352        }))
1353    }
1354
1355    #[allow(clippy::too_many_arguments)]
1356    pub(crate) async fn begin_route_bind_relay_for(
1357        &self,
1358        client_connection_id: ConnectionId,
1359        client_sink: FrameSink,
1360        client_negotiated_ver: u8,
1361        client_corr: u64,
1362        module_id: &str,
1363        principal: Principal,
1364        scope: Option<BoundScope>,
1365        project_root: Option<ProjectRootId>,
1366        harness: String,
1367        build_class: Option<String>,
1368        operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1369        deadline: Instant,
1370        busy_refusals: bool,
1371    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1372        // Reserve egress capacity before taking the forwarding lock. The permit is
1373        // held until the bind reaches one terminal state, so an accepted bind can
1374        // publish its table entry and RouteOpen response in one critical section.
1375        let client_permit =
1376            client_sink
1377                .reserve_owned()
1378                .await
1379                .map_err(|_| ForwardingError::ClientEgressClosed {
1380                    connection_id: client_connection_id,
1381                })?;
1382        self.begin_route_bind_relay_inner(
1383            client_connection_id,
1384            client_sink,
1385            client_negotiated_ver,
1386            client_corr,
1387            module_id,
1388            principal,
1389            scope,
1390            project_root,
1391            harness,
1392            build_class,
1393            operator_app,
1394            deadline,
1395            client_permit,
1396            busy_refusals,
1397        )
1398    }
1399
1400    #[cfg(test)]
1401    pub(crate) fn begin_route_bind_relay_for_test(
1402        &self,
1403        client_connection_id: ConnectionId,
1404        client_sink: FrameSink,
1405        client_corr: u64,
1406        module_id: &str,
1407    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1408        self.begin_route_bind_relay_with_busy_for_test(
1409            client_connection_id,
1410            client_sink,
1411            client_corr,
1412            module_id,
1413            false,
1414        )
1415    }
1416
1417    #[cfg(test)]
1418    pub(crate) fn begin_route_bind_relay_with_busy_for_test(
1419        &self,
1420        client_connection_id: ConnectionId,
1421        client_sink: FrameSink,
1422        client_corr: u64,
1423        module_id: &str,
1424        busy_refusals: bool,
1425    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1426        let permit =
1427            client_sink
1428                .try_reserve_owned()
1429                .map_err(|_| ForwardingError::ClientEgressClosed {
1430                    connection_id: client_connection_id,
1431                })?;
1432        self.begin_route_bind_relay_inner(
1433            client_connection_id,
1434            client_sink,
1435            subc_protocol::PROTOCOL_VERSION,
1436            client_corr,
1437            module_id,
1438            Principal::Direct,
1439            None,
1440            None,
1441            String::new(),
1442            None,
1443            None,
1444            Instant::now() + std::time::Duration::from_secs(60),
1445            permit,
1446            busy_refusals,
1447        )
1448    }
1449
1450    pub(crate) fn begin_module_control_rpc_for(
1451        &self,
1452        module_id: &str,
1453        expected_op: &str,
1454        deadline: Instant,
1455    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1456        self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1457    }
1458
1459    pub(crate) fn begin_health_probe_rpc_for(
1460        &self,
1461        module_id: &str,
1462        expected_op: &str,
1463        probe_started_at: Instant,
1464        deadline: Instant,
1465    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1466        self.begin_module_control_rpc_inner(
1467            module_id,
1468            expected_op,
1469            deadline,
1470            Some(probe_started_at),
1471            false,
1472        )
1473    }
1474
1475    pub(crate) fn begin_drain_health_probe_rpc_for(
1476        &self,
1477        module_id: &str,
1478        expected_op: &str,
1479        probe_started_at: Instant,
1480        deadline: Instant,
1481    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1482        self.begin_module_control_rpc_inner(
1483            module_id,
1484            expected_op,
1485            deadline,
1486            Some(probe_started_at),
1487            true,
1488        )
1489    }
1490
1491    /// A health probe addressed to one endpoint, in whichever slot it is.
1492    ///
1493    /// Every other control RPC resolves the id's ACTIVE endpoint, which is how
1494    /// a swap candidate (not active until cutover) and a superseded incumbent
1495    /// (not active after it) are unreachable by them. A swap needs to probe
1496    /// exactly those two: the candidate before promoting it, and the incumbent
1497    /// for its busy gauges while it drains.
1498    pub(crate) fn begin_endpoint_health_probe_rpc_for(
1499        &self,
1500        endpoint: ModuleEndpointId,
1501        expected_op: &str,
1502        probe_started_at: Instant,
1503        deadline: Instant,
1504    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1505        let inner = self.write_inner()?;
1506        let module = module_connection_for_endpoint_locked(&inner, endpoint)
1507            .cloned()
1508            .ok_or(ForwardingError::NoModuleConnection)?;
1509        let module_id = inner
1510            .module_id_by_endpoint
1511            .get(&endpoint)
1512            .cloned()
1513            .unwrap_or_default();
1514        // A draining incumbent is exactly what this probes for busy gauges, so
1515        // draining is allowed, as it is for the by-id drain probe.
1516        self.begin_control_rpc_locked(
1517            inner,
1518            &module_id,
1519            module,
1520            expected_op,
1521            deadline,
1522            Some(probe_started_at),
1523            true,
1524        )
1525    }
1526
1527    fn begin_module_control_rpc_inner(
1528        &self,
1529        module_id: &str,
1530        expected_op: &str,
1531        deadline: Instant,
1532        health_probe_started_at: Option<Instant>,
1533        allow_draining: bool,
1534    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1535        let inner = self.write_inner()?;
1536        let module = inner
1537            .modules_by_id
1538            .get(module_id)
1539            .cloned()
1540            .ok_or(ForwardingError::NoModuleConnection)?;
1541        self.begin_control_rpc_locked(
1542            inner,
1543            module_id,
1544            module,
1545            expected_op,
1546            deadline,
1547            health_probe_started_at,
1548            allow_draining,
1549        )
1550    }
1551
1552    #[allow(clippy::too_many_arguments)]
1553    fn begin_control_rpc_locked(
1554        &self,
1555        mut inner: RwLockWriteGuard<'_, ForwardingInner>,
1556        module_id: &str,
1557        module: ModuleConnection,
1558        expected_op: &str,
1559        deadline: Instant,
1560        health_probe_started_at: Option<Instant>,
1561        allow_draining: bool,
1562    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1563        if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1564            return Err(ForwardingError::ModuleReloading {
1565                module_id: module_id.to_string(),
1566            });
1567        }
1568        if inner
1569            .closing_connections
1570            .contains(&module.endpoint.connection_id)
1571        {
1572            return Err(ForwardingError::ConnectionClosing {
1573                connection_id: module.endpoint.connection_id,
1574            });
1575        }
1576        if health_probe_started_at.is_some() {
1577            // Recurring health probes are serial per endpoint. Once the next one
1578            // starts, an older answer can no longer improve the current snapshot,
1579            // so retaining more than the newest unanswered probe has no value.
1580            inner
1581                .health_probe_tombstones
1582                .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1583        }
1584        let corr = match inner.allocate_control_corr(module.endpoint) {
1585            Ok(corr) => corr,
1586            Err(err) => {
1587                drop(inner);
1588                self.request_connection_close(
1589                    module.endpoint.connection_id,
1590                    CloseReason::new(
1591                        "control_correlation_exhausted",
1592                        "daemon-originated channel-0 correlation space exhausted",
1593                    ),
1594                );
1595                return Err(err);
1596            }
1597        };
1598        let (sender, receiver) = oneshot::channel();
1599        inner.pending_control_rpcs.insert(
1600            (module.endpoint, corr),
1601            PendingModuleControlRpcEntry {
1602                expected_op: expected_op.to_string(),
1603                deadline,
1604                health_probe_started_at,
1605                sender,
1606            },
1607        );
1608
1609        Ok(PendingModuleControlRpc {
1610            endpoint: module.endpoint,
1611            module_sink: module.sink,
1612            negotiated_ver: module.negotiated_ver,
1613            corr,
1614            receiver,
1615        })
1616    }
1617
1618    #[allow(clippy::too_many_arguments)]
1619    fn begin_route_bind_relay_inner(
1620        &self,
1621        client_connection_id: ConnectionId,
1622        client_sink: FrameSink,
1623        client_negotiated_ver: u8,
1624        client_corr: u64,
1625        expected_module_id: &str,
1626        principal: Principal,
1627        scope: Option<BoundScope>,
1628        project_root: Option<ProjectRootId>,
1629        harness: String,
1630        build_class: Option<String>,
1631        operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1632        deadline: Instant,
1633        client_permit: crate::router::EgressPermit,
1634        busy_refusals: bool,
1635    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1636        let mut inner = self.write_inner()?;
1637        // HELLO and route.open can run on different tasks. Decide the role under
1638        // the same lock that installs endpoints and reserves client channels so
1639        // neither task can turn one connection into both kinds of owner.
1640        if inner
1641            .endpoint_by_connection
1642            .contains_key(&client_connection_id)
1643        {
1644            return Err(ForwardingError::ConnectionRoleConflict {
1645                connection_id: client_connection_id,
1646            });
1647        }
1648        if inner.closing_connections.contains(&client_connection_id) {
1649            return Err(ForwardingError::ConnectionClosing {
1650                connection_id: client_connection_id,
1651            });
1652        }
1653        let module = inner
1654            .modules_by_id
1655            .get(expected_module_id)
1656            .cloned()
1657            .ok_or(ForwardingError::NoModuleConnection)?;
1658        if inner.draining_endpoints.contains_key(&module.endpoint) {
1659            return Err(ForwardingError::ModuleReloading {
1660                module_id: expected_module_id.to_string(),
1661            });
1662        }
1663        if inner
1664            .closing_connections
1665            .contains(&module.endpoint.connection_id)
1666        {
1667            return Err(ForwardingError::ConnectionClosing {
1668                connection_id: module.endpoint.connection_id,
1669            });
1670        }
1671
1672        let corr = match inner.allocate_control_corr(module.endpoint) {
1673            Ok(corr) => corr,
1674            Err(err) => {
1675                drop(inner);
1676                self.request_connection_close(
1677                    module.endpoint.connection_id,
1678                    CloseReason::new(
1679                        "control_correlation_exhausted",
1680                        "daemon-originated channel-0 correlation space exhausted",
1681                    ),
1682                );
1683                return Err(err);
1684            }
1685        };
1686        let (client_channel, client_epoch, module_channel, module_epoch) =
1687            inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1688        let client_key = ClientRouteKey {
1689            connection_id: client_connection_id,
1690            channel: client_channel,
1691        };
1692        let module_key = ModuleRouteKey {
1693            endpoint: module.endpoint,
1694            channel: module_channel,
1695        };
1696        let reservation = RouteReservation {
1697            client_key,
1698            module_key,
1699            client_epoch,
1700            module_epoch,
1701            project_root,
1702        };
1703        let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
1704            request_window: None,
1705            watch_window: None,
1706            route_channel: client_channel,
1707            route_epoch: client_epoch,
1708        })
1709        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1710        let route_open_frame = Frame::build_with_version(
1711            client_negotiated_ver,
1712            FrameType::Response,
1713            Flags::new(false, Priority::Passive, false),
1714            0,
1715            0,
1716            client_corr,
1717            response_body,
1718        )
1719        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1720        let (sender, receiver) = oneshot::channel();
1721        inner.reserved_client.insert(client_key, module_key);
1722        inner.reserved_module.insert(module_key, client_key);
1723        inner.pending_relays.insert(
1724            (module.endpoint, corr),
1725            PendingRouteBindRelayEntry {
1726                reservation,
1727                client_sink,
1728                client_negotiated_ver,
1729                client_permit,
1730                route_open_frame,
1731                busy_refusals,
1732                request_window: window_for(&module.concurrency),
1733                watch_window: (busy_refusals && module.concurrency == Concurrency::ModuleManaged)
1734                    .then_some(DEFAULT_MODULE_MANAGED_WATCH_WINDOW),
1735                principal,
1736                scope,
1737                harness,
1738                build_class,
1739                operator_app,
1740                deadline,
1741                relay_enqueued: false,
1742                sender,
1743            },
1744        );
1745
1746        Ok(PendingRouteBindRelay {
1747            endpoint: module.endpoint,
1748            module_sink: module.sink,
1749            negotiated_ver: module.negotiated_ver,
1750            client_channel,
1751            client_epoch,
1752            module_channel,
1753            module_epoch,
1754            corr,
1755            receiver,
1756        })
1757    }
1758
1759    /// Called by `route.open` right after the route.bind entered the module's
1760    /// egress queue.
1761    ///
1762    /// The module reader runs concurrently with `route.open`, so a fast module
1763    /// can receive the bind, ack it, and have the reader commit the route
1764    /// before this runs. A missing entry therefore does not mean the bind was
1765    /// abandoned: only a removal recorded in `relays_released_before_enqueue`
1766    /// does. Telling the two apart is what stops `route.open` from sending a
1767    /// GOODBYE for a route that was just committed.
1768    pub(crate) fn mark_route_bind_relay_enqueued(
1769        &self,
1770        endpoint: ModuleEndpointId,
1771        corr: u64,
1772    ) -> Result<RouteBindRelayMark, ForwardingError> {
1773        let mut inner = self.write_inner()?;
1774        if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
1775            pending.relay_enqueued = true;
1776            return Ok(RouteBindRelayMark::Marked);
1777        }
1778        if inner
1779            .relays_released_before_enqueue
1780            .remove(&(endpoint, corr))
1781        {
1782            return Ok(RouteBindRelayMark::ReleasedWithoutReply);
1783        }
1784        Ok(RouteBindRelayMark::SettledByReply)
1785    }
1786
1787    pub(crate) fn release_client_route(
1788        &self,
1789        client_connection_id: ConnectionId,
1790        client_channel: u16,
1791        expected_epoch: u32,
1792    ) -> Result<RouteRelease, ForwardingError> {
1793        let mut inner = self.write_inner()?;
1794        let release = release_client_route_locked(
1795            &mut inner,
1796            ClientRouteKey {
1797                connection_id: client_connection_id,
1798                channel: client_channel,
1799            },
1800            expected_epoch,
1801        );
1802        self.record_route_release(&release);
1803        Ok(release)
1804    }
1805
1806    pub(crate) fn release_module_route(
1807        &self,
1808        module_connection_id: ConnectionId,
1809        module_channel: u16,
1810        expected_epoch: u32,
1811    ) -> Result<RouteRelease, ForwardingError> {
1812        let mut inner = self.write_inner()?;
1813        let Some(endpoint) = inner
1814            .endpoint_by_connection
1815            .get(&module_connection_id)
1816            .copied()
1817        else {
1818            return Ok(RouteRelease::Absent);
1819        };
1820        let release = release_module_route_locked(
1821            &mut inner,
1822            ModuleRouteKey {
1823                endpoint,
1824                channel: module_channel,
1825            },
1826            expected_epoch,
1827        );
1828        self.record_route_release(&release);
1829        Ok(release)
1830    }
1831
1832    pub(crate) fn abort_pending_relay(
1833        &self,
1834        endpoint: ModuleEndpointId,
1835        corr: u64,
1836        outcome: RouteBindRelayOutcome,
1837    ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
1838        let mut inner = self.write_inner()?;
1839        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1840            // If another path dropped the relay while `route.open` was still
1841            // sending the bind, and that open is now giving up without marking
1842            // it (its send failed or it was cancelled), forget the record so it
1843            // cannot outlive the open.
1844            inner
1845                .relays_released_before_enqueue
1846                .remove(&(endpoint, corr));
1847            return Ok(None);
1848        };
1849        release_reserved_route_locked(
1850            &mut inner,
1851            pending.reservation.client_key,
1852            pending.reservation.module_key,
1853        );
1854        let target = pending
1855            .relay_enqueued
1856            .then(|| abandoned_route_target(&inner, &pending.reservation));
1857        let _ = pending.sender.send(outcome);
1858        Ok(target.flatten())
1859    }
1860
1861    pub(crate) fn cancel_module_control_rpc(
1862        &self,
1863        endpoint: ModuleEndpointId,
1864        corr: u64,
1865    ) -> Result<(), ForwardingError> {
1866        self.write_inner()?
1867            .pending_control_rpcs
1868            .remove(&(endpoint, corr));
1869        Ok(())
1870    }
1871
1872    pub(crate) fn tombstone_health_probe_rpc(
1873        &self,
1874        endpoint: ModuleEndpointId,
1875        corr: u64,
1876    ) -> Result<bool, ForwardingError> {
1877        let key = (endpoint, corr);
1878        let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
1879        {
1880            let mut inner = self.write_inner()?;
1881            let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
1882                return Ok(false);
1883            };
1884            let Some(probe_started_at) = pending.health_probe_started_at else {
1885                inner.pending_control_rpcs.insert(key, pending);
1886                return Ok(false);
1887            };
1888            let module_id = inner
1889                .module_id_by_endpoint
1890                .get(&endpoint)
1891                .cloned()
1892                .unwrap_or_else(|| "unknown".to_string());
1893            inner.health_probe_tombstones.insert(
1894                key,
1895                HealthProbeTombstone {
1896                    expected_op: pending.expected_op,
1897                    module_id,
1898                    probe_started_at,
1899                    expires_at,
1900                },
1901            );
1902        }
1903        self.schedule_health_probe_tombstone_expiration(key, expires_at);
1904        Ok(true)
1905    }
1906
1907    fn schedule_health_probe_tombstone_expiration(
1908        &self,
1909        key: (ModuleEndpointId, u64),
1910        expires_at: Instant,
1911    ) {
1912        let inner = Arc::downgrade(&self.inner);
1913        tokio::spawn(async move {
1914            tokio::time::sleep_until(expires_at).await;
1915            let Some(inner) = inner.upgrade() else {
1916                return;
1917            };
1918            let Ok(mut inner) = inner.write() else {
1919                return;
1920            };
1921            let expired = inner
1922                .health_probe_tombstones
1923                .get(&key)
1924                .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
1925            if expired {
1926                inner.health_probe_tombstones.remove(&key);
1927            }
1928        });
1929    }
1930
1931    pub(crate) fn complete_pending_relay(
1932        &self,
1933        connection_id: ConnectionId,
1934        corr: u64,
1935        outcome: RouteBindRelayOutcome,
1936    ) -> Result<PendingRelayCompletion, ForwardingError> {
1937        let mut inner = self.write_inner()?;
1938        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1939            return Ok(PendingRelayCompletion {
1940                settled: false,
1941                abandoned: None,
1942            });
1943        };
1944        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1945            return Ok(PendingRelayCompletion {
1946                settled: false,
1947                abandoned: None,
1948            });
1949        };
1950
1951        if Instant::now() >= pending.deadline {
1952            release_reserved_route_locked(
1953                &mut inner,
1954                pending.reservation.client_key,
1955                pending.reservation.module_key,
1956            );
1957            let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
1958                .then(|| abandoned_route_target(&inner, &pending.reservation))
1959                .flatten();
1960            let _ = pending
1961                .sender
1962                .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody {
1963                    code: "module_timeout".to_string(),
1964                    message: "route.bind response arrived after its daemon deadline".to_string(),
1965                    detail: None,
1966                }));
1967            return Ok(PendingRelayCompletion {
1968                settled: true,
1969                abandoned,
1970            });
1971        }
1972
1973        // These readiness reads are not atomic with publication: a route may
1974        // commit after Dead. Taking a spawn transition lock here would invert
1975        // the publication lock order. Retirement, in one critical section,
1976        // stops every connection bound by the launch's claims and its own
1977        // registration channel from reading or sending frames, and shuts that
1978        // channel.
1979        // Once that step completes, no new request can cross the transient route
1980        // to or from the retired launch; connection teardown removes the route.
1981        match outcome {
1982            // Three shapes of "the client is not there to receive this route" that
1983            // must resolve identically: its egress is already closed, or it is
1984            // marked closing (its connection loop has been asked to end, but has
1985            // not drained yet, so the sink is still open), or its spawn is dead
1986            // before connection-close notifications have reached that loop.
1987            //
1988            // Only the first used to be caught here. The second fell through to
1989            // `commit_route_locked`, which refuses a closing client with
1990            // `ConnectionClosing` -- and this function is called from the MODULE
1991            // connection's frame handler, so that refusal ended the module's
1992            // connection instead of this one client's route. One dying client's
1993            // route.open then took down a connection carrying every other
1994            // client's routes to that module.
1995            //
1996            // The remedy for both is the same, which is why they share an arm:
1997            // give back the reserved handle pair, tell the waiting route.open the
1998            // route is gone, and report the module-side channel so the caller can
1999            // send a channel-scoped GOODBYE for the binding the module just
2000            // created. Nothing here touches the module connection.
2001            RouteBindRelayOutcome::Accepted
2002                if pending.client_sink.is_closed()
2003                    || spawn_connection_dead(
2004                        &inner,
2005                        pending.reservation.client_key.connection_id,
2006                    )
2007                    || inner
2008                        .closing_connections
2009                        .contains(&pending.reservation.client_key.connection_id) =>
2010            {
2011                let reason = if pending.client_sink.is_closed() {
2012                    "client egress closed before route publication"
2013                } else {
2014                    "client connection is closing before route publication"
2015                };
2016                release_reserved_route_locked(
2017                    &mut inner,
2018                    pending.reservation.client_key,
2019                    pending.reservation.module_key,
2020                );
2021                // Not gated on `relay_enqueued`: an ack proves the module
2022                // received the bind, and `route.open` sends nothing for a
2023                // relay a reply already settled, so this is the only GOODBYE.
2024                let abandoned = abandoned_route_target(&inner, &pending.reservation);
2025                let _ = pending
2026                    .sender
2027                    .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
2028                return Ok(PendingRelayCompletion {
2029                    settled: true,
2030                    abandoned,
2031                });
2032            }
2033            // The endpoint that acked this bind is no longer accepting new
2034            // routes, for one of two reasons. Either it was the module's active
2035            // endpoint when this relay was reserved and a blue/green swap has
2036            // since promoted a candidate (the replacement process registered
2037            // beside it) into the active slot, superseding it; or a daemon
2038            // drain has gated it, and the ack arrived before the ordered module
2039            // drain settled pending relays. The module did nothing wrong: it
2040            // bound the route it was asked to bind, and the endpoint still
2041            // carries every other client's routes until it is drained. So the
2042            // ack is settled here, while the pending entry still holds the
2043            // client's sender and the reserved handle pair: release the pair,
2044            // refuse the waiting route.open with `module_reloading`, which
2045            // callers retry (after a swap the retry reserves on the promoted
2046            // endpoint), and hand back the module-side channel so the caller
2047            // sends one channel-scoped GOODBYE for the binding the module just
2048            // created. Returning an error instead would end the module
2049            // connection all those other routes share; nothing here touches it.
2050            //
2051            // An endpoint that stopped being active any other way is stale
2052            // (replaced, but not by a swap and not draining). It still falls
2053            // through to `commit_route_locked`, which refuses it with
2054            // `StaleModuleEndpoint` exactly as before swaps existed.
2055            RouteBindRelayOutcome::Accepted
2056                if inner.superseded_endpoints.contains_key(&endpoint)
2057                    || spawn_connection_dead(&inner, connection_id)
2058                    || inner.draining_endpoints.contains_key(&endpoint) =>
2059            {
2060                release_reserved_route_locked(
2061                    &mut inner,
2062                    pending.reservation.client_key,
2063                    pending.reservation.module_key,
2064                );
2065                // Not gated on `relay_enqueued`: an ack proves the module
2066                // received the bind, whether or not the enqueue mark was set.
2067                let abandoned = abandoned_route_target(&inner, &pending.reservation);
2068                let module_id = inner
2069                    .module_id_by_endpoint
2070                    .get(&endpoint)
2071                    .cloned()
2072                    .unwrap_or_else(|| "unknown".to_string());
2073                let _ = pending
2074                    .sender
2075                    .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
2076                        "module_reloading",
2077                        format!("module_id '{module_id}' is reloading"),
2078                    )));
2079                return Ok(PendingRelayCompletion {
2080                    settled: true,
2081                    abandoned,
2082                });
2083            }
2084            // The scope the open was admitted under ended or changed after
2085            // admission. The module bound a route under a stamp that is no
2086            // longer true, so it must not become routable. Settled here like
2087            // the superseded arm above, never as an error from
2088            // `commit_route_locked`, which would close the module's whole
2089            // connection and every other client's routes to it: release the
2090            // pair, answer the waiting route.open by name, and hand back the
2091            // module-side channel for one channel-scoped GOODBYE.
2092            RouteBindRelayOutcome::Accepted
2093                if pending
2094                    .scope
2095                    .as_ref()
2096                    .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
2097            {
2098                let refusal = pending
2099                    .scope
2100                    .as_ref()
2101                    .and_then(|scope| scope_refusal_locked(&inner, scope))
2102                    .expect("guard matched a refusal under the same lock");
2103                release_reserved_route_locked(
2104                    &mut inner,
2105                    pending.reservation.client_key,
2106                    pending.reservation.module_key,
2107                );
2108                let abandoned = abandoned_route_target(&inner, &pending.reservation);
2109                let _ = pending
2110                    .sender
2111                    .send(RouteBindRelayOutcome::DaemonRejected(refusal));
2112                return Ok(PendingRelayCompletion {
2113                    settled: true,
2114                    abandoned,
2115                });
2116            }
2117            RouteBindRelayOutcome::Accepted => {
2118                #[cfg(all(test, target_os = "linux"))]
2119                if let Some(pause) = self.route_commit_pause.lock().unwrap().take() {
2120                    let _ = pause.reached.send(());
2121                    pause
2122                        .release
2123                        .recv_timeout(Duration::from_secs(5))
2124                        .expect("test must release bind publication");
2125                }
2126                let abandoned = commit_route_locked(&mut inner, pending)?;
2127                return Ok(PendingRelayCompletion {
2128                    settled: true,
2129                    abandoned,
2130                });
2131            }
2132            terminal => {
2133                release_reserved_route_locked(
2134                    &mut inner,
2135                    pending.reservation.client_key,
2136                    pending.reservation.module_key,
2137                );
2138                let _ = pending.sender.send(terminal);
2139            }
2140        }
2141        Ok(PendingRelayCompletion {
2142            settled: true,
2143            abandoned: None,
2144        })
2145    }
2146
2147    pub(crate) fn pending_module_control_op(
2148        &self,
2149        connection_id: ConnectionId,
2150        corr: u64,
2151    ) -> Result<Option<String>, ForwardingError> {
2152        let inner = self.read_inner()?;
2153        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2154            return Ok(None);
2155        };
2156        let key = (endpoint, corr);
2157        Ok(inner
2158            .pending_control_rpcs
2159            .get(&key)
2160            .map(|pending| pending.expected_op.clone())
2161            .or_else(|| {
2162                inner
2163                    .health_probe_tombstones
2164                    .get(&key)
2165                    .filter(|tombstone| tombstone.expires_at > Instant::now())
2166                    .map(|tombstone| tombstone.expected_op.clone())
2167            }))
2168    }
2169
2170    pub(crate) fn complete_module_control_rpc(
2171        &self,
2172        connection_id: ConnectionId,
2173        corr: u64,
2174        actual_op: Option<&str>,
2175        outcome: ModuleControlRpcOutcome,
2176    ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
2177        let now = Instant::now();
2178        let mut inner = self.write_inner()?;
2179        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2180            return Ok(ModuleControlRpcCompletion::Unknown);
2181        };
2182        let key = (endpoint, corr);
2183        if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2184            if now >= pending.deadline {
2185                let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
2186                    ModuleControlRpcCompletion::LateHealthAnswer {
2187                        module_id: inner
2188                            .module_id_by_endpoint
2189                            .get(&endpoint)
2190                            .cloned()
2191                            .unwrap_or_else(|| "unknown".to_string()),
2192                        latency: now.saturating_duration_since(probe_started_at),
2193                    }
2194                });
2195                let _ = pending
2196                    .sender
2197                    .send(ModuleControlRpcOutcome::DeadlineElapsed);
2198                return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
2199            }
2200            let outcome = match actual_op {
2201                Some(actual) if actual != pending.expected_op => {
2202                    ModuleControlRpcOutcome::UnexpectedOp {
2203                        expected: pending.expected_op,
2204                        actual: actual.to_string(),
2205                    }
2206                }
2207                _ => outcome,
2208            };
2209            let _ = pending.sender.send(outcome);
2210            return Ok(ModuleControlRpcCompletion::Settled);
2211        }
2212
2213        let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
2214            return Ok(ModuleControlRpcCompletion::Unknown);
2215        };
2216        if tombstone.expires_at <= now {
2217            return Ok(ModuleControlRpcCompletion::Unknown);
2218        }
2219        Ok(ModuleControlRpcCompletion::LateHealthAnswer {
2220            module_id: tombstone.module_id,
2221            latency: now.saturating_duration_since(tombstone.probe_started_at),
2222        })
2223    }
2224
2225    #[cfg(test)]
2226    pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
2227        Ok(self.read_inner()?.health_probe_tombstones.len())
2228    }
2229
2230    #[cfg(test)]
2231    pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
2232        Ok(self.read_inner()?.closing_connections.len())
2233    }
2234
2235    /// Reserved-but-uncommitted route handles, counted on both index sides, so
2236    /// a test can assert a reservation pair was actually given back.
2237    #[cfg(test)]
2238    pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
2239        let inner = self.read_inner()?;
2240        Ok((inner.reserved_client.len(), inner.reserved_module.len()))
2241    }
2242
2243    pub(crate) fn module_endpoint_for_connection(
2244        &self,
2245        connection_id: ConnectionId,
2246    ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
2247        Ok(self
2248            .read_inner()?
2249            .endpoint_by_connection
2250            .get(&connection_id)
2251            .copied())
2252    }
2253
2254    /// Looks up the module registered on a data-plane connection so route-drop
2255    /// diagnostics name the emitter instead of only reporting a daemon total.
2256    pub(crate) fn module_id_for_connection(
2257        &self,
2258        connection_id: ConnectionId,
2259    ) -> Result<Option<String>, ForwardingError> {
2260        let inner = self.read_inner()?;
2261        Ok(inner
2262            .endpoint_by_connection
2263            .get(&connection_id)
2264            .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
2265            .cloned())
2266    }
2267
2268    /// Whether the daemon ever allocated `(channel, epoch)` on this module
2269    /// connection. Epochs on a module channel are handed out as 1, 2, 3, ...
2270    /// and the last one handed out is remembered until the connection ends, so
2271    /// every epoch from 1 up to that one was a real route at some point. Used
2272    /// only on the drop path, to tell a module still sending on a route the
2273    /// daemon released from one sending on a route that never existed.
2274    pub(crate) fn module_route_epoch_was_allocated(
2275        &self,
2276        connection_id: ConnectionId,
2277        channel: u16,
2278        epoch: u32,
2279    ) -> Result<bool, ForwardingError> {
2280        let inner = self.read_inner()?;
2281        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2282            return Ok(false);
2283        };
2284        Ok(inner
2285            .module_slot_epochs
2286            .get(&ModuleRouteKey { endpoint, channel })
2287            .is_some_and(|last| epoch != 0 && epoch <= *last))
2288    }
2289
2290    pub(crate) fn has_live_module_connection(
2291        &self,
2292        module_id: &str,
2293    ) -> Result<bool, ForwardingError> {
2294        Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
2295    }
2296
2297    /// Admission and route teardown share the forwarding-then-flow lock order.
2298    /// No asynchronous work may split validation from recording the credits.
2299    pub(crate) fn try_admit_request(
2300        &self,
2301        route: &Arc<RouteBinding>,
2302        epoch: u32,
2303        corr: u64,
2304        subscription: bool,
2305    ) -> Result<RequestAdmission, ForwardingError> {
2306        let inner = self.read_inner()?;
2307        let key = ClientRouteKey {
2308            connection_id: route.client_connection_id,
2309            channel: route.client_channel,
2310        };
2311        if epoch != route.client_epoch
2312            || !inner
2313                .client_to_module
2314                .get(&key)
2315                .is_some_and(|current| Arc::ptr_eq(current, route))
2316        {
2317            return Ok(RequestAdmission::Stale);
2318        }
2319        let admission = route.flow.try_acquire_tagged(corr, subscription);
2320        Ok(match admission {
2321            RequestAdmission::Stale
2322                if inner
2323                    .draining_endpoints
2324                    .contains_key(&route.module_endpoint) =>
2325            {
2326                RequestAdmission::ModuleReloading
2327            }
2328            admission => admission,
2329        })
2330    }
2331
2332    pub(crate) fn lookup_data_route(
2333        &self,
2334        connection_id: ConnectionId,
2335        channel: u16,
2336        epoch: u32,
2337    ) -> Result<DataRoute, ForwardingError> {
2338        let inner = self.read_inner()?;
2339        let state = if let Some(endpoint) =
2340            inner.endpoint_by_connection.get(&connection_id).copied()
2341        {
2342            let key = ModuleRouteKey { endpoint, channel };
2343            match inner.module_to_client.get(&key) {
2344                Some(route) if route.module_epoch == epoch => {
2345                    DataRouteState::Bound(Arc::clone(route))
2346                }
2347                Some(_) => DataRouteState::EpochMismatch,
2348                None if inner.reserved_module.contains_key(&key)
2349                    && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
2350                {
2351                    DataRouteState::Reserved
2352                }
2353                None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
2354                None => DataRouteState::Absent,
2355            }
2356        } else {
2357            let key = ClientRouteKey {
2358                connection_id,
2359                channel,
2360            };
2361            match inner.client_to_module.get(&key) {
2362                Some(route) if route.client_epoch == epoch => {
2363                    DataRouteState::Bound(Arc::clone(route))
2364                }
2365                Some(_) => DataRouteState::EpochMismatch,
2366                None if inner.reserved_client.contains_key(&key)
2367                    && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
2368                {
2369                    DataRouteState::Reserved
2370                }
2371                None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
2372                None => DataRouteState::Absent,
2373            }
2374        };
2375        Ok(
2376            if inner.endpoint_by_connection.contains_key(&connection_id) {
2377                DataRoute::Module(state)
2378            } else {
2379                DataRoute::Client(state)
2380            },
2381        )
2382    }
2383
2384    #[cfg(test)]
2385    pub(crate) fn inject_client_slot_epoch(
2386        &self,
2387        connection_id: ConnectionId,
2388        channel: u16,
2389        last_epoch: u32,
2390    ) {
2391        let mut inner = self.write_inner().expect("forwarding lock");
2392        inner.client_slot_epochs.insert(
2393            ClientRouteKey {
2394                connection_id,
2395                channel,
2396            },
2397            last_epoch,
2398        );
2399        inner.next_client_channel.insert(connection_id, channel);
2400    }
2401
2402    #[cfg(test)]
2403    pub(crate) fn inject_module_slot_epoch(
2404        &self,
2405        endpoint: ModuleEndpointId,
2406        channel: u16,
2407        last_epoch: u32,
2408    ) {
2409        let mut inner = self.write_inner().expect("forwarding lock");
2410        inner
2411            .module_slot_epochs
2412            .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
2413        inner.next_module_channel.insert(endpoint, channel);
2414    }
2415
2416    #[cfg(test)]
2417    pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
2418        self.write_inner()
2419            .expect("forwarding lock")
2420            .next_control_corr
2421            .insert(endpoint, next_corr);
2422    }
2423
2424    pub(crate) fn cache_status(
2425        &self,
2426        endpoint: ModuleEndpointId,
2427        module_channel: u16,
2428        module_epoch: u32,
2429        status: String,
2430    ) -> Result<bool, ForwardingError> {
2431        let mut inner = self.write_inner()?;
2432        if !inner.module_id_by_endpoint.contains_key(&endpoint) {
2433            return Err(ForwardingError::StaleModuleEndpoint);
2434        }
2435
2436        let module_key = ModuleRouteKey {
2437            endpoint,
2438            channel: module_channel,
2439        };
2440        let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
2441            (route.module_epoch == module_epoch).then_some((
2442                ClientRouteKey {
2443                    connection_id: route.client_connection_id,
2444                    channel: route.client_channel,
2445                },
2446                route.client_epoch,
2447            ))
2448        } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
2449            (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
2450                client_key,
2451                inner
2452                    .client_slot_epochs
2453                    .get(&client_key)
2454                    .copied()
2455                    .unwrap_or(0),
2456            ))
2457        } else {
2458            None
2459        };
2460
2461        if let Some(handle) = handle {
2462            inner.status.insert(handle, status);
2463            Ok(true)
2464        } else {
2465            debug!(
2466                module_channel,
2467                module_epoch,
2468                generation = endpoint.generation,
2469                connection_id = endpoint.connection_id.get(),
2470                "dropping stale status update for module route handle"
2471            );
2472            Ok(false)
2473        }
2474    }
2475
2476    pub(crate) fn route_poll_snapshot(
2477        &self,
2478        client_connection_id: ConnectionId,
2479        client_channel: u16,
2480        client_epoch: u32,
2481    ) -> Result<RoutePollSnapshot, ForwardingError> {
2482        let inner = self.read_inner()?;
2483        let client_key = ClientRouteKey {
2484            connection_id: client_connection_id,
2485            channel: client_channel,
2486        };
2487        let Some(route) = inner.client_to_module.get(&client_key) else {
2488            return Ok(RoutePollSnapshot::Absent);
2489        };
2490        if route.client_epoch != client_epoch
2491            || !inner
2492                .module_id_by_endpoint
2493                .contains_key(&route.module_endpoint)
2494        {
2495            return Ok(RoutePollSnapshot::Absent);
2496        }
2497        Ok(RoutePollSnapshot::Bound {
2498            module_id: route.module_id.clone(),
2499            status: inner.status.get(&(client_key, client_epoch)).cloned(),
2500        })
2501    }
2502
2503    pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2504        Ok(self.read_inner()?.client_to_module.len())
2505    }
2506
2507    /// How many distinct client connections hold at least one committed route,
2508    /// alongside the largest number of routes any single connection holds.
2509    ///
2510    /// `connected_clients` alone cannot distinguish many clients with a route
2511    /// each from one client accumulating hundreds, and those have opposite
2512    /// causes. Reading it required an out-of-band `lsof` during a live
2513    /// investigation, and the count of connections was mistaken for a count of
2514    /// client processes — which sent two of us after cleanup paths that were
2515    /// working correctly.
2516    pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2517        let inner = self.read_inner()?;
2518        let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2519        for key in inner.client_to_module.keys() {
2520            *per_connection.entry(key.connection_id).or_insert(0) += 1;
2521        }
2522        let max = per_connection.values().copied().max().unwrap_or(0);
2523        Ok((per_connection.len(), max))
2524    }
2525
2526    pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2527        let inner = self.read_inner()?;
2528        Ok(inner
2529            .client_to_module
2530            .keys()
2531            .any(|key| key.channel == route_channel))
2532    }
2533
2534    /// Whether daemon-wide shutdown has begun draining all providers.
2535    /// `None` means normal operation; during daemon shutdown, return the
2536    /// endpoint's measured result, or false if no completed measurement exists.
2537    pub(crate) fn daemon_drain_result_for_connection(
2538        &self,
2539        connection_id: ConnectionId,
2540    ) -> Result<Option<bool>, ForwardingError> {
2541        let inner = self.read_inner()?;
2542        Ok(inner.daemon_draining.then(|| {
2543            inner
2544                .endpoint_by_connection
2545                .get(&connection_id)
2546                .and_then(|endpoint| inner.daemon_drain_results.get(endpoint))
2547                .copied()
2548                .unwrap_or(false)
2549        }))
2550    }
2551
2552    #[cfg(unix)]
2553    pub(crate) fn record_daemon_drain_result(
2554        &self,
2555        endpoint: ModuleEndpointId,
2556        drained: bool,
2557    ) -> Result<(), ForwardingError> {
2558        let mut inner = self.write_inner()?;
2559        // Only update an endpoint captured at the start of daemon shutdown.
2560        // Teardown removes its entry, so a late wait cannot recreate the result
2561        // or attribute it to a replacement connection.
2562        if let Some(result) = inner.daemon_drain_results.get_mut(&endpoint) {
2563            *result = drained;
2564        }
2565        Ok(())
2566    }
2567
2568    /// Gate every provider atomically, including registrations racing shutdown.
2569    /// No supervisor lock is held while taking the forwarding lock.
2570    #[cfg(unix)]
2571    pub(crate) fn begin_daemon_drain(
2572        &self,
2573    ) -> Result<Vec<(String, ModuleEndpointId)>, ForwardingError> {
2574        let mut inner = self.write_inner()?;
2575        if inner.daemon_draining {
2576            return Ok(Vec::new());
2577        }
2578        inner.daemon_draining = true;
2579        let modules = inner
2580            .modules_by_id
2581            .iter()
2582            .map(|(id, module)| (id.clone(), module.endpoint))
2583            .collect::<Vec<_>>();
2584        for (_, endpoint) in &modules {
2585            inner.daemon_drain_results.insert(*endpoint, false);
2586            inner
2587                .draining_endpoints
2588                .insert(*endpoint, RouteCloseReason::Restart);
2589        }
2590        // Swap candidates and superseded incumbents are not routable, but they
2591        // are live endpoints that could still be sent module-control work, so
2592        // the daemon-wide gate covers them too. Only the active endpoints are
2593        // returned to the supervisor for measurement, one per module id.
2594        let off_slot_endpoints = inner
2595            .candidates_by_id
2596            .values()
2597            .map(|module| module.endpoint)
2598            .chain(inner.superseded_endpoints.keys().copied())
2599            .collect::<Vec<_>>();
2600        for endpoint in off_slot_endpoints {
2601            inner
2602                .draining_endpoints
2603                .insert(endpoint, RouteCloseReason::Restart);
2604        }
2605        Ok(modules)
2606    }
2607
2608    /// Begin draining whatever endpoint is ACTIVE for `module_id`.
2609    ///
2610    /// After a swap's cutover the active endpoint is the promoted candidate, so
2611    /// this must not be used to drain the incumbent; use
2612    /// [`Self::begin_endpoint_drain`] with the endpoint `cutover_candidate`
2613    /// returned.
2614    pub(crate) fn begin_module_drain(
2615        &self,
2616        module_id: &str,
2617        reason: RouteCloseReason,
2618    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2619        let mut inner = self.write_inner()?;
2620        let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2621            return Ok(None);
2622        };
2623        Ok(Some(begin_drain_locked(
2624            &mut inner, module_id, module, reason,
2625        )))
2626    }
2627
2628    /// Begin draining one specific endpoint, whichever slot it is in.
2629    ///
2630    /// This is how a swap drains its incumbent after cutover: the incumbent's
2631    /// endpoint is captured by `cutover_candidate`, and resolving it by module id
2632    /// instead would find the promoted candidate and leave neither process
2633    /// routable. Returns `Ok(None)` when the endpoint is no longer registered.
2634    pub(crate) fn begin_endpoint_drain(
2635        &self,
2636        endpoint: ModuleEndpointId,
2637        reason: RouteCloseReason,
2638    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2639        let mut inner = self.write_inner()?;
2640        let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2641            return Ok(None);
2642        };
2643        let module_id = inner
2644            .module_id_by_endpoint
2645            .get(&endpoint)
2646            .cloned()
2647            .expect("an endpoint resolved to a module connection has a module id");
2648        Ok(Some(begin_drain_locked(
2649            &mut inner, &module_id, module, reason,
2650        )))
2651    }
2652}
2653
2654/// Mark `module`'s endpoint draining and settle everything still pending on it.
2655/// Shared by the by-id and by-endpoint drain entry points, which differ only in
2656/// how they find the endpoint.
2657fn begin_drain_locked(
2658    inner: &mut ForwardingInner,
2659    module_id: &str,
2660    module: ModuleConnection,
2661    reason: RouteCloseReason,
2662) -> ModuleDrainTarget {
2663    {
2664        let endpoint = module.endpoint;
2665        inner.draining_endpoints.insert(endpoint, reason);
2666
2667        let flows = inner
2668            .client_to_module
2669            .values()
2670            .filter(|route| route.module_endpoint == endpoint)
2671            .map(|route| Arc::clone(&route.flow))
2672            .collect::<Vec<_>>();
2673        let excluded_subscriptions = flows
2674            .into_iter()
2675            .map(|flow| flow.begin_drain())
2676            .fold(0u32, u32::saturating_add);
2677
2678        let pending_keys = inner
2679            .pending_relays
2680            .keys()
2681            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2682            .copied()
2683            .collect::<Vec<_>>();
2684        let mut abandoned_bindings = Vec::new();
2685        for key in pending_keys {
2686            let Some(pending) = inner.pending_relays.remove(&key) else {
2687                continue;
2688            };
2689            release_reserved_route_locked(
2690                inner,
2691                pending.reservation.client_key,
2692                pending.reservation.module_key,
2693            );
2694            if pending.relay_enqueued {
2695                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2696                    abandoned_bindings.push(target);
2697                }
2698            } else {
2699                // The bind may still be on its way into the module's queue;
2700                // a GOODBYE sent now could overtake it. `route.open` sends
2701                // it once its send completes.
2702                inner.relays_released_before_enqueue.insert(key);
2703            }
2704            let _ = pending
2705                .sender
2706                .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
2707                    "module_reloading",
2708                    format!("module_id '{module_id}' is reloading"),
2709                )));
2710        }
2711
2712        let pending_control_keys = inner
2713            .pending_control_rpcs
2714            .keys()
2715            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2716            .copied()
2717            .collect::<Vec<_>>();
2718        for key in pending_control_keys {
2719            if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2720                let _ = pending
2721                    .sender
2722                    .send(ModuleControlRpcOutcome::ModuleGone(format!(
2723                        "module '{module_id}' began draining during module-control RPC"
2724                    )));
2725            }
2726        }
2727
2728        ModuleDrainTarget {
2729            endpoint,
2730            sink: module.sink,
2731            negotiated_ver: module.negotiated_ver,
2732            abandoned_bindings,
2733            excluded_subscriptions,
2734        }
2735    }
2736}
2737
2738/// What a drain that timed out was still waiting on, for the log line that
2739/// reports the timeout. Per-request ages are not tracked.
2740#[derive(Debug, Default, PartialEq, Eq)]
2741pub(crate) struct DrainHoldouts {
2742    /// Requests still counted against the drain (subscriptions flagged as
2743    /// such are excluded from the drain and not counted here).
2744    pub(crate) requests: usize,
2745    /// Routes holding at least one of those requests.
2746    pub(crate) routes: usize,
2747    /// Every route on the endpoint, for scale.
2748    pub(crate) total_routes: usize,
2749    /// The client connections holding the most requests, largest first, at
2750    /// most three: enough to name the consumer without listing every route.
2751    pub(crate) top_connections: Vec<(u64, usize)>,
2752    /// The held requests themselves, as the module saw them, so its own log
2753    /// can say what each one was: `(module channel, corr)`, ordered by channel
2754    /// then corr, at most [`DRAIN_HELD_REQUESTS_LISTED`]. The daemon never reads
2755    /// request bodies, so it cannot name a request's method; the module can,
2756    /// from the channel and corr. A request is released only when the module
2757    /// sends a terminal frame (Response, Error or StreamEnd) with its corr on
2758    /// its route, so every pair listed here is one the module ended, if at all,
2759    /// without sending that frame.
2760    pub(crate) held: Vec<(u16, u64)>,
2761}
2762
2763/// How many held requests the drain-timeout line lists by channel and corr.
2764pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
2765
2766impl ForwardingTable {
2767    /// Summarise the requests one endpoint's drain is still waiting on.
2768    pub(crate) fn endpoint_drain_holdouts(
2769        &self,
2770        endpoint: ModuleEndpointId,
2771    ) -> Result<DrainHoldouts, ForwardingError> {
2772        let inner = self.read_inner()?;
2773        let mut holdouts = DrainHoldouts::default();
2774        let mut by_connection: HashMap<u64, usize> = HashMap::new();
2775        for (key, route) in &inner.client_to_module {
2776            if route.module_endpoint != endpoint {
2777                continue;
2778            }
2779            holdouts.total_routes += 1;
2780            let held = route.flow.drain_in_flight();
2781            if held == 0 {
2782                continue;
2783            }
2784            holdouts.requests += held;
2785            holdouts.routes += 1;
2786            *by_connection.entry(key.connection_id.get()).or_default() += held;
2787            holdouts.held.extend(
2788                route
2789                    .flow
2790                    .drain_held_corrs()
2791                    .into_iter()
2792                    .map(|corr| (route.module_channel, corr)),
2793            );
2794        }
2795        holdouts.held.sort_unstable();
2796        holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
2797        let mut connections = by_connection.into_iter().collect::<Vec<_>>();
2798        connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
2799        connections.truncate(3);
2800        holdouts.top_connections = connections;
2801        Ok(holdouts)
2802    }
2803
2804    pub(crate) fn endpoint_in_flight_count(
2805        &self,
2806        endpoint: ModuleEndpointId,
2807    ) -> Result<usize, ForwardingError> {
2808        let inner = self.read_inner()?;
2809        Ok(inner
2810            .client_to_module
2811            .values()
2812            .filter(|route| route.module_endpoint == endpoint)
2813            .map(|route| route.flow.drain_in_flight())
2814            .sum())
2815    }
2816
2817    pub(crate) fn endpoint_is_draining(
2818        &self,
2819        endpoint: ModuleEndpointId,
2820    ) -> Result<bool, ForwardingError> {
2821        Ok(self
2822            .read_inner()?
2823            .draining_endpoints
2824            .contains_key(&endpoint))
2825    }
2826
2827    pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
2828        let inner = self.read_inner()?;
2829        Ok(inner
2830            .modules_by_id
2831            .get(module_id)
2832            .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
2833    }
2834
2835    pub(crate) fn release_module_endpoint_routes(
2836        &self,
2837        endpoint: ModuleEndpointId,
2838    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2839        let mut inner = self.write_inner()?;
2840        let routes = inner
2841            .module_to_client
2842            .iter()
2843            .filter(|(module_key, _)| module_key.endpoint == endpoint)
2844            .map(|(module_key, route)| (*module_key, route.module_epoch))
2845            .collect::<Vec<_>>();
2846        let mut released = Vec::with_capacity(routes.len());
2847        for (module_key, epoch) in routes {
2848            if let RouteRelease::Removed(target) =
2849                release_module_route_locked(&mut inner, module_key, epoch)
2850            {
2851                released.push(target);
2852            }
2853        }
2854        Ok(released)
2855    }
2856
2857    /// Revoke old bind decisions atomically, including binds still in flight.
2858    /// The provider stays available: its acknowledged config governs new binds.
2859    pub(crate) fn rebind_module_policy(
2860        &self,
2861        module_id: &str,
2862        expected_connection: Option<ConnectionId>,
2863        authorized: impl FnOnce() -> bool,
2864    ) -> Result<Option<PolicyRevocation>, ForwardingError> {
2865        #[cfg(test)]
2866        if let Some(hold) = self.policy_rebind_before_write.lock().unwrap().take() {
2867            hold.wait();
2868            hold.wait();
2869        }
2870        let mut inner = self.write_inner()?;
2871        let module = inner
2872            .modules_by_id
2873            .get(module_id)
2874            .cloned()
2875            .ok_or(ForwardingError::NoModuleConnection)?;
2876        if expected_connection.is_some_and(|connection| module.endpoint.connection_id != connection)
2877            || !authorized()
2878        {
2879            return Ok(None);
2880        }
2881        let endpoint = module.endpoint;
2882        if inner.draining_endpoints.contains_key(&endpoint) {
2883            return Err(ForwardingError::ModuleReloading {
2884                module_id: module_id.to_owned(),
2885            });
2886        }
2887        let target = begin_drain_locked(&mut inner, module_id, module, RouteCloseReason::Reload);
2888        let routes = endpoint_routes_locked(&inner, endpoint);
2889        let mut goodbyes = target.abandoned_bindings;
2890        for route in &routes {
2891            let target = &route.goodbye_target;
2892            if let RouteRelease::Removed(module) = release_client_route_locked(
2893                &mut inner,
2894                ClientRouteKey {
2895                    connection_id: target.connection_id,
2896                    channel: target.channel,
2897                },
2898                target.epoch,
2899            ) {
2900                goodbyes.push(module);
2901                goodbyes.push(target.clone());
2902            }
2903        }
2904        inner.draining_endpoints.remove(&endpoint);
2905        Ok(Some((routes, goodbyes)))
2906    }
2907
2908    #[cfg(test)]
2909    pub(crate) fn pause_policy_rebind_before_write(&self, hold: Arc<std::sync::Barrier>) {
2910        *self.policy_rebind_before_write.lock().unwrap() = Some(hold);
2911    }
2912
2913    #[cfg(test)]
2914    pub(crate) fn with_forwarding_write_lock_for_test(&self, work: impl FnOnce()) {
2915        let _guard = self.write_inner().unwrap();
2916        work();
2917    }
2918
2919    /// Enumerate one endpoint's current routes without contacting the module.
2920    ///
2921    /// The read lock makes this safe while the endpoint drains: the returned
2922    /// `draining` marker describes the same table state that owns the route,
2923    /// rather than inferring liveness from a module that may be stopping.
2924    pub(crate) fn endpoint_routes(
2925        &self,
2926        endpoint: ModuleEndpointId,
2927    ) -> Result<Vec<EndpointRoute>, ForwardingError> {
2928        let inner = self.read_inner()?;
2929        Ok(endpoint_routes_locked(&inner, endpoint))
2930    }
2931
2932    /// Snapshot all live endpoint route sets under one forwarding-table read lock.
2933    pub(crate) fn route_census(
2934        &self,
2935        module_id: Option<&str>,
2936    ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
2937        let inner = self.read_inner()?;
2938        let mut endpoints = inner
2939            .modules_by_id
2940            .iter()
2941            .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
2942            .map(|(id, module)| (id.clone(), module.endpoint))
2943            .collect::<Vec<_>>();
2944        endpoints.sort_by(|left, right| left.0.cmp(&right.0));
2945        Ok(endpoints
2946            .into_iter()
2947            .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
2948            .collect())
2949    }
2950
2951    /// Snapshot only the endpoint currently routable under this module id.
2952    pub(crate) fn live_roots(
2953        &self,
2954        module_id: &str,
2955    ) -> Result<ModuleControlResponseToModule, ForwardingError> {
2956        let inner = self.read_inner()?;
2957        let endpoint = inner
2958            .modules_by_id
2959            .get(module_id)
2960            .map(|module| module.endpoint);
2961        let mut roots = BTreeMap::new();
2962        let mut unknown_root_bindings = 0;
2963        let mut total_bindings = 0;
2964        if let Some(endpoint) = endpoint {
2965            for binding in inner
2966                .module_to_client
2967                .values()
2968                .filter(|binding| binding.module_endpoint == endpoint)
2969            {
2970                total_bindings += 1;
2971                let workdir = binding
2972                    .scope
2973                    .as_ref()
2974                    .and_then(|scope| scope.workdir.as_deref())
2975                    .map(std::path::Path::new);
2976                if let Some(root) = &binding.project_root {
2977                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2978                    entry.0 += 1;
2979                } else if workdir.is_none() {
2980                    unknown_root_bindings += 1;
2981                }
2982                if let Some(workdir) = workdir.filter(|workdir| {
2983                    binding.project_root.as_ref().map(ProjectRootId::as_path) != Some(*workdir)
2984                }) {
2985                    roots.entry(workdir.to_path_buf()).or_insert((0, 0)).0 += 1;
2986                }
2987            }
2988            for pending in inner
2989                .pending_relays
2990                .values()
2991                .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
2992            {
2993                total_bindings += 1;
2994                if let Some(root) = &pending.reservation.project_root {
2995                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2996                    entry.1 += 1;
2997                } else {
2998                    unknown_root_bindings += 1;
2999                }
3000            }
3001        }
3002        Ok(ModuleControlResponseToModule::LiveRoots {
3003            roots: roots
3004                .into_iter()
3005                .map(|(project_root, (bound, pending))| LiveRoot {
3006                    project_root,
3007                    bound,
3008                    pending,
3009                })
3010                .collect(),
3011            unknown_root_bindings,
3012            total_bindings,
3013        })
3014    }
3015
3016    /// True if this connection already owns committed or reserved CLIENT routes.
3017    /// A module registers (HELLO) before serving and never opens client routes, so
3018    /// a connection that has client routes must not also become a module endpoint
3019    /// — otherwise one connection holds both client and module state and cleanup
3020    /// only releases one side.
3021    pub(crate) fn connection_has_client_routes(
3022        &self,
3023        connection_id: ConnectionId,
3024    ) -> Result<bool, ForwardingError> {
3025        let inner = self.read_inner()?;
3026        Ok(connection_has_client_routes_locked(&inner, connection_id))
3027    }
3028
3029    pub(crate) fn cleanup_connection(
3030        &self,
3031        connection_id: ConnectionId,
3032    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
3033        self.cleanup_connection_counted(connection_id)
3034            .map(|cleanup| cleanup.released)
3035    }
3036
3037    /// [`Self::cleanup_connection`], also reporting how many pending
3038    /// route.bind relays to the closed module were aborted. That count is the
3039    /// `abandoned` figure of the `route.closed` push sent for a lost module
3040    /// connection; it is always zero for a client connection.
3041    pub(crate) fn cleanup_connection_counted(
3042        &self,
3043        connection_id: ConnectionId,
3044    ) -> Result<ConnectionCleanup, ForwardingError> {
3045        self.frame_drop_warnings
3046            .lock()
3047            .unwrap_or_else(std::sync::PoisonError::into_inner)
3048            .remove(&connection_id);
3049        let mut inner = self.write_inner()?;
3050        inner.closing_connections.insert(connection_id);
3051        #[cfg(all(test, target_os = "linux"))]
3052        if let Some(pause) = self.cleanup_pause.lock().unwrap().take() {
3053            let _ = pause.reached.send(());
3054            pause
3055                .release
3056                .recv_timeout(Duration::from_secs(5))
3057                .expect("test must release connection cleanup");
3058        }
3059        #[cfg(all(test, target_os = "linux"))]
3060        if let Some(observe) = self.cleanup_tie_observer.lock().unwrap().take() {
3061            let pending = inner
3062                .pending_relays
3063                .values()
3064                .filter(|relay| relay.reservation.client_key.connection_id == connection_id)
3065                .count();
3066            let _ = observe.send((
3067                inner.spawn_connections.contains_key(&connection_id),
3068                pending,
3069            ));
3070        }
3071        let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
3072            remove_module_connection_locked(&mut inner, endpoint)
3073        } else {
3074            ConnectionCleanup {
3075                released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
3076                abandoned_relays: 0,
3077            }
3078        };
3079        // The closing mark refuses new work for a connection whose teardown is
3080        // still pending. This is the latest point at which lifting it is safe:
3081        // teardown has just removed every per-connection entry above, under
3082        // this same write lock, so no lookup can still find live state for the
3083        // id; and ids come from a monotonic counter that never reissues one,
3084        // so the id can never name a future connection either. Keeping the
3085        // mark past this point only grew the set by one entry per connection
3086        // for the life of the daemon.
3087        inner.closing_connections.remove(&connection_id);
3088        Ok(cleanup)
3089    }
3090
3091    fn cleanup_client_connection_locked(
3092        inner: &mut ForwardingInner,
3093        connection_id: ConnectionId,
3094    ) -> Vec<GoodbyeTarget> {
3095        let routes = inner
3096            .client_to_module
3097            .iter()
3098            .filter(|(key, _)| key.connection_id == connection_id)
3099            .map(|(key, route)| (*key, route.client_epoch))
3100            .collect::<Vec<_>>();
3101        let mut released = Vec::with_capacity(routes.len());
3102        for (client_key, epoch) in routes {
3103            if let RouteRelease::Removed(target) =
3104                release_client_route_locked(inner, client_key, epoch)
3105            {
3106                released.push(target);
3107            }
3108        }
3109
3110        let pending_keys = inner
3111            .pending_relays
3112            .iter()
3113            .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
3114            .map(|(key, _)| *key)
3115            .collect::<Vec<_>>();
3116        for key in pending_keys {
3117            let Some(pending) = inner.pending_relays.remove(&key) else {
3118                continue;
3119            };
3120            release_reserved_route_locked(
3121                inner,
3122                pending.reservation.client_key,
3123                pending.reservation.module_key,
3124            );
3125            if pending.relay_enqueued {
3126                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
3127                    released.push(target);
3128                }
3129            } else {
3130                // The bind may still be on its way into the module's queue;
3131                // a GOODBYE sent now could overtake it. `route.open` sends
3132                // it once its send completes.
3133                inner.relays_released_before_enqueue.insert(key);
3134            }
3135            let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3136                "client connection closed during route.bind relay".to_string(),
3137            ));
3138        }
3139
3140        let orphaned = inner
3141            .reserved_client
3142            .iter()
3143            .filter(|(key, _)| key.connection_id == connection_id)
3144            .map(|(client, module)| (*client, *module))
3145            .collect::<Vec<_>>();
3146        for (client_key, module_key) in orphaned {
3147            release_reserved_route_locked(inner, client_key, module_key);
3148        }
3149        inner.next_client_channel.remove(&connection_id);
3150        inner
3151            .client_slot_epochs
3152            .retain(|key, _| key.connection_id != connection_id);
3153        inner
3154            .last_published_epoch
3155            .retain(|key, _| key.connection_id != connection_id);
3156        inner
3157            .status
3158            .retain(|(key, _), _| key.connection_id != connection_id);
3159
3160        released
3161    }
3162
3163    /// Close a client connection whose egress queue refused a frame for the
3164    /// route `(connection_id, channel)` at `expected_epoch`. The whole
3165    /// connection closes because its queue is shared by every route on it.
3166    ///
3167    /// The first request that actually closes the connection logs one WARN with
3168    /// the diagnosis: which principals the connection's routes belonged to,
3169    /// which module's frame did not fit and on which client channel, and what
3170    /// the queue held at that moment.
3171    pub(crate) fn escalate_client_delivery_failure(
3172        &self,
3173        connection_id: ConnectionId,
3174        channel: u16,
3175        expected_epoch: u32,
3176        reason: CloseReason,
3177        undelivered: UndeliveredFrame<'_>,
3178    ) -> Result<bool, ForwardingError> {
3179        self.escalate_client_delivery_failure_diagnosed(
3180            connection_id,
3181            channel,
3182            expected_epoch,
3183            reason,
3184            undelivered,
3185            None,
3186        )
3187    }
3188
3189    pub(crate) fn escalate_client_delivery_failure_diagnosed(
3190        &self,
3191        connection_id: ConnectionId,
3192        channel: u16,
3193        expected_epoch: u32,
3194        reason: CloseReason,
3195        undelivered: UndeliveredFrame<'_>,
3196        diagnosis: Option<&crate::router::ClientDeliveryDiagnosis>,
3197    ) -> Result<bool, ForwardingError> {
3198        let principals = {
3199            let mut inner = self.write_inner()?;
3200            let key = ClientRouteKey {
3201                connection_id,
3202                channel,
3203            };
3204            if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
3205                None
3206            } else {
3207                inner.closing_connections.insert(connection_id);
3208                Some(connection_principals_locked(&inner, connection_id))
3209            }
3210        };
3211        let Some(principals) = principals else {
3212            // A concurrent release can remove the epoch before escalation.
3213            // The frame was still lost, even though this route can no longer
3214            // authorize closing the shared client connection.
3215            self.warn_frame_drop(
3216                connection_id,
3217                channel,
3218                expected_epoch,
3219                "module_to_client",
3220                diagnosis
3221                    .map(|failure| failure.cause)
3222                    .unwrap_or("delivery_failed"),
3223            );
3224            return Ok(false);
3225        };
3226        let backlog = undelivered.sink.backlog();
3227        let close_reason = reason.to_string();
3228        if self.request_connection_close(connection_id, reason) {
3229            warn!(
3230                connection_id = connection_id.get(),
3231                principals = %principals,
3232                module_id = undelivered.module_id.unwrap_or("unknown"),
3233                client_channel = channel,
3234                queued_bytes = backlog.queued_bytes,
3235                queued_frames = backlog.queued_frames,
3236                frame_type = ?diagnosis.map(|failure| failure.frame_type),
3237                frame_flags = ?diagnosis.map(|failure| failure.flags.0),
3238                close_cause = diagnosis.map(|failure| failure.cause).unwrap_or("delivery_failed"),
3239                oldest_queued_ms = backlog
3240                    .oldest_age
3241                    .map(|age| age.as_millis() as u64)
3242                    .unwrap_or(0),
3243                close_reason = %close_reason,
3244                "closing client connection: its egress queue could not take a frame"
3245            );
3246        }
3247        Ok(true)
3248    }
3249
3250    /// Publish the tags a scope sync changed and close the live routes its
3251    /// drain table selects, in one critical section.
3252    ///
3253    /// The caller holds the scope table's write lock (scope table, then this
3254    /// table, always). Publishing and selecting under one lock is what closes
3255    /// the race with a bind commit: a commit before this sees the old tag and
3256    /// its route is selected here like any other; a commit after it sees the
3257    /// new tag and is refused.
3258    ///
3259    /// Every client route index entry is scanned, which includes routes on a
3260    /// swap's superseded endpoint (the old process of a module being replaced
3261    /// blue/green, which keeps its existing routes until they drain): those
3262    /// stay indexed until drained, so ending a scope reaches them too.
3263    pub(crate) fn publish_scope_changes(
3264        &self,
3265        changes: &[ScopeTagChange],
3266    ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
3267        let mut inner = self.write_inner()?;
3268        let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
3269        for change in changes {
3270            let key = (change.owner.clone(), change.scope_ref.clone());
3271            match change.after {
3272                Some(tag) => {
3273                    inner.scope_tags.insert(key, tag);
3274                }
3275                None => {
3276                    inner.scope_tags.remove(&key);
3277                }
3278            }
3279            by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
3280        }
3281        let mut selected = Vec::new();
3282        for (client_key, route) in &inner.client_to_module {
3283            let Some(scope) = &route.scope else {
3284                continue;
3285            };
3286            let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
3287            else {
3288                continue;
3289            };
3290            if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
3291                continue;
3292            }
3293            let reason = match &change.drain {
3294                ScopeDrain::Nothing => continue,
3295                ScopeDrain::All(reason) => *reason,
3296                ScopeDrain::Carriers(narrowed) => {
3297                    let owner = Principal::Reserved {
3298                        module_id: scope.owner.clone(),
3299                    };
3300                    let hit = route.principal != owner
3301                        && narrowed.iter().any(|(principal, allowed)| {
3302                            *principal == route.principal
3303                                && allowed
3304                                    .as_ref()
3305                                    .is_none_or(|targets| !targets.contains(&route.module_id))
3306                        });
3307                    if !hit {
3308                        continue;
3309                    }
3310                    RouteCloseReason::ScopeCarrierRemoved
3311                }
3312            };
3313            selected.push((*client_key, route.client_epoch, reason));
3314        }
3315        let mut drained = Vec::new();
3316        for (client_key, client_epoch, reason) in selected {
3317            let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
3318                continue;
3319            };
3320            let release = release_client_route_locked(&mut inner, client_key, client_epoch);
3321            self.record_route_release(&release);
3322            if let RouteRelease::Removed(module) = release {
3323                drained.push(ScopeDrainedRoute {
3324                    reason,
3325                    scope: route
3326                        .scope
3327                        .clone()
3328                        .expect("selected scope drain has a bound scope"),
3329                    module_id: route.module_id.clone(),
3330                    client: GoodbyeTarget {
3331                        connection_id: route.client_connection_id,
3332                        sink: route.client_sink.clone(),
3333                        negotiated_ver: route.client_negotiated_ver,
3334                        channel: route.client_channel,
3335                        epoch: route.client_epoch,
3336                        kind: GoodbyeTargetKind::Client,
3337                        module_id: Some(route.module_id.clone()),
3338                    },
3339                    module,
3340                });
3341            }
3342        }
3343        Ok(drained)
3344    }
3345
3346    /// The published tag of one scope, for tests of the commit re-check.
3347    #[cfg(test)]
3348    pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
3349        self.read_inner()
3350            .ok()?
3351            .scope_tags
3352            .get(&(owner.to_string(), scope_ref.to_string()))
3353            .copied()
3354    }
3355
3356    fn record_route_release(&self, release: &RouteRelease) {
3357        match release {
3358            RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
3359            RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
3360            RouteRelease::Absent => {}
3361        }
3362    }
3363
3364    fn read_inner(&self) -> Result<RwLockReadGuard<'_, ForwardingInner>, ForwardingError> {
3365        self.inner.read().map_err(|_| ForwardingError::Poisoned)
3366    }
3367
3368    fn write_inner(&self) -> Result<RwLockWriteGuard<'_, ForwardingInner>, ForwardingError> {
3369        self.inner.write().map_err(|_| ForwardingError::Poisoned)
3370    }
3371
3372    fn lock_close_registry(
3373        &self,
3374    ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
3375        self.close_registry
3376            .lock()
3377            .unwrap_or_else(|poisoned| poisoned.into_inner())
3378    }
3379}
3380
3381impl ForwardingInner {
3382    fn allocate_route_slots(
3383        &mut self,
3384        connection_id: ConnectionId,
3385        endpoint: ModuleEndpointId,
3386    ) -> Result<(u16, u32, u16, u32), ForwardingError> {
3387        let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
3388        let mut client_channel = client_start;
3389        let client_channel = loop {
3390            let key = ClientRouteKey {
3391                connection_id,
3392                channel: client_channel,
3393            };
3394            let eligible = !self.client_to_module.contains_key(&key)
3395                && !self.reserved_client.contains_key(&key)
3396                && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3397            if eligible {
3398                break client_channel;
3399            }
3400            client_channel = next_channel(client_channel);
3401            if client_channel == client_start {
3402                return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
3403            }
3404        };
3405
3406        let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
3407        let mut module_channel = module_start;
3408        let module_channel = loop {
3409            let key = ModuleRouteKey {
3410                endpoint,
3411                channel: module_channel,
3412            };
3413            let eligible = !self.module_to_client.contains_key(&key)
3414                && !self.reserved_module.contains_key(&key)
3415                && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3416            if eligible {
3417                break module_channel;
3418            }
3419            module_channel = next_channel(module_channel);
3420            if module_channel == module_start {
3421                return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
3422            }
3423        };
3424
3425        let client_key = ClientRouteKey {
3426            connection_id,
3427            channel: client_channel,
3428        };
3429        let module_key = ModuleRouteKey {
3430            endpoint,
3431            channel: module_channel,
3432        };
3433        let client_epoch = self
3434            .client_slot_epochs
3435            .get(&client_key)
3436            .copied()
3437            .unwrap_or(0)
3438            + 1;
3439        let module_epoch = self
3440            .module_slot_epochs
3441            .get(&module_key)
3442            .copied()
3443            .unwrap_or(0)
3444            + 1;
3445        self.client_slot_epochs.insert(client_key, client_epoch);
3446        self.module_slot_epochs.insert(module_key, module_epoch);
3447        self.next_client_channel
3448            .insert(connection_id, next_channel(client_channel));
3449        self.next_module_channel
3450            .insert(endpoint, next_channel(module_channel));
3451        Ok((client_channel, client_epoch, module_channel, module_epoch))
3452    }
3453
3454    fn allocate_control_corr(
3455        &mut self,
3456        endpoint: ModuleEndpointId,
3457    ) -> Result<u64, ForwardingError> {
3458        let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
3459        if candidate == 0 {
3460            self.closing_connections.insert(endpoint.connection_id);
3461            return Err(ForwardingError::RelayCorrelationExhausted);
3462        }
3463        self.next_control_corr.insert(
3464            endpoint,
3465            if candidate == u64::MAX {
3466                0
3467            } else {
3468                candidate + 1
3469            },
3470        );
3471        Ok(candidate)
3472    }
3473}
3474
3475fn connection_has_client_routes_locked(
3476    inner: &ForwardingInner,
3477    connection_id: ConnectionId,
3478) -> bool {
3479    inner
3480        .client_to_module
3481        .keys()
3482        .any(|key| key.connection_id == connection_id)
3483        || inner
3484            .reserved_client
3485            .keys()
3486            .any(|key| key.connection_id == connection_id)
3487}
3488
3489fn check_module_connection_role_locked(
3490    inner: &ForwardingInner,
3491    connection_id: ConnectionId,
3492) -> Result<(), ForwardingError> {
3493    if inner.endpoint_by_connection.contains_key(&connection_id)
3494        || connection_has_client_routes_locked(inner, connection_id)
3495    {
3496        return Err(ForwardingError::ConnectionRoleConflict { connection_id });
3497    }
3498    Ok(())
3499}
3500
3501fn next_channel(channel: u16) -> u16 {
3502    let next = channel.wrapping_add(1);
3503    if next == 0 {
3504        1
3505    } else {
3506        next
3507    }
3508}
3509
3510fn endpoint_routes_locked(
3511    inner: &ForwardingInner,
3512    endpoint: ModuleEndpointId,
3513) -> Vec<EndpointRoute> {
3514    let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
3515    let draining = drain_reason.is_some();
3516    let mut routes = inner
3517        .module_to_client
3518        .iter()
3519        .filter(|(module_key, _)| module_key.endpoint == endpoint)
3520        .map(|(_, route)| EndpointRoute {
3521            goodbye_target: GoodbyeTarget {
3522                connection_id: route.client_connection_id,
3523                sink: route.client_sink.clone(),
3524                negotiated_ver: route.client_negotiated_ver,
3525                channel: route.client_channel,
3526                epoch: route.client_epoch,
3527                kind: GoodbyeTargetKind::Client,
3528                module_id: Some(route.module_id.clone()),
3529            },
3530            principal: route.principal.clone(),
3531            bound_at: route.bound_at,
3532            draining,
3533            drain_reason,
3534        })
3535        .collect::<Vec<_>>();
3536    routes.sort_by_key(|route| {
3537        (
3538            route.goodbye_target.connection_id.get(),
3539            route.goodbye_target.channel,
3540            route.goodbye_target.epoch,
3541        )
3542    });
3543    routes
3544}
3545
3546/// Why a bind admitted under `scope` may no longer commit, read from the
3547/// published tags: `scope_ended` when the scope is gone or at another epoch,
3548/// `scope_changed` (retryable) when only its content moved.
3549fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
3550    let key = (scope.owner.clone(), scope.scope_ref.clone());
3551    match inner.scope_tags.get(&key) {
3552        Some(current) if *current == scope.tag => None,
3553        Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
3554            subc_protocol::error_codes::SCOPE_CHANGED,
3555            format!(
3556                "scope '{}' of {} changed while the route was being bound; re-open it",
3557                scope.scope_ref, scope.owner
3558            ),
3559        )),
3560        _ => Some(ErrorBody::new(
3561            subc_protocol::error_codes::SCOPE_ENDED,
3562            format!(
3563                "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
3564                scope.scope_ref, scope.owner, scope.tag.scope_epoch
3565            ),
3566        )),
3567    }
3568}
3569
3570fn release_reserved_route_locked(
3571    inner: &mut ForwardingInner,
3572    client_key: ClientRouteKey,
3573    module_key: ModuleRouteKey,
3574) {
3575    if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
3576        inner.reserved_client.remove(&client_key);
3577    }
3578    if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
3579        inner.reserved_module.remove(&module_key);
3580    }
3581    inner.status.retain(|(key, _), _| *key != client_key);
3582}
3583
3584fn release_client_route_locked(
3585    inner: &mut ForwardingInner,
3586    client_key: ClientRouteKey,
3587    expected_epoch: u32,
3588) -> RouteRelease {
3589    let Some(route) = inner.client_to_module.get(&client_key) else {
3590        return RouteRelease::Absent;
3591    };
3592    if route.client_epoch != expected_epoch {
3593        return RouteRelease::Stale;
3594    }
3595    let route = inner
3596        .client_to_module
3597        .remove(&client_key)
3598        .expect("route checked under the same forwarding lock");
3599    route.flow.close();
3600    if let Some(scope) = &route.scope {
3601        scope.release_bind();
3602    }
3603    inner.module_to_client.remove(&ModuleRouteKey {
3604        endpoint: route.module_endpoint,
3605        channel: route.module_channel,
3606    });
3607    inner.operator_confirms.route_closed(ModuleRouteKey {
3608        endpoint: route.module_endpoint,
3609        channel: route.module_channel,
3610    });
3611    inner.status.remove(&(client_key, expected_epoch));
3612    RouteRelease::Removed(GoodbyeTarget {
3613        connection_id: route.module_endpoint.connection_id,
3614        sink: route.module_sink.clone(),
3615        negotiated_ver: route.module_negotiated_ver,
3616        channel: route.module_channel,
3617        epoch: route.module_epoch,
3618        kind: GoodbyeTargetKind::Module,
3619        module_id: Some(route.module_id.clone()),
3620    })
3621}
3622
3623fn release_module_route_locked(
3624    inner: &mut ForwardingInner,
3625    module_key: ModuleRouteKey,
3626    expected_epoch: u32,
3627) -> RouteRelease {
3628    let Some(route) = inner.module_to_client.get(&module_key) else {
3629        return RouteRelease::Absent;
3630    };
3631    if route.module_epoch != expected_epoch {
3632        return RouteRelease::Stale;
3633    }
3634    let route = inner
3635        .module_to_client
3636        .remove(&module_key)
3637        .expect("route checked under the same forwarding lock");
3638    inner.operator_confirms.route_closed(module_key);
3639    route.flow.close();
3640    if let Some(scope) = &route.scope {
3641        scope.release_bind();
3642    }
3643    let client_key = ClientRouteKey {
3644        connection_id: route.client_connection_id,
3645        channel: route.client_channel,
3646    };
3647    inner.client_to_module.remove(&client_key);
3648    inner.status.remove(&(client_key, route.client_epoch));
3649    RouteRelease::Removed(GoodbyeTarget {
3650        connection_id: route.client_connection_id,
3651        sink: route.client_sink.clone(),
3652        negotiated_ver: route.client_negotiated_ver,
3653        channel: route.client_channel,
3654        epoch: route.client_epoch,
3655        kind: GoodbyeTargetKind::Client,
3656        module_id: Some(route.module_id.clone()),
3657    })
3658}
3659
3660fn commit_route_locked(
3661    inner: &mut ForwardingInner,
3662    mut pending: PendingRouteBindRelayEntry,
3663) -> Result<Option<GoodbyeTarget>, ForwardingError> {
3664    let reservation = pending.reservation;
3665    if inner
3666        .closing_connections
3667        .contains(&reservation.client_key.connection_id)
3668    {
3669        return Err(ForwardingError::ConnectionClosing {
3670            connection_id: reservation.client_key.connection_id,
3671        });
3672    }
3673    let module_id = inner
3674        .module_id_by_endpoint
3675        .get(&reservation.module_key.endpoint)
3676        .cloned()
3677        .ok_or(ForwardingError::StaleModuleEndpoint)?;
3678    if inner
3679        .draining_endpoints
3680        .contains_key(&reservation.module_key.endpoint)
3681    {
3682        return Err(ForwardingError::ModuleReloading { module_id });
3683    }
3684    if pending.busy_refusals {
3685        let body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
3686            route_channel: reservation.client_key.channel,
3687            route_epoch: reservation.client_epoch,
3688            request_window: Some(pending.request_window),
3689            watch_window: pending.watch_window,
3690        })
3691        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
3692        pending.route_open_frame = Frame::build_with_version(
3693            pending.client_negotiated_ver,
3694            FrameType::Response,
3695            pending.route_open_frame.header.flags,
3696            0,
3697            0,
3698            pending.route_open_frame.header.corr,
3699            body,
3700        )
3701        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
3702    }
3703    if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
3704        || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
3705    {
3706        return Err(ForwardingError::UnknownReservation {
3707            client_channel: reservation.client_key.channel,
3708            module_channel: reservation.module_key.channel,
3709        });
3710    }
3711    let module = inner
3712        .modules_by_id
3713        .get(&module_id)
3714        .filter(|module| module.endpoint == reservation.module_key.endpoint)
3715        .cloned()
3716        .ok_or(ForwardingError::StaleModuleEndpoint)?;
3717    let binding = Arc::new(RouteBinding {
3718        client_connection_id: reservation.client_key.connection_id,
3719        client_sink: pending.client_sink,
3720        client_negotiated_ver: pending.client_negotiated_ver,
3721        client_channel: reservation.client_key.channel,
3722        client_epoch: reservation.client_epoch,
3723        module_id,
3724        module_endpoint: reservation.module_key.endpoint,
3725        module_sink: module.sink,
3726        module_negotiated_ver: module.negotiated_ver,
3727        module_channel: reservation.module_key.channel,
3728        module_epoch: reservation.module_epoch,
3729        principal: pending.principal,
3730        harness: pending.harness,
3731        build_class: pending.build_class,
3732        operator_app: pending.operator_app,
3733        project_root: reservation.project_root.clone(),
3734        bound_at: Instant::now(),
3735        flow: Arc::new(if pending.busy_refusals {
3736            ChannelFlow::with_admission(pending.request_window, true, pending.watch_window)
3737        } else {
3738            ChannelFlow::new(pending.request_window)
3739        }),
3740        scope: pending.scope,
3741    });
3742    inner
3743        .client_to_module
3744        .insert(reservation.client_key, Arc::clone(&binding));
3745    inner
3746        .module_to_client
3747        .insert(reservation.module_key, Arc::clone(&binding));
3748    let previous_published = inner
3749        .last_published_epoch
3750        .insert(reservation.client_key, reservation.client_epoch);
3751
3752    // This point is reachable only through an accepted route.bind acknowledgement.
3753    // Publish history before notifying the opener, so a subsequent
3754    // `scope.served_by` query sees the acknowledged bind.
3755    if let Some(scope) = &binding.scope {
3756        scope.record_served(&binding.module_id, module.verified_prefix_owner.as_deref());
3757    }
3758
3759    // Sending through the reserved slot cannot fail, but it reports a receiver
3760    // that closed after reservation and before this locked publication point.
3761    // The frame is stamped and charged to the queued-byte count here, when it
3762    // actually enters the queue, not when the slot was reserved.
3763    let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
3764    if client_writer_closed {
3765        // Not gated on `relay_enqueued`: only a module ack reaches this
3766        // commit, which proves the module holds the binding, and `route.open`
3767        // sends nothing for a relay a reply already settled.
3768        let abandoned = abandoned_route_target(inner, &reservation);
3769        if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
3770            route.flow.close();
3771        }
3772        inner.module_to_client.remove(&reservation.module_key);
3773        inner
3774            .status
3775            .remove(&(reservation.client_key, reservation.client_epoch));
3776        match previous_published {
3777            Some(epoch) => {
3778                inner
3779                    .last_published_epoch
3780                    .insert(reservation.client_key, epoch);
3781            }
3782            None => {
3783                inner.last_published_epoch.remove(&reservation.client_key);
3784            }
3785        }
3786        let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3787            "client egress closed during route publication".to_string(),
3788        ));
3789        return Ok(abandoned);
3790    }
3791
3792    if let Some(scope) = &binding.scope {
3793        scope.commit_presence();
3794    }
3795    let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
3796    Ok(None)
3797}
3798
3799fn spawn_connection_dead(inner: &ForwardingInner, connection: ConnectionId) -> bool {
3800    inner
3801        .spawn_connections
3802        .get(&connection)
3803        .is_some_and(|spawn| spawn.upgrade().is_none_or(|spawn| !spawn.launch_ready()))
3804}
3805
3806/// The live module connection registered under exactly `endpoint`, whichever
3807/// slot holds it: active, swap candidate, or superseded incumbent.
3808///
3809/// Resolving by endpoint rather than by module id is what keeps an incumbent
3810/// addressable after a swap promoted a candidate over its id. An endpoint that
3811/// is in none of the three (a stale one, replaced without a promotion) resolves
3812/// to nothing, as it always has.
3813fn module_connection_for_endpoint_locked(
3814    inner: &ForwardingInner,
3815    endpoint: ModuleEndpointId,
3816) -> Option<&ModuleConnection> {
3817    let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
3818    inner
3819        .modules_by_id
3820        .get(module_id)
3821        .filter(|module| module.endpoint == endpoint)
3822        .or_else(|| {
3823            inner
3824                .candidates_by_id
3825                .get(module_id)
3826                .filter(|module| module.endpoint == endpoint)
3827        })
3828        .or_else(|| inner.superseded_endpoints.get(&endpoint))
3829}
3830
3831fn abandoned_route_target(
3832    inner: &ForwardingInner,
3833    reservation: &RouteReservation,
3834) -> Option<GoodbyeTarget> {
3835    let module_id = inner
3836        .module_id_by_endpoint
3837        .get(&reservation.module_key.endpoint)?;
3838    let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
3839    (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
3840        connection_id: module.endpoint.connection_id,
3841        sink: module.sink.clone(),
3842        negotiated_ver: module.negotiated_ver,
3843        channel: reservation.module_key.channel,
3844        epoch: reservation.module_epoch,
3845        kind: GoodbyeTargetKind::Module,
3846        module_id: Some(module_id.clone()),
3847    })
3848}
3849
3850/// Queue a registering module's HELLO_ACK on its sink. Called with the
3851/// forwarding write lock held, before the endpoint is inserted, which is what
3852/// puts the ack ahead of any `route.bind` or control RPC routed to the module.
3853/// `try_send` never waits, so holding the lock across it is safe.
3854fn enqueue_hello_ack_locked(
3855    sink: &FrameSink,
3856    connection_id: ConnectionId,
3857    hello_ack: Option<Frame>,
3858) -> Result<(), ForwardingError> {
3859    let Some(hello_ack) = hello_ack else {
3860        return Ok(());
3861    };
3862    sink.try_send(hello_ack)
3863        .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
3864}
3865
3866fn remove_module_connection_locked(
3867    inner: &mut ForwardingInner,
3868    endpoint: ModuleEndpointId,
3869) -> ConnectionCleanup {
3870    // Commit the disconnect before its routes are released, so their teardown
3871    // cannot replace module_closed with route_closed.
3872    inner.operator_confirms.module_closed(endpoint);
3873    inner.draining_endpoints.remove(&endpoint);
3874    inner.daemon_drain_results.remove(&endpoint);
3875    let module_id = inner.module_id_by_endpoint.remove(&endpoint);
3876    if let Some(module_id) = module_id.as_ref() {
3877        if inner
3878            .modules_by_id
3879            .get(module_id)
3880            .is_some_and(|module| module.endpoint == endpoint)
3881        {
3882            inner.modules_by_id.remove(module_id);
3883        }
3884        if inner
3885            .candidates_by_id
3886            .get(module_id)
3887            .is_some_and(|module| module.endpoint == endpoint)
3888        {
3889            inner.candidates_by_id.remove(module_id);
3890        }
3891    }
3892    inner.superseded_endpoints.remove(&endpoint);
3893    inner.endpoint_by_connection.remove(&endpoint.connection_id);
3894    inner.next_module_channel.remove(&endpoint);
3895    inner.next_control_corr.remove(&endpoint);
3896    inner
3897        .health_probe_tombstones
3898        .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
3899    inner
3900        .module_slot_epochs
3901        .retain(|key, _| key.endpoint != endpoint);
3902    let reserved_module_keys: Vec<ModuleRouteKey> = inner
3903        .reserved_module
3904        .keys()
3905        .filter(|module_key| module_key.endpoint == endpoint)
3906        .copied()
3907        .collect();
3908    for module_key in reserved_module_keys {
3909        if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
3910            release_reserved_route_locked(inner, client_key, module_key);
3911        }
3912    }
3913
3914    let pending_keys: Vec<_> = inner
3915        .pending_relays
3916        .keys()
3917        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3918        .copied()
3919        .collect();
3920    let pending: Vec<_> = pending_keys
3921        .into_iter()
3922        .filter_map(|key| {
3923            inner
3924                .pending_relays
3925                .remove(&key)
3926                .map(|pending| (key, pending))
3927        })
3928        .collect();
3929    let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
3930    for (key, pending) in pending {
3931        // Recorded like every other removal without a reply, so a `route.open`
3932        // still sending the bind learns the relay was released rather than
3933        // settled by the module.
3934        if !pending.relay_enqueued {
3935            inner.relays_released_before_enqueue.insert(key);
3936        }
3937        let module_label = module_id.as_deref().unwrap_or("unknown");
3938        let _ = pending
3939            .sender
3940            .send(RouteBindRelayOutcome::ModuleGone(format!(
3941                "module '{module_label}' connection closed during route.bind relay"
3942            )));
3943    }
3944
3945    let pending_control_keys: Vec<_> = inner
3946        .pending_control_rpcs
3947        .keys()
3948        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3949        .copied()
3950        .collect();
3951    let pending_control: Vec<_> = pending_control_keys
3952        .into_iter()
3953        .filter_map(|key| inner.pending_control_rpcs.remove(&key))
3954        .collect();
3955    for pending in pending_control {
3956        let module_label = module_id.as_deref().unwrap_or("unknown");
3957        let _ = pending
3958            .sender
3959            .send(ModuleControlRpcOutcome::ModuleGone(format!(
3960                "module '{module_label}' connection closed during module-control RPC"
3961            )));
3962    }
3963
3964    let module_routes = inner
3965        .module_to_client
3966        .iter()
3967        .filter(|(module_key, _)| module_key.endpoint == endpoint)
3968        .map(|(module_key, route)| (*module_key, route.module_epoch))
3969        .collect::<Vec<_>>();
3970    let mut released = Vec::with_capacity(module_routes.len());
3971    for (module_key, epoch) in module_routes {
3972        if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
3973        {
3974            released.push(target);
3975        }
3976    }
3977    ConnectionCleanup {
3978        released,
3979        abandoned_relays,
3980    }
3981}
3982
3983#[derive(Debug, Clone, Copy)]
3984struct RequestCredit {
3985    subscription: bool,
3986    watch_credit: bool,
3987    excluded_from_drain: bool,
3988}
3989
3990#[derive(Debug, Default)]
3991struct CreditLedger {
3992    by_corr: HashMap<u64, Vec<RequestCredit>>,
3993}
3994
3995impl CreditLedger {
3996    fn acquire(&mut self, corr: u64, subscription: bool, watch_credit: bool) {
3997        self.by_corr.entry(corr).or_default().push(RequestCredit {
3998            subscription,
3999            watch_credit,
4000            excluded_from_drain: false,
4001        });
4002    }
4003
4004    fn release(&mut self, corr: u64) -> Option<RequestCredit> {
4005        let credits = self.by_corr.get_mut(&corr)?;
4006        let released = credits.pop();
4007        if credits.is_empty() {
4008            self.by_corr.remove(&corr);
4009        }
4010        released
4011    }
4012
4013    fn capture_subscription_exclusions(&mut self) -> u32 {
4014        let mut excluded = 0u32;
4015        for credit in self.by_corr.values_mut().flatten() {
4016            if credit.subscription && !credit.excluded_from_drain {
4017                credit.excluded_from_drain = true;
4018                excluded = excluded.saturating_add(1);
4019            }
4020        }
4021        excluded
4022    }
4023
4024    fn in_flight(&self) -> usize {
4025        self.by_corr.values().map(Vec::len).sum()
4026    }
4027
4028    fn drain_in_flight(&self) -> usize {
4029        self.by_corr
4030            .values()
4031            .flatten()
4032            .filter(|credit| !credit.excluded_from_drain)
4033            .count()
4034    }
4035
4036    /// The correlation ids of the requests a drain still waits on, one entry
4037    /// per held credit, in ascending order.
4038    fn drain_held_corrs(&self) -> Vec<u64> {
4039        let mut corrs = self
4040            .by_corr
4041            .iter()
4042            .flat_map(|(corr, credits)| {
4043                credits
4044                    .iter()
4045                    .filter(|credit| !credit.excluded_from_drain)
4046                    .map(move |_| *corr)
4047            })
4048            .collect::<Vec<_>>();
4049        corrs.sort_unstable();
4050        corrs
4051    }
4052}
4053
4054#[derive(Debug, Default)]
4055struct ChannelFlowState {
4056    closed: bool,
4057    credits: CreditLedger,
4058    last_busy_warn: Option<tokio::time::Instant>,
4059}
4060
4061/// Per-channel request-credit accounting shared by the client and module route halves.
4062#[derive(Debug)]
4063pub(crate) struct ChannelFlow {
4064    sem: Semaphore,
4065    window: usize,
4066    busy_refusals: bool,
4067    watch_sem: Option<Semaphore>,
4068    watch_window: Option<usize>,
4069    state: Mutex<ChannelFlowState>,
4070}
4071
4072impl ChannelFlow {
4073    pub(crate) fn new(window: usize) -> Self {
4074        Self::with_admission(window, false, None)
4075    }
4076
4077    fn with_admission(window: usize, busy_refusals: bool, watch_window: Option<usize>) -> Self {
4078        debug_assert!(window > 0, "flow-control window must be non-zero");
4079        Self {
4080            sem: Semaphore::new(window),
4081            window,
4082            busy_refusals,
4083            watch_sem: watch_window.map(Semaphore::new),
4084            watch_window,
4085            state: Mutex::new(ChannelFlowState::default()),
4086        }
4087    }
4088
4089    pub(crate) fn busy_refusals(&self) -> bool {
4090        self.busy_refusals
4091    }
4092
4093    fn try_acquire_tagged(&self, corr: u64, subscription: bool) -> RequestAdmission {
4094        let mut state = self
4095            .state
4096            .lock()
4097            .unwrap_or_else(|poisoned| poisoned.into_inner());
4098        if state.closed {
4099            return RequestAdmission::Stale;
4100        }
4101        let busy = |state: &ChannelFlowState| {
4102            RequestAdmission::Busy(subc_protocol::RouteBusyDetail {
4103                reason: "no_credit".into(),
4104                request_in_flight: state.credits.in_flight(),
4105                watch_in_flight: state
4106                    .credits
4107                    .by_corr
4108                    .values()
4109                    .flatten()
4110                    .filter(|credit| credit.subscription)
4111                    .count(),
4112                request_window: self.window,
4113                watch_window: self.watch_window,
4114            })
4115        };
4116        let Ok(total) = self.sem.try_acquire() else {
4117            return busy(&state);
4118        };
4119        // A watch takes the total credit first. Dropping it on watch-lane
4120        // exhaustion rolls back before any ledger entry becomes visible.
4121        let watch = if subscription {
4122            if let Some(sem) = &self.watch_sem {
4123                match sem.try_acquire() {
4124                    Ok(permit) => Some(permit),
4125                    Err(_) => return busy(&state),
4126                }
4127            } else {
4128                None
4129            }
4130        } else {
4131            None
4132        };
4133        state.credits.acquire(corr, subscription, watch.is_some());
4134        total.forget();
4135        if let Some(watch) = watch {
4136            watch.forget();
4137        }
4138        RequestAdmission::Admitted
4139    }
4140
4141    pub(crate) fn should_warn_busy(&self) -> bool {
4142        let mut state = self
4143            .state
4144            .lock()
4145            .unwrap_or_else(|poisoned| poisoned.into_inner());
4146        let now = tokio::time::Instant::now();
4147        if state
4148            .last_busy_warn
4149            .is_some_and(|last| now.duration_since(last) < std::time::Duration::from_secs(60))
4150        {
4151            return false;
4152        }
4153        state.last_busy_warn = Some(now);
4154        true
4155    }
4156
4157    #[cfg(test)]
4158    pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
4159        self.acquire_tagged(0, false).await
4160    }
4161
4162    pub(crate) async fn acquire_tagged(
4163        &self,
4164        corr: u64,
4165        subscription: bool,
4166    ) -> Result<(), ChannelFlowClosed> {
4167        let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
4168        let mut state = self
4169            .state
4170            .lock()
4171            .unwrap_or_else(|poisoned| poisoned.into_inner());
4172        if state.closed {
4173            return Err(ChannelFlowClosed);
4174        }
4175        state.credits.acquire(corr, subscription, false);
4176        permit.forget();
4177        Ok(())
4178    }
4179
4180    #[cfg(test)]
4181    pub(crate) fn release(&self) {
4182        self.release_corr(0);
4183    }
4184
4185    pub(crate) fn release_corr(&self, corr: u64) {
4186        let mut state = self
4187            .state
4188            .lock()
4189            .unwrap_or_else(|poisoned| poisoned.into_inner());
4190        let Some(released) = state.credits.release(corr) else {
4191            // Protocol-conforming modules emit exactly one terminal per request.
4192            // This guard is a best-effort safety net against window growth, not a
4193            // security boundary against malicious peers.
4194            warn!(
4195                window = self.window,
4196                available = self.sem.available_permits(),
4197                "flow-control over-release ignored"
4198            );
4199            return;
4200        };
4201        if !self.sem.is_closed() {
4202            self.sem.add_permits(1);
4203        }
4204        if released.watch_credit {
4205            if let Some(watch_sem) = self.watch_sem.as_ref().filter(|sem| !sem.is_closed()) {
4206                watch_sem.add_permits(1);
4207            }
4208        }
4209    }
4210
4211    #[cfg(test)]
4212    pub(crate) fn in_flight(&self) -> usize {
4213        self.state
4214            .lock()
4215            .unwrap_or_else(|poisoned| poisoned.into_inner())
4216            .credits
4217            .in_flight()
4218    }
4219
4220    pub(crate) fn drain_in_flight(&self) -> usize {
4221        self.state
4222            .lock()
4223            .unwrap_or_else(|poisoned| poisoned.into_inner())
4224            .credits
4225            .drain_in_flight()
4226    }
4227
4228    pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
4229        self.state
4230            .lock()
4231            .unwrap_or_else(|poisoned| poisoned.into_inner())
4232            .credits
4233            .drain_held_corrs()
4234    }
4235
4236    #[cfg(test)]
4237    pub(crate) fn available_permits(&self) -> usize {
4238        self.sem.available_permits()
4239    }
4240
4241    pub(crate) fn begin_drain(&self) -> u32 {
4242        let mut state = self
4243            .state
4244            .lock()
4245            .unwrap_or_else(|poisoned| poisoned.into_inner());
4246        state.closed = true;
4247        self.sem.close();
4248        if let Some(sem) = &self.watch_sem {
4249            sem.close();
4250        }
4251        state.credits.capture_subscription_exclusions()
4252    }
4253
4254    pub(crate) fn close(&self) {
4255        let mut state = self
4256            .state
4257            .lock()
4258            .unwrap_or_else(|poisoned| poisoned.into_inner());
4259        state.closed = true;
4260        self.sem.close();
4261        if let Some(sem) = &self.watch_sem {
4262            sem.close();
4263        }
4264    }
4265}
4266
4267#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4268pub(crate) struct ChannelFlowClosed;
4269
4270#[derive(Debug, Clone, PartialEq, Eq)]
4271pub(crate) enum RequestAdmission {
4272    Admitted,
4273    Busy(subc_protocol::RouteBusyDetail),
4274    ModuleReloading,
4275    Stale,
4276}
4277
4278impl fmt::Display for ChannelFlowClosed {
4279    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4280        write!(f, "flow-control window closed")
4281    }
4282}
4283
4284impl Error for ChannelFlowClosed {}
4285
4286#[cfg(test)]
4287#[path = "route_busy_flow_tests.rs"]
4288mod route_busy_flow_tests;
4289
4290fn window_for(concurrency: &Concurrency) -> usize {
4291    match concurrency {
4292        Concurrency::Serial => 1,
4293        Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
4294        Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
4295    }
4296}
4297
4298#[derive(Debug, Clone, PartialEq, Eq)]
4299pub enum ForwardingError {
4300    /// A connection cannot hold both client routes and a module endpoint, or
4301    /// replace a module identity while that endpoint is still registered.
4302    ConnectionRoleConflict {
4303        connection_id: ConnectionId,
4304    },
4305    NoModuleConnection,
4306    ModuleReloading {
4307        module_id: String,
4308    },
4309    StaleModuleEndpoint,
4310    UnknownReservation {
4311        client_channel: u16,
4312        module_channel: u16,
4313    },
4314    ClientRouteChannelExhausted {
4315        connection_id: ConnectionId,
4316    },
4317    ModuleRouteChannelExhausted {
4318        endpoint: ModuleEndpointId,
4319    },
4320    RelayCorrelationExhausted,
4321    ConnectionClosing {
4322        connection_id: ConnectionId,
4323    },
4324    ClientEgressClosed {
4325        connection_id: ConnectionId,
4326    },
4327    RouteOpenBuild(String),
4328    /// A swap candidate is already registered for this module id.
4329    CandidateSlotOccupied {
4330        module_id: String,
4331    },
4332    /// A registering module's HELLO_ACK could not be queued because its
4333    /// outbound queue is closed or full.
4334    ModuleEgressUnavailable {
4335        connection_id: ConnectionId,
4336    },
4337    Poisoned,
4338}
4339
4340impl fmt::Display for ForwardingError {
4341    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4342        match self {
4343            Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
4344            Self::NoModuleConnection => write!(f, "no module connection is registered"),
4345            Self::ModuleReloading { module_id } => {
4346                write!(f, "module_id '{module_id}' is reloading")
4347            }
4348            Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
4349            Self::UnknownReservation {
4350                client_channel,
4351                module_channel,
4352            } => write!(
4353                f,
4354                "route reservation client channel {client_channel} / module channel {module_channel} was not found"
4355            ),
4356            Self::ClientRouteChannelExhausted { connection_id } => write!(
4357                f,
4358                "no client route channels are available for connection {}",
4359                connection_id.get()
4360            ),
4361            Self::ModuleRouteChannelExhausted { endpoint } => write!(
4362                f,
4363                "no module route channels are available for endpoint generation {} on connection {}",
4364                endpoint.generation,
4365                endpoint.connection_id.get()
4366            ),
4367            Self::RelayCorrelationExhausted => {
4368                write!(f, "module control correlation ids are exhausted")
4369            }
4370            Self::ConnectionClosing { connection_id } => write!(
4371                f,
4372                "connection {} is closing and cannot accept route allocation",
4373                connection_id.get()
4374            ),
4375            Self::ClientEgressClosed { connection_id } => write!(
4376                f,
4377                "client connection {} egress is closed",
4378                connection_id.get()
4379            ),
4380            Self::RouteOpenBuild(message) => {
4381                write!(f, "failed to prebuild route.open response: {message}")
4382            }
4383            Self::CandidateSlotOccupied { module_id } => write!(
4384                f,
4385                "module_id '{module_id}' already has a swap candidate registered"
4386            ),
4387            Self::ModuleEgressUnavailable { connection_id } => write!(
4388                f,
4389                "module connection {} egress is unavailable; HELLO_ACK could not be queued",
4390                connection_id.get()
4391            ),
4392            Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
4393        }
4394    }
4395}
4396
4397impl Error for ForwardingError {}
4398
4399#[cfg(test)]
4400mod tests {
4401    use std::time::Duration;
4402
4403    use super::*;
4404    use tokio::sync::mpsc;
4405
4406    #[test]
4407    fn ordinary_long_running_request_is_not_excluded_from_drain() {
4408        let mut ledger = CreditLedger::default();
4409        ledger.acquire(1, false, false);
4410
4411        assert_eq!(ledger.capture_subscription_exclusions(), 0);
4412        assert_eq!(ledger.drain_in_flight(), 1);
4413    }
4414
4415    #[test]
4416    fn bit_set_subscription_is_excluded_and_counted() {
4417        let mut ledger = CreditLedger::default();
4418        ledger.acquire(1, true, false);
4419
4420        assert_eq!(ledger.capture_subscription_exclusions(), 1);
4421        assert_eq!(ledger.drain_in_flight(), 0);
4422    }
4423
4424    #[test]
4425    fn subscription_opened_after_drain_snapshot_is_not_excluded() {
4426        let mut ledger = CreditLedger::default();
4427        ledger.acquire(1, true, false);
4428        assert_eq!(ledger.capture_subscription_exclusions(), 1);
4429
4430        ledger.acquire(2, true, false);
4431
4432        assert_eq!(ledger.drain_in_flight(), 1);
4433    }
4434
4435    #[test]
4436    fn drain_with_no_subscriptions_reports_zero_excluded() {
4437        let mut ledger = CreditLedger::default();
4438        assert_eq!(ledger.capture_subscription_exclusions(), 0);
4439    }
4440
4441    fn test_hello_ack(corr: u64) -> Frame {
4442        Frame::build(
4443            FrameType::HelloAck,
4444            Flags::new(false, Priority::Passive, false),
4445            0,
4446            0,
4447            corr,
4448            Vec::new(),
4449        )
4450        .unwrap()
4451    }
4452
4453    /// An acked registration whose HELLO_ACK cannot be queued must leave no
4454    /// endpoint behind: a routable module that never got its ack would read a
4455    /// route.bind first and exit. A closed queue and a full one both refuse.
4456    #[test]
4457    fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
4458        let forwarding = ForwardingTable::default();
4459
4460        let (closed_tx, closed_rx) = mpsc::channel(8);
4461        drop(closed_rx);
4462        let closed = ConnectionId::new(1);
4463        assert_eq!(
4464            forwarding.register_module_connection_acked(
4465                closed,
4466                "closed".to_string(),
4467                2,
4468                Concurrency::ModuleManaged,
4469                FrameSink::new(closed_tx),
4470                ModuleHelloAck {
4471                    frame: test_hello_ack(1),
4472                    verified_prefix_owner: None
4473                },
4474            ),
4475            Err(ForwardingError::ModuleEgressUnavailable {
4476                connection_id: closed
4477            })
4478        );
4479
4480        let (full_tx, _full_rx) = mpsc::channel(1);
4481        let full_sink = FrameSink::new(full_tx);
4482        full_sink.try_send(test_hello_ack(99)).unwrap();
4483        let full = ConnectionId::new(2);
4484        assert_eq!(
4485            forwarding.register_module_connection_acked(
4486                full,
4487                "full".to_string(),
4488                2,
4489                Concurrency::ModuleManaged,
4490                full_sink.clone(),
4491                ModuleHelloAck {
4492                    frame: test_hello_ack(2),
4493                    verified_prefix_owner: None
4494                },
4495            ),
4496            Err(ForwardingError::ModuleEgressUnavailable {
4497                connection_id: full
4498            })
4499        );
4500        assert_eq!(
4501            forwarding.register_candidate_module_connection_acked(
4502                full,
4503                "full".to_string(),
4504                2,
4505                Concurrency::ModuleManaged,
4506                full_sink,
4507                test_hello_ack(3),
4508            ),
4509            Err(ForwardingError::ModuleEgressUnavailable {
4510                connection_id: full
4511            })
4512        );
4513
4514        for (connection, module_id) in [(closed, "closed"), (full, "full")] {
4515            assert_eq!(
4516                forwarding
4517                    .module_endpoint_for_connection(connection)
4518                    .unwrap(),
4519                None
4520            );
4521            let (client_tx, _client_rx) = mpsc::channel(8);
4522            assert_eq!(
4523                forwarding
4524                    .begin_route_bind_relay_for_test(
4525                        ConnectionId::new(50),
4526                        FrameSink::new(client_tx),
4527                        1,
4528                        module_id,
4529                    )
4530                    .err(),
4531                Some(ForwardingError::NoModuleConnection)
4532            );
4533        }
4534        assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
4535    }
4536
4537    /// Both acked registration forms put the HELLO_ACK on the module's queue
4538    /// by the time the endpoint can be resolved.
4539    #[test]
4540    fn acked_registration_queues_the_hello_ack_first() {
4541        let forwarding = ForwardingTable::default();
4542        let (active_tx, mut active_rx) = mpsc::channel(8);
4543        forwarding
4544            .register_module_connection_acked(
4545                ConnectionId::new(1),
4546                "acked".to_string(),
4547                2,
4548                Concurrency::ModuleManaged,
4549                FrameSink::new(active_tx),
4550                ModuleHelloAck {
4551                    frame: test_hello_ack(11),
4552                    verified_prefix_owner: None,
4553                },
4554            )
4555            .unwrap();
4556        let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
4557        forwarding
4558            .register_candidate_module_connection_acked(
4559                ConnectionId::new(2),
4560                "acked".to_string(),
4561                2,
4562                Concurrency::ModuleManaged,
4563                FrameSink::new(candidate_tx),
4564                test_hello_ack(12),
4565            )
4566            .unwrap();
4567
4568        let active_first = active_rx.try_recv().unwrap().frame;
4569        assert_eq!(active_first.header.ty, FrameType::HelloAck);
4570        assert_eq!(active_first.header.corr, 11);
4571        let candidate_first = candidate_rx.try_recv().unwrap().frame;
4572        assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
4573        assert_eq!(candidate_first.header.corr, 12);
4574    }
4575
4576    #[test]
4577    fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
4578        let forwarding = ForwardingTable::default();
4579        let module_connection = ConnectionId::new(10);
4580        let exhausted_client = ConnectionId::new(20);
4581        let second_client = ConnectionId::new(30);
4582        let (module_tx, _module_rx) = mpsc::channel(1);
4583        let endpoint = forwarding
4584            .register_module_connection(
4585                module_connection,
4586                "route-limit-provider".to_string(),
4587                1,
4588                Concurrency::ModuleManaged,
4589                FrameSink::new(module_tx),
4590            )
4591            .unwrap();
4592
4593        {
4594            let mut inner = forwarding.inner.write().unwrap();
4595            for channel in 1..=u16::MAX {
4596                inner.reserved_client.insert(
4597                    ClientRouteKey {
4598                        connection_id: exhausted_client,
4599                        channel,
4600                    },
4601                    ModuleRouteKey {
4602                        endpoint,
4603                        channel: 1,
4604                    },
4605                );
4606            }
4607        }
4608
4609        let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
4610        let err = forwarding
4611            .begin_route_bind_relay_for_test(
4612                exhausted_client,
4613                FrameSink::new(exhausted_tx),
4614                1,
4615                "route-limit-provider",
4616            )
4617            .unwrap_err();
4618        assert!(matches!(
4619            err,
4620            ForwardingError::ClientRouteChannelExhausted { connection_id }
4621                if connection_id == exhausted_client
4622        ));
4623
4624        let (second_tx, _second_rx) = mpsc::channel(1);
4625        let pending = forwarding
4626            .begin_route_bind_relay_for_test(
4627                second_client,
4628                FrameSink::new(second_tx),
4629                2,
4630                "route-limit-provider",
4631            )
4632            .unwrap();
4633        assert_eq!(pending.client_channel, 1);
4634    }
4635
4636    #[test]
4637    fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
4638        let forwarding = ForwardingTable::default();
4639        let module_connection = ConnectionId::new(40);
4640        let client = ConnectionId::new(50);
4641        let (module_tx, _module_rx) = mpsc::channel(1);
4642        forwarding
4643            .register_module_connection(
4644                module_connection,
4645                "slot-reuse-provider".to_string(),
4646                1,
4647                Concurrency::ModuleManaged,
4648                FrameSink::new(module_tx),
4649            )
4650            .unwrap();
4651
4652        let (client_tx, _client_rx) = mpsc::channel(1);
4653        let client_sink = FrameSink::new(client_tx);
4654        let mut wrapped_channel = None;
4655        for index in 0..=usize::from(u16::MAX) {
4656            let pending = forwarding
4657                .begin_route_bind_relay_for_test(
4658                    client,
4659                    client_sink.clone(),
4660                    index as u64 + 1,
4661                    "slot-reuse-provider",
4662                )
4663                .unwrap();
4664            if index == usize::from(u16::MAX) {
4665                wrapped_channel = Some(pending.module_channel);
4666            }
4667            forwarding
4668                .abort_pending_relay(
4669                    pending.endpoint,
4670                    pending.corr,
4671                    RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
4672                )
4673                .unwrap();
4674        }
4675
4676        assert_eq!(wrapped_channel, Some(1));
4677    }
4678
4679    #[test]
4680    fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
4681        let forwarding = ForwardingTable::default();
4682        let client = ConnectionId::new(60);
4683        forwarding
4684            .inner
4685            .write()
4686            .unwrap()
4687            .next_client_channel
4688            .insert(client, 41);
4689
4690        let released = forwarding.cleanup_connection(client).unwrap();
4691
4692        assert!(released.is_empty());
4693        assert!(!forwarding
4694            .inner
4695            .read()
4696            .unwrap()
4697            .next_client_channel
4698            .contains_key(&client));
4699    }
4700
4701    #[test]
4702    fn stale_module_cleanup_preserves_fast_reconnect_successor() {
4703        let forwarding = ForwardingTable::default();
4704        let module_id = "fast-reconnect-provider";
4705        let first_connection = ConnectionId::new(70);
4706        let second_connection = ConnectionId::new(80);
4707        let (first_tx, _first_rx) = mpsc::channel(1);
4708        let first_endpoint = forwarding
4709            .register_module_connection(
4710                first_connection,
4711                module_id.to_string(),
4712                1,
4713                Concurrency::ModuleManaged,
4714                FrameSink::new(first_tx),
4715            )
4716            .unwrap();
4717        let (second_tx, _second_rx) = mpsc::channel(1);
4718        let second_endpoint = forwarding
4719            .register_module_connection(
4720                second_connection,
4721                module_id.to_string(),
4722                1,
4723                Concurrency::ModuleManaged,
4724                FrameSink::new(second_tx),
4725            )
4726            .unwrap();
4727        assert_ne!(first_endpoint, second_endpoint);
4728
4729        let released = forwarding.cleanup_connection(first_connection).unwrap();
4730
4731        assert!(released.is_empty());
4732        assert_eq!(
4733            forwarding
4734                .inner
4735                .read()
4736                .unwrap()
4737                .modules_by_id
4738                .get(module_id)
4739                .map(|module| module.endpoint),
4740            Some(second_endpoint)
4741        );
4742        assert!(forwarding.has_live_module_connection(module_id).unwrap());
4743        let control_rpc = forwarding
4744            .begin_module_control_rpc_for(
4745                module_id,
4746                "health.check",
4747                Instant::now() + Duration::from_secs(1),
4748            )
4749            .unwrap();
4750        assert_eq!(control_rpc.endpoint, second_endpoint);
4751    }
4752
4753    fn route_fixture(
4754        module_id: &str,
4755    ) -> (
4756        ForwardingTable,
4757        ConnectionId,
4758        ModuleEndpointId,
4759        ConnectionId,
4760        FrameSink,
4761        mpsc::Receiver<crate::router::OutboundFrame>,
4762    ) {
4763        let forwarding = ForwardingTable::default();
4764        let module_connection = ConnectionId::new(100);
4765        let client_connection = ConnectionId::new(200);
4766        let (module_tx, _module_rx) = mpsc::channel(8);
4767        let endpoint = forwarding
4768            .register_module_connection(
4769                module_connection,
4770                module_id.to_string(),
4771                2,
4772                Concurrency::ModuleManaged,
4773                FrameSink::new(module_tx),
4774            )
4775            .unwrap();
4776        let (client_tx, client_rx) = mpsc::channel(8);
4777        (
4778            forwarding,
4779            module_connection,
4780            endpoint,
4781            client_connection,
4782            FrameSink::new(client_tx),
4783            client_rx,
4784        )
4785    }
4786
4787    #[test]
4788    #[cfg(unix)]
4789    fn daemon_drain_gates_current_and_racing_provider_registrations() {
4790        let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
4791        assert_eq!(
4792            forwarding.begin_daemon_drain().unwrap(),
4793            [("provider".to_string(), endpoint)]
4794        );
4795        assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
4796        assert!(matches!(
4797            forwarding.register_module_connection(
4798                ConnectionId::new(300),
4799                "late-provider".into(),
4800                2,
4801                Concurrency::ModuleManaged,
4802                sink,
4803            ),
4804            Err(ForwardingError::ConnectionClosing { .. })
4805        ));
4806    }
4807
4808    #[test]
4809    #[cfg(unix)]
4810    fn daemon_drain_results_are_endpoint_fenced_and_removed_on_cleanup() {
4811        let (forwarding, connection, endpoint, _, _, _) = route_fixture("provider");
4812        assert_eq!(
4813            forwarding
4814                .daemon_drain_result_for_connection(connection)
4815                .unwrap(),
4816            None
4817        );
4818        forwarding
4819            .record_daemon_drain_result(endpoint, true)
4820            .unwrap();
4821        forwarding.begin_daemon_drain().unwrap();
4822        assert_eq!(
4823            forwarding
4824                .daemon_drain_result_for_connection(connection)
4825                .unwrap(),
4826            Some(false)
4827        );
4828        let replacement = ModuleEndpointId {
4829            generation: endpoint.generation + 1,
4830            ..endpoint
4831        };
4832        forwarding
4833            .record_daemon_drain_result(replacement, true)
4834            .unwrap();
4835        assert_eq!(
4836            forwarding
4837                .daemon_drain_result_for_connection(connection)
4838                .unwrap(),
4839            Some(false)
4840        );
4841        forwarding
4842            .record_daemon_drain_result(endpoint, true)
4843            .unwrap();
4844        assert_eq!(
4845            forwarding
4846                .daemon_drain_result_for_connection(connection)
4847                .unwrap(),
4848            Some(true)
4849        );
4850        assert_eq!(
4851            forwarding
4852                .daemon_drain_result_for_connection(ConnectionId::new(300))
4853                .unwrap(),
4854            Some(false)
4855        );
4856        forwarding.cleanup_connection(connection).unwrap();
4857        forwarding
4858            .record_daemon_drain_result(endpoint, true)
4859            .unwrap();
4860        assert_eq!(
4861            forwarding
4862                .daemon_drain_result_for_connection(connection)
4863                .unwrap(),
4864            Some(false)
4865        );
4866        assert!(forwarding
4867            .read_inner()
4868            .unwrap()
4869            .daemon_drain_results
4870            .is_empty());
4871    }
4872
4873    #[test]
4874    #[cfg(unix)]
4875    fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
4876        let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
4877            route_fixture("provider");
4878        let mut pending =
4879            begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
4880        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
4881        forwarding.begin_daemon_drain().unwrap();
4882        let completion = forwarding
4883            .complete_pending_relay(
4884                module_connection,
4885                pending.corr,
4886                RouteBindRelayOutcome::Accepted,
4887            )
4888            .expect("draining admission is not a fatal module error");
4889        assert!(completion.settled);
4890        let abandoned = completion
4891            .abandoned
4892            .expect("late accepted bind needs a route GOODBYE");
4893        assert_eq!(abandoned.channel, pending.module_channel);
4894        assert!(matches!(pending.receiver.try_recv().unwrap(),
4895            RouteBindRelayOutcome::DaemonRejected(error) if error.code == "module_reloading"));
4896        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4897        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4898        assert!(
4899            rx.try_recv().is_err(),
4900            "no route.open may be published during drain"
4901        );
4902        assert_eq!(
4903            forwarding
4904                .module_endpoint_for_connection(module_connection)
4905                .unwrap(),
4906            Some(endpoint)
4907        );
4908        assert!(
4909            !forwarding
4910                .complete_pending_relay(
4911                    module_connection,
4912                    pending.corr,
4913                    RouteBindRelayOutcome::Accepted,
4914                )
4915                .unwrap()
4916                .settled
4917        );
4918    }
4919
4920    fn test_ping(corr: u64) -> Frame {
4921        Frame::build(
4922            FrameType::Ping,
4923            Flags::new(false, Priority::Passive, false),
4924            0,
4925            0,
4926            corr,
4927            Vec::new(),
4928        )
4929        .unwrap()
4930    }
4931
4932    fn begin_test_route(
4933        forwarding: &ForwardingTable,
4934        client_connection: ConnectionId,
4935        client_sink: FrameSink,
4936        corr: u64,
4937        module_id: &str,
4938    ) -> PendingRouteBindRelay {
4939        forwarding
4940            .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
4941            .unwrap()
4942    }
4943
4944    #[test]
4945    fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
4946        for candidate in [false, true] {
4947            for register_first in [false, true] {
4948                let (forwarding, module_connection, _, client_connection, sink, _rx) =
4949                    route_fixture("target");
4950                let register = || {
4951                    if candidate {
4952                        forwarding.register_candidate_module_connection(
4953                            client_connection,
4954                            "source".into(),
4955                            2,
4956                            Concurrency::ModuleManaged,
4957                            sink.clone(),
4958                        )
4959                    } else {
4960                        forwarding.register_module_connection(
4961                            client_connection,
4962                            "source".into(),
4963                            2,
4964                            Concurrency::ModuleManaged,
4965                            sink.clone(),
4966                        )
4967                    }
4968                };
4969                if register_first {
4970                    register().unwrap();
4971                    assert!(
4972                        forwarding
4973                            .begin_route_bind_relay_for_test(
4974                                client_connection,
4975                                sink.clone(),
4976                                1,
4977                                "target",
4978                            )
4979                            .is_err(),
4980                        "a deferred route.open cannot reserve after HELLO"
4981                    );
4982                    assert!(forwarding
4983                        .cleanup_connection(client_connection)
4984                        .unwrap()
4985                        .is_empty());
4986                } else {
4987                    let pending =
4988                        begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
4989                    assert!(
4990                        register().is_err(),
4991                        "HELLO cannot register after a route reservation"
4992                    );
4993                    forwarding
4994                        .complete_pending_relay(
4995                            module_connection,
4996                            pending.corr,
4997                            RouteBindRelayOutcome::Accepted,
4998                        )
4999                        .unwrap();
5000                    assert_eq!(
5001                        forwarding
5002                            .cleanup_connection(client_connection)
5003                            .unwrap()
5004                            .len(),
5005                        1
5006                    );
5007                }
5008                assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5009                assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5010            }
5011        }
5012    }
5013
5014    /// A pending route.open reserves its queue slot with `reserve_owned` and
5015    /// sends its prebuilt response through that slot when the module accepts.
5016    /// A queue already holding many data frames (far more than the old 64-frame
5017    /// queue) must not stop the open from reserving or completing, and the
5018    /// response must be charged to the queue's byte count when it is sent.
5019    #[tokio::test]
5020    async fn pending_route_open_completes_behind_queued_data_frames() {
5021        assert_eq!(
5022            crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
5023            8,
5024            "the per-connection pending route.open limit is its own constant"
5025        );
5026        let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
5027            route_fixture("open-behind-data");
5028        let (sink, mut client_rx) = crate::server::connection_egress();
5029        const DATA_FRAMES: usize = 1_000;
5030        let data = |corr: u64| {
5031            Frame::build(
5032                FrameType::StreamData,
5033                Flags::new(false, Priority::Interactive, false),
5034                9,
5035                1,
5036                corr,
5037                vec![b'x'; 200],
5038            )
5039            .unwrap()
5040        };
5041        for corr in 0..DATA_FRAMES as u64 {
5042            sink.try_send(data(corr)).unwrap();
5043        }
5044        let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
5045        assert_eq!(sink.backlog().queued_bytes, data_bytes);
5046
5047        let pending = tokio::time::timeout(
5048            Duration::from_secs(5),
5049            forwarding.begin_route_bind_relay_for(
5050                client,
5051                sink.clone(),
5052                subc_protocol::PROTOCOL_VERSION,
5053                4_242,
5054                "open-behind-data",
5055                Principal::Direct,
5056                None,
5057                None,
5058                "fixture".into(),
5059                None,
5060                None,
5061                Instant::now() + Duration::from_secs(60),
5062                false,
5063            ),
5064        )
5065        .await
5066        .expect("reserving the route.open slot must not wait behind data frames")
5067        .unwrap();
5068        forwarding
5069            .complete_pending_relay(
5070                module_connection,
5071                pending.corr,
5072                RouteBindRelayOutcome::Accepted,
5073            )
5074            .unwrap();
5075
5076        let backlog = sink.backlog();
5077        assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
5078        assert!(
5079            backlog.queued_bytes > data_bytes,
5080            "the route.open response must be counted in queued bytes: {backlog:?}"
5081        );
5082        for corr in 0..DATA_FRAMES as u64 {
5083            assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
5084        }
5085        let open = client_rx.recv().await.unwrap();
5086        assert_eq!(open.header.corr, 4_242);
5087        assert_eq!(open.header.ty, FrameType::Response);
5088        drop(open);
5089        assert_eq!(sink.backlog().queued_bytes, 0);
5090        assert_eq!(sink.backlog().queued_frames, 0);
5091    }
5092
5093    /// The drain-timeout line reports these numbers, so they must count the
5094    /// requests the drain is waiting on and no others: a route with nothing in
5095    /// flight is not a holdout, and a flagged subscription is excluded from the
5096    /// drain and so from the count.
5097    #[tokio::test]
5098    async fn drain_holdouts_count_held_requests_and_name_the_connection() {
5099        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5100            route_fixture("holdouts");
5101        let mut bound = |corr| {
5102            let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
5103            forwarding
5104                .complete_pending_relay(
5105                    module_connection,
5106                    route.corr,
5107                    RouteBindRelayOutcome::Accepted,
5108                )
5109                .unwrap();
5110            client_rx.try_recv().unwrap();
5111            match forwarding
5112                .lookup_data_route(client, route.client_channel, route.client_epoch)
5113                .unwrap()
5114            {
5115                DataRoute::Client(DataRouteState::Bound(binding)) => binding,
5116                other => panic!("expected live route, got {other:?}"),
5117            }
5118        };
5119        let holding = bound(61);
5120        let _idle = bound(62);
5121        holding.flow.acquire_tagged(7, false).await.unwrap();
5122        holding.flow.acquire_tagged(2, false).await.unwrap();
5123        holding.flow.acquire_tagged(3, true).await.unwrap();
5124        forwarding
5125            .begin_module_drain("holdouts", RouteCloseReason::Restart)
5126            .unwrap();
5127
5128        let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
5129        assert_eq!(
5130            holdouts,
5131            DrainHoldouts {
5132                requests: 2,
5133                routes: 1,
5134                total_routes: 2,
5135                top_connections: vec![(client.get(), 2)],
5136                // The held requests by the module's channel and corr, ascending,
5137                // without the excluded subscription (corr 3).
5138                held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
5139            }
5140        );
5141    }
5142
5143    #[test]
5144    fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
5145        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
5146            route_fixture("census");
5147        let pending = begin_test_route(&forwarding, client, sink, 1, "census");
5148        forwarding
5149            .complete_pending_relay(
5150                module_connection,
5151                pending.corr,
5152                RouteBindRelayOutcome::Accepted,
5153            )
5154            .unwrap();
5155
5156        let routes = forwarding.endpoint_routes(endpoint).unwrap();
5157        assert_eq!(routes.len(), 1);
5158        assert!(matches!(routes[0].principal, Principal::Direct));
5159        assert_eq!(routes[0].goodbye_target.connection_id, client);
5160        assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
5161        assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
5162        assert!(!routes[0].draining);
5163
5164        forwarding
5165            .begin_module_drain("census", RouteCloseReason::Restart)
5166            .unwrap();
5167        let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
5168        assert_eq!(draining_routes.len(), 1);
5169        assert!(draining_routes[0].draining);
5170    }
5171
5172    #[test]
5173    fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
5174        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
5175        let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
5176        assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
5177        forwarding
5178            .abort_pending_relay(
5179                first.endpoint,
5180                first.corr,
5181                RouteBindRelayOutcome::ModuleGone("abort".into()),
5182            )
5183            .unwrap();
5184        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
5185        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
5186
5187        let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
5188        assert_eq!(second.client_channel, first.client_channel);
5189        assert_eq!(second.module_channel, first.module_channel);
5190        assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
5191    }
5192
5193    #[test]
5194    fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
5195        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5196            route_fixture("epoch-release");
5197        let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
5198        forwarding
5199            .complete_pending_relay(
5200                module_connection,
5201                first.corr,
5202                RouteBindRelayOutcome::Accepted,
5203            )
5204            .unwrap();
5205        assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
5206        assert!(matches!(
5207            forwarding
5208                .release_client_route(client, first.client_channel, first.client_epoch)
5209                .unwrap(),
5210            RouteRelease::Removed(_)
5211        ));
5212        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
5213        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
5214
5215        let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
5216        forwarding
5217            .complete_pending_relay(
5218                module_connection,
5219                second.corr,
5220                RouteBindRelayOutcome::Accepted,
5221            )
5222            .unwrap();
5223        assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
5224        assert!(matches!(
5225            forwarding
5226                .release_client_route(client, second.client_channel, first.client_epoch)
5227                .unwrap(),
5228            RouteRelease::Stale
5229        ));
5230        assert!(!forwarding
5231            .cache_status(
5232                endpoint,
5233                second.module_channel,
5234                first.module_epoch,
5235                "stale".into(),
5236            )
5237            .unwrap());
5238        assert!(forwarding
5239            .cache_status(
5240                endpoint,
5241                second.module_channel,
5242                second.module_epoch,
5243                "current".into(),
5244            )
5245            .unwrap());
5246        match forwarding
5247            .route_poll_snapshot(client, second.client_channel, second.client_epoch)
5248            .unwrap()
5249        {
5250            RoutePollSnapshot::Bound { status, .. } => {
5251                assert_eq!(status.as_deref(), Some("current"));
5252            }
5253            RoutePollSnapshot::Absent => panic!("successor binding was removed"),
5254        }
5255        let counters = forwarding.counters().snapshot();
5256        assert_eq!(counters["route_released_epoch_fenced"], 1);
5257        assert_eq!(counters["route_release_stale_skipped"], 1);
5258    }
5259
5260    #[test]
5261    fn max_epoch_reservation_retires_only_that_slot() {
5262        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
5263        forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
5264        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
5265        let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
5266        assert_eq!(
5267            (final_use.client_channel, final_use.client_epoch),
5268            (7, u32::MAX)
5269        );
5270        assert_eq!(
5271            (final_use.module_channel, final_use.module_epoch),
5272            (9, u32::MAX)
5273        );
5274        forwarding
5275            .abort_pending_relay(
5276                endpoint,
5277                final_use.corr,
5278                RouteBindRelayOutcome::ModuleGone("abort".into()),
5279            )
5280            .unwrap();
5281        forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
5282        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
5283        let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
5284        assert_ne!(next.client_channel, 7);
5285        assert_ne!(next.module_channel, 9);
5286        assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
5287    }
5288
5289    #[test]
5290    fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
5291        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
5292            route_fixture("corr-shared");
5293        let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
5294        assert_eq!(bind.corr, 1);
5295        forwarding
5296            .abort_pending_relay(
5297                endpoint,
5298                bind.corr,
5299                RouteBindRelayOutcome::ModuleGone("abort".into()),
5300            )
5301            .unwrap();
5302        let rpc = forwarding
5303            .begin_module_control_rpc_for(
5304                "corr-shared",
5305                "health.check",
5306                Instant::now() - Duration::from_millis(1),
5307            )
5308            .unwrap();
5309        assert_eq!(rpc.corr, 2);
5310        assert_eq!(
5311            forwarding
5312                .complete_module_control_rpc(
5313                    module_connection,
5314                    rpc.corr,
5315                    Some("health.check"),
5316                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
5317                        status: subc_protocol::session::HealthStatus::Ok,
5318                        detail: None,
5319                        metrics: None,
5320                    }),
5321                )
5322                .unwrap(),
5323            ModuleControlRpcCompletion::Settled
5324        );
5325        assert!(matches!(
5326            rpc.receiver.blocking_recv().unwrap(),
5327            ModuleControlRpcOutcome::DeadlineElapsed
5328        ));
5329    }
5330
5331    #[tokio::test(start_paused = true)]
5332    async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
5333        let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
5334        let probe_started_at = Instant::now();
5335        let rpc = forwarding
5336            .begin_health_probe_rpc_for(
5337                "tombstone-ttl",
5338                "health.check",
5339                probe_started_at,
5340                probe_started_at + Duration::from_secs(5),
5341            )
5342            .unwrap();
5343        assert!(forwarding
5344            .tombstone_health_probe_rpc(endpoint, rpc.corr)
5345            .unwrap());
5346        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
5347
5348        tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
5349        tokio::task::yield_now().await;
5350
5351        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
5352    }
5353
5354    #[test]
5355    fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
5356        let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
5357        let mut close = forwarding.register_connection_close(endpoint.connection_id);
5358        forwarding.inject_control_corr(endpoint, u64::MAX);
5359        let final_rpc = forwarding
5360            .begin_module_control_rpc_for(
5361                "corr-max",
5362                "health.check",
5363                Instant::now() + Duration::from_secs(1),
5364            )
5365            .unwrap();
5366        assert_eq!(final_rpc.corr, u64::MAX);
5367        forwarding
5368            .cancel_module_control_rpc(endpoint, final_rpc.corr)
5369            .unwrap();
5370        assert!(matches!(
5371            forwarding.begin_module_control_rpc_for(
5372                "corr-max",
5373                "health.check",
5374                Instant::now() + Duration::from_secs(1),
5375            ),
5376            Err(ForwardingError::RelayCorrelationExhausted)
5377        ));
5378        assert!(close.try_recv().is_ok());
5379    }
5380
5381    #[test]
5382    fn publication_epoch_controls_delivery_failure_escalation() {
5383        fn setup_successor(
5384            commit_successor: Option<bool>,
5385        ) -> (ForwardingTable, ConnectionId, u16, u32) {
5386            let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5387                route_fixture("escalation");
5388            let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
5389            forwarding
5390                .complete_pending_relay(
5391                    module_connection,
5392                    first.corr,
5393                    RouteBindRelayOutcome::Accepted,
5394                )
5395                .unwrap();
5396            client_rx.try_recv().unwrap();
5397            assert!(matches!(
5398                forwarding
5399                    .release_client_route(client, first.client_channel, first.client_epoch)
5400                    .unwrap(),
5401                RouteRelease::Removed(_)
5402            ));
5403            if let Some(commit_successor) = commit_successor {
5404                forwarding.inject_client_slot_epoch(
5405                    client,
5406                    first.client_channel,
5407                    first.client_epoch,
5408                );
5409                forwarding.inject_module_slot_epoch(
5410                    endpoint,
5411                    first.module_channel,
5412                    first.module_epoch,
5413                );
5414                let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
5415                if commit_successor {
5416                    forwarding
5417                        .complete_pending_relay(
5418                            module_connection,
5419                            successor.corr,
5420                            RouteBindRelayOutcome::Accepted,
5421                        )
5422                        .unwrap();
5423                    client_rx.try_recv().unwrap();
5424                } else {
5425                    forwarding
5426                        .abort_pending_relay(
5427                            endpoint,
5428                            successor.corr,
5429                            RouteBindRelayOutcome::ModuleGone("abort".into()),
5430                        )
5431                        .unwrap();
5432                }
5433            }
5434            (forwarding, client, first.client_channel, first.client_epoch)
5435        }
5436
5437        let probe_sink = FrameSink::new(mpsc::channel(1).0);
5438        let (no_successor, client, channel, epoch) = setup_successor(None);
5439        let mut close = no_successor.register_connection_close(client);
5440        assert!(no_successor
5441            .escalate_client_delivery_failure(
5442                client,
5443                channel,
5444                epoch,
5445                CloseReason::new("delivery", "failed"),
5446                UndeliveredFrame {
5447                    module_id: None,
5448                    sink: &probe_sink,
5449                },
5450            )
5451            .unwrap());
5452        assert!(close.try_recv().is_ok());
5453
5454        let (aborted, client, channel, epoch) = setup_successor(Some(false));
5455        let mut close = aborted.register_connection_close(client);
5456        assert!(aborted
5457            .escalate_client_delivery_failure(
5458                client,
5459                channel,
5460                epoch,
5461                CloseReason::new("delivery", "failed"),
5462                UndeliveredFrame {
5463                    module_id: None,
5464                    sink: &probe_sink,
5465                },
5466            )
5467            .unwrap());
5468        assert!(close.try_recv().is_ok());
5469
5470        let (published, client, channel, epoch) = setup_successor(Some(true));
5471        let mut close = published.register_connection_close(client);
5472        assert!(!published
5473            .escalate_client_delivery_failure(
5474                client,
5475                channel,
5476                epoch,
5477                CloseReason::new("delivery", "stale failure"),
5478                UndeliveredFrame {
5479                    module_id: None,
5480                    sink: &probe_sink,
5481                },
5482            )
5483            .unwrap());
5484        assert!(close.try_recv().is_err());
5485    }
5486
5487    #[test]
5488    fn route_concentration_separates_client_count_from_routes_per_client() {
5489        // The distinction this asserts is the one a bare connection count cannot
5490        // make: two connections holding one route each and one connection
5491        // holding two are the same total, and have opposite causes.
5492        let (forwarding, module_connection, _, client, sink, _client_rx) =
5493            route_fixture("concentration");
5494        assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
5495
5496        for corr in [70_u64, 71] {
5497            let pending =
5498                begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
5499            forwarding
5500                .complete_pending_relay(
5501                    module_connection,
5502                    pending.corr,
5503                    RouteBindRelayOutcome::Accepted,
5504                )
5505                .unwrap();
5506        }
5507
5508        // One connection, two routes — not two connections with a route each.
5509        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5510        assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
5511    }
5512
5513    #[test]
5514    fn cleanup_and_accepted_resolution_have_one_lock_winner() {
5515        let (forwarding, module_connection, _, client, sink, mut client_rx) =
5516            route_fixture("cleanup-race");
5517        let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
5518        forwarding
5519            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5520            .unwrap();
5521        let released = forwarding.cleanup_connection(client).unwrap();
5522        assert_eq!(released.len(), 1);
5523        let completion = forwarding
5524            .complete_pending_relay(
5525                module_connection,
5526                pending.corr,
5527                RouteBindRelayOutcome::Accepted,
5528            )
5529            .unwrap();
5530        assert!(!completion.settled);
5531        assert!(client_rx.try_recv().is_err());
5532        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5533
5534        let (forwarding, module_connection, _, client, sink, mut client_rx) =
5535            route_fixture("accepted-race");
5536        let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
5537        forwarding
5538            .complete_pending_relay(
5539                module_connection,
5540                pending.corr,
5541                RouteBindRelayOutcome::Accepted,
5542            )
5543            .unwrap();
5544        assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
5545        let released = forwarding.cleanup_connection(client).unwrap();
5546        assert_eq!(released.len(), 1);
5547        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5548    }
5549
5550    #[test]
5551    fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
5552        let (forwarding, module_connection, _, client, sink, mut client_rx) =
5553            route_fixture("drain-gap");
5554        let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
5555        forwarding
5556            .complete_pending_relay(
5557                module_connection,
5558                live.corr,
5559                RouteBindRelayOutcome::Accepted,
5560            )
5561            .unwrap();
5562        client_rx.try_recv().unwrap();
5563        let binding = match forwarding
5564            .lookup_data_route(client, live.client_channel, live.client_epoch)
5565            .unwrap()
5566        {
5567            DataRoute::Client(DataRouteState::Bound(binding)) => binding,
5568            other => panic!("expected live route, got {other:?}"),
5569        };
5570
5571        let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
5572        forwarding
5573            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5574            .unwrap();
5575        let control_rpc = forwarding
5576            .begin_module_control_rpc_for(
5577                "drain-gap",
5578                "health.check",
5579                Instant::now() + Duration::from_secs(1),
5580            )
5581            .unwrap();
5582        let target = forwarding
5583            .begin_module_drain("drain-gap", RouteCloseReason::Reload)
5584            .unwrap()
5585            .unwrap();
5586        assert!(matches!(
5587            control_rpc.receiver.blocking_recv().unwrap(),
5588            ModuleControlRpcOutcome::ModuleGone(_)
5589        ));
5590        assert_eq!(target.abandoned_bindings.len(), 1);
5591        assert!(binding.flow.sem.is_closed());
5592        assert!(
5593            !forwarding
5594                .complete_pending_relay(
5595                    module_connection,
5596                    pending.corr,
5597                    RouteBindRelayOutcome::Accepted,
5598                )
5599                .unwrap()
5600                .settled
5601        );
5602        assert!(matches!(
5603            forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
5604            Err(ForwardingError::ModuleReloading { .. })
5605        ));
5606        let released = forwarding
5607            .release_module_endpoint_routes(target.endpoint)
5608            .unwrap();
5609        assert_eq!(released.len(), 1);
5610        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5611    }
5612
5613    /// A client can be marked closing while its egress is still open: the daemon
5614    /// asks a connection to close (here through the production path, a module
5615    /// frame that its egress refused) and the connection loop tears down a moment
5616    /// later. A route.bind ack that lands inside that window is answered on the
5617    /// MODULE connection's frame handler, so resolving it must not produce an
5618    /// error -- an error there ends the module connection, and that connection
5619    /// carries every other client's routes to the module.
5620    #[test]
5621    fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
5622        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5623            route_fixture("closing-client");
5624
5625        // A published route on this client: escalate_client_delivery_failure only
5626        // marks a connection closing for a route it has already published.
5627        let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
5628        forwarding
5629            .complete_pending_relay(
5630                module_connection,
5631                live.corr,
5632                RouteBindRelayOutcome::Accepted,
5633            )
5634            .unwrap();
5635        client_rx.try_recv().unwrap();
5636
5637        // A second route.open from the same client, relayed and awaiting its ack.
5638        let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
5639        forwarding
5640            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5641            .unwrap();
5642
5643        // The window: closing, but the sink is still open.
5644        assert!(forwarding
5645            .escalate_client_delivery_failure(
5646                client,
5647                live.client_channel,
5648                live.client_epoch,
5649                CloseReason::new(
5650                    "module_to_client_delivery_failed",
5651                    "client egress refused a module frame",
5652                ),
5653                UndeliveredFrame {
5654                    module_id: None,
5655                    sink: &sink,
5656                },
5657            )
5658            .unwrap());
5659        assert!(!sink.is_closed());
5660
5661        let completion = forwarding
5662            .complete_pending_relay(
5663                module_connection,
5664                pending.corr,
5665                RouteBindRelayOutcome::Accepted,
5666            )
5667            .expect("a closing client must not turn a module's ack into an error");
5668
5669        assert!(completion.settled);
5670        let abandoned = completion
5671            .abandoned
5672            .expect("the module must be told to drop the binding it just created");
5673        assert_eq!(abandoned.connection_id, module_connection);
5674        assert_eq!(abandoned.channel, pending.module_channel);
5675        assert_eq!(abandoned.epoch, pending.module_epoch);
5676        assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
5677        assert!(matches!(
5678            pending.receiver.blocking_recv().unwrap(),
5679            RouteBindRelayOutcome::ModuleGone(_)
5680        ));
5681        // No route was published to a client that is on its way out, and the
5682        // reserved handle pair went back.
5683        assert!(client_rx.try_recv().is_err());
5684        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
5685
5686        // The module endpoint is untouched: still live, and still able to take a
5687        // route from another client.
5688        assert!(forwarding
5689            .has_live_module_connection("closing-client")
5690            .unwrap());
5691        let cotenant = ConnectionId::new(201);
5692        let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
5693        let cotenant_route = begin_test_route(
5694            &forwarding,
5695            cotenant,
5696            FrameSink::new(cotenant_tx),
5697            62,
5698            "closing-client",
5699        );
5700        assert_eq!(cotenant_route.endpoint, endpoint);
5701        forwarding
5702            .complete_pending_relay(
5703                module_connection,
5704                cotenant_route.corr,
5705                RouteBindRelayOutcome::Accepted,
5706            )
5707            .unwrap();
5708        assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
5709        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5710    }
5711
5712    #[test]
5713    fn pending_route_permit_is_released_on_rejection_and_abort() {
5714        let forwarding = ForwardingTable::default();
5715        let module_connection = ConnectionId::new(300);
5716        let client = ConnectionId::new(301);
5717        let (module_tx, _module_rx) = mpsc::channel(1);
5718        let endpoint = forwarding
5719            .register_module_connection(
5720                module_connection,
5721                "permit".into(),
5722                2,
5723                Concurrency::ModuleManaged,
5724                FrameSink::new(module_tx),
5725            )
5726            .unwrap();
5727        let (client_tx, mut client_rx) = mpsc::channel(1);
5728        let sink = FrameSink::new(client_tx);
5729        let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
5730        assert!(sink.try_send(test_ping(999)).is_err());
5731        forwarding
5732            .complete_pending_relay(
5733                module_connection,
5734                rejected.corr,
5735                RouteBindRelayOutcome::Rejected(ErrorBody {
5736                    code: "no".into(),
5737                    message: "rejected".into(),
5738                    detail: None,
5739                }),
5740            )
5741            .unwrap();
5742        sink.try_send(test_ping(1000)).unwrap();
5743        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
5744
5745        let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
5746        assert!(sink.try_send(test_ping(1001)).is_err());
5747        forwarding
5748            .abort_pending_relay(
5749                endpoint,
5750                aborted.corr,
5751                RouteBindRelayOutcome::ModuleGone("abort".into()),
5752            )
5753            .unwrap();
5754        sink.try_send(test_ping(1002)).unwrap();
5755        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
5756
5757        let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
5758        forwarding
5759            .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
5760            .unwrap();
5761        drop(client_rx);
5762        let completion = forwarding
5763            .complete_pending_relay(
5764                module_connection,
5765                receiver_closed.corr,
5766                RouteBindRelayOutcome::Accepted,
5767            )
5768            .unwrap();
5769        assert!(completion.abandoned.is_some());
5770        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5771    }
5772
5773    /// Every connection teardown passes through `cleanup_connection`, and
5774    /// connection ids come from a monotonic counter that never hands an id out
5775    /// twice. If the closing mark survives teardown, the set grows by one entry
5776    /// per connection for the life of the daemon -- the self-watchdog alone
5777    /// reconnects once a minute.
5778    #[test]
5779    fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
5780        let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
5781            route_fixture("closing-set-leak");
5782
5783        const CONNECTIONS: u64 = 32;
5784        for index in 0..CONNECTIONS {
5785            let client = ConnectionId::new(1000 + index);
5786            let (client_tx, _client_rx) = mpsc::channel(8);
5787            let route = begin_test_route(
5788                &forwarding,
5789                client,
5790                FrameSink::new(client_tx),
5791                index + 1,
5792                "closing-set-leak",
5793            );
5794            forwarding
5795                .complete_pending_relay(
5796                    module_connection,
5797                    route.corr,
5798                    RouteBindRelayOutcome::Accepted,
5799                )
5800                .unwrap();
5801            forwarding.cleanup_connection(client).unwrap();
5802        }
5803        forwarding.cleanup_connection(module_connection).unwrap();
5804
5805        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5806    }
5807
5808    /// The closing mark exists to refuse new work for a connection that is on
5809    /// its way out but whose teardown has not run yet: the daemon asks the
5810    /// connection loop to end, and only when the loop reacts does
5811    /// `cleanup_connection` strip the connection's state. Inside that window an
5812    /// operation for the dying connection must still be refused; only after
5813    /// cleanup completes may the mark go.
5814    #[test]
5815    fn closing_connection_is_refused_new_work_until_cleanup_completes() {
5816        let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
5817            route_fixture("closing-gate");
5818
5819        // A published route: escalate_client_delivery_failure only marks a
5820        // connection closing for a route it has already published.
5821        let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
5822        forwarding
5823            .complete_pending_relay(
5824                module_connection,
5825                live.corr,
5826                RouteBindRelayOutcome::Accepted,
5827            )
5828            .unwrap();
5829        client_rx.try_recv().unwrap();
5830
5831        // Mark the connection closing through the production path without
5832        // running teardown, pinning the window open.
5833        assert!(forwarding
5834            .escalate_client_delivery_failure(
5835                client,
5836                live.client_channel,
5837                live.client_epoch,
5838                CloseReason::new(
5839                    "module_to_client_delivery_failed",
5840                    "client egress refused a module frame",
5841                ),
5842                UndeliveredFrame {
5843                    module_id: None,
5844                    sink: &sink,
5845                },
5846            )
5847            .unwrap());
5848        assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
5849
5850        // A late route.open for the closing client is refused ...
5851        assert!(matches!(
5852            forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
5853            Err(ForwardingError::ConnectionClosing { connection_id })
5854                if connection_id == client
5855        ));
5856        // ... and so is a late attempt to register the connection as a module.
5857        let (late_tx, _late_rx) = mpsc::channel(1);
5858        assert!(matches!(
5859            forwarding.register_module_connection(
5860                client,
5861                "late-module".into(),
5862                2,
5863                Concurrency::ModuleManaged,
5864                FrameSink::new(late_tx),
5865            ),
5866            Err(ForwardingError::ConnectionClosing { connection_id })
5867                if connection_id == client
5868        ));
5869
5870        // Teardown is the point that lifts the mark: it has just removed every
5871        // per-connection entry under the same lock, so the gate has nothing
5872        // left to protect for this id.
5873        forwarding.cleanup_connection(client).unwrap();
5874        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5875    }
5876}
5877
5878/// Blue/green swap slots: a candidate registered beside the active endpoint,
5879/// promoted by `cutover_candidate`, with the old incumbent drained by endpoint.
5880#[cfg(test)]
5881mod swap_slot_tests {
5882    use std::time::Duration;
5883
5884    use super::*;
5885    use tokio::sync::mpsc;
5886
5887    const MODULE_ID: &str = "swapped";
5888
5889    struct SwapFixture {
5890        forwarding: ForwardingTable,
5891        incumbent_connection: ConnectionId,
5892        incumbent: ModuleEndpointId,
5893        candidate_connection: ConnectionId,
5894        candidate: ModuleEndpointId,
5895        _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
5896    }
5897
5898    fn swap_fixture() -> SwapFixture {
5899        let forwarding = ForwardingTable::default();
5900        let incumbent_connection = ConnectionId::new(100);
5901        let candidate_connection = ConnectionId::new(110);
5902        let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
5903        let incumbent = forwarding
5904            .register_module_connection(
5905                incumbent_connection,
5906                MODULE_ID.to_string(),
5907                2,
5908                Concurrency::ModuleManaged,
5909                FrameSink::new(incumbent_tx),
5910            )
5911            .unwrap();
5912        let (candidate_tx, candidate_rx) = mpsc::channel(8);
5913        let candidate = forwarding
5914            .register_candidate_module_connection(
5915                candidate_connection,
5916                MODULE_ID.to_string(),
5917                2,
5918                Concurrency::ModuleManaged,
5919                FrameSink::new(candidate_tx),
5920            )
5921            .unwrap();
5922        SwapFixture {
5923            forwarding,
5924            incumbent_connection,
5925            incumbent,
5926            candidate_connection,
5927            candidate,
5928            _module_rxs: vec![incumbent_rx, candidate_rx],
5929        }
5930    }
5931
5932    fn client(
5933        raw: u64,
5934    ) -> (
5935        ConnectionId,
5936        FrameSink,
5937        mpsc::Receiver<crate::router::OutboundFrame>,
5938    ) {
5939        let (tx, rx) = mpsc::channel(8);
5940        (ConnectionId::new(raw), FrameSink::new(tx), rx)
5941    }
5942
5943    fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
5944        forwarding
5945            .read_inner()
5946            .unwrap()
5947            .client_to_module
5948            .values()
5949            .map(|route| route.module_endpoint)
5950            .collect()
5951    }
5952
5953    #[test]
5954    fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
5955        let fixture = swap_fixture();
5956        let forwarding = &fixture.forwarding;
5957        assert_ne!(fixture.incumbent, fixture.candidate);
5958
5959        // Every by-id consumer still resolves the incumbent.
5960        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5961        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5962        let (client_connection, client_sink, _client_rx) = client(200);
5963        let pending = forwarding
5964            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5965            .unwrap();
5966        assert_eq!(pending.endpoint, fixture.incumbent);
5967        let rpc = forwarding
5968            .begin_module_control_rpc_for(
5969                MODULE_ID,
5970                "health.check",
5971                Instant::now() + Duration::from_secs(1),
5972            )
5973            .unwrap();
5974        assert_eq!(rpc.endpoint, fixture.incumbent);
5975        let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
5976        assert_eq!(census.len(), 1, "the census lists one endpoint per id");
5977
5978        // Connection-keyed lookups see the candidate, so its own frames resolve.
5979        assert_eq!(
5980            forwarding
5981                .module_endpoint_for_connection(fixture.candidate_connection)
5982                .unwrap(),
5983            Some(fixture.candidate)
5984        );
5985        assert_eq!(
5986            forwarding
5987                .module_id_for_connection(fixture.candidate_connection)
5988                .unwrap()
5989                .as_deref(),
5990            Some(MODULE_ID)
5991        );
5992
5993        // One candidate per id.
5994        let (other_tx, _other_rx) = mpsc::channel(1);
5995        assert_eq!(
5996            forwarding.register_candidate_module_connection(
5997                ConnectionId::new(120),
5998                MODULE_ID.to_string(),
5999                2,
6000                Concurrency::ModuleManaged,
6001                FrameSink::new(other_tx),
6002            ),
6003            Err(ForwardingError::CandidateSlotOccupied {
6004                module_id: MODULE_ID.to_string()
6005            })
6006        );
6007    }
6008
6009    /// The cutover linearization point. A relay reserved on the incumbent before
6010    /// cutover must never become a route on the incumbent, and every relay
6011    /// reserved after it must land on the promoted candidate.
6012    #[test]
6013    fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
6014        let fixture = swap_fixture();
6015        let forwarding = &fixture.forwarding;
6016        let (early_client, early_sink, _early_rx) = client(200);
6017        let mut early = forwarding
6018            .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
6019            .unwrap();
6020        assert_eq!(early.endpoint, fixture.incumbent);
6021        assert_eq!(
6022            forwarding
6023                .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
6024                .unwrap(),
6025            RouteBindRelayMark::Marked
6026        );
6027
6028        let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
6029        assert_eq!(
6030            cutover,
6031            ForwardingCutover {
6032                promoted: fixture.candidate,
6033                incumbent: Some(fixture.incumbent),
6034            }
6035        );
6036
6037        // A route.open reserved after cutover goes to the promoted candidate.
6038        let (late_client, late_sink, _late_rx) = client(201);
6039        let late = forwarding
6040            .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
6041            .unwrap();
6042        assert_eq!(
6043            late.endpoint, fixture.candidate,
6044            "a route.open after cutover was reserved on the incumbent"
6045        );
6046
6047        // The incumbent acks the early relay after cutover.
6048        let completion = forwarding
6049            .complete_pending_relay(
6050                fixture.incumbent_connection,
6051                early.corr,
6052                RouteBindRelayOutcome::Accepted,
6053            )
6054            .expect("a superseded endpoint's ack is not an error on its connection");
6055        assert!(completion.settled);
6056        assert!(
6057            !committed_endpoints(forwarding).contains(&fixture.incumbent),
6058            "a relay reserved before cutover committed a route on the incumbent"
6059        );
6060        let goodbye = completion
6061            .abandoned
6062            .expect("the incumbent is told to drop the binding it just created");
6063        assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
6064        assert_eq!(goodbye.channel, early.module_channel);
6065        assert_eq!(goodbye.epoch, early.module_epoch);
6066        assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
6067        match early.receiver.try_recv() {
6068            Ok(RouteBindRelayOutcome::DaemonRejected(body)) => {
6069                assert_eq!(body.code, "module_reloading")
6070            }
6071            other => panic!("expected a retryable module_reloading answer, got {other:?}"),
6072        }
6073        assert!(matches!(
6074            forwarding
6075                .lookup_data_route(early_client, early.client_channel, early.client_epoch)
6076                .unwrap(),
6077            DataRoute::Client(DataRouteState::Absent)
6078        ));
6079
6080        // The late relay commits on the candidate; only its pair stays reserved
6081        // until then.
6082        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
6083        forwarding
6084            .complete_pending_relay(
6085                fixture.candidate_connection,
6086                late.corr,
6087                RouteBindRelayOutcome::Accepted,
6088            )
6089            .unwrap();
6090        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
6091        assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
6092    }
6093
6094    #[test]
6095    fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
6096        let fixture = swap_fixture();
6097        let forwarding = &fixture.forwarding;
6098        // One bound route and one in-flight relay on the incumbent.
6099        let (bound_client, bound_sink, _bound_rx) = client(200);
6100        let bound = forwarding
6101            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
6102            .unwrap();
6103        forwarding
6104            .complete_pending_relay(
6105                fixture.incumbent_connection,
6106                bound.corr,
6107                RouteBindRelayOutcome::Accepted,
6108            )
6109            .unwrap();
6110        let (pending_client, pending_sink, _pending_rx) = client(201);
6111        let mut in_flight = forwarding
6112            .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
6113            .unwrap();
6114        forwarding
6115            .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
6116            .unwrap();
6117
6118        let incumbent = forwarding
6119            .cutover_candidate(MODULE_ID)
6120            .unwrap()
6121            .unwrap()
6122            .incumbent
6123            .unwrap();
6124        let target = forwarding
6125            .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
6126            .unwrap()
6127            .expect("the superseded incumbent is still registered");
6128
6129        assert_eq!(target.endpoint, fixture.incumbent);
6130        assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
6131        assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
6132        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
6133        assert_eq!(target.abandoned_bindings.len(), 1);
6134        assert_eq!(
6135            target.abandoned_bindings[0].channel,
6136            in_flight.module_channel
6137        );
6138        assert!(matches!(
6139            in_flight.receiver.try_recv(),
6140            Ok(RouteBindRelayOutcome::DaemonRejected(body)) if body.code == "module_reloading"
6141        ));
6142        assert_eq!(
6143            forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
6144            1,
6145            "the incumbent's bound route stays until its drain finishes"
6146        );
6147
6148        let (next_client, next_sink, _next_rx) = client(202);
6149        let next = forwarding
6150            .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
6151            .expect("the promoted candidate keeps accepting routes");
6152        assert_eq!(next.endpoint, fixture.candidate);
6153    }
6154
6155    /// An endpoint replaced WITHOUT a promotion (a successor registered over it
6156    /// as an ordinary active HELLO) is stale, not superseded, and its ack still
6157    /// fails the acking connection exactly as it did before swap slots existed:
6158    /// `StaleModuleEndpoint`, the reservation indexes already stripped, and the
6159    /// waiting client's sender dropped unanswered.
6160    #[test]
6161    fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
6162        let forwarding = ForwardingTable::default();
6163        let first_connection = ConnectionId::new(70);
6164        let (first_tx, _first_rx) = mpsc::channel(8);
6165        forwarding
6166            .register_module_connection(
6167                first_connection,
6168                MODULE_ID.to_string(),
6169                2,
6170                Concurrency::ModuleManaged,
6171                FrameSink::new(first_tx),
6172            )
6173            .unwrap();
6174        let (client_connection, client_sink, _client_rx) = client(200);
6175        let mut pending = forwarding
6176            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
6177            .unwrap();
6178        let (second_tx, _second_rx) = mpsc::channel(8);
6179        forwarding
6180            .register_module_connection(
6181                ConnectionId::new(80),
6182                MODULE_ID.to_string(),
6183                2,
6184                Concurrency::ModuleManaged,
6185                FrameSink::new(second_tx),
6186            )
6187            .unwrap();
6188
6189        assert_eq!(
6190            forwarding
6191                .complete_pending_relay(
6192                    first_connection,
6193                    pending.corr,
6194                    RouteBindRelayOutcome::Accepted
6195                )
6196                .unwrap_err(),
6197            ForwardingError::StaleModuleEndpoint
6198        );
6199        assert!(committed_endpoints(&forwarding).is_empty());
6200        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
6201        assert!(matches!(
6202            pending.receiver.try_recv(),
6203            Err(oneshot::error::TryRecvError::Closed)
6204        ));
6205    }
6206
6207    #[test]
6208    fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
6209        // A candidate whose connection drops leaves the incumbent routable.
6210        let fixture = swap_fixture();
6211        let forwarding = &fixture.forwarding;
6212        assert!(forwarding
6213            .cleanup_connection(fixture.candidate_connection)
6214            .unwrap()
6215            .is_empty());
6216        assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
6217        let (client_connection, client_sink, _client_rx) = client(200);
6218        assert_eq!(
6219            forwarding
6220                .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
6221                .unwrap()
6222                .endpoint,
6223            fixture.incumbent
6224        );
6225
6226        // After a cutover, the incumbent's teardown releases its own routes and
6227        // leaves the promoted candidate in place.
6228        let fixture = swap_fixture();
6229        let forwarding = &fixture.forwarding;
6230        let (bound_client, bound_sink, _bound_rx) = client(200);
6231        let bound = forwarding
6232            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
6233            .unwrap();
6234        forwarding
6235            .complete_pending_relay(
6236                fixture.incumbent_connection,
6237                bound.corr,
6238                RouteBindRelayOutcome::Accepted,
6239            )
6240            .unwrap();
6241        forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
6242        let released = forwarding
6243            .cleanup_connection(fixture.incumbent_connection)
6244            .unwrap();
6245        assert_eq!(released.len(), 1);
6246        assert_eq!(released[0].connection_id, bound_client);
6247        assert!(forwarding
6248            .read_inner()
6249            .unwrap()
6250            .superseded_endpoints
6251            .is_empty());
6252        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
6253        let (next_client, next_sink, _next_rx) = client(201);
6254        assert_eq!(
6255            forwarding
6256                .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
6257                .unwrap()
6258                .endpoint,
6259            fixture.candidate
6260        );
6261    }
6262}