Skip to main content

subc_daemon/
forwarding.rs

1use std::{
2    collections::{BTreeMap, HashMap, HashSet},
3    error::Error,
4    fmt,
5    sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
6    time::Duration,
7};
8
9use subc_control::{ClientControlResponse, RouteCloseReason};
10use subc_protocol::{
11    manifest::Concurrency,
12    session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
13    ErrorBody, Flags, FrameType, Principal, Priority,
14};
15use tokio::sync::{oneshot, Semaphore};
16use tokio::time::Instant;
17use tracing::{debug, info, warn};
18
19use crate::{
20    control::{RouteBindBreakers, RouteBindConcurrency},
21    observability::DaemonCounters,
22    registry::ConnectionId,
23    router::FrameSink,
24    scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
25    Frame, ProjectRootId,
26};
27
28/// Default per-channel request-credit window for modules that schedule internally.
29const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
30
31/// High per-channel cap for stateless modules; this is an OOM guard, not scheduling policy.
32const STATELESS_PARALLEL_WINDOW: usize = 1024;
33
34/// A stopped probe cycle cannot retain its last unanswered correlation forever.
35/// Active endpoints replace the tombstone on their next serial health probe;
36/// this backstop covers endpoints that stop probing altogether.
37const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
38
39/// Module connection identity used in forwarding keys.
40///
41/// The generation is bumped every time a module connection is registered so a future restart cannot
42/// accidentally reuse a stale `(connection_id, route_channel)` binding.
43#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
44pub struct ModuleEndpointId {
45    pub connection_id: ConnectionId,
46    pub generation: u64,
47}
48
49/// Client-local route key. A route channel is unique only within one client connection.
50#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
51pub(crate) struct ClientRouteKey {
52    pub connection_id: ConnectionId,
53    pub channel: u16,
54}
55
56/// Module-local route key. A route channel is unique only within one live module endpoint.
57#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
58pub(crate) struct ModuleRouteKey {
59    pub endpoint: ModuleEndpointId,
60    pub channel: u16,
61}
62
63#[derive(Debug)]
64pub(crate) struct RouteBinding {
65    pub client_connection_id: ConnectionId,
66    pub client_sink: FrameSink,
67    pub client_negotiated_ver: u8,
68    pub client_channel: u16,
69    pub client_epoch: u32,
70    pub module_id: String,
71    pub module_endpoint: ModuleEndpointId,
72    pub module_sink: FrameSink,
73    pub module_negotiated_ver: u8,
74    pub module_channel: u16,
75    pub module_epoch: u32,
76    pub principal: Principal,
77    pub project_root: Option<ProjectRootId>,
78    pub bound_at: Instant,
79    pub flow: Arc<ChannelFlow>,
80    /// The scope the route was admitted under, with the tag its bind was
81    /// stamped at. A scope change finds the routes to close by it.
82    pub scope: Option<BoundScope>,
83}
84
85#[derive(Debug, Clone)]
86pub(crate) enum DataRoute {
87    Client(DataRouteState),
88    Module(DataRouteState),
89}
90
91#[derive(Debug, Clone)]
92pub(crate) enum DataRouteState {
93    Bound(Arc<RouteBinding>),
94    Reserved,
95    EpochMismatch,
96    Absent,
97}
98
99/// Which kind of peer a route GOODBYE is being delivered to. This decides what
100/// happens when the GOODBYE cannot be enqueued (egress full/closed):
101/// - `Client`: escalate to closing that client connection (a socket close is a
102///   stronger teardown signal, and a full client egress means it is the slow
103///   client we would drop anyway).
104/// - `Module`: never close; deliver late instead (see
105///   [`send_module_route_goodbye`]), and drop only if the module still has no
106///   room after [`LATE_MODULE_GOODBYE_DEADLINE`]. A client-disconnect notifies the
107///   SHARED module that one client's route is gone; closing the module on its
108///   egress backpressure would tear down every co-tenant client (the exact
109///   cross-tenant blast radius this never-close rule exists to prevent — observed when a
110///   flooding dead client filled BOTH its own and the module's egress, so its
111///   route-gone GOODBYE to the module failed and closed the shared connection).
112///   subc has already removed the route from its forwarding state and drops
113///   stale module frames for the released channel (see router.rs), so subc's
114///   own routing is correct. The residual: under SUSTAINED module-egress
115///   backpressure a module-targeted route-gone notification can be lost, which a
116///   module using it for client-refcounting (e.g. AFT's session accounting)
117///   would miss. This is INTENTIONALLY ACCEPTED, not a gap. A consuming module
118///   must bound stale bindings with its own idle-activity reaper (last-touched
119///   TTL) independent of route-gone signals — AFT does exactly this, so a lost
120///   GOODBYE degrades to "the
121///   binding stays warm until its idle TTL" (bounded wasted resources), never an
122///   unbounded leak; disk-durable replay is unaffected. A dedicated reliable
123///   module control lane was evaluated and deliberately NOT
124///   built: it would add starvation-avoidance machinery to the thin core for a
125///   bounded warm-resource window that is not a correctness issue. never-close
126///   is the invariant that matters here.
127#[derive(Debug, Clone, Copy, PartialEq, Eq)]
128pub(crate) enum GoodbyeTargetKind {
129    Client,
130    Module,
131}
132
133#[derive(Debug, Clone)]
134pub(crate) struct GoodbyeTarget {
135    pub connection_id: ConnectionId,
136    pub sink: FrameSink,
137    pub negotiated_ver: u8,
138    pub channel: u16,
139    pub epoch: u32,
140    pub kind: GoodbyeTargetKind,
141    /// The module on the other end of the route: for a module-targeted relay
142    /// the receiving module (attributing a dropped relay), for a client target
143    /// the module whose route is going away (naming it when an undeliverable
144    /// relay closes the client).
145    pub module_id: Option<String>,
146}
147
148/// The frame a client connection's egress queue refused, for the diagnosis
149/// logged when that refusal closes the connection.
150#[derive(Debug, Clone, Copy)]
151pub(crate) struct UndeliveredFrame<'a> {
152    /// The module on the other end of the route the frame belonged to, when known.
153    pub module_id: Option<&'a str>,
154    /// The refusing connection's sink, read for what its queue held.
155    pub sink: &'a FrameSink,
156}
157
158/// How a route's principal appears in logs: `direct`, or `reserved:<module>`.
159pub(crate) fn principal_label(principal: &Principal) -> String {
160    match principal {
161        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
162        Principal::Direct => "direct".to_string(),
163        other => format!("{other:?}"),
164    }
165}
166
167/// The distinct principals of the routes bound on one client connection,
168/// sorted and comma-joined, or `none` when it has no bound route.
169fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
170    let labels = inner
171        .client_to_module
172        .iter()
173        .filter(|(key, _)| key.connection_id == connection_id)
174        .map(|(_, route)| principal_label(&route.principal))
175        .collect::<std::collections::BTreeSet<_>>();
176    if labels.is_empty() {
177        "none".to_string()
178    } else {
179        labels.into_iter().collect::<Vec<_>>().join(",")
180    }
181}
182
183impl GoodbyeTarget {
184    /// True only when an undeliverable GOODBYE should escalate to closing the
185    /// target connection. Never escalate for module recipients.
186    pub(crate) fn close_on_delivery_failure(&self) -> bool {
187        matches!(self.kind, GoodbyeTargetKind::Client)
188    }
189}
190
191/// How long a route GOODBYE that a module's egress queue refused keeps waiting
192/// for room before it is given up and counted as dropped.
193///
194/// A module that stops reading for a moment (a GC pause, a long synchronous
195/// handler, a reconnect herd it is still working through) should still learn
196/// that the route is gone, because a module that never hears the GOODBYE keeps
197/// the route and keeps sending on it for as long as its connection lives. The
198/// value is the default module drain timeout: the daemon already treats that as
199/// the longest it is reasonable to wait on a module that is busy but alive, and
200/// a module that cannot free 21 bytes of egress in that time is wedged rather
201/// than slow. Resolving each module's own configured drain timeout here would
202/// need a registry lookup on a path that holds only the module's sink.
203pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
204
205/// Send a route GOODBYE to a module without ever closing the module's shared
206/// connection, which would sever every other route it serves.
207///
208/// The frame is enqueued at once when the module's egress queue has room. When
209/// it does not, a detached task waits for room (bounded by
210/// [`LATE_MODULE_GOODBYE_DEADLINE`]) and sends it then. Arriving late is safe:
211/// the daemon has already released the route, and module SDKs tear a route
212/// down only when both the channel AND the epoch match the route installed on
213/// that channel, so a late GOODBYE for a channel since reused at a newer epoch
214/// is ignored. The GOODBYE is counted in `goodbye_relay_module_dropped` only
215/// when the connection is closed or the deadline passes.
216pub(crate) fn send_module_route_goodbye(
217    counters: &DaemonCounters,
218    sink: &FrameSink,
219    frame: Frame,
220    module_id: Option<&str>,
221    context: &'static str,
222) {
223    let channel = frame.header.channel;
224    let epoch = frame.header.epoch;
225    let Err(err) = sink.try_send(frame.clone()) else {
226        return;
227    };
228    // Waiting is pointless once the module's writer is gone, and impossible
229    // outside a Tokio runtime (cleanup that runs from a destructor at shutdown).
230    let runtime = match tokio::runtime::Handle::try_current() {
231        Ok(runtime) if !sink.is_closed() => runtime,
232        _ => {
233            counters.increment_goodbye_relay_module_dropped(module_id);
234            warn!(
235            module_id = module_id.unwrap_or("unknown"),
236            route_channel = channel,
237            route_epoch = epoch,
238            error = %err,
239            context,
240            "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
241            );
242            return;
243        }
244    };
245    debug!(
246        module_id = module_id.unwrap_or("unknown"),
247        route_channel = channel,
248        route_epoch = epoch,
249        error = %err,
250        context,
251        "module egress queue refused route GOODBYE; delivering it once the module frees room"
252    );
253    let counters = counters.clone();
254    let sink = sink.clone();
255    let module_id = module_id.map(str::to_string);
256    runtime.spawn(async move {
257        let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
258        let why = match outcome {
259            Ok(Ok(())) => {
260                debug!(
261                    module_id = module_id.as_deref().unwrap_or("unknown"),
262                    route_channel = channel,
263                    route_epoch = epoch,
264                    context,
265                    "late route GOODBYE delivered to module"
266                );
267                return;
268            }
269            Ok(Err(err)) => err.to_string(),
270            Err(_) => format!(
271                "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
272            ),
273        };
274        counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
275        warn!(
276            module_id = module_id.as_deref().unwrap_or("unknown"),
277            route_channel = channel,
278            route_epoch = epoch,
279            error = %why,
280            context,
281            "route GOODBYE to module dropped under backpressure; not closing shared module connection"
282        );
283    });
284}
285
286/// One route currently served by a module endpoint.
287///
288/// `goodbye_target` is deliberately retained alongside the census projection so
289/// a draining caller can address exactly the same route set that this read
290/// reports, without a second forwarding-table pass.
291#[derive(Debug, Clone)]
292pub(crate) struct EndpointRoute {
293    pub goodbye_target: GoodbyeTarget,
294    pub principal: Principal,
295    pub bound_at: Instant,
296    pub draining: bool,
297    /// WHY the endpoint is draining, when it is. Carried per-route so the
298    /// census can answer "closing because of what" without a second lookup;
299    /// `None` exactly when `draining` is false (one source: the drain map).
300    pub drain_reason: Option<RouteCloseReason>,
301}
302
303/// One live route a scope change closed: both ends to send GOODBYE to, and the
304/// reason for the client's `route.closed`.
305#[derive(Debug, Clone)]
306pub(crate) struct ScopeDrainedRoute {
307    pub reason: RouteCloseReason,
308    pub scope: BoundScope,
309    pub module_id: String,
310    pub client: GoodbyeTarget,
311    pub module: GoodbyeTarget,
312}
313
314#[derive(Debug)]
315pub(crate) struct PendingRouteBindRelay {
316    pub endpoint: ModuleEndpointId,
317    pub module_sink: FrameSink,
318    pub negotiated_ver: u8,
319    pub client_channel: u16,
320    pub client_epoch: u32,
321    pub module_channel: u16,
322    pub module_epoch: u32,
323    pub corr: u64,
324    pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
325}
326
327#[derive(Debug, Clone)]
328pub(crate) struct ModuleDrainTarget {
329    pub endpoint: ModuleEndpointId,
330    pub sink: FrameSink,
331    pub negotiated_ver: u8,
332    pub abandoned_bindings: Vec<GoodbyeTarget>,
333    pub excluded_subscriptions: u32,
334}
335
336/// One registered module connection, as [`ForwardingTable::module_connections`]
337/// reports it: enough to send it a frame and then close it.
338#[cfg(unix)]
339#[derive(Debug, Clone)]
340pub(crate) struct ModuleConnectionTarget {
341    pub module_id: String,
342    pub endpoint: ModuleEndpointId,
343    pub sink: FrameSink,
344    pub negotiated_ver: u8,
345}
346
347#[derive(Debug, Clone)]
348pub(crate) enum RouteBindRelayOutcome {
349    Accepted,
350    Rejected(ErrorBody),
351    ModuleGone(String),
352}
353
354/// What [`ForwardingTable::cutover_candidate`] did.
355#[derive(Debug, Clone, Copy, PartialEq, Eq)]
356pub(crate) struct ForwardingCutover {
357    /// The endpoint now in the active slot (the former candidate).
358    pub promoted: ModuleEndpointId,
359    /// The endpoint demoted out of the active slot, to be drained by endpoint.
360    /// `None` when the incumbent's connection was already gone.
361    pub incumbent: Option<ModuleEndpointId>,
362}
363
364/// What tearing down one connection released.
365#[derive(Debug)]
366pub(crate) struct ConnectionCleanup {
367    /// Routes whose other end must be sent GOODBYE.
368    pub released: Vec<GoodbyeTarget>,
369    /// Pending route.bind relays to the closed module that were aborted: opens
370    /// that had not been bound yet. Zero when a client connection closed.
371    pub abandoned_relays: u32,
372}
373
374#[derive(Debug, Clone)]
375pub(crate) struct PendingRelayCompletion {
376    pub settled: bool,
377    pub abandoned: Option<GoodbyeTarget>,
378}
379
380#[derive(Debug)]
381pub(crate) struct PendingModuleControlRpc {
382    pub endpoint: ModuleEndpointId,
383    pub module_sink: FrameSink,
384    pub negotiated_ver: u8,
385    pub corr: u64,
386    pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
387}
388
389#[derive(Debug, Clone)]
390pub(crate) enum ModuleControlRpcOutcome {
391    Response(ModuleControlResponse),
392    Rejected(ErrorBody),
393    ModuleGone(String),
394    MalformedResponse(String),
395    UnexpectedOp { expected: String, actual: String },
396    DeadlineElapsed,
397}
398
399#[derive(Debug, Clone, PartialEq, Eq)]
400pub(crate) enum ModuleControlRpcCompletion {
401    Unknown,
402    Settled,
403    LateHealthAnswer {
404        module_id: String,
405        latency: Duration,
406    },
407}
408
409#[derive(Debug)]
410struct PendingModuleControlRpcEntry {
411    expected_op: String,
412    deadline: Instant,
413    health_probe_started_at: Option<Instant>,
414    sender: oneshot::Sender<ModuleControlRpcOutcome>,
415}
416
417#[derive(Debug)]
418struct HealthProbeTombstone {
419    expected_op: String,
420    module_id: String,
421    probe_started_at: Instant,
422    expires_at: Instant,
423}
424
425#[derive(Debug, Clone)]
426struct RouteReservation {
427    client_key: ClientRouteKey,
428    module_key: ModuleRouteKey,
429    client_epoch: u32,
430    module_epoch: u32,
431    project_root: Option<ProjectRootId>,
432}
433
434#[derive(Debug)]
435struct PendingRouteBindRelayEntry {
436    reservation: RouteReservation,
437    client_sink: FrameSink,
438    client_negotiated_ver: u8,
439    client_permit: crate::router::EgressPermit,
440    route_open_frame: Frame,
441    principal: Principal,
442    /// The scope admitted at `route.open`, with the tag captured then. Commit
443    /// compares it with the published tag.
444    scope: Option<BoundScope>,
445    deadline: Instant,
446    /// Set once `route.open` has finished putting the route.bind on the
447    /// module's egress queue. A path that drops this entry WITHOUT a module
448    /// reply (client teardown, module drain, the open's own abort) sends the
449    /// module a GOODBYE for the bind only when this is set, because only then
450    /// is the GOODBYE guaranteed to queue behind the bind. When it is not set,
451    /// the removal is recorded in `relays_released_before_enqueue` instead and
452    /// `route.open` sends that GOODBYE itself once its send has completed.
453    ///
454    /// A module reply does not consult it: an ack proves the module received
455    /// the bind, so the reply path decides about the GOODBYE on its own.
456    relay_enqueued: bool,
457    sender: oneshot::Sender<RouteBindRelayOutcome>,
458}
459
460/// What `mark_route_bind_relay_enqueued` found once `route.open` had put the
461/// route.bind on the module's queue.
462#[derive(Debug, Clone, Copy, PartialEq, Eq)]
463pub(crate) enum RouteBindRelayMark {
464    /// The relay is still pending; later removals handle their own GOODBYE.
465    Marked,
466    /// The module already answered and the reply path settled the relay:
467    /// it committed the route, or it refused it and sent any GOODBYE the
468    /// module needed. `route.open` must send nothing.
469    SettledByReply,
470    /// The relay was dropped without a module reply while the bind was still
471    /// being queued, so nothing has told the module to drop the binding it
472    /// is about to create. `route.open` must send that GOODBYE.
473    ReleasedWithoutReply,
474}
475
476#[derive(Debug, Clone)]
477pub(crate) enum RouteRelease {
478    Removed(GoodbyeTarget),
479    Stale,
480    Absent,
481}
482
483#[derive(Debug, Clone)]
484pub(crate) enum RoutePollSnapshot {
485    Bound {
486        module_id: String,
487        status: Option<String>,
488    },
489    Absent,
490}
491
492#[derive(Debug, Clone)]
493struct ModuleConnection {
494    endpoint: ModuleEndpointId,
495    sink: FrameSink,
496    negotiated_ver: u8,
497    concurrency: Concurrency,
498}
499
500#[derive(Debug, Default)]
501struct ForwardingInner {
502    operator_confirms: Arc<crate::operator_confirm::OperatorConfirms>,
503    daemon_draining: bool,
504    /// Whether forwarding reached quiescence during daemon shutdown, for each
505    /// active endpoint present when shutdown began. A pending wait is false
506    /// because it has not proved quiescence. Endpoint generations prevent a
507    /// result from describing a replacement connection with the same module id.
508    daemon_drain_results: HashMap<ModuleEndpointId, bool>,
509    /// The ACTIVE slot: the one endpoint per module id that routing resolves.
510    /// Every by-id lookup (relay reservation, drain-by-id, liveness, census,
511    /// live roots, module-control RPCs) reads this map and nothing else.
512    modules_by_id: HashMap<String, ModuleConnection>,
513    /// The CANDIDATE slot of a blue/green swap: a second process registered
514    /// under an id that already has an active endpoint. It has a full endpoint
515    /// identity (so its own connection can be looked up and torn down) but no
516    /// by-id lookup sees it, so nothing is routed to it until `cutover_candidate`
517    /// promotes it.
518    candidates_by_id: HashMap<String, ModuleConnection>,
519    /// Former active endpoints demoted by `cutover_candidate`, until their
520    /// connection is removed. Membership is what distinguishes an endpoint that
521    /// was SUPERSEDED by a promotion (still alive, still carrying bound routes,
522    /// about to be drained) from one that is merely STALE (replaced some other
523    /// way, which is treated as a fault on the acking connection).
524    superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
525    endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
526    module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
527    /// Endpoints mid-drain, keyed to the reason the drain was begun with. The
528    /// value serves the census ("draining because restart"); membership alone
529    /// still answers every admission-gate check.
530    draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
531    closing_connections: HashSet<ConnectionId>,
532    next_generation: u64,
533    reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
534    reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
535    next_client_channel: HashMap<ConnectionId, u16>,
536    next_module_channel: HashMap<ModuleEndpointId, u16>,
537    client_slot_epochs: HashMap<ClientRouteKey, u32>,
538    module_slot_epochs: HashMap<ModuleRouteKey, u32>,
539    last_published_epoch: HashMap<ClientRouteKey, u32>,
540    client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
541    module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
542    status: HashMap<(ClientRouteKey, u32), String>,
543    pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
544    /// Pending relays removed without a module reply before `route.open`
545    /// marked them enqueued. `route.open` consumes its key when it marks the
546    /// relay (and sends the GOODBYE the removal could not send safely) or when
547    /// it aborts, so an entry lives only while that open is mid-send.
548    relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
549    /// Each scope's current `(scope_epoch, version)`, keyed `(owner, ref)`,
550    /// published by a scope sync in the same step that changes the record. A
551    /// bind commit reads it here so it never takes the scope table's lock
552    /// while holding this one. Absent means the scope is not live.
553    scope_tags: HashMap<(String, String), ScopeTag>,
554    next_control_corr: HashMap<ModuleEndpointId, u64>,
555    pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
556    health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
557}
558
559#[derive(Debug, Clone)]
560pub(crate) struct CloseReason {
561    code: &'static str,
562    message: String,
563}
564
565impl CloseReason {
566    pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
567        Self {
568            code,
569            message: message.into(),
570        }
571    }
572}
573
574impl fmt::Display for CloseReason {
575    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
576        write!(f, "{}: {}", self.code, self.message)
577    }
578}
579
580pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
581
582/// Dynamic forwarding state shared by the control plane and data-plane router.
583#[derive(Debug, Default)]
584pub struct ForwardingTable {
585    inner: Arc<RwLock<ForwardingInner>>,
586    #[cfg(feature = "test-support")]
587    route_closing_hold: Mutex<Option<oneshot::Receiver<()>>>,
588    close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
589    counters: DaemonCounters,
590    /// Per-target-module bind-relay breaker state. It lives here, beside the
591    /// module connections it describes, because this is where a module
592    /// connection's identity is established and therefore where a stale
593    /// verdict has to be discarded.
594    route_bind_breakers: RouteBindBreakers,
595    /// Current route.bind relays keyed by target module. Admission is shared
596    /// across every client connection that points at the same endpoint.
597    route_bind_concurrency: RouteBindConcurrency,
598    /// Start and end of each module's `route.open` outage. Held here for the
599    /// same reason as the breakers: every handler built over this table must
600    /// see one outage per module, or each would log its own opening line.
601    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
602}
603
604impl ForwardingTable {
605    /// Install scripted operator confirmation for an in-process integration harness,
606    /// without bootstrapping a daemon process. Only available with `test-support`.
607    ///
608    /// The script has one response per line: `approve`, `decline`, `unavailable`,
609    /// `hang` (until withdrawn), or `approve when <path>` (wait for the path to
610    /// exist). Unreadable or exhausted scripts fail closed. If supplied, `events`
611    /// receives the provider's JSON-lines event log. `limits: None` uses
612    /// [`OperatorLimits::default`](crate::operator_confirm::OperatorLimits); use
613    /// `Some(OperatorLimits { queue_wait, prompt_timeout, stuck_grace })` to
614    /// override the queue wait, prompt timeout and stuck-provider grace periods.
615    ///
616    /// The current executable's file name **must start with `ckdev-`**. Otherwise
617    /// this logs a warning and keeps the OS provider, just like bootstrap.
618    /// `cortexkit-test-support`'s `ckdev_binary` copies a harness executable under
619    /// a qualifying name; run that copy to enable scripted confirmation. See
620    /// `crates/subc-client-rs/tests/real_daemon/operator_confirm.rs` for the
621    /// reference end-to-end harness.
622    ///
623    /// Configure the table shared by the harness's `ControlHandler` and `Router`
624    /// before serving any requests. This also configures privileged confirmations
625    /// that use the same table.
626    ///
627    /// # Panics
628    ///
629    /// Panics if operator-confirm processing has already started.
630    ///
631    /// # Example
632    ///
633    /// ```no_run
634    /// use std::{path::Path, sync::Arc};
635    /// use subc_daemon::{ControlHandler, ForwardingTable, Registry};
636    ///
637    /// let registry = Arc::new(Registry::default());
638    /// let forwarding = Arc::new(ForwardingTable::default());
639    /// forwarding.configure_operator_script(
640    ///     Path::new("operator.script"),
641    ///     Some(Path::new("operator.events")),
642    ///     None,
643    /// );
644    /// let handler = ControlHandler::with_forwarding(registry, forwarding);
645    /// ```
646    #[cfg(feature = "test-support")]
647    pub fn configure_operator_script(
648        &self,
649        script: &std::path::Path,
650        events: Option<&std::path::Path>,
651        limits: Option<crate::operator_confirm::OperatorLimits>,
652    ) {
653        self.operator_confirms()
654            .configure_script(script, events, limits);
655    }
656
657    /// Hold the next supervisor drain between enqueueing `module.draining` and
658    /// `route.closing`, so a test can forward a response in that window. Dropping
659    /// the returned sender also releases the hold, including on a test panic.
660    #[cfg(feature = "test-support")]
661    pub fn hold_route_closing_for_test(&self) -> oneshot::Sender<()> {
662        let (release, hold) = oneshot::channel();
663        *self.route_closing_hold.lock().unwrap() = Some(hold);
664        release
665    }
666
667    #[cfg(feature = "test-support")]
668    pub(crate) async fn wait_before_route_closing_for_test(&self) {
669        let hold = self.route_closing_hold.lock().unwrap().take();
670        if let Some(hold) = hold {
671            let _ = hold.await;
672        }
673    }
674
675    pub(crate) fn operator_confirms(&self) -> Arc<crate::operator_confirm::OperatorConfirms> {
676        Arc::clone(
677            &self
678                .inner
679                .read()
680                .unwrap_or_else(|p| p.into_inner())
681                .operator_confirms,
682        )
683    }
684
685    #[cfg(test)]
686    pub(crate) fn inject_operator_principal(&self, key: ModuleRouteKey, principal: Principal) {
687        let mut inner = self.write_inner().unwrap();
688        let old = inner.module_to_client.remove(&key).unwrap();
689        let client = ClientRouteKey {
690            connection_id: old.client_connection_id,
691            channel: old.client_channel,
692        };
693        inner.client_to_module.remove(&client);
694        let mut binding = Arc::try_unwrap(old).expect("test binding has no outstanding readers");
695        binding.principal = principal;
696        let binding = Arc::new(binding);
697        inner.client_to_module.insert(client, Arc::clone(&binding));
698        inner.module_to_client.insert(key, binding);
699    }
700
701    /// Keep the binding protected through confirmation admission. Route release
702    /// takes the write lock and then the same confirm-state lock, so it cannot
703    /// miss an admitted request or leave it waiting on a vanished route.
704    pub(crate) fn with_operator_route<T>(
705        &self,
706        connection_id: ConnectionId,
707        channel: u16,
708        epoch: u32,
709        admit: impl FnOnce(Option<&RouteBinding>) -> T,
710    ) -> Result<T, ForwardingError> {
711        let inner = self.read_inner()?;
712        let binding = inner
713            .endpoint_by_connection
714            .get(&connection_id)
715            .and_then(|endpoint| {
716                inner.module_to_client.get(&ModuleRouteKey {
717                    endpoint: *endpoint,
718                    channel,
719                })
720            })
721            .filter(|binding| binding.module_epoch == epoch);
722        Ok(admit(binding.map(Arc::as_ref)))
723    }
724
725    pub(crate) fn counters(&self) -> DaemonCounters {
726        self.counters.clone()
727    }
728
729    pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
730        self.route_bind_breakers.clone()
731    }
732
733    pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
734        self.route_bind_concurrency.clone()
735    }
736
737    pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
738        Arc::clone(&self.route_outages)
739    }
740
741    pub(crate) fn register_connection_close(
742        &self,
743        connection_id: ConnectionId,
744    ) -> ConnectionCloseReceiver {
745        let (sender, receiver) = oneshot::channel();
746        let replaced = self
747            .lock_close_registry()
748            .insert(connection_id, sender)
749            .is_some();
750        if replaced {
751            warn!(
752                connection_id = connection_id.get(),
753                "replaced existing connection close registration"
754            );
755        }
756        receiver
757    }
758
759    pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
760        self.lock_close_registry().remove(&connection_id);
761    }
762
763    /// Close every established connection, modules and clients alike, as the
764    /// last step of an announced daemon shutdown. By the time this runs each
765    /// registered module has already been sent a module GOODBYE (see
766    /// `Supervisor::end_children_for_daemon_shutdown`), so the EOF that follows
767    /// is a planned stop; EOF with no GOODBYE before it stays reserved for a
768    /// daemon that went away unannounced. Closing while the daemon is still
769    /// alive lets it wait for the modules' own teardowns before it ends
770    /// whatever is left.
771    #[cfg(unix)]
772    pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
773        let senders: Vec<_> = self.lock_close_registry().drain().collect();
774        let count = senders.len();
775        for (_, sender) in senders {
776            let _ = sender.send(reason.clone());
777        }
778        count
779    }
780
781    /// Every registered module connection, whichever slot it occupies (active,
782    /// swap candidate, or superseded incumbent), once each. Daemon shutdown
783    /// uses this to tell each of them it is a planned stop before closing it.
784    #[cfg(unix)]
785    pub(crate) fn module_connections(
786        &self,
787    ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
788        let inner = self.read_inner()?;
789        let mut seen = HashSet::new();
790        Ok(inner
791            .modules_by_id
792            .values()
793            .chain(inner.candidates_by_id.values())
794            .chain(inner.superseded_endpoints.values())
795            .filter(|module| seen.insert(module.endpoint))
796            .map(|module| ModuleConnectionTarget {
797                module_id: inner
798                    .module_id_by_endpoint
799                    .get(&module.endpoint)
800                    .cloned()
801                    .unwrap_or_default(),
802                endpoint: module.endpoint,
803                sink: module.sink.clone(),
804                negotiated_ver: module.negotiated_ver,
805            })
806            .collect())
807    }
808
809    /// Ask a registered connection to close. Returns true only for the request
810    /// that actually reached it; later requests for the same connection, and
811    /// requests for a connection that is not registered, return false.
812    pub(crate) fn request_connection_close(
813        &self,
814        connection_id: ConnectionId,
815        reason: CloseReason,
816    ) -> bool {
817        let sender = self.lock_close_registry().remove(&connection_id);
818        if let Some(sender) = sender {
819            debug!(
820                connection_id = connection_id.get(),
821                close_reason = %reason,
822                "requesting connection close"
823            );
824            let _ = sender.send(reason);
825            true
826        } else {
827            debug!(
828                connection_id = connection_id.get(),
829                close_reason = %reason,
830                "connection close request ignored for inactive connection"
831            );
832            false
833        }
834    }
835
836    pub fn register_module_connection(
837        &self,
838        connection_id: ConnectionId,
839        module_id: String,
840        negotiated_ver: u8,
841        concurrency: Concurrency,
842        sink: FrameSink,
843    ) -> Result<ModuleEndpointId, ForwardingError> {
844        self.register_module_connection_inner(
845            connection_id,
846            module_id,
847            negotiated_ver,
848            concurrency,
849            sink,
850            None,
851        )
852    }
853
854    /// Register a module connection and queue its HELLO_ACK as the first frame
855    /// on its sink, in the same write-lock critical section that makes the
856    /// endpoint visible.
857    ///
858    /// A module reads HELLO_ACK as the first frame after its HELLO and exits on
859    /// anything else. Once the endpoint is in `modules_by_id`, a `route.open`
860    /// on any other connection can queue a `route.bind` request onto this sink,
861    /// so the ack must already be queued by then. Every lookup that can queue a
862    /// frame for the module takes this lock, so nothing can get in ahead of it.
863    ///
864    /// If the ack cannot be queued (the sink is closed or full), the
865    /// registration fails with nothing inserted.
866    pub(crate) fn register_module_connection_acked(
867        &self,
868        connection_id: ConnectionId,
869        module_id: String,
870        negotiated_ver: u8,
871        concurrency: Concurrency,
872        sink: FrameSink,
873        hello_ack: Frame,
874    ) -> Result<ModuleEndpointId, ForwardingError> {
875        self.register_module_connection_inner(
876            connection_id,
877            module_id,
878            negotiated_ver,
879            concurrency,
880            sink,
881            Some(hello_ack),
882        )
883    }
884
885    fn register_module_connection_inner(
886        &self,
887        connection_id: ConnectionId,
888        module_id: String,
889        negotiated_ver: u8,
890        concurrency: Concurrency,
891        sink: FrameSink,
892        hello_ack: Option<Frame>,
893    ) -> Result<ModuleEndpointId, ForwardingError> {
894        let mut inner = self.write_inner()?;
895        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
896            return Err(ForwardingError::ConnectionClosing { connection_id });
897        }
898        check_module_connection_role_locked(&inner, connection_id)?;
899        // Every refusal check has passed and nothing has been mutated yet, so
900        // a failed enqueue leaves the table exactly as it was.
901        enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
902
903        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
904        let endpoint = ModuleEndpointId {
905            connection_id,
906            generation: inner.next_generation,
907        };
908        inner.endpoint_by_connection.insert(connection_id, endpoint);
909        inner
910            .module_id_by_endpoint
911            .insert(endpoint, module_id.clone());
912        inner.next_module_channel.insert(endpoint, 1);
913        inner.next_control_corr.insert(endpoint, 1);
914        inner.modules_by_id.insert(
915            module_id.clone(),
916            ModuleConnection {
917                endpoint,
918                sink,
919                negotiated_ver,
920                concurrency,
921            },
922        );
923        drop(inner);
924
925        // A new module connection has arrived under this id, so anything the
926        // bind-relay breaker learned was learned about a process that is no
927        // longer the one behind this name. See
928        // `RouteBindBreakers::reset_for_new_module_connection`.
929        //
930        // Keyed on ARRIVAL rather than on teardown deliberately: a connection
931        // going away is not evidence about anything, and a module whose
932        // connection drops without coming back should keep its verdict until
933        // something actually registers in its place. This also covers the
934        // unclean replacements -- a module killed mid-bind, or one whose
935        // connection was closing -- because registration is the single path by
936        // which any module connection becomes usable.
937        if let Some(discarded) = self
938            .route_bind_breakers
939            .reset_for_new_module_connection(&module_id)
940        {
941            info!(
942                module_id = %module_id,
943                discarded_consecutive_timeouts = discarded,
944                "route.bind breaker state discarded: a new module connection replaced the process it described"
945            );
946        }
947        Ok(endpoint)
948    }
949
950    /// Register a blue/green swap candidate for `module_id` into the candidate
951    /// slot, alongside whatever endpoint is active for the id.
952    ///
953    /// Unlike [`Self::register_module_connection`], this never touches the
954    /// active slot and never resets the bind-relay breaker: the incumbent is
955    /// still the process serving the id, so what the breaker learned about it
956    /// is still true. The breaker is reset at cutover instead, when the process
957    /// behind the name actually changes. A second candidate for the same id is
958    /// refused.
959    ///
960    /// Production registers candidates through
961    /// [`Self::register_candidate_module_connection_acked`]; this ack-less form
962    /// is for tests that build forwarding state directly.
963    #[cfg(test)]
964    pub(crate) fn register_candidate_module_connection(
965        &self,
966        connection_id: ConnectionId,
967        module_id: String,
968        negotiated_ver: u8,
969        concurrency: Concurrency,
970        sink: FrameSink,
971    ) -> Result<ModuleEndpointId, ForwardingError> {
972        self.register_candidate_module_connection_inner(
973            connection_id,
974            module_id,
975            negotiated_ver,
976            concurrency,
977            sink,
978            None,
979        )
980    }
981
982    /// The swap-candidate counterpart of
983    /// [`Self::register_module_connection_acked`]: the HELLO_ACK is queued on
984    /// the candidate's sink before its endpoint is inserted. No by-id lookup
985    /// sees a candidate, but its own connection's endpoint does become
986    /// resolvable here, and at cutover it becomes routable; the ack has to be
987    /// ahead of anything either can queue.
988    pub(crate) fn register_candidate_module_connection_acked(
989        &self,
990        connection_id: ConnectionId,
991        module_id: String,
992        negotiated_ver: u8,
993        concurrency: Concurrency,
994        sink: FrameSink,
995        hello_ack: Frame,
996    ) -> Result<ModuleEndpointId, ForwardingError> {
997        self.register_candidate_module_connection_inner(
998            connection_id,
999            module_id,
1000            negotiated_ver,
1001            concurrency,
1002            sink,
1003            Some(hello_ack),
1004        )
1005    }
1006
1007    fn register_candidate_module_connection_inner(
1008        &self,
1009        connection_id: ConnectionId,
1010        module_id: String,
1011        negotiated_ver: u8,
1012        concurrency: Concurrency,
1013        sink: FrameSink,
1014        hello_ack: Option<Frame>,
1015    ) -> Result<ModuleEndpointId, ForwardingError> {
1016        let mut inner = self.write_inner()?;
1017        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1018            return Err(ForwardingError::ConnectionClosing { connection_id });
1019        }
1020        check_module_connection_role_locked(&inner, connection_id)?;
1021        if inner.candidates_by_id.contains_key(&module_id) {
1022            return Err(ForwardingError::CandidateSlotOccupied { module_id });
1023        }
1024        // As in `register_module_connection_inner`: after every refusal check,
1025        // before any mutation.
1026        enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1027
1028        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1029        let endpoint = ModuleEndpointId {
1030            connection_id,
1031            generation: inner.next_generation,
1032        };
1033        inner.endpoint_by_connection.insert(connection_id, endpoint);
1034        inner
1035            .module_id_by_endpoint
1036            .insert(endpoint, module_id.clone());
1037        inner.next_module_channel.insert(endpoint, 1);
1038        inner.next_control_corr.insert(endpoint, 1);
1039        inner.candidates_by_id.insert(
1040            module_id,
1041            ModuleConnection {
1042                endpoint,
1043                sink,
1044                negotiated_ver,
1045                concurrency,
1046            },
1047        );
1048        Ok(endpoint)
1049    }
1050
1051    /// Promote `module_id`'s candidate to the active slot, in ONE forwarding
1052    /// write-lock critical section.
1053    ///
1054    /// That single section is the linearization point of a swap. Every relay
1055    /// reservation resolves the active slot under the same lock, so each one
1056    /// lands wholly before cutover (on the incumbent) or wholly after it (on the
1057    /// promoted candidate), never on a mix. A relay reserved on the incumbent
1058    /// before cutover can no longer commit: `commit_route_locked` requires the
1059    /// reservation's endpoint to be the active one, and an ack for it arriving
1060    /// later is answered by the superseded-endpoint arm of
1061    /// `complete_pending_relay` instead of ending the incumbent's connection.
1062    ///
1063    /// Both endpoints keep their identities: nothing is re-keyed, so the
1064    /// incumbent's bound routes and its pending correlation keys stay exactly
1065    /// where they are until it is drained with [`Self::begin_endpoint_drain`],
1066    /// using the incumbent endpoint this returns. Returns `Ok(None)` when there
1067    /// is no candidate for the id.
1068    pub(crate) fn cutover_candidate(
1069        &self,
1070        module_id: &str,
1071    ) -> Result<Option<ForwardingCutover>, ForwardingError> {
1072        let mut inner = self.write_inner()?;
1073        if inner.daemon_draining {
1074            return Err(ForwardingError::ModuleReloading {
1075                module_id: module_id.to_string(),
1076            });
1077        }
1078        let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
1079            return Ok(None);
1080        };
1081        let promoted = candidate.endpoint;
1082        let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
1083        let incumbent = incumbent.map(|incumbent| {
1084            let endpoint = incumbent.endpoint;
1085            inner.superseded_endpoints.insert(endpoint, incumbent);
1086            endpoint
1087        });
1088        drop(inner);
1089
1090        // A different process now answers for this id; see the matching reset
1091        // in `register_module_connection` for why a verdict about the old one
1092        // must not carry over.
1093        if let Some(discarded) = self
1094            .route_bind_breakers
1095            .reset_for_new_module_connection(module_id)
1096        {
1097            info!(
1098                module_id = %module_id,
1099                discarded_consecutive_timeouts = discarded,
1100                "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
1101            );
1102        }
1103        Ok(Some(ForwardingCutover {
1104            promoted,
1105            incumbent,
1106        }))
1107    }
1108
1109    #[allow(clippy::too_many_arguments)]
1110    pub(crate) async fn begin_route_bind_relay_for(
1111        &self,
1112        client_connection_id: ConnectionId,
1113        client_sink: FrameSink,
1114        client_negotiated_ver: u8,
1115        client_corr: u64,
1116        module_id: &str,
1117        principal: Principal,
1118        scope: Option<BoundScope>,
1119        project_root: Option<ProjectRootId>,
1120        deadline: Instant,
1121    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1122        // Reserve egress capacity before taking the forwarding lock. The permit is
1123        // held until the bind reaches one terminal state, so an accepted bind can
1124        // publish its table entry and RouteOpen response in one critical section.
1125        let client_permit =
1126            client_sink
1127                .reserve_owned()
1128                .await
1129                .map_err(|_| ForwardingError::ClientEgressClosed {
1130                    connection_id: client_connection_id,
1131                })?;
1132        self.begin_route_bind_relay_inner(
1133            client_connection_id,
1134            client_sink,
1135            client_negotiated_ver,
1136            client_corr,
1137            module_id,
1138            principal,
1139            scope,
1140            project_root,
1141            deadline,
1142            client_permit,
1143        )
1144    }
1145
1146    #[cfg(test)]
1147    pub(crate) fn begin_route_bind_relay_for_test(
1148        &self,
1149        client_connection_id: ConnectionId,
1150        client_sink: FrameSink,
1151        client_corr: u64,
1152        module_id: &str,
1153    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1154        let permit =
1155            client_sink
1156                .try_reserve_owned()
1157                .map_err(|_| ForwardingError::ClientEgressClosed {
1158                    connection_id: client_connection_id,
1159                })?;
1160        self.begin_route_bind_relay_inner(
1161            client_connection_id,
1162            client_sink,
1163            subc_protocol::PROTOCOL_VERSION,
1164            client_corr,
1165            module_id,
1166            Principal::Direct,
1167            None,
1168            None,
1169            Instant::now() + std::time::Duration::from_secs(60),
1170            permit,
1171        )
1172    }
1173
1174    pub(crate) fn begin_module_control_rpc_for(
1175        &self,
1176        module_id: &str,
1177        expected_op: &str,
1178        deadline: Instant,
1179    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1180        self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1181    }
1182
1183    pub(crate) fn begin_health_probe_rpc_for(
1184        &self,
1185        module_id: &str,
1186        expected_op: &str,
1187        probe_started_at: Instant,
1188        deadline: Instant,
1189    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1190        self.begin_module_control_rpc_inner(
1191            module_id,
1192            expected_op,
1193            deadline,
1194            Some(probe_started_at),
1195            false,
1196        )
1197    }
1198
1199    pub(crate) fn begin_drain_health_probe_rpc_for(
1200        &self,
1201        module_id: &str,
1202        expected_op: &str,
1203        probe_started_at: Instant,
1204        deadline: Instant,
1205    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1206        self.begin_module_control_rpc_inner(
1207            module_id,
1208            expected_op,
1209            deadline,
1210            Some(probe_started_at),
1211            true,
1212        )
1213    }
1214
1215    /// A health probe addressed to one endpoint, in whichever slot it is.
1216    ///
1217    /// Every other control RPC resolves the id's ACTIVE endpoint, which is how
1218    /// a swap candidate (not active until cutover) and a superseded incumbent
1219    /// (not active after it) are unreachable by them. A swap needs to probe
1220    /// exactly those two: the candidate before promoting it, and the incumbent
1221    /// for its busy gauges while it drains.
1222    pub(crate) fn begin_endpoint_health_probe_rpc_for(
1223        &self,
1224        endpoint: ModuleEndpointId,
1225        expected_op: &str,
1226        probe_started_at: Instant,
1227        deadline: Instant,
1228    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1229        let inner = self.write_inner()?;
1230        let module = module_connection_for_endpoint_locked(&inner, endpoint)
1231            .cloned()
1232            .ok_or(ForwardingError::NoModuleConnection)?;
1233        let module_id = inner
1234            .module_id_by_endpoint
1235            .get(&endpoint)
1236            .cloned()
1237            .unwrap_or_default();
1238        // A draining incumbent is exactly what this probes for busy gauges, so
1239        // draining is allowed, as it is for the by-id drain probe.
1240        self.begin_control_rpc_locked(
1241            inner,
1242            &module_id,
1243            module,
1244            expected_op,
1245            deadline,
1246            Some(probe_started_at),
1247            true,
1248        )
1249    }
1250
1251    fn begin_module_control_rpc_inner(
1252        &self,
1253        module_id: &str,
1254        expected_op: &str,
1255        deadline: Instant,
1256        health_probe_started_at: Option<Instant>,
1257        allow_draining: bool,
1258    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1259        let inner = self.write_inner()?;
1260        let module = inner
1261            .modules_by_id
1262            .get(module_id)
1263            .cloned()
1264            .ok_or(ForwardingError::NoModuleConnection)?;
1265        self.begin_control_rpc_locked(
1266            inner,
1267            module_id,
1268            module,
1269            expected_op,
1270            deadline,
1271            health_probe_started_at,
1272            allow_draining,
1273        )
1274    }
1275
1276    #[allow(clippy::too_many_arguments)]
1277    fn begin_control_rpc_locked(
1278        &self,
1279        mut inner: RwLockWriteGuard<'_, ForwardingInner>,
1280        module_id: &str,
1281        module: ModuleConnection,
1282        expected_op: &str,
1283        deadline: Instant,
1284        health_probe_started_at: Option<Instant>,
1285        allow_draining: bool,
1286    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1287        if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1288            return Err(ForwardingError::ModuleReloading {
1289                module_id: module_id.to_string(),
1290            });
1291        }
1292        if inner
1293            .closing_connections
1294            .contains(&module.endpoint.connection_id)
1295        {
1296            return Err(ForwardingError::ConnectionClosing {
1297                connection_id: module.endpoint.connection_id,
1298            });
1299        }
1300        if health_probe_started_at.is_some() {
1301            // Recurring health probes are serial per endpoint. Once the next one
1302            // starts, an older answer can no longer improve the current snapshot,
1303            // so retaining more than the newest unanswered probe has no value.
1304            inner
1305                .health_probe_tombstones
1306                .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1307        }
1308        let corr = match inner.allocate_control_corr(module.endpoint) {
1309            Ok(corr) => corr,
1310            Err(err) => {
1311                drop(inner);
1312                self.request_connection_close(
1313                    module.endpoint.connection_id,
1314                    CloseReason::new(
1315                        "control_correlation_exhausted",
1316                        "daemon-originated channel-0 correlation space exhausted",
1317                    ),
1318                );
1319                return Err(err);
1320            }
1321        };
1322        let (sender, receiver) = oneshot::channel();
1323        inner.pending_control_rpcs.insert(
1324            (module.endpoint, corr),
1325            PendingModuleControlRpcEntry {
1326                expected_op: expected_op.to_string(),
1327                deadline,
1328                health_probe_started_at,
1329                sender,
1330            },
1331        );
1332
1333        Ok(PendingModuleControlRpc {
1334            endpoint: module.endpoint,
1335            module_sink: module.sink,
1336            negotiated_ver: module.negotiated_ver,
1337            corr,
1338            receiver,
1339        })
1340    }
1341
1342    #[allow(clippy::too_many_arguments)]
1343    fn begin_route_bind_relay_inner(
1344        &self,
1345        client_connection_id: ConnectionId,
1346        client_sink: FrameSink,
1347        client_negotiated_ver: u8,
1348        client_corr: u64,
1349        expected_module_id: &str,
1350        principal: Principal,
1351        scope: Option<BoundScope>,
1352        project_root: Option<ProjectRootId>,
1353        deadline: Instant,
1354        client_permit: crate::router::EgressPermit,
1355    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1356        let mut inner = self.write_inner()?;
1357        // HELLO and route.open can run on different tasks. Decide the role under
1358        // the same lock that installs endpoints and reserves client channels so
1359        // neither task can turn one connection into both kinds of owner.
1360        if inner
1361            .endpoint_by_connection
1362            .contains_key(&client_connection_id)
1363        {
1364            return Err(ForwardingError::ConnectionRoleConflict {
1365                connection_id: client_connection_id,
1366            });
1367        }
1368        if inner.closing_connections.contains(&client_connection_id) {
1369            return Err(ForwardingError::ConnectionClosing {
1370                connection_id: client_connection_id,
1371            });
1372        }
1373        let module = inner
1374            .modules_by_id
1375            .get(expected_module_id)
1376            .cloned()
1377            .ok_or(ForwardingError::NoModuleConnection)?;
1378        if inner.draining_endpoints.contains_key(&module.endpoint) {
1379            return Err(ForwardingError::ModuleReloading {
1380                module_id: expected_module_id.to_string(),
1381            });
1382        }
1383        if inner
1384            .closing_connections
1385            .contains(&module.endpoint.connection_id)
1386        {
1387            return Err(ForwardingError::ConnectionClosing {
1388                connection_id: module.endpoint.connection_id,
1389            });
1390        }
1391
1392        let corr = match inner.allocate_control_corr(module.endpoint) {
1393            Ok(corr) => corr,
1394            Err(err) => {
1395                drop(inner);
1396                self.request_connection_close(
1397                    module.endpoint.connection_id,
1398                    CloseReason::new(
1399                        "control_correlation_exhausted",
1400                        "daemon-originated channel-0 correlation space exhausted",
1401                    ),
1402                );
1403                return Err(err);
1404            }
1405        };
1406        let (client_channel, client_epoch, module_channel, module_epoch) =
1407            inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1408        let client_key = ClientRouteKey {
1409            connection_id: client_connection_id,
1410            channel: client_channel,
1411        };
1412        let module_key = ModuleRouteKey {
1413            endpoint: module.endpoint,
1414            channel: module_channel,
1415        };
1416        let reservation = RouteReservation {
1417            client_key,
1418            module_key,
1419            client_epoch,
1420            module_epoch,
1421            project_root,
1422        };
1423        let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
1424            route_channel: client_channel,
1425            route_epoch: client_epoch,
1426        })
1427        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1428        let route_open_frame = Frame::build_with_version(
1429            client_negotiated_ver,
1430            FrameType::Response,
1431            Flags::new(false, Priority::Passive, false),
1432            0,
1433            0,
1434            client_corr,
1435            response_body,
1436        )
1437        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1438        let (sender, receiver) = oneshot::channel();
1439        inner.reserved_client.insert(client_key, module_key);
1440        inner.reserved_module.insert(module_key, client_key);
1441        inner.pending_relays.insert(
1442            (module.endpoint, corr),
1443            PendingRouteBindRelayEntry {
1444                reservation,
1445                client_sink,
1446                client_negotiated_ver,
1447                client_permit,
1448                route_open_frame,
1449                principal,
1450                scope,
1451                deadline,
1452                relay_enqueued: false,
1453                sender,
1454            },
1455        );
1456
1457        Ok(PendingRouteBindRelay {
1458            endpoint: module.endpoint,
1459            module_sink: module.sink,
1460            negotiated_ver: module.negotiated_ver,
1461            client_channel,
1462            client_epoch,
1463            module_channel,
1464            module_epoch,
1465            corr,
1466            receiver,
1467        })
1468    }
1469
1470    /// Called by `route.open` right after the route.bind entered the module's
1471    /// egress queue.
1472    ///
1473    /// The module reader runs concurrently with `route.open`, so a fast module
1474    /// can receive the bind, ack it, and have the reader commit the route
1475    /// before this runs. A missing entry therefore does not mean the bind was
1476    /// abandoned: only a removal recorded in `relays_released_before_enqueue`
1477    /// does. Telling the two apart is what stops `route.open` from sending a
1478    /// GOODBYE for a route that was just committed.
1479    pub(crate) fn mark_route_bind_relay_enqueued(
1480        &self,
1481        endpoint: ModuleEndpointId,
1482        corr: u64,
1483    ) -> Result<RouteBindRelayMark, ForwardingError> {
1484        let mut inner = self.write_inner()?;
1485        if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
1486            pending.relay_enqueued = true;
1487            return Ok(RouteBindRelayMark::Marked);
1488        }
1489        if inner
1490            .relays_released_before_enqueue
1491            .remove(&(endpoint, corr))
1492        {
1493            return Ok(RouteBindRelayMark::ReleasedWithoutReply);
1494        }
1495        Ok(RouteBindRelayMark::SettledByReply)
1496    }
1497
1498    pub(crate) fn release_client_route(
1499        &self,
1500        client_connection_id: ConnectionId,
1501        client_channel: u16,
1502        expected_epoch: u32,
1503    ) -> Result<RouteRelease, ForwardingError> {
1504        let mut inner = self.write_inner()?;
1505        let release = release_client_route_locked(
1506            &mut inner,
1507            ClientRouteKey {
1508                connection_id: client_connection_id,
1509                channel: client_channel,
1510            },
1511            expected_epoch,
1512        );
1513        self.record_route_release(&release);
1514        Ok(release)
1515    }
1516
1517    pub(crate) fn release_module_route(
1518        &self,
1519        module_connection_id: ConnectionId,
1520        module_channel: u16,
1521        expected_epoch: u32,
1522    ) -> Result<RouteRelease, ForwardingError> {
1523        let mut inner = self.write_inner()?;
1524        let Some(endpoint) = inner
1525            .endpoint_by_connection
1526            .get(&module_connection_id)
1527            .copied()
1528        else {
1529            return Ok(RouteRelease::Absent);
1530        };
1531        let release = release_module_route_locked(
1532            &mut inner,
1533            ModuleRouteKey {
1534                endpoint,
1535                channel: module_channel,
1536            },
1537            expected_epoch,
1538        );
1539        self.record_route_release(&release);
1540        Ok(release)
1541    }
1542
1543    pub(crate) fn abort_pending_relay(
1544        &self,
1545        endpoint: ModuleEndpointId,
1546        corr: u64,
1547        outcome: RouteBindRelayOutcome,
1548    ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
1549        let mut inner = self.write_inner()?;
1550        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1551            // If another path dropped the relay while `route.open` was still
1552            // sending the bind, and that open is now giving up without marking
1553            // it (its send failed or it was cancelled), forget the record so it
1554            // cannot outlive the open.
1555            inner
1556                .relays_released_before_enqueue
1557                .remove(&(endpoint, corr));
1558            return Ok(None);
1559        };
1560        release_reserved_route_locked(
1561            &mut inner,
1562            pending.reservation.client_key,
1563            pending.reservation.module_key,
1564        );
1565        let target = pending
1566            .relay_enqueued
1567            .then(|| abandoned_route_target(&inner, &pending.reservation));
1568        let _ = pending.sender.send(outcome);
1569        Ok(target.flatten())
1570    }
1571
1572    pub(crate) fn cancel_module_control_rpc(
1573        &self,
1574        endpoint: ModuleEndpointId,
1575        corr: u64,
1576    ) -> Result<(), ForwardingError> {
1577        self.write_inner()?
1578            .pending_control_rpcs
1579            .remove(&(endpoint, corr));
1580        Ok(())
1581    }
1582
1583    pub(crate) fn tombstone_health_probe_rpc(
1584        &self,
1585        endpoint: ModuleEndpointId,
1586        corr: u64,
1587    ) -> Result<bool, ForwardingError> {
1588        let key = (endpoint, corr);
1589        let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
1590        {
1591            let mut inner = self.write_inner()?;
1592            let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
1593                return Ok(false);
1594            };
1595            let Some(probe_started_at) = pending.health_probe_started_at else {
1596                inner.pending_control_rpcs.insert(key, pending);
1597                return Ok(false);
1598            };
1599            let module_id = inner
1600                .module_id_by_endpoint
1601                .get(&endpoint)
1602                .cloned()
1603                .unwrap_or_else(|| "unknown".to_string());
1604            inner.health_probe_tombstones.insert(
1605                key,
1606                HealthProbeTombstone {
1607                    expected_op: pending.expected_op,
1608                    module_id,
1609                    probe_started_at,
1610                    expires_at,
1611                },
1612            );
1613        }
1614        self.schedule_health_probe_tombstone_expiration(key, expires_at);
1615        Ok(true)
1616    }
1617
1618    fn schedule_health_probe_tombstone_expiration(
1619        &self,
1620        key: (ModuleEndpointId, u64),
1621        expires_at: Instant,
1622    ) {
1623        let inner = Arc::downgrade(&self.inner);
1624        tokio::spawn(async move {
1625            tokio::time::sleep_until(expires_at).await;
1626            let Some(inner) = inner.upgrade() else {
1627                return;
1628            };
1629            let Ok(mut inner) = inner.write() else {
1630                return;
1631            };
1632            let expired = inner
1633                .health_probe_tombstones
1634                .get(&key)
1635                .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
1636            if expired {
1637                inner.health_probe_tombstones.remove(&key);
1638            }
1639        });
1640    }
1641
1642    pub(crate) fn complete_pending_relay(
1643        &self,
1644        connection_id: ConnectionId,
1645        corr: u64,
1646        outcome: RouteBindRelayOutcome,
1647    ) -> Result<PendingRelayCompletion, ForwardingError> {
1648        let mut inner = self.write_inner()?;
1649        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1650            return Ok(PendingRelayCompletion {
1651                settled: false,
1652                abandoned: None,
1653            });
1654        };
1655        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1656            return Ok(PendingRelayCompletion {
1657                settled: false,
1658                abandoned: None,
1659            });
1660        };
1661
1662        if Instant::now() >= pending.deadline {
1663            release_reserved_route_locked(
1664                &mut inner,
1665                pending.reservation.client_key,
1666                pending.reservation.module_key,
1667            );
1668            let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
1669                .then(|| abandoned_route_target(&inner, &pending.reservation))
1670                .flatten();
1671            let _ = pending
1672                .sender
1673                .send(RouteBindRelayOutcome::Rejected(ErrorBody {
1674                    code: "module_timeout".to_string(),
1675                    message: "route.bind response arrived after its daemon deadline".to_string(),
1676                    detail: None,
1677                }));
1678            return Ok(PendingRelayCompletion {
1679                settled: true,
1680                abandoned,
1681            });
1682        }
1683
1684        match outcome {
1685            // Two shapes of "the client is not there to receive this route" that
1686            // must resolve identically: its egress is already closed, or it is
1687            // marked closing (its connection loop has been asked to end, but has
1688            // not drained yet, so the sink is still open).
1689            //
1690            // Only the first used to be caught here. The second fell through to
1691            // `commit_route_locked`, which refuses a closing client with
1692            // `ConnectionClosing` -- and this function is called from the MODULE
1693            // connection's frame handler, so that refusal ended the module's
1694            // connection instead of this one client's route. One dying client's
1695            // route.open then took down a connection carrying every other
1696            // client's routes to that module.
1697            //
1698            // The remedy for both is the same, which is why they share an arm:
1699            // give back the reserved handle pair, tell the waiting route.open the
1700            // route is gone, and report the module-side channel so the caller can
1701            // send a channel-scoped GOODBYE for the binding the module just
1702            // created. Nothing here touches the module connection.
1703            RouteBindRelayOutcome::Accepted
1704                if pending.client_sink.is_closed()
1705                    || inner
1706                        .closing_connections
1707                        .contains(&pending.reservation.client_key.connection_id) =>
1708            {
1709                let reason = if pending.client_sink.is_closed() {
1710                    "client egress closed before route publication"
1711                } else {
1712                    "client connection is closing before route publication"
1713                };
1714                release_reserved_route_locked(
1715                    &mut inner,
1716                    pending.reservation.client_key,
1717                    pending.reservation.module_key,
1718                );
1719                // Not gated on `relay_enqueued`: an ack proves the module
1720                // received the bind, and `route.open` sends nothing for a
1721                // relay a reply already settled, so this is the only GOODBYE.
1722                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1723                let _ = pending
1724                    .sender
1725                    .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
1726                return Ok(PendingRelayCompletion {
1727                    settled: true,
1728                    abandoned,
1729                });
1730            }
1731            // The endpoint that acked this bind is no longer accepting new
1732            // routes, for one of two reasons. Either it was the module's active
1733            // endpoint when this relay was reserved and a blue/green swap has
1734            // since promoted a candidate (the replacement process registered
1735            // beside it) into the active slot, superseding it; or a daemon
1736            // drain has gated it, and the ack arrived before the ordered module
1737            // drain settled pending relays. The module did nothing wrong: it
1738            // bound the route it was asked to bind, and the endpoint still
1739            // carries every other client's routes until it is drained. So the
1740            // ack is settled here, while the pending entry still holds the
1741            // client's sender and the reserved handle pair: release the pair,
1742            // refuse the waiting route.open with `module_reloading`, which
1743            // callers retry (after a swap the retry reserves on the promoted
1744            // endpoint), and hand back the module-side channel so the caller
1745            // sends one channel-scoped GOODBYE for the binding the module just
1746            // created. Returning an error instead would end the module
1747            // connection all those other routes share; nothing here touches it.
1748            //
1749            // An endpoint that stopped being active any other way is stale
1750            // (replaced, but not by a swap and not draining). It still falls
1751            // through to `commit_route_locked`, which refuses it with
1752            // `StaleModuleEndpoint` exactly as before swaps existed.
1753            RouteBindRelayOutcome::Accepted
1754                if inner.superseded_endpoints.contains_key(&endpoint)
1755                    || inner.draining_endpoints.contains_key(&endpoint) =>
1756            {
1757                release_reserved_route_locked(
1758                    &mut inner,
1759                    pending.reservation.client_key,
1760                    pending.reservation.module_key,
1761                );
1762                // Not gated on `relay_enqueued`: an ack proves the module
1763                // received the bind, whether or not the enqueue mark was set.
1764                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1765                let module_id = inner
1766                    .module_id_by_endpoint
1767                    .get(&endpoint)
1768                    .cloned()
1769                    .unwrap_or_else(|| "unknown".to_string());
1770                let _ = pending
1771                    .sender
1772                    .send(RouteBindRelayOutcome::Rejected(ErrorBody::new(
1773                        "module_reloading",
1774                        format!("module_id '{module_id}' is reloading"),
1775                    )));
1776                return Ok(PendingRelayCompletion {
1777                    settled: true,
1778                    abandoned,
1779                });
1780            }
1781            // The scope the open was admitted under ended or changed after
1782            // admission. The module bound a route under a stamp that is no
1783            // longer true, so it must not become routable. Settled here like
1784            // the superseded arm above, never as an error from
1785            // `commit_route_locked`, which would close the module's whole
1786            // connection and every other client's routes to it: release the
1787            // pair, answer the waiting route.open by name, and hand back the
1788            // module-side channel for one channel-scoped GOODBYE.
1789            RouteBindRelayOutcome::Accepted
1790                if pending
1791                    .scope
1792                    .as_ref()
1793                    .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
1794            {
1795                let refusal = pending
1796                    .scope
1797                    .as_ref()
1798                    .and_then(|scope| scope_refusal_locked(&inner, scope))
1799                    .expect("guard matched a refusal under the same lock");
1800                release_reserved_route_locked(
1801                    &mut inner,
1802                    pending.reservation.client_key,
1803                    pending.reservation.module_key,
1804                );
1805                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1806                let _ = pending
1807                    .sender
1808                    .send(RouteBindRelayOutcome::Rejected(refusal));
1809                return Ok(PendingRelayCompletion {
1810                    settled: true,
1811                    abandoned,
1812                });
1813            }
1814            RouteBindRelayOutcome::Accepted => {
1815                let abandoned = commit_route_locked(&mut inner, pending)?;
1816                return Ok(PendingRelayCompletion {
1817                    settled: true,
1818                    abandoned,
1819                });
1820            }
1821            terminal => {
1822                release_reserved_route_locked(
1823                    &mut inner,
1824                    pending.reservation.client_key,
1825                    pending.reservation.module_key,
1826                );
1827                let _ = pending.sender.send(terminal);
1828            }
1829        }
1830        Ok(PendingRelayCompletion {
1831            settled: true,
1832            abandoned: None,
1833        })
1834    }
1835
1836    pub(crate) fn pending_module_control_op(
1837        &self,
1838        connection_id: ConnectionId,
1839        corr: u64,
1840    ) -> Result<Option<String>, ForwardingError> {
1841        let inner = self.read_inner()?;
1842        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1843            return Ok(None);
1844        };
1845        let key = (endpoint, corr);
1846        Ok(inner
1847            .pending_control_rpcs
1848            .get(&key)
1849            .map(|pending| pending.expected_op.clone())
1850            .or_else(|| {
1851                inner
1852                    .health_probe_tombstones
1853                    .get(&key)
1854                    .filter(|tombstone| tombstone.expires_at > Instant::now())
1855                    .map(|tombstone| tombstone.expected_op.clone())
1856            }))
1857    }
1858
1859    pub(crate) fn complete_module_control_rpc(
1860        &self,
1861        connection_id: ConnectionId,
1862        corr: u64,
1863        actual_op: Option<&str>,
1864        outcome: ModuleControlRpcOutcome,
1865    ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
1866        let now = Instant::now();
1867        let mut inner = self.write_inner()?;
1868        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1869            return Ok(ModuleControlRpcCompletion::Unknown);
1870        };
1871        let key = (endpoint, corr);
1872        if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
1873            if now >= pending.deadline {
1874                let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
1875                    ModuleControlRpcCompletion::LateHealthAnswer {
1876                        module_id: inner
1877                            .module_id_by_endpoint
1878                            .get(&endpoint)
1879                            .cloned()
1880                            .unwrap_or_else(|| "unknown".to_string()),
1881                        latency: now.saturating_duration_since(probe_started_at),
1882                    }
1883                });
1884                let _ = pending
1885                    .sender
1886                    .send(ModuleControlRpcOutcome::DeadlineElapsed);
1887                return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
1888            }
1889            let outcome = match actual_op {
1890                Some(actual) if actual != pending.expected_op => {
1891                    ModuleControlRpcOutcome::UnexpectedOp {
1892                        expected: pending.expected_op,
1893                        actual: actual.to_string(),
1894                    }
1895                }
1896                _ => outcome,
1897            };
1898            let _ = pending.sender.send(outcome);
1899            return Ok(ModuleControlRpcCompletion::Settled);
1900        }
1901
1902        let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
1903            return Ok(ModuleControlRpcCompletion::Unknown);
1904        };
1905        if tombstone.expires_at <= now {
1906            return Ok(ModuleControlRpcCompletion::Unknown);
1907        }
1908        Ok(ModuleControlRpcCompletion::LateHealthAnswer {
1909            module_id: tombstone.module_id,
1910            latency: now.saturating_duration_since(tombstone.probe_started_at),
1911        })
1912    }
1913
1914    #[cfg(test)]
1915    pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
1916        Ok(self.read_inner()?.health_probe_tombstones.len())
1917    }
1918
1919    #[cfg(test)]
1920    pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
1921        Ok(self.read_inner()?.closing_connections.len())
1922    }
1923
1924    /// Reserved-but-uncommitted route handles, counted on both index sides, so
1925    /// a test can assert a reservation pair was actually given back.
1926    #[cfg(test)]
1927    pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
1928        let inner = self.read_inner()?;
1929        Ok((inner.reserved_client.len(), inner.reserved_module.len()))
1930    }
1931
1932    pub(crate) fn module_endpoint_for_connection(
1933        &self,
1934        connection_id: ConnectionId,
1935    ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
1936        Ok(self
1937            .read_inner()?
1938            .endpoint_by_connection
1939            .get(&connection_id)
1940            .copied())
1941    }
1942
1943    /// Looks up the module registered on a data-plane connection so route-drop
1944    /// diagnostics name the emitter instead of only reporting a daemon total.
1945    pub(crate) fn module_id_for_connection(
1946        &self,
1947        connection_id: ConnectionId,
1948    ) -> Result<Option<String>, ForwardingError> {
1949        let inner = self.read_inner()?;
1950        Ok(inner
1951            .endpoint_by_connection
1952            .get(&connection_id)
1953            .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
1954            .cloned())
1955    }
1956
1957    /// Whether the daemon ever allocated `(channel, epoch)` on this module
1958    /// connection. Epochs on a module channel are handed out as 1, 2, 3, ...
1959    /// and the last one handed out is remembered until the connection ends, so
1960    /// every epoch from 1 up to that one was a real route at some point. Used
1961    /// only on the drop path, to tell a module still sending on a route the
1962    /// daemon released from one sending on a route that never existed.
1963    pub(crate) fn module_route_epoch_was_allocated(
1964        &self,
1965        connection_id: ConnectionId,
1966        channel: u16,
1967        epoch: u32,
1968    ) -> Result<bool, ForwardingError> {
1969        let inner = self.read_inner()?;
1970        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1971            return Ok(false);
1972        };
1973        Ok(inner
1974            .module_slot_epochs
1975            .get(&ModuleRouteKey { endpoint, channel })
1976            .is_some_and(|last| epoch != 0 && epoch <= *last))
1977    }
1978
1979    pub(crate) fn has_live_module_connection(
1980        &self,
1981        module_id: &str,
1982    ) -> Result<bool, ForwardingError> {
1983        Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
1984    }
1985
1986    pub(crate) fn lookup_data_route(
1987        &self,
1988        connection_id: ConnectionId,
1989        channel: u16,
1990        epoch: u32,
1991    ) -> Result<DataRoute, ForwardingError> {
1992        let inner = self.read_inner()?;
1993        let state = if let Some(endpoint) =
1994            inner.endpoint_by_connection.get(&connection_id).copied()
1995        {
1996            let key = ModuleRouteKey { endpoint, channel };
1997            match inner.module_to_client.get(&key) {
1998                Some(route) if route.module_epoch == epoch => {
1999                    DataRouteState::Bound(Arc::clone(route))
2000                }
2001                Some(_) => DataRouteState::EpochMismatch,
2002                None if inner.reserved_module.contains_key(&key)
2003                    && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
2004                {
2005                    DataRouteState::Reserved
2006                }
2007                None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
2008                None => DataRouteState::Absent,
2009            }
2010        } else {
2011            let key = ClientRouteKey {
2012                connection_id,
2013                channel,
2014            };
2015            match inner.client_to_module.get(&key) {
2016                Some(route) if route.client_epoch == epoch => {
2017                    DataRouteState::Bound(Arc::clone(route))
2018                }
2019                Some(_) => DataRouteState::EpochMismatch,
2020                None if inner.reserved_client.contains_key(&key)
2021                    && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
2022                {
2023                    DataRouteState::Reserved
2024                }
2025                None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
2026                None => DataRouteState::Absent,
2027            }
2028        };
2029        Ok(
2030            if inner.endpoint_by_connection.contains_key(&connection_id) {
2031                DataRoute::Module(state)
2032            } else {
2033                DataRoute::Client(state)
2034            },
2035        )
2036    }
2037
2038    #[cfg(test)]
2039    pub(crate) fn inject_client_slot_epoch(
2040        &self,
2041        connection_id: ConnectionId,
2042        channel: u16,
2043        last_epoch: u32,
2044    ) {
2045        let mut inner = self.write_inner().expect("forwarding lock");
2046        inner.client_slot_epochs.insert(
2047            ClientRouteKey {
2048                connection_id,
2049                channel,
2050            },
2051            last_epoch,
2052        );
2053        inner.next_client_channel.insert(connection_id, channel);
2054    }
2055
2056    #[cfg(test)]
2057    pub(crate) fn inject_module_slot_epoch(
2058        &self,
2059        endpoint: ModuleEndpointId,
2060        channel: u16,
2061        last_epoch: u32,
2062    ) {
2063        let mut inner = self.write_inner().expect("forwarding lock");
2064        inner
2065            .module_slot_epochs
2066            .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
2067        inner.next_module_channel.insert(endpoint, channel);
2068    }
2069
2070    #[cfg(test)]
2071    pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
2072        self.write_inner()
2073            .expect("forwarding lock")
2074            .next_control_corr
2075            .insert(endpoint, next_corr);
2076    }
2077
2078    pub(crate) fn cache_status(
2079        &self,
2080        endpoint: ModuleEndpointId,
2081        module_channel: u16,
2082        module_epoch: u32,
2083        status: String,
2084    ) -> Result<bool, ForwardingError> {
2085        let mut inner = self.write_inner()?;
2086        if !inner.module_id_by_endpoint.contains_key(&endpoint) {
2087            return Err(ForwardingError::StaleModuleEndpoint);
2088        }
2089
2090        let module_key = ModuleRouteKey {
2091            endpoint,
2092            channel: module_channel,
2093        };
2094        let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
2095            (route.module_epoch == module_epoch).then_some((
2096                ClientRouteKey {
2097                    connection_id: route.client_connection_id,
2098                    channel: route.client_channel,
2099                },
2100                route.client_epoch,
2101            ))
2102        } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
2103            (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
2104                client_key,
2105                inner
2106                    .client_slot_epochs
2107                    .get(&client_key)
2108                    .copied()
2109                    .unwrap_or(0),
2110            ))
2111        } else {
2112            None
2113        };
2114
2115        if let Some(handle) = handle {
2116            inner.status.insert(handle, status);
2117            Ok(true)
2118        } else {
2119            debug!(
2120                module_channel,
2121                module_epoch,
2122                generation = endpoint.generation,
2123                connection_id = endpoint.connection_id.get(),
2124                "dropping stale status update for module route handle"
2125            );
2126            Ok(false)
2127        }
2128    }
2129
2130    pub(crate) fn route_poll_snapshot(
2131        &self,
2132        client_connection_id: ConnectionId,
2133        client_channel: u16,
2134        client_epoch: u32,
2135    ) -> Result<RoutePollSnapshot, ForwardingError> {
2136        let inner = self.read_inner()?;
2137        let client_key = ClientRouteKey {
2138            connection_id: client_connection_id,
2139            channel: client_channel,
2140        };
2141        let Some(route) = inner.client_to_module.get(&client_key) else {
2142            return Ok(RoutePollSnapshot::Absent);
2143        };
2144        if route.client_epoch != client_epoch
2145            || !inner
2146                .module_id_by_endpoint
2147                .contains_key(&route.module_endpoint)
2148        {
2149            return Ok(RoutePollSnapshot::Absent);
2150        }
2151        Ok(RoutePollSnapshot::Bound {
2152            module_id: route.module_id.clone(),
2153            status: inner.status.get(&(client_key, client_epoch)).cloned(),
2154        })
2155    }
2156
2157    pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2158        Ok(self.read_inner()?.client_to_module.len())
2159    }
2160
2161    /// How many distinct client connections hold at least one committed route,
2162    /// alongside the largest number of routes any single connection holds.
2163    ///
2164    /// `connected_clients` alone cannot distinguish many clients with a route
2165    /// each from one client accumulating hundreds, and those have opposite
2166    /// causes. Reading it required an out-of-band `lsof` during a live
2167    /// investigation, and the count of connections was mistaken for a count of
2168    /// client processes — which sent two of us after cleanup paths that were
2169    /// working correctly.
2170    pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2171        let inner = self.read_inner()?;
2172        let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2173        for key in inner.client_to_module.keys() {
2174            *per_connection.entry(key.connection_id).or_insert(0) += 1;
2175        }
2176        let max = per_connection.values().copied().max().unwrap_or(0);
2177        Ok((per_connection.len(), max))
2178    }
2179
2180    pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2181        let inner = self.read_inner()?;
2182        Ok(inner
2183            .client_to_module
2184            .keys()
2185            .any(|key| key.channel == route_channel))
2186    }
2187
2188    /// Whether daemon-wide shutdown has begun draining all providers.
2189    /// `None` means normal operation; during daemon shutdown, return the
2190    /// endpoint's measured result, or false if no completed measurement exists.
2191    pub(crate) fn daemon_drain_result_for_connection(
2192        &self,
2193        connection_id: ConnectionId,
2194    ) -> Result<Option<bool>, ForwardingError> {
2195        let inner = self.read_inner()?;
2196        Ok(inner.daemon_draining.then(|| {
2197            inner
2198                .endpoint_by_connection
2199                .get(&connection_id)
2200                .and_then(|endpoint| inner.daemon_drain_results.get(endpoint))
2201                .copied()
2202                .unwrap_or(false)
2203        }))
2204    }
2205
2206    #[cfg(unix)]
2207    pub(crate) fn record_daemon_drain_result(
2208        &self,
2209        endpoint: ModuleEndpointId,
2210        drained: bool,
2211    ) -> Result<(), ForwardingError> {
2212        let mut inner = self.write_inner()?;
2213        // Only update an endpoint captured at the start of daemon shutdown.
2214        // Teardown removes its entry, so a late wait cannot recreate the result
2215        // or attribute it to a replacement connection.
2216        if let Some(result) = inner.daemon_drain_results.get_mut(&endpoint) {
2217            *result = drained;
2218        }
2219        Ok(())
2220    }
2221
2222    /// Gate every provider atomically, including registrations racing shutdown.
2223    /// No supervisor lock is held while taking the forwarding lock.
2224    #[cfg(unix)]
2225    pub(crate) fn begin_daemon_drain(
2226        &self,
2227    ) -> Result<Vec<(String, ModuleEndpointId)>, ForwardingError> {
2228        let mut inner = self.write_inner()?;
2229        if inner.daemon_draining {
2230            return Ok(Vec::new());
2231        }
2232        inner.daemon_draining = true;
2233        let modules = inner
2234            .modules_by_id
2235            .iter()
2236            .map(|(id, module)| (id.clone(), module.endpoint))
2237            .collect::<Vec<_>>();
2238        for (_, endpoint) in &modules {
2239            inner.daemon_drain_results.insert(*endpoint, false);
2240            inner
2241                .draining_endpoints
2242                .insert(*endpoint, RouteCloseReason::Restart);
2243        }
2244        // Swap candidates and superseded incumbents are not routable, but they
2245        // are live endpoints that could still be sent module-control work, so
2246        // the daemon-wide gate covers them too. Only the active endpoints are
2247        // returned to the supervisor for measurement, one per module id.
2248        let off_slot_endpoints = inner
2249            .candidates_by_id
2250            .values()
2251            .map(|module| module.endpoint)
2252            .chain(inner.superseded_endpoints.keys().copied())
2253            .collect::<Vec<_>>();
2254        for endpoint in off_slot_endpoints {
2255            inner
2256                .draining_endpoints
2257                .insert(endpoint, RouteCloseReason::Restart);
2258        }
2259        Ok(modules)
2260    }
2261
2262    /// Begin draining whatever endpoint is ACTIVE for `module_id`.
2263    ///
2264    /// After a swap's cutover the active endpoint is the promoted candidate, so
2265    /// this must not be used to drain the incumbent; use
2266    /// [`Self::begin_endpoint_drain`] with the endpoint `cutover_candidate`
2267    /// returned.
2268    pub(crate) fn begin_module_drain(
2269        &self,
2270        module_id: &str,
2271        reason: RouteCloseReason,
2272    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2273        let mut inner = self.write_inner()?;
2274        let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2275            return Ok(None);
2276        };
2277        Ok(Some(begin_drain_locked(
2278            &mut inner, module_id, module, reason,
2279        )))
2280    }
2281
2282    /// Begin draining one specific endpoint, whichever slot it is in.
2283    ///
2284    /// This is how a swap drains its incumbent after cutover: the incumbent's
2285    /// endpoint is captured by `cutover_candidate`, and resolving it by module id
2286    /// instead would find the promoted candidate and leave neither process
2287    /// routable. Returns `Ok(None)` when the endpoint is no longer registered.
2288    pub(crate) fn begin_endpoint_drain(
2289        &self,
2290        endpoint: ModuleEndpointId,
2291        reason: RouteCloseReason,
2292    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2293        let mut inner = self.write_inner()?;
2294        let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2295            return Ok(None);
2296        };
2297        let module_id = inner
2298            .module_id_by_endpoint
2299            .get(&endpoint)
2300            .cloned()
2301            .expect("an endpoint resolved to a module connection has a module id");
2302        Ok(Some(begin_drain_locked(
2303            &mut inner, &module_id, module, reason,
2304        )))
2305    }
2306}
2307
2308/// Mark `module`'s endpoint draining and settle everything still pending on it.
2309/// Shared by the by-id and by-endpoint drain entry points, which differ only in
2310/// how they find the endpoint.
2311fn begin_drain_locked(
2312    inner: &mut ForwardingInner,
2313    module_id: &str,
2314    module: ModuleConnection,
2315    reason: RouteCloseReason,
2316) -> ModuleDrainTarget {
2317    {
2318        let endpoint = module.endpoint;
2319        inner.draining_endpoints.insert(endpoint, reason);
2320
2321        let flows = inner
2322            .client_to_module
2323            .values()
2324            .filter(|route| route.module_endpoint == endpoint)
2325            .map(|route| Arc::clone(&route.flow))
2326            .collect::<Vec<_>>();
2327        let excluded_subscriptions = flows
2328            .into_iter()
2329            .map(|flow| flow.begin_drain())
2330            .fold(0u32, u32::saturating_add);
2331
2332        let pending_keys = inner
2333            .pending_relays
2334            .keys()
2335            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2336            .copied()
2337            .collect::<Vec<_>>();
2338        let mut abandoned_bindings = Vec::new();
2339        for key in pending_keys {
2340            let Some(pending) = inner.pending_relays.remove(&key) else {
2341                continue;
2342            };
2343            release_reserved_route_locked(
2344                inner,
2345                pending.reservation.client_key,
2346                pending.reservation.module_key,
2347            );
2348            if pending.relay_enqueued {
2349                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2350                    abandoned_bindings.push(target);
2351                }
2352            } else {
2353                // The bind may still be on its way into the module's queue;
2354                // a GOODBYE sent now could overtake it. `route.open` sends
2355                // it once its send completes.
2356                inner.relays_released_before_enqueue.insert(key);
2357            }
2358            let _ = pending
2359                .sender
2360                .send(RouteBindRelayOutcome::Rejected(ErrorBody::new(
2361                    "module_reloading",
2362                    format!("module_id '{module_id}' is reloading"),
2363                )));
2364        }
2365
2366        let pending_control_keys = inner
2367            .pending_control_rpcs
2368            .keys()
2369            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2370            .copied()
2371            .collect::<Vec<_>>();
2372        for key in pending_control_keys {
2373            if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2374                let _ = pending
2375                    .sender
2376                    .send(ModuleControlRpcOutcome::ModuleGone(format!(
2377                        "module '{module_id}' began draining during module-control RPC"
2378                    )));
2379            }
2380        }
2381
2382        ModuleDrainTarget {
2383            endpoint,
2384            sink: module.sink,
2385            negotiated_ver: module.negotiated_ver,
2386            abandoned_bindings,
2387            excluded_subscriptions,
2388        }
2389    }
2390}
2391
2392/// What a drain that timed out was still waiting on, for the log line that
2393/// reports the timeout. Per-request ages are not tracked.
2394#[derive(Debug, Default, PartialEq, Eq)]
2395pub(crate) struct DrainHoldouts {
2396    /// Requests still counted against the drain (subscriptions flagged as
2397    /// such are excluded from the drain and not counted here).
2398    pub(crate) requests: usize,
2399    /// Routes holding at least one of those requests.
2400    pub(crate) routes: usize,
2401    /// Every route on the endpoint, for scale.
2402    pub(crate) total_routes: usize,
2403    /// The client connections holding the most requests, largest first, at
2404    /// most three: enough to name the consumer without listing every route.
2405    pub(crate) top_connections: Vec<(u64, usize)>,
2406    /// The held requests themselves, as the module saw them, so its own log
2407    /// can say what each one was: `(module channel, corr)`, ordered by channel
2408    /// then corr, at most [`DRAIN_HELD_REQUESTS_LISTED`]. The daemon never reads
2409    /// request bodies, so it cannot name a request's method; the module can,
2410    /// from the channel and corr. A request is released only when the module
2411    /// sends a terminal frame (Response, Error or StreamEnd) with its corr on
2412    /// its route, so every pair listed here is one the module ended, if at all,
2413    /// without sending that frame.
2414    pub(crate) held: Vec<(u16, u64)>,
2415}
2416
2417/// How many held requests the drain-timeout line lists by channel and corr.
2418pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
2419
2420impl ForwardingTable {
2421    /// Summarise the requests one endpoint's drain is still waiting on.
2422    pub(crate) fn endpoint_drain_holdouts(
2423        &self,
2424        endpoint: ModuleEndpointId,
2425    ) -> Result<DrainHoldouts, ForwardingError> {
2426        let inner = self.read_inner()?;
2427        let mut holdouts = DrainHoldouts::default();
2428        let mut by_connection: HashMap<u64, usize> = HashMap::new();
2429        for (key, route) in &inner.client_to_module {
2430            if route.module_endpoint != endpoint {
2431                continue;
2432            }
2433            holdouts.total_routes += 1;
2434            let held = route.flow.drain_in_flight();
2435            if held == 0 {
2436                continue;
2437            }
2438            holdouts.requests += held;
2439            holdouts.routes += 1;
2440            *by_connection.entry(key.connection_id.get()).or_default() += held;
2441            holdouts.held.extend(
2442                route
2443                    .flow
2444                    .drain_held_corrs()
2445                    .into_iter()
2446                    .map(|corr| (route.module_channel, corr)),
2447            );
2448        }
2449        holdouts.held.sort_unstable();
2450        holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
2451        let mut connections = by_connection.into_iter().collect::<Vec<_>>();
2452        connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
2453        connections.truncate(3);
2454        holdouts.top_connections = connections;
2455        Ok(holdouts)
2456    }
2457
2458    pub(crate) fn endpoint_in_flight_count(
2459        &self,
2460        endpoint: ModuleEndpointId,
2461    ) -> Result<usize, ForwardingError> {
2462        let inner = self.read_inner()?;
2463        Ok(inner
2464            .client_to_module
2465            .values()
2466            .filter(|route| route.module_endpoint == endpoint)
2467            .map(|route| route.flow.drain_in_flight())
2468            .sum())
2469    }
2470
2471    pub(crate) fn endpoint_is_draining(
2472        &self,
2473        endpoint: ModuleEndpointId,
2474    ) -> Result<bool, ForwardingError> {
2475        Ok(self
2476            .read_inner()?
2477            .draining_endpoints
2478            .contains_key(&endpoint))
2479    }
2480
2481    pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
2482        let inner = self.read_inner()?;
2483        Ok(inner
2484            .modules_by_id
2485            .get(module_id)
2486            .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
2487    }
2488
2489    pub(crate) fn release_module_endpoint_routes(
2490        &self,
2491        endpoint: ModuleEndpointId,
2492    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2493        let mut inner = self.write_inner()?;
2494        let routes = inner
2495            .module_to_client
2496            .iter()
2497            .filter(|(module_key, _)| module_key.endpoint == endpoint)
2498            .map(|(module_key, route)| (*module_key, route.module_epoch))
2499            .collect::<Vec<_>>();
2500        let mut released = Vec::with_capacity(routes.len());
2501        for (module_key, epoch) in routes {
2502            if let RouteRelease::Removed(target) =
2503                release_module_route_locked(&mut inner, module_key, epoch)
2504            {
2505                released.push(target);
2506            }
2507        }
2508        Ok(released)
2509    }
2510
2511    /// Enumerate one endpoint's current routes without contacting the module.
2512    ///
2513    /// The read lock makes this safe while the endpoint drains: the returned
2514    /// `draining` marker describes the same table state that owns the route,
2515    /// rather than inferring liveness from a module that may be stopping.
2516    pub(crate) fn endpoint_routes(
2517        &self,
2518        endpoint: ModuleEndpointId,
2519    ) -> Result<Vec<EndpointRoute>, ForwardingError> {
2520        let inner = self.read_inner()?;
2521        Ok(endpoint_routes_locked(&inner, endpoint))
2522    }
2523
2524    /// Snapshot all live endpoint route sets under one forwarding-table read lock.
2525    pub(crate) fn route_census(
2526        &self,
2527        module_id: Option<&str>,
2528    ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
2529        let inner = self.read_inner()?;
2530        let mut endpoints = inner
2531            .modules_by_id
2532            .iter()
2533            .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
2534            .map(|(id, module)| (id.clone(), module.endpoint))
2535            .collect::<Vec<_>>();
2536        endpoints.sort_by(|left, right| left.0.cmp(&right.0));
2537        Ok(endpoints
2538            .into_iter()
2539            .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
2540            .collect())
2541    }
2542
2543    /// Snapshot only the endpoint currently routable under this module id.
2544    pub(crate) fn live_roots(
2545        &self,
2546        module_id: &str,
2547    ) -> Result<ModuleControlResponseToModule, ForwardingError> {
2548        let inner = self.read_inner()?;
2549        let endpoint = inner
2550            .modules_by_id
2551            .get(module_id)
2552            .map(|module| module.endpoint);
2553        let mut roots = BTreeMap::new();
2554        let mut unknown_root_bindings = 0;
2555        let mut total_bindings = 0;
2556        if let Some(endpoint) = endpoint {
2557            for binding in inner
2558                .module_to_client
2559                .values()
2560                .filter(|binding| binding.module_endpoint == endpoint)
2561            {
2562                total_bindings += 1;
2563                if let Some(root) = &binding.project_root {
2564                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2565                    entry.0 += 1;
2566                } else {
2567                    unknown_root_bindings += 1;
2568                }
2569            }
2570            for pending in inner
2571                .pending_relays
2572                .values()
2573                .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
2574            {
2575                total_bindings += 1;
2576                if let Some(root) = &pending.reservation.project_root {
2577                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2578                    entry.1 += 1;
2579                } else {
2580                    unknown_root_bindings += 1;
2581                }
2582            }
2583        }
2584        Ok(ModuleControlResponseToModule::LiveRoots {
2585            roots: roots
2586                .into_iter()
2587                .map(|(project_root, (bound, pending))| LiveRoot {
2588                    project_root,
2589                    bound,
2590                    pending,
2591                })
2592                .collect(),
2593            unknown_root_bindings,
2594            total_bindings,
2595        })
2596    }
2597
2598    /// True if this connection already owns committed or reserved CLIENT routes.
2599    /// A module registers (HELLO) before serving and never opens client routes, so
2600    /// a connection that has client routes must not also become a module endpoint
2601    /// — otherwise one connection holds both client and module state and cleanup
2602    /// only releases one side.
2603    pub(crate) fn connection_has_client_routes(
2604        &self,
2605        connection_id: ConnectionId,
2606    ) -> Result<bool, ForwardingError> {
2607        let inner = self.read_inner()?;
2608        Ok(connection_has_client_routes_locked(&inner, connection_id))
2609    }
2610
2611    pub(crate) fn cleanup_connection(
2612        &self,
2613        connection_id: ConnectionId,
2614    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2615        self.cleanup_connection_counted(connection_id)
2616            .map(|cleanup| cleanup.released)
2617    }
2618
2619    /// [`Self::cleanup_connection`], also reporting how many pending
2620    /// route.bind relays to the closed module were aborted. That count is the
2621    /// `abandoned` figure of the `route.closed` push sent for a lost module
2622    /// connection; it is always zero for a client connection.
2623    pub(crate) fn cleanup_connection_counted(
2624        &self,
2625        connection_id: ConnectionId,
2626    ) -> Result<ConnectionCleanup, ForwardingError> {
2627        let mut inner = self.write_inner()?;
2628        inner.closing_connections.insert(connection_id);
2629        let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
2630            remove_module_connection_locked(&mut inner, endpoint)
2631        } else {
2632            ConnectionCleanup {
2633                released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
2634                abandoned_relays: 0,
2635            }
2636        };
2637        // The closing mark refuses new work for a connection whose teardown is
2638        // still pending. This is the latest point at which lifting it is safe:
2639        // teardown has just removed every per-connection entry above, under
2640        // this same write lock, so no lookup can still find live state for the
2641        // id; and ids come from a monotonic counter that never reissues one,
2642        // so the id can never name a future connection either. Keeping the
2643        // mark past this point only grew the set by one entry per connection
2644        // for the life of the daemon.
2645        inner.closing_connections.remove(&connection_id);
2646        Ok(cleanup)
2647    }
2648
2649    fn cleanup_client_connection_locked(
2650        inner: &mut ForwardingInner,
2651        connection_id: ConnectionId,
2652    ) -> Vec<GoodbyeTarget> {
2653        let routes = inner
2654            .client_to_module
2655            .iter()
2656            .filter(|(key, _)| key.connection_id == connection_id)
2657            .map(|(key, route)| (*key, route.client_epoch))
2658            .collect::<Vec<_>>();
2659        let mut released = Vec::with_capacity(routes.len());
2660        for (client_key, epoch) in routes {
2661            if let RouteRelease::Removed(target) =
2662                release_client_route_locked(inner, client_key, epoch)
2663            {
2664                released.push(target);
2665            }
2666        }
2667
2668        let pending_keys = inner
2669            .pending_relays
2670            .iter()
2671            .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
2672            .map(|(key, _)| *key)
2673            .collect::<Vec<_>>();
2674        for key in pending_keys {
2675            let Some(pending) = inner.pending_relays.remove(&key) else {
2676                continue;
2677            };
2678            release_reserved_route_locked(
2679                inner,
2680                pending.reservation.client_key,
2681                pending.reservation.module_key,
2682            );
2683            if pending.relay_enqueued {
2684                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2685                    released.push(target);
2686                }
2687            } else {
2688                // The bind may still be on its way into the module's queue;
2689                // a GOODBYE sent now could overtake it. `route.open` sends
2690                // it once its send completes.
2691                inner.relays_released_before_enqueue.insert(key);
2692            }
2693            let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
2694                "client connection closed during route.bind relay".to_string(),
2695            ));
2696        }
2697
2698        let orphaned = inner
2699            .reserved_client
2700            .iter()
2701            .filter(|(key, _)| key.connection_id == connection_id)
2702            .map(|(client, module)| (*client, *module))
2703            .collect::<Vec<_>>();
2704        for (client_key, module_key) in orphaned {
2705            release_reserved_route_locked(inner, client_key, module_key);
2706        }
2707        inner.next_client_channel.remove(&connection_id);
2708        inner
2709            .client_slot_epochs
2710            .retain(|key, _| key.connection_id != connection_id);
2711        inner
2712            .last_published_epoch
2713            .retain(|key, _| key.connection_id != connection_id);
2714        inner
2715            .status
2716            .retain(|(key, _), _| key.connection_id != connection_id);
2717
2718        released
2719    }
2720
2721    /// Close a client connection whose egress queue refused a frame for the
2722    /// route `(connection_id, channel)` at `expected_epoch`. The whole
2723    /// connection closes because its queue is shared by every route on it.
2724    ///
2725    /// The first request that actually closes the connection logs one WARN with
2726    /// the diagnosis: which principals the connection's routes belonged to,
2727    /// which module's frame did not fit and on which client channel, and what
2728    /// the queue held at that moment.
2729    pub(crate) fn escalate_client_delivery_failure(
2730        &self,
2731        connection_id: ConnectionId,
2732        channel: u16,
2733        expected_epoch: u32,
2734        reason: CloseReason,
2735        undelivered: UndeliveredFrame<'_>,
2736    ) -> Result<bool, ForwardingError> {
2737        let principals = {
2738            let mut inner = self.write_inner()?;
2739            let key = ClientRouteKey {
2740                connection_id,
2741                channel,
2742            };
2743            if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
2744                None
2745            } else {
2746                inner.closing_connections.insert(connection_id);
2747                Some(connection_principals_locked(&inner, connection_id))
2748            }
2749        };
2750        let Some(principals) = principals else {
2751            return Ok(false);
2752        };
2753        let backlog = undelivered.sink.backlog();
2754        let close_reason = reason.to_string();
2755        if self.request_connection_close(connection_id, reason) {
2756            warn!(
2757                connection_id = connection_id.get(),
2758                principals = %principals,
2759                module_id = undelivered.module_id.unwrap_or("unknown"),
2760                client_channel = channel,
2761                queued_bytes = backlog.queued_bytes,
2762                queued_frames = backlog.queued_frames,
2763                oldest_queued_ms = backlog
2764                    .oldest_age
2765                    .map(|age| age.as_millis() as u64)
2766                    .unwrap_or(0),
2767                close_reason = %close_reason,
2768                "closing client connection: its egress queue could not take a frame"
2769            );
2770        }
2771        Ok(true)
2772    }
2773
2774    /// Publish the tags a scope sync changed and close the live routes its
2775    /// drain table selects, in one critical section.
2776    ///
2777    /// The caller holds the scope table's write lock (scope table, then this
2778    /// table, always). Publishing and selecting under one lock is what closes
2779    /// the race with a bind commit: a commit before this sees the old tag and
2780    /// its route is selected here like any other; a commit after it sees the
2781    /// new tag and is refused.
2782    ///
2783    /// Every client route index entry is scanned, which includes routes on a
2784    /// swap's superseded endpoint (the old process of a module being replaced
2785    /// blue/green, which keeps its existing routes until they drain): those
2786    /// stay indexed until drained, so ending a scope reaches them too.
2787    pub(crate) fn publish_scope_changes(
2788        &self,
2789        changes: &[ScopeTagChange],
2790    ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
2791        let mut inner = self.write_inner()?;
2792        let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
2793        for change in changes {
2794            let key = (change.owner.clone(), change.scope_ref.clone());
2795            match change.after {
2796                Some(tag) => {
2797                    inner.scope_tags.insert(key, tag);
2798                }
2799                None => {
2800                    inner.scope_tags.remove(&key);
2801                }
2802            }
2803            by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
2804        }
2805        let mut selected = Vec::new();
2806        for (client_key, route) in &inner.client_to_module {
2807            let Some(scope) = &route.scope else {
2808                continue;
2809            };
2810            let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
2811            else {
2812                continue;
2813            };
2814            if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
2815                continue;
2816            }
2817            let reason = match &change.drain {
2818                ScopeDrain::Nothing => continue,
2819                ScopeDrain::All(reason) => *reason,
2820                ScopeDrain::Carriers(narrowed) => {
2821                    let owner = Principal::Reserved {
2822                        module_id: scope.owner.clone(),
2823                    };
2824                    let hit = route.principal != owner
2825                        && narrowed.iter().any(|(principal, allowed)| {
2826                            *principal == route.principal
2827                                && allowed
2828                                    .as_ref()
2829                                    .is_none_or(|targets| !targets.contains(&route.module_id))
2830                        });
2831                    if !hit {
2832                        continue;
2833                    }
2834                    RouteCloseReason::ScopeCarrierRemoved
2835                }
2836            };
2837            selected.push((*client_key, route.client_epoch, reason));
2838        }
2839        let mut drained = Vec::new();
2840        for (client_key, client_epoch, reason) in selected {
2841            let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
2842                continue;
2843            };
2844            let release = release_client_route_locked(&mut inner, client_key, client_epoch);
2845            self.record_route_release(&release);
2846            if let RouteRelease::Removed(module) = release {
2847                drained.push(ScopeDrainedRoute {
2848                    reason,
2849                    scope: route
2850                        .scope
2851                        .clone()
2852                        .expect("selected scope drain has a bound scope"),
2853                    module_id: route.module_id.clone(),
2854                    client: GoodbyeTarget {
2855                        connection_id: route.client_connection_id,
2856                        sink: route.client_sink.clone(),
2857                        negotiated_ver: route.client_negotiated_ver,
2858                        channel: route.client_channel,
2859                        epoch: route.client_epoch,
2860                        kind: GoodbyeTargetKind::Client,
2861                        module_id: Some(route.module_id.clone()),
2862                    },
2863                    module,
2864                });
2865            }
2866        }
2867        Ok(drained)
2868    }
2869
2870    /// The published tag of one scope, for tests of the commit re-check.
2871    #[cfg(test)]
2872    pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
2873        self.read_inner()
2874            .ok()?
2875            .scope_tags
2876            .get(&(owner.to_string(), scope_ref.to_string()))
2877            .copied()
2878    }
2879
2880    fn record_route_release(&self, release: &RouteRelease) {
2881        match release {
2882            RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
2883            RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
2884            RouteRelease::Absent => {}
2885        }
2886    }
2887
2888    fn read_inner(&self) -> Result<RwLockReadGuard<'_, ForwardingInner>, ForwardingError> {
2889        self.inner.read().map_err(|_| ForwardingError::Poisoned)
2890    }
2891
2892    fn write_inner(&self) -> Result<RwLockWriteGuard<'_, ForwardingInner>, ForwardingError> {
2893        self.inner.write().map_err(|_| ForwardingError::Poisoned)
2894    }
2895
2896    fn lock_close_registry(
2897        &self,
2898    ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
2899        self.close_registry
2900            .lock()
2901            .unwrap_or_else(|poisoned| poisoned.into_inner())
2902    }
2903}
2904
2905impl ForwardingInner {
2906    fn allocate_route_slots(
2907        &mut self,
2908        connection_id: ConnectionId,
2909        endpoint: ModuleEndpointId,
2910    ) -> Result<(u16, u32, u16, u32), ForwardingError> {
2911        let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
2912        let mut client_channel = client_start;
2913        let client_channel = loop {
2914            let key = ClientRouteKey {
2915                connection_id,
2916                channel: client_channel,
2917            };
2918            let eligible = !self.client_to_module.contains_key(&key)
2919                && !self.reserved_client.contains_key(&key)
2920                && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2921            if eligible {
2922                break client_channel;
2923            }
2924            client_channel = next_channel(client_channel);
2925            if client_channel == client_start {
2926                return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
2927            }
2928        };
2929
2930        let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
2931        let mut module_channel = module_start;
2932        let module_channel = loop {
2933            let key = ModuleRouteKey {
2934                endpoint,
2935                channel: module_channel,
2936            };
2937            let eligible = !self.module_to_client.contains_key(&key)
2938                && !self.reserved_module.contains_key(&key)
2939                && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2940            if eligible {
2941                break module_channel;
2942            }
2943            module_channel = next_channel(module_channel);
2944            if module_channel == module_start {
2945                return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
2946            }
2947        };
2948
2949        let client_key = ClientRouteKey {
2950            connection_id,
2951            channel: client_channel,
2952        };
2953        let module_key = ModuleRouteKey {
2954            endpoint,
2955            channel: module_channel,
2956        };
2957        let client_epoch = self
2958            .client_slot_epochs
2959            .get(&client_key)
2960            .copied()
2961            .unwrap_or(0)
2962            + 1;
2963        let module_epoch = self
2964            .module_slot_epochs
2965            .get(&module_key)
2966            .copied()
2967            .unwrap_or(0)
2968            + 1;
2969        self.client_slot_epochs.insert(client_key, client_epoch);
2970        self.module_slot_epochs.insert(module_key, module_epoch);
2971        self.next_client_channel
2972            .insert(connection_id, next_channel(client_channel));
2973        self.next_module_channel
2974            .insert(endpoint, next_channel(module_channel));
2975        Ok((client_channel, client_epoch, module_channel, module_epoch))
2976    }
2977
2978    fn allocate_control_corr(
2979        &mut self,
2980        endpoint: ModuleEndpointId,
2981    ) -> Result<u64, ForwardingError> {
2982        let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
2983        if candidate == 0 {
2984            self.closing_connections.insert(endpoint.connection_id);
2985            return Err(ForwardingError::RelayCorrelationExhausted);
2986        }
2987        self.next_control_corr.insert(
2988            endpoint,
2989            if candidate == u64::MAX {
2990                0
2991            } else {
2992                candidate + 1
2993            },
2994        );
2995        Ok(candidate)
2996    }
2997}
2998
2999fn connection_has_client_routes_locked(
3000    inner: &ForwardingInner,
3001    connection_id: ConnectionId,
3002) -> bool {
3003    inner
3004        .client_to_module
3005        .keys()
3006        .any(|key| key.connection_id == connection_id)
3007        || inner
3008            .reserved_client
3009            .keys()
3010            .any(|key| key.connection_id == connection_id)
3011}
3012
3013fn check_module_connection_role_locked(
3014    inner: &ForwardingInner,
3015    connection_id: ConnectionId,
3016) -> Result<(), ForwardingError> {
3017    if inner.endpoint_by_connection.contains_key(&connection_id)
3018        || connection_has_client_routes_locked(inner, connection_id)
3019    {
3020        return Err(ForwardingError::ConnectionRoleConflict { connection_id });
3021    }
3022    Ok(())
3023}
3024
3025fn next_channel(channel: u16) -> u16 {
3026    let next = channel.wrapping_add(1);
3027    if next == 0 {
3028        1
3029    } else {
3030        next
3031    }
3032}
3033
3034fn endpoint_routes_locked(
3035    inner: &ForwardingInner,
3036    endpoint: ModuleEndpointId,
3037) -> Vec<EndpointRoute> {
3038    let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
3039    let draining = drain_reason.is_some();
3040    let mut routes = inner
3041        .module_to_client
3042        .iter()
3043        .filter(|(module_key, _)| module_key.endpoint == endpoint)
3044        .map(|(_, route)| EndpointRoute {
3045            goodbye_target: GoodbyeTarget {
3046                connection_id: route.client_connection_id,
3047                sink: route.client_sink.clone(),
3048                negotiated_ver: route.client_negotiated_ver,
3049                channel: route.client_channel,
3050                epoch: route.client_epoch,
3051                kind: GoodbyeTargetKind::Client,
3052                module_id: Some(route.module_id.clone()),
3053            },
3054            principal: route.principal.clone(),
3055            bound_at: route.bound_at,
3056            draining,
3057            drain_reason,
3058        })
3059        .collect::<Vec<_>>();
3060    routes.sort_by_key(|route| {
3061        (
3062            route.goodbye_target.connection_id.get(),
3063            route.goodbye_target.channel,
3064            route.goodbye_target.epoch,
3065        )
3066    });
3067    routes
3068}
3069
3070/// Why a bind admitted under `scope` may no longer commit, read from the
3071/// published tags: `scope_ended` when the scope is gone or at another epoch,
3072/// `scope_changed` (retryable) when only its content moved.
3073fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
3074    let key = (scope.owner.clone(), scope.scope_ref.clone());
3075    match inner.scope_tags.get(&key) {
3076        Some(current) if *current == scope.tag => None,
3077        Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
3078            subc_protocol::error_codes::SCOPE_CHANGED,
3079            format!(
3080                "scope '{}' of {} changed while the route was being bound; re-open it",
3081                scope.scope_ref, scope.owner
3082            ),
3083        )),
3084        _ => Some(ErrorBody::new(
3085            subc_protocol::error_codes::SCOPE_ENDED,
3086            format!(
3087                "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
3088                scope.scope_ref, scope.owner, scope.tag.scope_epoch
3089            ),
3090        )),
3091    }
3092}
3093
3094fn release_reserved_route_locked(
3095    inner: &mut ForwardingInner,
3096    client_key: ClientRouteKey,
3097    module_key: ModuleRouteKey,
3098) {
3099    if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
3100        inner.reserved_client.remove(&client_key);
3101    }
3102    if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
3103        inner.reserved_module.remove(&module_key);
3104    }
3105    inner.status.retain(|(key, _), _| *key != client_key);
3106}
3107
3108fn release_client_route_locked(
3109    inner: &mut ForwardingInner,
3110    client_key: ClientRouteKey,
3111    expected_epoch: u32,
3112) -> RouteRelease {
3113    let Some(route) = inner.client_to_module.get(&client_key) else {
3114        return RouteRelease::Absent;
3115    };
3116    if route.client_epoch != expected_epoch {
3117        return RouteRelease::Stale;
3118    }
3119    let route = inner
3120        .client_to_module
3121        .remove(&client_key)
3122        .expect("route checked under the same forwarding lock");
3123    route.flow.close();
3124    if let Some(scope) = &route.scope {
3125        scope.release_bind();
3126    }
3127    inner.module_to_client.remove(&ModuleRouteKey {
3128        endpoint: route.module_endpoint,
3129        channel: route.module_channel,
3130    });
3131    inner.operator_confirms.route_closed(ModuleRouteKey {
3132        endpoint: route.module_endpoint,
3133        channel: route.module_channel,
3134    });
3135    inner.status.remove(&(client_key, expected_epoch));
3136    RouteRelease::Removed(GoodbyeTarget {
3137        connection_id: route.module_endpoint.connection_id,
3138        sink: route.module_sink.clone(),
3139        negotiated_ver: route.module_negotiated_ver,
3140        channel: route.module_channel,
3141        epoch: route.module_epoch,
3142        kind: GoodbyeTargetKind::Module,
3143        module_id: Some(route.module_id.clone()),
3144    })
3145}
3146
3147fn release_module_route_locked(
3148    inner: &mut ForwardingInner,
3149    module_key: ModuleRouteKey,
3150    expected_epoch: u32,
3151) -> RouteRelease {
3152    let Some(route) = inner.module_to_client.get(&module_key) else {
3153        return RouteRelease::Absent;
3154    };
3155    if route.module_epoch != expected_epoch {
3156        return RouteRelease::Stale;
3157    }
3158    let route = inner
3159        .module_to_client
3160        .remove(&module_key)
3161        .expect("route checked under the same forwarding lock");
3162    inner.operator_confirms.route_closed(module_key);
3163    route.flow.close();
3164    if let Some(scope) = &route.scope {
3165        scope.release_bind();
3166    }
3167    let client_key = ClientRouteKey {
3168        connection_id: route.client_connection_id,
3169        channel: route.client_channel,
3170    };
3171    inner.client_to_module.remove(&client_key);
3172    inner.status.remove(&(client_key, route.client_epoch));
3173    RouteRelease::Removed(GoodbyeTarget {
3174        connection_id: route.client_connection_id,
3175        sink: route.client_sink.clone(),
3176        negotiated_ver: route.client_negotiated_ver,
3177        channel: route.client_channel,
3178        epoch: route.client_epoch,
3179        kind: GoodbyeTargetKind::Client,
3180        module_id: Some(route.module_id.clone()),
3181    })
3182}
3183
3184fn commit_route_locked(
3185    inner: &mut ForwardingInner,
3186    pending: PendingRouteBindRelayEntry,
3187) -> Result<Option<GoodbyeTarget>, ForwardingError> {
3188    let reservation = pending.reservation;
3189    if inner
3190        .closing_connections
3191        .contains(&reservation.client_key.connection_id)
3192    {
3193        return Err(ForwardingError::ConnectionClosing {
3194            connection_id: reservation.client_key.connection_id,
3195        });
3196    }
3197    let module_id = inner
3198        .module_id_by_endpoint
3199        .get(&reservation.module_key.endpoint)
3200        .cloned()
3201        .ok_or(ForwardingError::StaleModuleEndpoint)?;
3202    if inner
3203        .draining_endpoints
3204        .contains_key(&reservation.module_key.endpoint)
3205    {
3206        return Err(ForwardingError::ModuleReloading { module_id });
3207    }
3208    if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
3209        || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
3210    {
3211        return Err(ForwardingError::UnknownReservation {
3212            client_channel: reservation.client_key.channel,
3213            module_channel: reservation.module_key.channel,
3214        });
3215    }
3216    let module = inner
3217        .modules_by_id
3218        .get(&module_id)
3219        .filter(|module| module.endpoint == reservation.module_key.endpoint)
3220        .cloned()
3221        .ok_or(ForwardingError::StaleModuleEndpoint)?;
3222    let binding = Arc::new(RouteBinding {
3223        client_connection_id: reservation.client_key.connection_id,
3224        client_sink: pending.client_sink,
3225        client_negotiated_ver: pending.client_negotiated_ver,
3226        client_channel: reservation.client_key.channel,
3227        client_epoch: reservation.client_epoch,
3228        module_id,
3229        module_endpoint: reservation.module_key.endpoint,
3230        module_sink: module.sink,
3231        module_negotiated_ver: module.negotiated_ver,
3232        module_channel: reservation.module_key.channel,
3233        module_epoch: reservation.module_epoch,
3234        principal: pending.principal,
3235        project_root: reservation.project_root.clone(),
3236        bound_at: Instant::now(),
3237        flow: Arc::new(ChannelFlow::new(window_for(&module.concurrency))),
3238        scope: pending.scope,
3239    });
3240    inner
3241        .client_to_module
3242        .insert(reservation.client_key, Arc::clone(&binding));
3243    inner
3244        .module_to_client
3245        .insert(reservation.module_key, Arc::clone(&binding));
3246    let previous_published = inner
3247        .last_published_epoch
3248        .insert(reservation.client_key, reservation.client_epoch);
3249
3250    // Sending through the reserved slot cannot fail, but it reports a receiver
3251    // that closed after reservation and before this locked publication point.
3252    // The frame is stamped and charged to the queued-byte count here, when it
3253    // actually enters the queue, not when the slot was reserved.
3254    let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
3255    if client_writer_closed {
3256        // Not gated on `relay_enqueued`: only a module ack reaches this
3257        // commit, which proves the module holds the binding, and `route.open`
3258        // sends nothing for a relay a reply already settled.
3259        let abandoned = abandoned_route_target(inner, &reservation);
3260        if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
3261            route.flow.close();
3262        }
3263        inner.module_to_client.remove(&reservation.module_key);
3264        inner
3265            .status
3266            .remove(&(reservation.client_key, reservation.client_epoch));
3267        match previous_published {
3268            Some(epoch) => {
3269                inner
3270                    .last_published_epoch
3271                    .insert(reservation.client_key, epoch);
3272            }
3273            None => {
3274                inner.last_published_epoch.remove(&reservation.client_key);
3275            }
3276        }
3277        let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3278            "client egress closed during route publication".to_string(),
3279        ));
3280        return Ok(abandoned);
3281    }
3282
3283    if let Some(scope) = &binding.scope {
3284        scope.commit_presence();
3285    }
3286    let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
3287    Ok(None)
3288}
3289
3290/// The live module connection registered under exactly `endpoint`, whichever
3291/// slot holds it: active, swap candidate, or superseded incumbent.
3292///
3293/// Resolving by endpoint rather than by module id is what keeps an incumbent
3294/// addressable after a swap promoted a candidate over its id. An endpoint that
3295/// is in none of the three (a stale one, replaced without a promotion) resolves
3296/// to nothing, as it always has.
3297fn module_connection_for_endpoint_locked(
3298    inner: &ForwardingInner,
3299    endpoint: ModuleEndpointId,
3300) -> Option<&ModuleConnection> {
3301    let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
3302    inner
3303        .modules_by_id
3304        .get(module_id)
3305        .filter(|module| module.endpoint == endpoint)
3306        .or_else(|| {
3307            inner
3308                .candidates_by_id
3309                .get(module_id)
3310                .filter(|module| module.endpoint == endpoint)
3311        })
3312        .or_else(|| inner.superseded_endpoints.get(&endpoint))
3313}
3314
3315fn abandoned_route_target(
3316    inner: &ForwardingInner,
3317    reservation: &RouteReservation,
3318) -> Option<GoodbyeTarget> {
3319    let module_id = inner
3320        .module_id_by_endpoint
3321        .get(&reservation.module_key.endpoint)?;
3322    let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
3323    (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
3324        connection_id: module.endpoint.connection_id,
3325        sink: module.sink.clone(),
3326        negotiated_ver: module.negotiated_ver,
3327        channel: reservation.module_key.channel,
3328        epoch: reservation.module_epoch,
3329        kind: GoodbyeTargetKind::Module,
3330        module_id: Some(module_id.clone()),
3331    })
3332}
3333
3334/// Queue a registering module's HELLO_ACK on its sink. Called with the
3335/// forwarding write lock held, before the endpoint is inserted, which is what
3336/// puts the ack ahead of any `route.bind` or control RPC routed to the module.
3337/// `try_send` never waits, so holding the lock across it is safe.
3338fn enqueue_hello_ack_locked(
3339    sink: &FrameSink,
3340    connection_id: ConnectionId,
3341    hello_ack: Option<Frame>,
3342) -> Result<(), ForwardingError> {
3343    let Some(hello_ack) = hello_ack else {
3344        return Ok(());
3345    };
3346    sink.try_send(hello_ack)
3347        .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
3348}
3349
3350fn remove_module_connection_locked(
3351    inner: &mut ForwardingInner,
3352    endpoint: ModuleEndpointId,
3353) -> ConnectionCleanup {
3354    // Commit the disconnect before its routes are released, so their teardown
3355    // cannot replace module_closed with route_closed.
3356    inner.operator_confirms.module_closed(endpoint);
3357    inner.draining_endpoints.remove(&endpoint);
3358    inner.daemon_drain_results.remove(&endpoint);
3359    let module_id = inner.module_id_by_endpoint.remove(&endpoint);
3360    if let Some(module_id) = module_id.as_ref() {
3361        if inner
3362            .modules_by_id
3363            .get(module_id)
3364            .is_some_and(|module| module.endpoint == endpoint)
3365        {
3366            inner.modules_by_id.remove(module_id);
3367        }
3368        if inner
3369            .candidates_by_id
3370            .get(module_id)
3371            .is_some_and(|module| module.endpoint == endpoint)
3372        {
3373            inner.candidates_by_id.remove(module_id);
3374        }
3375    }
3376    inner.superseded_endpoints.remove(&endpoint);
3377    inner.endpoint_by_connection.remove(&endpoint.connection_id);
3378    inner.next_module_channel.remove(&endpoint);
3379    inner.next_control_corr.remove(&endpoint);
3380    inner
3381        .health_probe_tombstones
3382        .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
3383    inner
3384        .module_slot_epochs
3385        .retain(|key, _| key.endpoint != endpoint);
3386    let reserved_module_keys: Vec<ModuleRouteKey> = inner
3387        .reserved_module
3388        .keys()
3389        .filter(|module_key| module_key.endpoint == endpoint)
3390        .copied()
3391        .collect();
3392    for module_key in reserved_module_keys {
3393        if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
3394            release_reserved_route_locked(inner, client_key, module_key);
3395        }
3396    }
3397
3398    let pending_keys: Vec<_> = inner
3399        .pending_relays
3400        .keys()
3401        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3402        .copied()
3403        .collect();
3404    let pending: Vec<_> = pending_keys
3405        .into_iter()
3406        .filter_map(|key| {
3407            inner
3408                .pending_relays
3409                .remove(&key)
3410                .map(|pending| (key, pending))
3411        })
3412        .collect();
3413    let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
3414    for (key, pending) in pending {
3415        // Recorded like every other removal without a reply, so a `route.open`
3416        // still sending the bind learns the relay was released rather than
3417        // settled by the module.
3418        if !pending.relay_enqueued {
3419            inner.relays_released_before_enqueue.insert(key);
3420        }
3421        let module_label = module_id.as_deref().unwrap_or("unknown");
3422        let _ = pending
3423            .sender
3424            .send(RouteBindRelayOutcome::ModuleGone(format!(
3425                "module '{module_label}' connection closed during route.bind relay"
3426            )));
3427    }
3428
3429    let pending_control_keys: Vec<_> = inner
3430        .pending_control_rpcs
3431        .keys()
3432        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3433        .copied()
3434        .collect();
3435    let pending_control: Vec<_> = pending_control_keys
3436        .into_iter()
3437        .filter_map(|key| inner.pending_control_rpcs.remove(&key))
3438        .collect();
3439    for pending in pending_control {
3440        let module_label = module_id.as_deref().unwrap_or("unknown");
3441        let _ = pending
3442            .sender
3443            .send(ModuleControlRpcOutcome::ModuleGone(format!(
3444                "module '{module_label}' connection closed during module-control RPC"
3445            )));
3446    }
3447
3448    let module_routes = inner
3449        .module_to_client
3450        .iter()
3451        .filter(|(module_key, _)| module_key.endpoint == endpoint)
3452        .map(|(module_key, route)| (*module_key, route.module_epoch))
3453        .collect::<Vec<_>>();
3454    let mut released = Vec::with_capacity(module_routes.len());
3455    for (module_key, epoch) in module_routes {
3456        if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
3457        {
3458            released.push(target);
3459        }
3460    }
3461    ConnectionCleanup {
3462        released,
3463        abandoned_relays,
3464    }
3465}
3466
3467#[derive(Debug, Clone, Copy)]
3468struct RequestCredit {
3469    subscription: bool,
3470    excluded_from_drain: bool,
3471}
3472
3473#[derive(Debug, Default)]
3474struct CreditLedger {
3475    by_corr: HashMap<u64, Vec<RequestCredit>>,
3476}
3477
3478impl CreditLedger {
3479    fn acquire(&mut self, corr: u64, subscription: bool) {
3480        self.by_corr.entry(corr).or_default().push(RequestCredit {
3481            subscription,
3482            excluded_from_drain: false,
3483        });
3484    }
3485
3486    fn release(&mut self, corr: u64) -> bool {
3487        let Some(credits) = self.by_corr.get_mut(&corr) else {
3488            return false;
3489        };
3490        let released = credits.pop().is_some();
3491        if credits.is_empty() {
3492            self.by_corr.remove(&corr);
3493        }
3494        released
3495    }
3496
3497    fn capture_subscription_exclusions(&mut self) -> u32 {
3498        let mut excluded = 0u32;
3499        for credit in self.by_corr.values_mut().flatten() {
3500            if credit.subscription && !credit.excluded_from_drain {
3501                credit.excluded_from_drain = true;
3502                excluded = excluded.saturating_add(1);
3503            }
3504        }
3505        excluded
3506    }
3507
3508    #[cfg(test)]
3509    fn in_flight(&self) -> usize {
3510        self.by_corr.values().map(Vec::len).sum()
3511    }
3512
3513    fn drain_in_flight(&self) -> usize {
3514        self.by_corr
3515            .values()
3516            .flatten()
3517            .filter(|credit| !credit.excluded_from_drain)
3518            .count()
3519    }
3520
3521    /// The correlation ids of the requests a drain still waits on, one entry
3522    /// per held credit, in ascending order.
3523    fn drain_held_corrs(&self) -> Vec<u64> {
3524        let mut corrs = self
3525            .by_corr
3526            .iter()
3527            .flat_map(|(corr, credits)| {
3528                credits
3529                    .iter()
3530                    .filter(|credit| !credit.excluded_from_drain)
3531                    .map(move |_| *corr)
3532            })
3533            .collect::<Vec<_>>();
3534        corrs.sort_unstable();
3535        corrs
3536    }
3537}
3538
3539#[derive(Debug, Default)]
3540struct ChannelFlowState {
3541    closed: bool,
3542    credits: CreditLedger,
3543}
3544
3545/// Per-channel request-credit accounting shared by the client and module route halves.
3546#[derive(Debug)]
3547pub(crate) struct ChannelFlow {
3548    sem: Semaphore,
3549    window: usize,
3550    state: Mutex<ChannelFlowState>,
3551}
3552
3553impl ChannelFlow {
3554    pub(crate) fn new(window: usize) -> Self {
3555        debug_assert!(window > 0, "flow-control window must be non-zero");
3556        Self {
3557            sem: Semaphore::new(window),
3558            window,
3559            state: Mutex::new(ChannelFlowState::default()),
3560        }
3561    }
3562
3563    #[cfg(test)]
3564    pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
3565        self.acquire_tagged(0, false).await
3566    }
3567
3568    pub(crate) async fn acquire_tagged(
3569        &self,
3570        corr: u64,
3571        subscription: bool,
3572    ) -> Result<(), ChannelFlowClosed> {
3573        let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
3574        let mut state = self
3575            .state
3576            .lock()
3577            .unwrap_or_else(|poisoned| poisoned.into_inner());
3578        if state.closed {
3579            return Err(ChannelFlowClosed);
3580        }
3581        state.credits.acquire(corr, subscription);
3582        permit.forget();
3583        Ok(())
3584    }
3585
3586    #[cfg(test)]
3587    pub(crate) fn release(&self) {
3588        self.release_corr(0);
3589    }
3590
3591    pub(crate) fn release_corr(&self, corr: u64) {
3592        let released = self
3593            .state
3594            .lock()
3595            .unwrap_or_else(|poisoned| poisoned.into_inner())
3596            .credits
3597            .release(corr);
3598        if !released {
3599            // Protocol-conforming modules emit exactly one terminal per request.
3600            // This guard is a best-effort safety net against window growth, not a
3601            // security boundary against malicious peers.
3602            warn!(
3603                window = self.window,
3604                available = self.sem.available_permits(),
3605                "flow-control over-release ignored"
3606            );
3607            return;
3608        }
3609        if !self.sem.is_closed() {
3610            self.sem.add_permits(1);
3611        }
3612    }
3613
3614    #[cfg(test)]
3615    pub(crate) fn in_flight(&self) -> usize {
3616        self.state
3617            .lock()
3618            .unwrap_or_else(|poisoned| poisoned.into_inner())
3619            .credits
3620            .in_flight()
3621    }
3622
3623    pub(crate) fn drain_in_flight(&self) -> usize {
3624        self.state
3625            .lock()
3626            .unwrap_or_else(|poisoned| poisoned.into_inner())
3627            .credits
3628            .drain_in_flight()
3629    }
3630
3631    pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
3632        self.state
3633            .lock()
3634            .unwrap_or_else(|poisoned| poisoned.into_inner())
3635            .credits
3636            .drain_held_corrs()
3637    }
3638
3639    #[cfg(test)]
3640    pub(crate) fn available_permits(&self) -> usize {
3641        self.sem.available_permits()
3642    }
3643
3644    pub(crate) fn begin_drain(&self) -> u32 {
3645        let mut state = self
3646            .state
3647            .lock()
3648            .unwrap_or_else(|poisoned| poisoned.into_inner());
3649        state.closed = true;
3650        self.sem.close();
3651        state.credits.capture_subscription_exclusions()
3652    }
3653
3654    pub(crate) fn close(&self) {
3655        self.state
3656            .lock()
3657            .unwrap_or_else(|poisoned| poisoned.into_inner())
3658            .closed = true;
3659        self.sem.close();
3660    }
3661}
3662
3663#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3664pub(crate) struct ChannelFlowClosed;
3665
3666impl fmt::Display for ChannelFlowClosed {
3667    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3668        write!(f, "flow-control window closed")
3669    }
3670}
3671
3672impl Error for ChannelFlowClosed {}
3673
3674fn window_for(concurrency: &Concurrency) -> usize {
3675    match concurrency {
3676        Concurrency::Serial => 1,
3677        Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
3678        Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
3679    }
3680}
3681
3682#[derive(Debug, Clone, PartialEq, Eq)]
3683pub enum ForwardingError {
3684    /// A connection cannot hold both client routes and a module endpoint, or
3685    /// replace a module identity while that endpoint is still registered.
3686    ConnectionRoleConflict {
3687        connection_id: ConnectionId,
3688    },
3689    NoModuleConnection,
3690    ModuleReloading {
3691        module_id: String,
3692    },
3693    StaleModuleEndpoint,
3694    UnknownReservation {
3695        client_channel: u16,
3696        module_channel: u16,
3697    },
3698    ClientRouteChannelExhausted {
3699        connection_id: ConnectionId,
3700    },
3701    ModuleRouteChannelExhausted {
3702        endpoint: ModuleEndpointId,
3703    },
3704    RelayCorrelationExhausted,
3705    ConnectionClosing {
3706        connection_id: ConnectionId,
3707    },
3708    ClientEgressClosed {
3709        connection_id: ConnectionId,
3710    },
3711    RouteOpenBuild(String),
3712    /// A swap candidate is already registered for this module id.
3713    CandidateSlotOccupied {
3714        module_id: String,
3715    },
3716    /// A registering module's HELLO_ACK could not be queued because its
3717    /// outbound queue is closed or full.
3718    ModuleEgressUnavailable {
3719        connection_id: ConnectionId,
3720    },
3721    Poisoned,
3722}
3723
3724impl fmt::Display for ForwardingError {
3725    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3726        match self {
3727            Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
3728            Self::NoModuleConnection => write!(f, "no module connection is registered"),
3729            Self::ModuleReloading { module_id } => {
3730                write!(f, "module_id '{module_id}' is reloading")
3731            }
3732            Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
3733            Self::UnknownReservation {
3734                client_channel,
3735                module_channel,
3736            } => write!(
3737                f,
3738                "route reservation client channel {client_channel} / module channel {module_channel} was not found"
3739            ),
3740            Self::ClientRouteChannelExhausted { connection_id } => write!(
3741                f,
3742                "no client route channels are available for connection {}",
3743                connection_id.get()
3744            ),
3745            Self::ModuleRouteChannelExhausted { endpoint } => write!(
3746                f,
3747                "no module route channels are available for endpoint generation {} on connection {}",
3748                endpoint.generation,
3749                endpoint.connection_id.get()
3750            ),
3751            Self::RelayCorrelationExhausted => {
3752                write!(f, "module control correlation ids are exhausted")
3753            }
3754            Self::ConnectionClosing { connection_id } => write!(
3755                f,
3756                "connection {} is closing and cannot accept route allocation",
3757                connection_id.get()
3758            ),
3759            Self::ClientEgressClosed { connection_id } => write!(
3760                f,
3761                "client connection {} egress is closed",
3762                connection_id.get()
3763            ),
3764            Self::RouteOpenBuild(message) => {
3765                write!(f, "failed to prebuild route.open response: {message}")
3766            }
3767            Self::CandidateSlotOccupied { module_id } => write!(
3768                f,
3769                "module_id '{module_id}' already has a swap candidate registered"
3770            ),
3771            Self::ModuleEgressUnavailable { connection_id } => write!(
3772                f,
3773                "module connection {} egress is unavailable; HELLO_ACK could not be queued",
3774                connection_id.get()
3775            ),
3776            Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
3777        }
3778    }
3779}
3780
3781impl Error for ForwardingError {}
3782
3783#[cfg(test)]
3784mod tests {
3785    use std::time::Duration;
3786
3787    use super::*;
3788    use tokio::sync::mpsc;
3789
3790    #[test]
3791    fn ordinary_long_running_request_is_not_excluded_from_drain() {
3792        let mut ledger = CreditLedger::default();
3793        ledger.acquire(1, false);
3794
3795        assert_eq!(ledger.capture_subscription_exclusions(), 0);
3796        assert_eq!(ledger.drain_in_flight(), 1);
3797    }
3798
3799    #[test]
3800    fn bit_set_subscription_is_excluded_and_counted() {
3801        let mut ledger = CreditLedger::default();
3802        ledger.acquire(1, true);
3803
3804        assert_eq!(ledger.capture_subscription_exclusions(), 1);
3805        assert_eq!(ledger.drain_in_flight(), 0);
3806    }
3807
3808    #[test]
3809    fn subscription_opened_after_drain_snapshot_is_not_excluded() {
3810        let mut ledger = CreditLedger::default();
3811        ledger.acquire(1, true);
3812        assert_eq!(ledger.capture_subscription_exclusions(), 1);
3813
3814        ledger.acquire(2, true);
3815
3816        assert_eq!(ledger.drain_in_flight(), 1);
3817    }
3818
3819    #[test]
3820    fn drain_with_no_subscriptions_reports_zero_excluded() {
3821        let mut ledger = CreditLedger::default();
3822        assert_eq!(ledger.capture_subscription_exclusions(), 0);
3823    }
3824
3825    fn test_hello_ack(corr: u64) -> Frame {
3826        Frame::build(
3827            FrameType::HelloAck,
3828            Flags::new(false, Priority::Passive, false),
3829            0,
3830            0,
3831            corr,
3832            Vec::new(),
3833        )
3834        .unwrap()
3835    }
3836
3837    /// An acked registration whose HELLO_ACK cannot be queued must leave no
3838    /// endpoint behind: a routable module that never got its ack would read a
3839    /// route.bind first and exit. A closed queue and a full one both refuse.
3840    #[test]
3841    fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
3842        let forwarding = ForwardingTable::default();
3843
3844        let (closed_tx, closed_rx) = mpsc::channel(8);
3845        drop(closed_rx);
3846        let closed = ConnectionId::new(1);
3847        assert_eq!(
3848            forwarding.register_module_connection_acked(
3849                closed,
3850                "closed".to_string(),
3851                2,
3852                Concurrency::ModuleManaged,
3853                FrameSink::new(closed_tx),
3854                test_hello_ack(1),
3855            ),
3856            Err(ForwardingError::ModuleEgressUnavailable {
3857                connection_id: closed
3858            })
3859        );
3860
3861        let (full_tx, _full_rx) = mpsc::channel(1);
3862        let full_sink = FrameSink::new(full_tx);
3863        full_sink.try_send(test_hello_ack(99)).unwrap();
3864        let full = ConnectionId::new(2);
3865        assert_eq!(
3866            forwarding.register_module_connection_acked(
3867                full,
3868                "full".to_string(),
3869                2,
3870                Concurrency::ModuleManaged,
3871                full_sink.clone(),
3872                test_hello_ack(2),
3873            ),
3874            Err(ForwardingError::ModuleEgressUnavailable {
3875                connection_id: full
3876            })
3877        );
3878        assert_eq!(
3879            forwarding.register_candidate_module_connection_acked(
3880                full,
3881                "full".to_string(),
3882                2,
3883                Concurrency::ModuleManaged,
3884                full_sink,
3885                test_hello_ack(3),
3886            ),
3887            Err(ForwardingError::ModuleEgressUnavailable {
3888                connection_id: full
3889            })
3890        );
3891
3892        for (connection, module_id) in [(closed, "closed"), (full, "full")] {
3893            assert_eq!(
3894                forwarding
3895                    .module_endpoint_for_connection(connection)
3896                    .unwrap(),
3897                None
3898            );
3899            let (client_tx, _client_rx) = mpsc::channel(8);
3900            assert_eq!(
3901                forwarding
3902                    .begin_route_bind_relay_for_test(
3903                        ConnectionId::new(50),
3904                        FrameSink::new(client_tx),
3905                        1,
3906                        module_id,
3907                    )
3908                    .err(),
3909                Some(ForwardingError::NoModuleConnection)
3910            );
3911        }
3912        assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
3913    }
3914
3915    /// Both acked registration forms put the HELLO_ACK on the module's queue
3916    /// by the time the endpoint can be resolved.
3917    #[test]
3918    fn acked_registration_queues_the_hello_ack_first() {
3919        let forwarding = ForwardingTable::default();
3920        let (active_tx, mut active_rx) = mpsc::channel(8);
3921        forwarding
3922            .register_module_connection_acked(
3923                ConnectionId::new(1),
3924                "acked".to_string(),
3925                2,
3926                Concurrency::ModuleManaged,
3927                FrameSink::new(active_tx),
3928                test_hello_ack(11),
3929            )
3930            .unwrap();
3931        let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
3932        forwarding
3933            .register_candidate_module_connection_acked(
3934                ConnectionId::new(2),
3935                "acked".to_string(),
3936                2,
3937                Concurrency::ModuleManaged,
3938                FrameSink::new(candidate_tx),
3939                test_hello_ack(12),
3940            )
3941            .unwrap();
3942
3943        let active_first = active_rx.try_recv().unwrap().frame;
3944        assert_eq!(active_first.header.ty, FrameType::HelloAck);
3945        assert_eq!(active_first.header.corr, 11);
3946        let candidate_first = candidate_rx.try_recv().unwrap().frame;
3947        assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
3948        assert_eq!(candidate_first.header.corr, 12);
3949    }
3950
3951    #[test]
3952    fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
3953        let forwarding = ForwardingTable::default();
3954        let module_connection = ConnectionId::new(10);
3955        let exhausted_client = ConnectionId::new(20);
3956        let second_client = ConnectionId::new(30);
3957        let (module_tx, _module_rx) = mpsc::channel(1);
3958        let endpoint = forwarding
3959            .register_module_connection(
3960                module_connection,
3961                "route-limit-provider".to_string(),
3962                1,
3963                Concurrency::ModuleManaged,
3964                FrameSink::new(module_tx),
3965            )
3966            .unwrap();
3967
3968        {
3969            let mut inner = forwarding.inner.write().unwrap();
3970            for channel in 1..=u16::MAX {
3971                inner.reserved_client.insert(
3972                    ClientRouteKey {
3973                        connection_id: exhausted_client,
3974                        channel,
3975                    },
3976                    ModuleRouteKey {
3977                        endpoint,
3978                        channel: 1,
3979                    },
3980                );
3981            }
3982        }
3983
3984        let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
3985        let err = forwarding
3986            .begin_route_bind_relay_for_test(
3987                exhausted_client,
3988                FrameSink::new(exhausted_tx),
3989                1,
3990                "route-limit-provider",
3991            )
3992            .unwrap_err();
3993        assert!(matches!(
3994            err,
3995            ForwardingError::ClientRouteChannelExhausted { connection_id }
3996                if connection_id == exhausted_client
3997        ));
3998
3999        let (second_tx, _second_rx) = mpsc::channel(1);
4000        let pending = forwarding
4001            .begin_route_bind_relay_for_test(
4002                second_client,
4003                FrameSink::new(second_tx),
4004                2,
4005                "route-limit-provider",
4006            )
4007            .unwrap();
4008        assert_eq!(pending.client_channel, 1);
4009    }
4010
4011    #[test]
4012    fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
4013        let forwarding = ForwardingTable::default();
4014        let module_connection = ConnectionId::new(40);
4015        let client = ConnectionId::new(50);
4016        let (module_tx, _module_rx) = mpsc::channel(1);
4017        forwarding
4018            .register_module_connection(
4019                module_connection,
4020                "slot-reuse-provider".to_string(),
4021                1,
4022                Concurrency::ModuleManaged,
4023                FrameSink::new(module_tx),
4024            )
4025            .unwrap();
4026
4027        let (client_tx, _client_rx) = mpsc::channel(1);
4028        let client_sink = FrameSink::new(client_tx);
4029        let mut wrapped_channel = None;
4030        for index in 0..=usize::from(u16::MAX) {
4031            let pending = forwarding
4032                .begin_route_bind_relay_for_test(
4033                    client,
4034                    client_sink.clone(),
4035                    index as u64 + 1,
4036                    "slot-reuse-provider",
4037                )
4038                .unwrap();
4039            if index == usize::from(u16::MAX) {
4040                wrapped_channel = Some(pending.module_channel);
4041            }
4042            forwarding
4043                .abort_pending_relay(
4044                    pending.endpoint,
4045                    pending.corr,
4046                    RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
4047                )
4048                .unwrap();
4049        }
4050
4051        assert_eq!(wrapped_channel, Some(1));
4052    }
4053
4054    #[test]
4055    fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
4056        let forwarding = ForwardingTable::default();
4057        let client = ConnectionId::new(60);
4058        forwarding
4059            .inner
4060            .write()
4061            .unwrap()
4062            .next_client_channel
4063            .insert(client, 41);
4064
4065        let released = forwarding.cleanup_connection(client).unwrap();
4066
4067        assert!(released.is_empty());
4068        assert!(!forwarding
4069            .inner
4070            .read()
4071            .unwrap()
4072            .next_client_channel
4073            .contains_key(&client));
4074    }
4075
4076    #[test]
4077    fn stale_module_cleanup_preserves_fast_reconnect_successor() {
4078        let forwarding = ForwardingTable::default();
4079        let module_id = "fast-reconnect-provider";
4080        let first_connection = ConnectionId::new(70);
4081        let second_connection = ConnectionId::new(80);
4082        let (first_tx, _first_rx) = mpsc::channel(1);
4083        let first_endpoint = forwarding
4084            .register_module_connection(
4085                first_connection,
4086                module_id.to_string(),
4087                1,
4088                Concurrency::ModuleManaged,
4089                FrameSink::new(first_tx),
4090            )
4091            .unwrap();
4092        let (second_tx, _second_rx) = mpsc::channel(1);
4093        let second_endpoint = forwarding
4094            .register_module_connection(
4095                second_connection,
4096                module_id.to_string(),
4097                1,
4098                Concurrency::ModuleManaged,
4099                FrameSink::new(second_tx),
4100            )
4101            .unwrap();
4102        assert_ne!(first_endpoint, second_endpoint);
4103
4104        let released = forwarding.cleanup_connection(first_connection).unwrap();
4105
4106        assert!(released.is_empty());
4107        assert_eq!(
4108            forwarding
4109                .inner
4110                .read()
4111                .unwrap()
4112                .modules_by_id
4113                .get(module_id)
4114                .map(|module| module.endpoint),
4115            Some(second_endpoint)
4116        );
4117        assert!(forwarding.has_live_module_connection(module_id).unwrap());
4118        let control_rpc = forwarding
4119            .begin_module_control_rpc_for(
4120                module_id,
4121                "health.check",
4122                Instant::now() + Duration::from_secs(1),
4123            )
4124            .unwrap();
4125        assert_eq!(control_rpc.endpoint, second_endpoint);
4126    }
4127
4128    fn route_fixture(
4129        module_id: &str,
4130    ) -> (
4131        ForwardingTable,
4132        ConnectionId,
4133        ModuleEndpointId,
4134        ConnectionId,
4135        FrameSink,
4136        mpsc::Receiver<crate::router::OutboundFrame>,
4137    ) {
4138        let forwarding = ForwardingTable::default();
4139        let module_connection = ConnectionId::new(100);
4140        let client_connection = ConnectionId::new(200);
4141        let (module_tx, _module_rx) = mpsc::channel(8);
4142        let endpoint = forwarding
4143            .register_module_connection(
4144                module_connection,
4145                module_id.to_string(),
4146                2,
4147                Concurrency::ModuleManaged,
4148                FrameSink::new(module_tx),
4149            )
4150            .unwrap();
4151        let (client_tx, client_rx) = mpsc::channel(8);
4152        (
4153            forwarding,
4154            module_connection,
4155            endpoint,
4156            client_connection,
4157            FrameSink::new(client_tx),
4158            client_rx,
4159        )
4160    }
4161
4162    #[test]
4163    #[cfg(unix)]
4164    fn daemon_drain_gates_current_and_racing_provider_registrations() {
4165        let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
4166        assert_eq!(
4167            forwarding.begin_daemon_drain().unwrap(),
4168            [("provider".to_string(), endpoint)]
4169        );
4170        assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
4171        assert!(matches!(
4172            forwarding.register_module_connection(
4173                ConnectionId::new(300),
4174                "late-provider".into(),
4175                2,
4176                Concurrency::ModuleManaged,
4177                sink,
4178            ),
4179            Err(ForwardingError::ConnectionClosing { .. })
4180        ));
4181    }
4182
4183    #[test]
4184    #[cfg(unix)]
4185    fn daemon_drain_results_are_endpoint_fenced_and_removed_on_cleanup() {
4186        let (forwarding, connection, endpoint, _, _, _) = route_fixture("provider");
4187        assert_eq!(
4188            forwarding
4189                .daemon_drain_result_for_connection(connection)
4190                .unwrap(),
4191            None
4192        );
4193        forwarding
4194            .record_daemon_drain_result(endpoint, true)
4195            .unwrap();
4196        forwarding.begin_daemon_drain().unwrap();
4197        assert_eq!(
4198            forwarding
4199                .daemon_drain_result_for_connection(connection)
4200                .unwrap(),
4201            Some(false)
4202        );
4203        let replacement = ModuleEndpointId {
4204            generation: endpoint.generation + 1,
4205            ..endpoint
4206        };
4207        forwarding
4208            .record_daemon_drain_result(replacement, true)
4209            .unwrap();
4210        assert_eq!(
4211            forwarding
4212                .daemon_drain_result_for_connection(connection)
4213                .unwrap(),
4214            Some(false)
4215        );
4216        forwarding
4217            .record_daemon_drain_result(endpoint, true)
4218            .unwrap();
4219        assert_eq!(
4220            forwarding
4221                .daemon_drain_result_for_connection(connection)
4222                .unwrap(),
4223            Some(true)
4224        );
4225        assert_eq!(
4226            forwarding
4227                .daemon_drain_result_for_connection(ConnectionId::new(300))
4228                .unwrap(),
4229            Some(false)
4230        );
4231        forwarding.cleanup_connection(connection).unwrap();
4232        forwarding
4233            .record_daemon_drain_result(endpoint, true)
4234            .unwrap();
4235        assert_eq!(
4236            forwarding
4237                .daemon_drain_result_for_connection(connection)
4238                .unwrap(),
4239            Some(false)
4240        );
4241        assert!(forwarding
4242            .read_inner()
4243            .unwrap()
4244            .daemon_drain_results
4245            .is_empty());
4246    }
4247
4248    #[test]
4249    #[cfg(unix)]
4250    fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
4251        let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
4252            route_fixture("provider");
4253        let mut pending =
4254            begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
4255        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
4256        forwarding.begin_daemon_drain().unwrap();
4257        let completion = forwarding
4258            .complete_pending_relay(
4259                module_connection,
4260                pending.corr,
4261                RouteBindRelayOutcome::Accepted,
4262            )
4263            .expect("draining admission is not a fatal module error");
4264        assert!(completion.settled);
4265        let abandoned = completion
4266            .abandoned
4267            .expect("late accepted bind needs a route GOODBYE");
4268        assert_eq!(abandoned.channel, pending.module_channel);
4269        assert!(matches!(pending.receiver.try_recv().unwrap(),
4270            RouteBindRelayOutcome::Rejected(error) if error.code == "module_reloading"));
4271        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4272        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4273        assert!(
4274            rx.try_recv().is_err(),
4275            "no route.open may be published during drain"
4276        );
4277        assert_eq!(
4278            forwarding
4279                .module_endpoint_for_connection(module_connection)
4280                .unwrap(),
4281            Some(endpoint)
4282        );
4283        assert!(
4284            !forwarding
4285                .complete_pending_relay(
4286                    module_connection,
4287                    pending.corr,
4288                    RouteBindRelayOutcome::Accepted,
4289                )
4290                .unwrap()
4291                .settled
4292        );
4293    }
4294
4295    fn test_ping(corr: u64) -> Frame {
4296        Frame::build(
4297            FrameType::Ping,
4298            Flags::new(false, Priority::Passive, false),
4299            0,
4300            0,
4301            corr,
4302            Vec::new(),
4303        )
4304        .unwrap()
4305    }
4306
4307    fn begin_test_route(
4308        forwarding: &ForwardingTable,
4309        client_connection: ConnectionId,
4310        client_sink: FrameSink,
4311        corr: u64,
4312        module_id: &str,
4313    ) -> PendingRouteBindRelay {
4314        forwarding
4315            .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
4316            .unwrap()
4317    }
4318
4319    #[test]
4320    fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
4321        for candidate in [false, true] {
4322            for register_first in [false, true] {
4323                let (forwarding, module_connection, _, client_connection, sink, _rx) =
4324                    route_fixture("target");
4325                let register = || {
4326                    if candidate {
4327                        forwarding.register_candidate_module_connection(
4328                            client_connection,
4329                            "source".into(),
4330                            2,
4331                            Concurrency::ModuleManaged,
4332                            sink.clone(),
4333                        )
4334                    } else {
4335                        forwarding.register_module_connection(
4336                            client_connection,
4337                            "source".into(),
4338                            2,
4339                            Concurrency::ModuleManaged,
4340                            sink.clone(),
4341                        )
4342                    }
4343                };
4344                if register_first {
4345                    register().unwrap();
4346                    assert!(
4347                        forwarding
4348                            .begin_route_bind_relay_for_test(
4349                                client_connection,
4350                                sink.clone(),
4351                                1,
4352                                "target",
4353                            )
4354                            .is_err(),
4355                        "a deferred route.open cannot reserve after HELLO"
4356                    );
4357                    assert!(forwarding
4358                        .cleanup_connection(client_connection)
4359                        .unwrap()
4360                        .is_empty());
4361                } else {
4362                    let pending =
4363                        begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
4364                    assert!(
4365                        register().is_err(),
4366                        "HELLO cannot register after a route reservation"
4367                    );
4368                    forwarding
4369                        .complete_pending_relay(
4370                            module_connection,
4371                            pending.corr,
4372                            RouteBindRelayOutcome::Accepted,
4373                        )
4374                        .unwrap();
4375                    assert_eq!(
4376                        forwarding
4377                            .cleanup_connection(client_connection)
4378                            .unwrap()
4379                            .len(),
4380                        1
4381                    );
4382                }
4383                assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4384                assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4385            }
4386        }
4387    }
4388
4389    /// A pending route.open reserves its queue slot with `reserve_owned` and
4390    /// sends its prebuilt response through that slot when the module accepts.
4391    /// A queue already holding many data frames (far more than the old 64-frame
4392    /// queue) must not stop the open from reserving or completing, and the
4393    /// response must be charged to the queue's byte count when it is sent.
4394    #[tokio::test]
4395    async fn pending_route_open_completes_behind_queued_data_frames() {
4396        assert_eq!(
4397            crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
4398            8,
4399            "the per-connection pending route.open limit is its own constant"
4400        );
4401        let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
4402            route_fixture("open-behind-data");
4403        let (sink, mut client_rx) = crate::server::connection_egress();
4404        const DATA_FRAMES: usize = 1_000;
4405        let data = |corr: u64| {
4406            Frame::build(
4407                FrameType::StreamData,
4408                Flags::new(false, Priority::Interactive, false),
4409                9,
4410                1,
4411                corr,
4412                vec![b'x'; 200],
4413            )
4414            .unwrap()
4415        };
4416        for corr in 0..DATA_FRAMES as u64 {
4417            sink.try_send(data(corr)).unwrap();
4418        }
4419        let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
4420        assert_eq!(sink.backlog().queued_bytes, data_bytes);
4421
4422        let pending = tokio::time::timeout(
4423            Duration::from_secs(5),
4424            forwarding.begin_route_bind_relay_for(
4425                client,
4426                sink.clone(),
4427                subc_protocol::PROTOCOL_VERSION,
4428                4_242,
4429                "open-behind-data",
4430                Principal::Direct,
4431                None,
4432                None,
4433                Instant::now() + Duration::from_secs(60),
4434            ),
4435        )
4436        .await
4437        .expect("reserving the route.open slot must not wait behind data frames")
4438        .unwrap();
4439        forwarding
4440            .complete_pending_relay(
4441                module_connection,
4442                pending.corr,
4443                RouteBindRelayOutcome::Accepted,
4444            )
4445            .unwrap();
4446
4447        let backlog = sink.backlog();
4448        assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
4449        assert!(
4450            backlog.queued_bytes > data_bytes,
4451            "the route.open response must be counted in queued bytes: {backlog:?}"
4452        );
4453        for corr in 0..DATA_FRAMES as u64 {
4454            assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
4455        }
4456        let open = client_rx.recv().await.unwrap();
4457        assert_eq!(open.header.corr, 4_242);
4458        assert_eq!(open.header.ty, FrameType::Response);
4459        drop(open);
4460        assert_eq!(sink.backlog().queued_bytes, 0);
4461        assert_eq!(sink.backlog().queued_frames, 0);
4462    }
4463
4464    /// The drain-timeout line reports these numbers, so they must count the
4465    /// requests the drain is waiting on and no others: a route with nothing in
4466    /// flight is not a holdout, and a flagged subscription is excluded from the
4467    /// drain and so from the count.
4468    #[tokio::test]
4469    async fn drain_holdouts_count_held_requests_and_name_the_connection() {
4470        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4471            route_fixture("holdouts");
4472        let mut bound = |corr| {
4473            let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
4474            forwarding
4475                .complete_pending_relay(
4476                    module_connection,
4477                    route.corr,
4478                    RouteBindRelayOutcome::Accepted,
4479                )
4480                .unwrap();
4481            client_rx.try_recv().unwrap();
4482            match forwarding
4483                .lookup_data_route(client, route.client_channel, route.client_epoch)
4484                .unwrap()
4485            {
4486                DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4487                other => panic!("expected live route, got {other:?}"),
4488            }
4489        };
4490        let holding = bound(61);
4491        let _idle = bound(62);
4492        holding.flow.acquire_tagged(7, false).await.unwrap();
4493        holding.flow.acquire_tagged(2, false).await.unwrap();
4494        holding.flow.acquire_tagged(3, true).await.unwrap();
4495        forwarding
4496            .begin_module_drain("holdouts", RouteCloseReason::Restart)
4497            .unwrap();
4498
4499        let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
4500        assert_eq!(
4501            holdouts,
4502            DrainHoldouts {
4503                requests: 2,
4504                routes: 1,
4505                total_routes: 2,
4506                top_connections: vec![(client.get(), 2)],
4507                // The held requests by the module's channel and corr, ascending,
4508                // without the excluded subscription (corr 3).
4509                held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
4510            }
4511        );
4512    }
4513
4514    #[test]
4515    fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
4516        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4517            route_fixture("census");
4518        let pending = begin_test_route(&forwarding, client, sink, 1, "census");
4519        forwarding
4520            .complete_pending_relay(
4521                module_connection,
4522                pending.corr,
4523                RouteBindRelayOutcome::Accepted,
4524            )
4525            .unwrap();
4526
4527        let routes = forwarding.endpoint_routes(endpoint).unwrap();
4528        assert_eq!(routes.len(), 1);
4529        assert!(matches!(routes[0].principal, Principal::Direct));
4530        assert_eq!(routes[0].goodbye_target.connection_id, client);
4531        assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
4532        assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
4533        assert!(!routes[0].draining);
4534
4535        forwarding
4536            .begin_module_drain("census", RouteCloseReason::Restart)
4537            .unwrap();
4538        let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
4539        assert_eq!(draining_routes.len(), 1);
4540        assert!(draining_routes[0].draining);
4541    }
4542
4543    #[test]
4544    fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
4545        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
4546        let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
4547        assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
4548        forwarding
4549            .abort_pending_relay(
4550                first.endpoint,
4551                first.corr,
4552                RouteBindRelayOutcome::ModuleGone("abort".into()),
4553            )
4554            .unwrap();
4555        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4556        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4557
4558        let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
4559        assert_eq!(second.client_channel, first.client_channel);
4560        assert_eq!(second.module_channel, first.module_channel);
4561        assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
4562    }
4563
4564    #[test]
4565    fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
4566        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4567            route_fixture("epoch-release");
4568        let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
4569        forwarding
4570            .complete_pending_relay(
4571                module_connection,
4572                first.corr,
4573                RouteBindRelayOutcome::Accepted,
4574            )
4575            .unwrap();
4576        assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
4577        assert!(matches!(
4578            forwarding
4579                .release_client_route(client, first.client_channel, first.client_epoch)
4580                .unwrap(),
4581            RouteRelease::Removed(_)
4582        ));
4583        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4584        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4585
4586        let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
4587        forwarding
4588            .complete_pending_relay(
4589                module_connection,
4590                second.corr,
4591                RouteBindRelayOutcome::Accepted,
4592            )
4593            .unwrap();
4594        assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
4595        assert!(matches!(
4596            forwarding
4597                .release_client_route(client, second.client_channel, first.client_epoch)
4598                .unwrap(),
4599            RouteRelease::Stale
4600        ));
4601        assert!(!forwarding
4602            .cache_status(
4603                endpoint,
4604                second.module_channel,
4605                first.module_epoch,
4606                "stale".into(),
4607            )
4608            .unwrap());
4609        assert!(forwarding
4610            .cache_status(
4611                endpoint,
4612                second.module_channel,
4613                second.module_epoch,
4614                "current".into(),
4615            )
4616            .unwrap());
4617        match forwarding
4618            .route_poll_snapshot(client, second.client_channel, second.client_epoch)
4619            .unwrap()
4620        {
4621            RoutePollSnapshot::Bound { status, .. } => {
4622                assert_eq!(status.as_deref(), Some("current"));
4623            }
4624            RoutePollSnapshot::Absent => panic!("successor binding was removed"),
4625        }
4626        let counters = forwarding.counters().snapshot();
4627        assert_eq!(counters["route_released_epoch_fenced"], 1);
4628        assert_eq!(counters["route_release_stale_skipped"], 1);
4629    }
4630
4631    #[test]
4632    fn max_epoch_reservation_retires_only_that_slot() {
4633        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
4634        forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
4635        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
4636        let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
4637        assert_eq!(
4638            (final_use.client_channel, final_use.client_epoch),
4639            (7, u32::MAX)
4640        );
4641        assert_eq!(
4642            (final_use.module_channel, final_use.module_epoch),
4643            (9, u32::MAX)
4644        );
4645        forwarding
4646            .abort_pending_relay(
4647                endpoint,
4648                final_use.corr,
4649                RouteBindRelayOutcome::ModuleGone("abort".into()),
4650            )
4651            .unwrap();
4652        forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
4653        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
4654        let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
4655        assert_ne!(next.client_channel, 7);
4656        assert_ne!(next.module_channel, 9);
4657        assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
4658    }
4659
4660    #[test]
4661    fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
4662        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4663            route_fixture("corr-shared");
4664        let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
4665        assert_eq!(bind.corr, 1);
4666        forwarding
4667            .abort_pending_relay(
4668                endpoint,
4669                bind.corr,
4670                RouteBindRelayOutcome::ModuleGone("abort".into()),
4671            )
4672            .unwrap();
4673        let rpc = forwarding
4674            .begin_module_control_rpc_for(
4675                "corr-shared",
4676                "health.check",
4677                Instant::now() - Duration::from_millis(1),
4678            )
4679            .unwrap();
4680        assert_eq!(rpc.corr, 2);
4681        assert_eq!(
4682            forwarding
4683                .complete_module_control_rpc(
4684                    module_connection,
4685                    rpc.corr,
4686                    Some("health.check"),
4687                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
4688                        status: subc_protocol::session::HealthStatus::Ok,
4689                        detail: None,
4690                        metrics: None,
4691                    }),
4692                )
4693                .unwrap(),
4694            ModuleControlRpcCompletion::Settled
4695        );
4696        assert!(matches!(
4697            rpc.receiver.blocking_recv().unwrap(),
4698            ModuleControlRpcOutcome::DeadlineElapsed
4699        ));
4700    }
4701
4702    #[tokio::test(start_paused = true)]
4703    async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
4704        let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
4705        let probe_started_at = Instant::now();
4706        let rpc = forwarding
4707            .begin_health_probe_rpc_for(
4708                "tombstone-ttl",
4709                "health.check",
4710                probe_started_at,
4711                probe_started_at + Duration::from_secs(5),
4712            )
4713            .unwrap();
4714        assert!(forwarding
4715            .tombstone_health_probe_rpc(endpoint, rpc.corr)
4716            .unwrap());
4717        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
4718
4719        tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
4720        tokio::task::yield_now().await;
4721
4722        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
4723    }
4724
4725    #[test]
4726    fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
4727        let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
4728        let mut close = forwarding.register_connection_close(endpoint.connection_id);
4729        forwarding.inject_control_corr(endpoint, u64::MAX);
4730        let final_rpc = forwarding
4731            .begin_module_control_rpc_for(
4732                "corr-max",
4733                "health.check",
4734                Instant::now() + Duration::from_secs(1),
4735            )
4736            .unwrap();
4737        assert_eq!(final_rpc.corr, u64::MAX);
4738        forwarding
4739            .cancel_module_control_rpc(endpoint, final_rpc.corr)
4740            .unwrap();
4741        assert!(matches!(
4742            forwarding.begin_module_control_rpc_for(
4743                "corr-max",
4744                "health.check",
4745                Instant::now() + Duration::from_secs(1),
4746            ),
4747            Err(ForwardingError::RelayCorrelationExhausted)
4748        ));
4749        assert!(close.try_recv().is_ok());
4750    }
4751
4752    #[test]
4753    fn publication_epoch_controls_delivery_failure_escalation() {
4754        fn setup_successor(
4755            commit_successor: Option<bool>,
4756        ) -> (ForwardingTable, ConnectionId, u16, u32) {
4757            let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4758                route_fixture("escalation");
4759            let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
4760            forwarding
4761                .complete_pending_relay(
4762                    module_connection,
4763                    first.corr,
4764                    RouteBindRelayOutcome::Accepted,
4765                )
4766                .unwrap();
4767            client_rx.try_recv().unwrap();
4768            assert!(matches!(
4769                forwarding
4770                    .release_client_route(client, first.client_channel, first.client_epoch)
4771                    .unwrap(),
4772                RouteRelease::Removed(_)
4773            ));
4774            if let Some(commit_successor) = commit_successor {
4775                forwarding.inject_client_slot_epoch(
4776                    client,
4777                    first.client_channel,
4778                    first.client_epoch,
4779                );
4780                forwarding.inject_module_slot_epoch(
4781                    endpoint,
4782                    first.module_channel,
4783                    first.module_epoch,
4784                );
4785                let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
4786                if commit_successor {
4787                    forwarding
4788                        .complete_pending_relay(
4789                            module_connection,
4790                            successor.corr,
4791                            RouteBindRelayOutcome::Accepted,
4792                        )
4793                        .unwrap();
4794                    client_rx.try_recv().unwrap();
4795                } else {
4796                    forwarding
4797                        .abort_pending_relay(
4798                            endpoint,
4799                            successor.corr,
4800                            RouteBindRelayOutcome::ModuleGone("abort".into()),
4801                        )
4802                        .unwrap();
4803                }
4804            }
4805            (forwarding, client, first.client_channel, first.client_epoch)
4806        }
4807
4808        let probe_sink = FrameSink::new(mpsc::channel(1).0);
4809        let (no_successor, client, channel, epoch) = setup_successor(None);
4810        let mut close = no_successor.register_connection_close(client);
4811        assert!(no_successor
4812            .escalate_client_delivery_failure(
4813                client,
4814                channel,
4815                epoch,
4816                CloseReason::new("delivery", "failed"),
4817                UndeliveredFrame {
4818                    module_id: None,
4819                    sink: &probe_sink,
4820                },
4821            )
4822            .unwrap());
4823        assert!(close.try_recv().is_ok());
4824
4825        let (aborted, client, channel, epoch) = setup_successor(Some(false));
4826        let mut close = aborted.register_connection_close(client);
4827        assert!(aborted
4828            .escalate_client_delivery_failure(
4829                client,
4830                channel,
4831                epoch,
4832                CloseReason::new("delivery", "failed"),
4833                UndeliveredFrame {
4834                    module_id: None,
4835                    sink: &probe_sink,
4836                },
4837            )
4838            .unwrap());
4839        assert!(close.try_recv().is_ok());
4840
4841        let (published, client, channel, epoch) = setup_successor(Some(true));
4842        let mut close = published.register_connection_close(client);
4843        assert!(!published
4844            .escalate_client_delivery_failure(
4845                client,
4846                channel,
4847                epoch,
4848                CloseReason::new("delivery", "stale failure"),
4849                UndeliveredFrame {
4850                    module_id: None,
4851                    sink: &probe_sink,
4852                },
4853            )
4854            .unwrap());
4855        assert!(close.try_recv().is_err());
4856    }
4857
4858    #[test]
4859    fn route_concentration_separates_client_count_from_routes_per_client() {
4860        // The distinction this asserts is the one a bare connection count cannot
4861        // make: two connections holding one route each and one connection
4862        // holding two are the same total, and have opposite causes.
4863        let (forwarding, module_connection, _, client, sink, _client_rx) =
4864            route_fixture("concentration");
4865        assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
4866
4867        for corr in [70_u64, 71] {
4868            let pending =
4869                begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
4870            forwarding
4871                .complete_pending_relay(
4872                    module_connection,
4873                    pending.corr,
4874                    RouteBindRelayOutcome::Accepted,
4875                )
4876                .unwrap();
4877        }
4878
4879        // One connection, two routes — not two connections with a route each.
4880        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
4881        assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
4882    }
4883
4884    #[test]
4885    fn cleanup_and_accepted_resolution_have_one_lock_winner() {
4886        let (forwarding, module_connection, _, client, sink, mut client_rx) =
4887            route_fixture("cleanup-race");
4888        let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
4889        forwarding
4890            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4891            .unwrap();
4892        let released = forwarding.cleanup_connection(client).unwrap();
4893        assert_eq!(released.len(), 1);
4894        let completion = forwarding
4895            .complete_pending_relay(
4896                module_connection,
4897                pending.corr,
4898                RouteBindRelayOutcome::Accepted,
4899            )
4900            .unwrap();
4901        assert!(!completion.settled);
4902        assert!(client_rx.try_recv().is_err());
4903        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4904
4905        let (forwarding, module_connection, _, client, sink, mut client_rx) =
4906            route_fixture("accepted-race");
4907        let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
4908        forwarding
4909            .complete_pending_relay(
4910                module_connection,
4911                pending.corr,
4912                RouteBindRelayOutcome::Accepted,
4913            )
4914            .unwrap();
4915        assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
4916        let released = forwarding.cleanup_connection(client).unwrap();
4917        assert_eq!(released.len(), 1);
4918        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4919    }
4920
4921    #[test]
4922    fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
4923        let (forwarding, module_connection, _, client, sink, mut client_rx) =
4924            route_fixture("drain-gap");
4925        let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
4926        forwarding
4927            .complete_pending_relay(
4928                module_connection,
4929                live.corr,
4930                RouteBindRelayOutcome::Accepted,
4931            )
4932            .unwrap();
4933        client_rx.try_recv().unwrap();
4934        let binding = match forwarding
4935            .lookup_data_route(client, live.client_channel, live.client_epoch)
4936            .unwrap()
4937        {
4938            DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4939            other => panic!("expected live route, got {other:?}"),
4940        };
4941
4942        let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
4943        forwarding
4944            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4945            .unwrap();
4946        let control_rpc = forwarding
4947            .begin_module_control_rpc_for(
4948                "drain-gap",
4949                "health.check",
4950                Instant::now() + Duration::from_secs(1),
4951            )
4952            .unwrap();
4953        let target = forwarding
4954            .begin_module_drain("drain-gap", RouteCloseReason::Reload)
4955            .unwrap()
4956            .unwrap();
4957        assert!(matches!(
4958            control_rpc.receiver.blocking_recv().unwrap(),
4959            ModuleControlRpcOutcome::ModuleGone(_)
4960        ));
4961        assert_eq!(target.abandoned_bindings.len(), 1);
4962        assert!(binding.flow.sem.is_closed());
4963        assert!(
4964            !forwarding
4965                .complete_pending_relay(
4966                    module_connection,
4967                    pending.corr,
4968                    RouteBindRelayOutcome::Accepted,
4969                )
4970                .unwrap()
4971                .settled
4972        );
4973        assert!(matches!(
4974            forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
4975            Err(ForwardingError::ModuleReloading { .. })
4976        ));
4977        let released = forwarding
4978            .release_module_endpoint_routes(target.endpoint)
4979            .unwrap();
4980        assert_eq!(released.len(), 1);
4981        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4982    }
4983
4984    /// A client can be marked closing while its egress is still open: the daemon
4985    /// asks a connection to close (here through the production path, a module
4986    /// frame that its egress refused) and the connection loop tears down a moment
4987    /// later. A route.bind ack that lands inside that window is answered on the
4988    /// MODULE connection's frame handler, so resolving it must not produce an
4989    /// error -- an error there ends the module connection, and that connection
4990    /// carries every other client's routes to the module.
4991    #[test]
4992    fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
4993        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4994            route_fixture("closing-client");
4995
4996        // A published route on this client: escalate_client_delivery_failure only
4997        // marks a connection closing for a route it has already published.
4998        let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
4999        forwarding
5000            .complete_pending_relay(
5001                module_connection,
5002                live.corr,
5003                RouteBindRelayOutcome::Accepted,
5004            )
5005            .unwrap();
5006        client_rx.try_recv().unwrap();
5007
5008        // A second route.open from the same client, relayed and awaiting its ack.
5009        let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
5010        forwarding
5011            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5012            .unwrap();
5013
5014        // The window: closing, but the sink is still open.
5015        assert!(forwarding
5016            .escalate_client_delivery_failure(
5017                client,
5018                live.client_channel,
5019                live.client_epoch,
5020                CloseReason::new(
5021                    "module_to_client_delivery_failed",
5022                    "client egress refused a module frame",
5023                ),
5024                UndeliveredFrame {
5025                    module_id: None,
5026                    sink: &sink,
5027                },
5028            )
5029            .unwrap());
5030        assert!(!sink.is_closed());
5031
5032        let completion = forwarding
5033            .complete_pending_relay(
5034                module_connection,
5035                pending.corr,
5036                RouteBindRelayOutcome::Accepted,
5037            )
5038            .expect("a closing client must not turn a module's ack into an error");
5039
5040        assert!(completion.settled);
5041        let abandoned = completion
5042            .abandoned
5043            .expect("the module must be told to drop the binding it just created");
5044        assert_eq!(abandoned.connection_id, module_connection);
5045        assert_eq!(abandoned.channel, pending.module_channel);
5046        assert_eq!(abandoned.epoch, pending.module_epoch);
5047        assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
5048        assert!(matches!(
5049            pending.receiver.blocking_recv().unwrap(),
5050            RouteBindRelayOutcome::ModuleGone(_)
5051        ));
5052        // No route was published to a client that is on its way out, and the
5053        // reserved handle pair went back.
5054        assert!(client_rx.try_recv().is_err());
5055        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
5056
5057        // The module endpoint is untouched: still live, and still able to take a
5058        // route from another client.
5059        assert!(forwarding
5060            .has_live_module_connection("closing-client")
5061            .unwrap());
5062        let cotenant = ConnectionId::new(201);
5063        let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
5064        let cotenant_route = begin_test_route(
5065            &forwarding,
5066            cotenant,
5067            FrameSink::new(cotenant_tx),
5068            62,
5069            "closing-client",
5070        );
5071        assert_eq!(cotenant_route.endpoint, endpoint);
5072        forwarding
5073            .complete_pending_relay(
5074                module_connection,
5075                cotenant_route.corr,
5076                RouteBindRelayOutcome::Accepted,
5077            )
5078            .unwrap();
5079        assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
5080        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5081    }
5082
5083    #[test]
5084    fn pending_route_permit_is_released_on_rejection_and_abort() {
5085        let forwarding = ForwardingTable::default();
5086        let module_connection = ConnectionId::new(300);
5087        let client = ConnectionId::new(301);
5088        let (module_tx, _module_rx) = mpsc::channel(1);
5089        let endpoint = forwarding
5090            .register_module_connection(
5091                module_connection,
5092                "permit".into(),
5093                2,
5094                Concurrency::ModuleManaged,
5095                FrameSink::new(module_tx),
5096            )
5097            .unwrap();
5098        let (client_tx, mut client_rx) = mpsc::channel(1);
5099        let sink = FrameSink::new(client_tx);
5100        let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
5101        assert!(sink.try_send(test_ping(999)).is_err());
5102        forwarding
5103            .complete_pending_relay(
5104                module_connection,
5105                rejected.corr,
5106                RouteBindRelayOutcome::Rejected(ErrorBody {
5107                    code: "no".into(),
5108                    message: "rejected".into(),
5109                    detail: None,
5110                }),
5111            )
5112            .unwrap();
5113        sink.try_send(test_ping(1000)).unwrap();
5114        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
5115
5116        let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
5117        assert!(sink.try_send(test_ping(1001)).is_err());
5118        forwarding
5119            .abort_pending_relay(
5120                endpoint,
5121                aborted.corr,
5122                RouteBindRelayOutcome::ModuleGone("abort".into()),
5123            )
5124            .unwrap();
5125        sink.try_send(test_ping(1002)).unwrap();
5126        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
5127
5128        let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
5129        forwarding
5130            .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
5131            .unwrap();
5132        drop(client_rx);
5133        let completion = forwarding
5134            .complete_pending_relay(
5135                module_connection,
5136                receiver_closed.corr,
5137                RouteBindRelayOutcome::Accepted,
5138            )
5139            .unwrap();
5140        assert!(completion.abandoned.is_some());
5141        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5142    }
5143
5144    /// Every connection teardown passes through `cleanup_connection`, and
5145    /// connection ids come from a monotonic counter that never hands an id out
5146    /// twice. If the closing mark survives teardown, the set grows by one entry
5147    /// per connection for the life of the daemon -- the self-watchdog alone
5148    /// reconnects once a minute.
5149    #[test]
5150    fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
5151        let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
5152            route_fixture("closing-set-leak");
5153
5154        const CONNECTIONS: u64 = 32;
5155        for index in 0..CONNECTIONS {
5156            let client = ConnectionId::new(1000 + index);
5157            let (client_tx, _client_rx) = mpsc::channel(8);
5158            let route = begin_test_route(
5159                &forwarding,
5160                client,
5161                FrameSink::new(client_tx),
5162                index + 1,
5163                "closing-set-leak",
5164            );
5165            forwarding
5166                .complete_pending_relay(
5167                    module_connection,
5168                    route.corr,
5169                    RouteBindRelayOutcome::Accepted,
5170                )
5171                .unwrap();
5172            forwarding.cleanup_connection(client).unwrap();
5173        }
5174        forwarding.cleanup_connection(module_connection).unwrap();
5175
5176        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5177    }
5178
5179    /// The closing mark exists to refuse new work for a connection that is on
5180    /// its way out but whose teardown has not run yet: the daemon asks the
5181    /// connection loop to end, and only when the loop reacts does
5182    /// `cleanup_connection` strip the connection's state. Inside that window an
5183    /// operation for the dying connection must still be refused; only after
5184    /// cleanup completes may the mark go.
5185    #[test]
5186    fn closing_connection_is_refused_new_work_until_cleanup_completes() {
5187        let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
5188            route_fixture("closing-gate");
5189
5190        // A published route: escalate_client_delivery_failure only marks a
5191        // connection closing for a route it has already published.
5192        let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
5193        forwarding
5194            .complete_pending_relay(
5195                module_connection,
5196                live.corr,
5197                RouteBindRelayOutcome::Accepted,
5198            )
5199            .unwrap();
5200        client_rx.try_recv().unwrap();
5201
5202        // Mark the connection closing through the production path without
5203        // running teardown, pinning the window open.
5204        assert!(forwarding
5205            .escalate_client_delivery_failure(
5206                client,
5207                live.client_channel,
5208                live.client_epoch,
5209                CloseReason::new(
5210                    "module_to_client_delivery_failed",
5211                    "client egress refused a module frame",
5212                ),
5213                UndeliveredFrame {
5214                    module_id: None,
5215                    sink: &sink,
5216                },
5217            )
5218            .unwrap());
5219        assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
5220
5221        // A late route.open for the closing client is refused ...
5222        assert!(matches!(
5223            forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
5224            Err(ForwardingError::ConnectionClosing { connection_id })
5225                if connection_id == client
5226        ));
5227        // ... and so is a late attempt to register the connection as a module.
5228        let (late_tx, _late_rx) = mpsc::channel(1);
5229        assert!(matches!(
5230            forwarding.register_module_connection(
5231                client,
5232                "late-module".into(),
5233                2,
5234                Concurrency::ModuleManaged,
5235                FrameSink::new(late_tx),
5236            ),
5237            Err(ForwardingError::ConnectionClosing { connection_id })
5238                if connection_id == client
5239        ));
5240
5241        // Teardown is the point that lifts the mark: it has just removed every
5242        // per-connection entry under the same lock, so the gate has nothing
5243        // left to protect for this id.
5244        forwarding.cleanup_connection(client).unwrap();
5245        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5246    }
5247}
5248
5249/// Blue/green swap slots: a candidate registered beside the active endpoint,
5250/// promoted by `cutover_candidate`, with the old incumbent drained by endpoint.
5251#[cfg(test)]
5252mod swap_slot_tests {
5253    use std::time::Duration;
5254
5255    use super::*;
5256    use tokio::sync::mpsc;
5257
5258    const MODULE_ID: &str = "swapped";
5259
5260    struct SwapFixture {
5261        forwarding: ForwardingTable,
5262        incumbent_connection: ConnectionId,
5263        incumbent: ModuleEndpointId,
5264        candidate_connection: ConnectionId,
5265        candidate: ModuleEndpointId,
5266        _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
5267    }
5268
5269    fn swap_fixture() -> SwapFixture {
5270        let forwarding = ForwardingTable::default();
5271        let incumbent_connection = ConnectionId::new(100);
5272        let candidate_connection = ConnectionId::new(110);
5273        let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
5274        let incumbent = forwarding
5275            .register_module_connection(
5276                incumbent_connection,
5277                MODULE_ID.to_string(),
5278                2,
5279                Concurrency::ModuleManaged,
5280                FrameSink::new(incumbent_tx),
5281            )
5282            .unwrap();
5283        let (candidate_tx, candidate_rx) = mpsc::channel(8);
5284        let candidate = forwarding
5285            .register_candidate_module_connection(
5286                candidate_connection,
5287                MODULE_ID.to_string(),
5288                2,
5289                Concurrency::ModuleManaged,
5290                FrameSink::new(candidate_tx),
5291            )
5292            .unwrap();
5293        SwapFixture {
5294            forwarding,
5295            incumbent_connection,
5296            incumbent,
5297            candidate_connection,
5298            candidate,
5299            _module_rxs: vec![incumbent_rx, candidate_rx],
5300        }
5301    }
5302
5303    fn client(
5304        raw: u64,
5305    ) -> (
5306        ConnectionId,
5307        FrameSink,
5308        mpsc::Receiver<crate::router::OutboundFrame>,
5309    ) {
5310        let (tx, rx) = mpsc::channel(8);
5311        (ConnectionId::new(raw), FrameSink::new(tx), rx)
5312    }
5313
5314    fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
5315        forwarding
5316            .read_inner()
5317            .unwrap()
5318            .client_to_module
5319            .values()
5320            .map(|route| route.module_endpoint)
5321            .collect()
5322    }
5323
5324    #[test]
5325    fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
5326        let fixture = swap_fixture();
5327        let forwarding = &fixture.forwarding;
5328        assert_ne!(fixture.incumbent, fixture.candidate);
5329
5330        // Every by-id consumer still resolves the incumbent.
5331        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5332        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5333        let (client_connection, client_sink, _client_rx) = client(200);
5334        let pending = forwarding
5335            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5336            .unwrap();
5337        assert_eq!(pending.endpoint, fixture.incumbent);
5338        let rpc = forwarding
5339            .begin_module_control_rpc_for(
5340                MODULE_ID,
5341                "health.check",
5342                Instant::now() + Duration::from_secs(1),
5343            )
5344            .unwrap();
5345        assert_eq!(rpc.endpoint, fixture.incumbent);
5346        let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
5347        assert_eq!(census.len(), 1, "the census lists one endpoint per id");
5348
5349        // Connection-keyed lookups see the candidate, so its own frames resolve.
5350        assert_eq!(
5351            forwarding
5352                .module_endpoint_for_connection(fixture.candidate_connection)
5353                .unwrap(),
5354            Some(fixture.candidate)
5355        );
5356        assert_eq!(
5357            forwarding
5358                .module_id_for_connection(fixture.candidate_connection)
5359                .unwrap()
5360                .as_deref(),
5361            Some(MODULE_ID)
5362        );
5363
5364        // One candidate per id.
5365        let (other_tx, _other_rx) = mpsc::channel(1);
5366        assert_eq!(
5367            forwarding.register_candidate_module_connection(
5368                ConnectionId::new(120),
5369                MODULE_ID.to_string(),
5370                2,
5371                Concurrency::ModuleManaged,
5372                FrameSink::new(other_tx),
5373            ),
5374            Err(ForwardingError::CandidateSlotOccupied {
5375                module_id: MODULE_ID.to_string()
5376            })
5377        );
5378    }
5379
5380    /// The cutover linearization point. A relay reserved on the incumbent before
5381    /// cutover must never become a route on the incumbent, and every relay
5382    /// reserved after it must land on the promoted candidate.
5383    #[test]
5384    fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
5385        let fixture = swap_fixture();
5386        let forwarding = &fixture.forwarding;
5387        let (early_client, early_sink, _early_rx) = client(200);
5388        let mut early = forwarding
5389            .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
5390            .unwrap();
5391        assert_eq!(early.endpoint, fixture.incumbent);
5392        assert_eq!(
5393            forwarding
5394                .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
5395                .unwrap(),
5396            RouteBindRelayMark::Marked
5397        );
5398
5399        let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5400        assert_eq!(
5401            cutover,
5402            ForwardingCutover {
5403                promoted: fixture.candidate,
5404                incumbent: Some(fixture.incumbent),
5405            }
5406        );
5407
5408        // A route.open reserved after cutover goes to the promoted candidate.
5409        let (late_client, late_sink, _late_rx) = client(201);
5410        let late = forwarding
5411            .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
5412            .unwrap();
5413        assert_eq!(
5414            late.endpoint, fixture.candidate,
5415            "a route.open after cutover was reserved on the incumbent"
5416        );
5417
5418        // The incumbent acks the early relay after cutover.
5419        let completion = forwarding
5420            .complete_pending_relay(
5421                fixture.incumbent_connection,
5422                early.corr,
5423                RouteBindRelayOutcome::Accepted,
5424            )
5425            .expect("a superseded endpoint's ack is not an error on its connection");
5426        assert!(completion.settled);
5427        assert!(
5428            !committed_endpoints(forwarding).contains(&fixture.incumbent),
5429            "a relay reserved before cutover committed a route on the incumbent"
5430        );
5431        let goodbye = completion
5432            .abandoned
5433            .expect("the incumbent is told to drop the binding it just created");
5434        assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
5435        assert_eq!(goodbye.channel, early.module_channel);
5436        assert_eq!(goodbye.epoch, early.module_epoch);
5437        assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
5438        match early.receiver.try_recv() {
5439            Ok(RouteBindRelayOutcome::Rejected(body)) => assert_eq!(body.code, "module_reloading"),
5440            other => panic!("expected a retryable module_reloading answer, got {other:?}"),
5441        }
5442        assert!(matches!(
5443            forwarding
5444                .lookup_data_route(early_client, early.client_channel, early.client_epoch)
5445                .unwrap(),
5446            DataRoute::Client(DataRouteState::Absent)
5447        ));
5448
5449        // The late relay commits on the candidate; only its pair stays reserved
5450        // until then.
5451        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
5452        forwarding
5453            .complete_pending_relay(
5454                fixture.candidate_connection,
5455                late.corr,
5456                RouteBindRelayOutcome::Accepted,
5457            )
5458            .unwrap();
5459        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5460        assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
5461    }
5462
5463    #[test]
5464    fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
5465        let fixture = swap_fixture();
5466        let forwarding = &fixture.forwarding;
5467        // One bound route and one in-flight relay on the incumbent.
5468        let (bound_client, bound_sink, _bound_rx) = client(200);
5469        let bound = forwarding
5470            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5471            .unwrap();
5472        forwarding
5473            .complete_pending_relay(
5474                fixture.incumbent_connection,
5475                bound.corr,
5476                RouteBindRelayOutcome::Accepted,
5477            )
5478            .unwrap();
5479        let (pending_client, pending_sink, _pending_rx) = client(201);
5480        let mut in_flight = forwarding
5481            .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
5482            .unwrap();
5483        forwarding
5484            .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
5485            .unwrap();
5486
5487        let incumbent = forwarding
5488            .cutover_candidate(MODULE_ID)
5489            .unwrap()
5490            .unwrap()
5491            .incumbent
5492            .unwrap();
5493        let target = forwarding
5494            .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
5495            .unwrap()
5496            .expect("the superseded incumbent is still registered");
5497
5498        assert_eq!(target.endpoint, fixture.incumbent);
5499        assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
5500        assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
5501        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5502        assert_eq!(target.abandoned_bindings.len(), 1);
5503        assert_eq!(
5504            target.abandoned_bindings[0].channel,
5505            in_flight.module_channel
5506        );
5507        assert!(matches!(
5508            in_flight.receiver.try_recv(),
5509            Ok(RouteBindRelayOutcome::Rejected(body)) if body.code == "module_reloading"
5510        ));
5511        assert_eq!(
5512            forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
5513            1,
5514            "the incumbent's bound route stays until its drain finishes"
5515        );
5516
5517        let (next_client, next_sink, _next_rx) = client(202);
5518        let next = forwarding
5519            .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
5520            .expect("the promoted candidate keeps accepting routes");
5521        assert_eq!(next.endpoint, fixture.candidate);
5522    }
5523
5524    /// An endpoint replaced WITHOUT a promotion (a successor registered over it
5525    /// as an ordinary active HELLO) is stale, not superseded, and its ack still
5526    /// fails the acking connection exactly as it did before swap slots existed:
5527    /// `StaleModuleEndpoint`, the reservation indexes already stripped, and the
5528    /// waiting client's sender dropped unanswered.
5529    #[test]
5530    fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
5531        let forwarding = ForwardingTable::default();
5532        let first_connection = ConnectionId::new(70);
5533        let (first_tx, _first_rx) = mpsc::channel(8);
5534        forwarding
5535            .register_module_connection(
5536                first_connection,
5537                MODULE_ID.to_string(),
5538                2,
5539                Concurrency::ModuleManaged,
5540                FrameSink::new(first_tx),
5541            )
5542            .unwrap();
5543        let (client_connection, client_sink, _client_rx) = client(200);
5544        let mut pending = forwarding
5545            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5546            .unwrap();
5547        let (second_tx, _second_rx) = mpsc::channel(8);
5548        forwarding
5549            .register_module_connection(
5550                ConnectionId::new(80),
5551                MODULE_ID.to_string(),
5552                2,
5553                Concurrency::ModuleManaged,
5554                FrameSink::new(second_tx),
5555            )
5556            .unwrap();
5557
5558        assert_eq!(
5559            forwarding
5560                .complete_pending_relay(
5561                    first_connection,
5562                    pending.corr,
5563                    RouteBindRelayOutcome::Accepted
5564                )
5565                .unwrap_err(),
5566            ForwardingError::StaleModuleEndpoint
5567        );
5568        assert!(committed_endpoints(&forwarding).is_empty());
5569        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5570        assert!(matches!(
5571            pending.receiver.try_recv(),
5572            Err(oneshot::error::TryRecvError::Closed)
5573        ));
5574    }
5575
5576    #[test]
5577    fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
5578        // A candidate whose connection drops leaves the incumbent routable.
5579        let fixture = swap_fixture();
5580        let forwarding = &fixture.forwarding;
5581        assert!(forwarding
5582            .cleanup_connection(fixture.candidate_connection)
5583            .unwrap()
5584            .is_empty());
5585        assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
5586        let (client_connection, client_sink, _client_rx) = client(200);
5587        assert_eq!(
5588            forwarding
5589                .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5590                .unwrap()
5591                .endpoint,
5592            fixture.incumbent
5593        );
5594
5595        // After a cutover, the incumbent's teardown releases its own routes and
5596        // leaves the promoted candidate in place.
5597        let fixture = swap_fixture();
5598        let forwarding = &fixture.forwarding;
5599        let (bound_client, bound_sink, _bound_rx) = client(200);
5600        let bound = forwarding
5601            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5602            .unwrap();
5603        forwarding
5604            .complete_pending_relay(
5605                fixture.incumbent_connection,
5606                bound.corr,
5607                RouteBindRelayOutcome::Accepted,
5608            )
5609            .unwrap();
5610        forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5611        let released = forwarding
5612            .cleanup_connection(fixture.incumbent_connection)
5613            .unwrap();
5614        assert_eq!(released.len(), 1);
5615        assert_eq!(released[0].connection_id, bound_client);
5616        assert!(forwarding
5617            .read_inner()
5618            .unwrap()
5619            .superseded_endpoints
5620            .is_empty());
5621        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5622        let (next_client, next_sink, _next_rx) = client(201);
5623        assert_eq!(
5624            forwarding
5625                .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
5626                .unwrap()
5627                .endpoint,
5628            fixture.candidate
5629        );
5630    }
5631}