Skip to main content

subc_daemon/
forwarding.rs

1use std::{
2    collections::{BTreeMap, HashMap, HashSet},
3    error::Error,
4    fmt,
5    sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
6    time::Duration,
7};
8
9use subc_control::{ClientControlResponse, RouteCloseReason};
10use subc_protocol::{
11    manifest::Concurrency,
12    session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
13    ErrorBody, Flags, FrameType, Principal, Priority,
14};
15use tokio::sync::{oneshot, Semaphore};
16use tokio::time::Instant;
17use tracing::{debug, info, warn};
18
19use crate::{
20    control::{RouteBindBreakers, RouteBindConcurrency},
21    observability::DaemonCounters,
22    registry::ConnectionId,
23    router::FrameSink,
24    scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
25    Frame, ProjectRootId,
26};
27
28/// Default per-channel request-credit window for modules that schedule internally.
29const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
30
31/// High per-channel cap for stateless modules; this is an OOM guard, not scheduling policy.
32const STATELESS_PARALLEL_WINDOW: usize = 1024;
33
34/// A stopped probe cycle cannot retain its last unanswered correlation forever.
35/// Active endpoints replace the tombstone on their next serial health probe;
36/// this backstop covers endpoints that stop probing altogether.
37const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
38
39/// Module connection identity used in forwarding keys.
40///
41/// The generation is bumped every time a module connection is registered so a future restart cannot
42/// accidentally reuse a stale `(connection_id, route_channel)` binding.
43#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
44pub struct ModuleEndpointId {
45    pub connection_id: ConnectionId,
46    pub generation: u64,
47}
48
49/// Client-local route key. A route channel is unique only within one client connection.
50#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
51pub(crate) struct ClientRouteKey {
52    pub connection_id: ConnectionId,
53    pub channel: u16,
54}
55
56/// Module-local route key. A route channel is unique only within one live module endpoint.
57#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
58pub(crate) struct ModuleRouteKey {
59    pub endpoint: ModuleEndpointId,
60    pub channel: u16,
61}
62
63#[derive(Debug)]
64pub(crate) struct RouteBinding {
65    pub client_connection_id: ConnectionId,
66    pub client_sink: FrameSink,
67    pub client_negotiated_ver: u8,
68    pub client_channel: u16,
69    pub client_epoch: u32,
70    pub module_id: String,
71    pub module_endpoint: ModuleEndpointId,
72    pub module_sink: FrameSink,
73    pub module_negotiated_ver: u8,
74    pub module_channel: u16,
75    pub module_epoch: u32,
76    pub principal: Principal,
77    pub project_root: Option<ProjectRootId>,
78    pub bound_at: Instant,
79    pub flow: Arc<ChannelFlow>,
80    /// The scope the route was admitted under, with the tag its bind was
81    /// stamped at. A scope change finds the routes to close by it.
82    pub scope: Option<BoundScope>,
83}
84
85#[derive(Debug, Clone)]
86pub(crate) enum DataRoute {
87    Client(DataRouteState),
88    Module(DataRouteState),
89}
90
91#[derive(Debug, Clone)]
92pub(crate) enum DataRouteState {
93    Bound(Arc<RouteBinding>),
94    Reserved,
95    EpochMismatch,
96    Absent,
97}
98
99/// Which kind of peer a route GOODBYE is being delivered to. This decides what
100/// happens when the GOODBYE cannot be enqueued (egress full/closed):
101/// - `Client`: escalate to closing that client connection (a socket close is a
102///   stronger teardown signal, and a full client egress means it is the slow
103///   client we would drop anyway).
104/// - `Module`: never close; deliver late instead (see
105///   [`send_module_route_goodbye`]), and drop only if the module still has no
106///   room after [`LATE_MODULE_GOODBYE_DEADLINE`]. A client-disconnect notifies the
107///   SHARED module that one client's route is gone; closing the module on its
108///   egress backpressure would tear down every co-tenant client (the exact
109///   cross-tenant blast radius this never-close rule exists to prevent — observed when a
110///   flooding dead client filled BOTH its own and the module's egress, so its
111///   route-gone GOODBYE to the module failed and closed the shared connection).
112///   subc has already removed the route from its forwarding state and drops
113///   stale module frames for the released channel (see router.rs), so subc's
114///   own routing is correct. The residual: under SUSTAINED module-egress
115///   backpressure a module-targeted route-gone notification can be lost, which a
116///   module using it for client-refcounting (e.g. AFT's session accounting)
117///   would miss. This is INTENTIONALLY ACCEPTED, not a gap. A consuming module
118///   must bound stale bindings with its own idle-activity reaper (last-touched
119///   TTL) independent of route-gone signals — AFT does exactly this, so a lost
120///   GOODBYE degrades to "the
121///   binding stays warm until its idle TTL" (bounded wasted resources), never an
122///   unbounded leak; disk-durable replay is unaffected. A dedicated reliable
123///   module control lane was evaluated and deliberately NOT
124///   built: it would add starvation-avoidance machinery to the thin core for a
125///   bounded warm-resource window that is not a correctness issue. never-close
126///   is the invariant that matters here.
127#[derive(Debug, Clone, Copy, PartialEq, Eq)]
128pub(crate) enum GoodbyeTargetKind {
129    Client,
130    Module,
131}
132
133#[derive(Debug, Clone)]
134pub(crate) struct GoodbyeTarget {
135    pub connection_id: ConnectionId,
136    pub sink: FrameSink,
137    pub negotiated_ver: u8,
138    pub channel: u16,
139    pub epoch: u32,
140    pub kind: GoodbyeTargetKind,
141    /// The module on the other end of the route: for a module-targeted relay
142    /// the receiving module (attributing a dropped relay), for a client target
143    /// the module whose route is going away (naming it when an undeliverable
144    /// relay closes the client).
145    pub module_id: Option<String>,
146}
147
148/// The frame a client connection's egress queue refused, for the diagnosis
149/// logged when that refusal closes the connection.
150#[derive(Debug, Clone, Copy)]
151pub(crate) struct UndeliveredFrame<'a> {
152    /// The module on the other end of the route the frame belonged to, when known.
153    pub module_id: Option<&'a str>,
154    /// The refusing connection's sink, read for what its queue held.
155    pub sink: &'a FrameSink,
156}
157
158/// How a route's principal appears in logs: `direct`, or `reserved:<module>`.
159pub(crate) fn principal_label(principal: &Principal) -> String {
160    match principal {
161        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
162        Principal::Direct => "direct".to_string(),
163        other => format!("{other:?}"),
164    }
165}
166
167/// The distinct principals of the routes bound on one client connection,
168/// sorted and comma-joined, or `none` when it has no bound route.
169fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
170    let labels = inner
171        .client_to_module
172        .iter()
173        .filter(|(key, _)| key.connection_id == connection_id)
174        .map(|(_, route)| principal_label(&route.principal))
175        .collect::<std::collections::BTreeSet<_>>();
176    if labels.is_empty() {
177        "none".to_string()
178    } else {
179        labels.into_iter().collect::<Vec<_>>().join(",")
180    }
181}
182
183impl GoodbyeTarget {
184    /// True only when an undeliverable GOODBYE should escalate to closing the
185    /// target connection. Never escalate for module recipients.
186    pub(crate) fn close_on_delivery_failure(&self) -> bool {
187        matches!(self.kind, GoodbyeTargetKind::Client)
188    }
189}
190
191/// How long a route GOODBYE that a module's egress queue refused keeps waiting
192/// for room before it is given up and counted as dropped.
193///
194/// A module that stops reading for a moment (a GC pause, a long synchronous
195/// handler, a reconnect herd it is still working through) should still learn
196/// that the route is gone, because a module that never hears the GOODBYE keeps
197/// the route and keeps sending on it for as long as its connection lives. The
198/// value is the default module drain timeout: the daemon already treats that as
199/// the longest it is reasonable to wait on a module that is busy but alive, and
200/// a module that cannot free 21 bytes of egress in that time is wedged rather
201/// than slow. Resolving each module's own configured drain timeout here would
202/// need a registry lookup on a path that holds only the module's sink.
203pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
204
205/// Send a route GOODBYE to a module without ever closing the module's shared
206/// connection, which would sever every other route it serves.
207///
208/// The frame is enqueued at once when the module's egress queue has room. When
209/// it does not, a detached task waits for room (bounded by
210/// [`LATE_MODULE_GOODBYE_DEADLINE`]) and sends it then. Arriving late is safe:
211/// the daemon has already released the route, and module SDKs tear a route
212/// down only when both the channel AND the epoch match the route installed on
213/// that channel, so a late GOODBYE for a channel since reused at a newer epoch
214/// is ignored. The GOODBYE is counted in `goodbye_relay_module_dropped` only
215/// when the connection is closed or the deadline passes.
216pub(crate) fn send_module_route_goodbye(
217    counters: &DaemonCounters,
218    sink: &FrameSink,
219    frame: Frame,
220    module_id: Option<&str>,
221    context: &'static str,
222) {
223    let channel = frame.header.channel;
224    let epoch = frame.header.epoch;
225    let Err(err) = sink.try_send(frame.clone()) else {
226        return;
227    };
228    // Waiting is pointless once the module's writer is gone, and impossible
229    // outside a Tokio runtime (cleanup that runs from a destructor at shutdown).
230    let runtime = match tokio::runtime::Handle::try_current() {
231        Ok(runtime) if !sink.is_closed() => runtime,
232        _ => {
233            counters.increment_goodbye_relay_module_dropped(module_id);
234            warn!(
235            module_id = module_id.unwrap_or("unknown"),
236            route_channel = channel,
237            route_epoch = epoch,
238            error = %err,
239            context,
240            "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
241            );
242            return;
243        }
244    };
245    debug!(
246        module_id = module_id.unwrap_or("unknown"),
247        route_channel = channel,
248        route_epoch = epoch,
249        error = %err,
250        context,
251        "module egress queue refused route GOODBYE; delivering it once the module frees room"
252    );
253    let counters = counters.clone();
254    let sink = sink.clone();
255    let module_id = module_id.map(str::to_string);
256    runtime.spawn(async move {
257        let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
258        let why = match outcome {
259            Ok(Ok(())) => {
260                debug!(
261                    module_id = module_id.as_deref().unwrap_or("unknown"),
262                    route_channel = channel,
263                    route_epoch = epoch,
264                    context,
265                    "late route GOODBYE delivered to module"
266                );
267                return;
268            }
269            Ok(Err(err)) => err.to_string(),
270            Err(_) => format!(
271                "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
272            ),
273        };
274        counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
275        warn!(
276            module_id = module_id.as_deref().unwrap_or("unknown"),
277            route_channel = channel,
278            route_epoch = epoch,
279            error = %why,
280            context,
281            "route GOODBYE to module dropped under backpressure; not closing shared module connection"
282        );
283    });
284}
285
286/// One route currently served by a module endpoint.
287///
288/// `goodbye_target` is deliberately retained alongside the census projection so
289/// a draining caller can address exactly the same route set that this read
290/// reports, without a second forwarding-table pass.
291#[derive(Debug, Clone)]
292pub(crate) struct EndpointRoute {
293    pub goodbye_target: GoodbyeTarget,
294    pub principal: Principal,
295    pub bound_at: Instant,
296    pub draining: bool,
297    /// WHY the endpoint is draining, when it is. Carried per-route so the
298    /// census can answer "closing because of what" without a second lookup;
299    /// `None` exactly when `draining` is false (one source: the drain map).
300    pub drain_reason: Option<RouteCloseReason>,
301}
302
303/// One live route a scope change closed: both ends to send GOODBYE to, and the
304/// reason for the client's `route.closed`.
305#[derive(Debug, Clone)]
306pub(crate) struct ScopeDrainedRoute {
307    pub reason: RouteCloseReason,
308    pub scope: BoundScope,
309    pub module_id: String,
310    pub client: GoodbyeTarget,
311    pub module: GoodbyeTarget,
312}
313
314#[derive(Debug)]
315pub(crate) struct PendingRouteBindRelay {
316    pub endpoint: ModuleEndpointId,
317    pub module_sink: FrameSink,
318    pub negotiated_ver: u8,
319    pub client_channel: u16,
320    pub client_epoch: u32,
321    pub module_channel: u16,
322    pub module_epoch: u32,
323    pub corr: u64,
324    pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
325}
326
327#[derive(Debug, Clone)]
328pub(crate) struct ModuleDrainTarget {
329    pub endpoint: ModuleEndpointId,
330    pub sink: FrameSink,
331    pub negotiated_ver: u8,
332    pub abandoned_bindings: Vec<GoodbyeTarget>,
333    pub excluded_subscriptions: u32,
334}
335
336/// One registered module connection, as [`ForwardingTable::module_connections`]
337/// reports it: enough to send it a frame and then close it.
338#[cfg(unix)]
339#[derive(Debug, Clone)]
340pub(crate) struct ModuleConnectionTarget {
341    pub module_id: String,
342    pub endpoint: ModuleEndpointId,
343    pub sink: FrameSink,
344    pub negotiated_ver: u8,
345}
346
347#[derive(Debug, Clone)]
348pub(crate) enum RouteBindRelayOutcome {
349    Accepted,
350    /// The module refused the bind with its own body. Relayed to the client
351    /// without the daemon-origin bit: the code and message are module-chosen,
352    /// so the frame must not claim daemon authorship, even when the module
353    /// picks a daemon code like `module_reloading`.
354    Rejected(ErrorBody),
355    /// The daemon refused the bind itself — a drain or superseded endpoint
356    /// (`module_reloading`), a reply that arrived after the deadline
357    /// (`module_timeout`), or a scope that ended mid-bind. Carries the
358    /// daemon-origin bit because the daemon wrote it. The module may still
359    /// have received the bind (a late reply, or a drain that began after it
360    /// accepted); the daemon then ends that binding with a GOODBYE, so no
361    /// call runs on it.
362    DaemonRejected(ErrorBody),
363    ModuleGone(String),
364}
365
366/// What [`ForwardingTable::cutover_candidate`] did.
367#[derive(Debug, Clone, Copy, PartialEq, Eq)]
368pub(crate) struct ForwardingCutover {
369    /// The endpoint now in the active slot (the former candidate).
370    pub promoted: ModuleEndpointId,
371    /// The endpoint demoted out of the active slot, to be drained by endpoint.
372    /// `None` when the incumbent's connection was already gone.
373    pub incumbent: Option<ModuleEndpointId>,
374}
375
376/// What tearing down one connection released.
377#[derive(Debug)]
378pub(crate) struct ConnectionCleanup {
379    /// Routes whose other end must be sent GOODBYE.
380    pub released: Vec<GoodbyeTarget>,
381    /// Pending route.bind relays to the closed module that were aborted: opens
382    /// that had not been bound yet. Zero when a client connection closed.
383    pub abandoned_relays: u32,
384}
385
386#[derive(Debug, Clone)]
387pub(crate) struct PendingRelayCompletion {
388    pub settled: bool,
389    pub abandoned: Option<GoodbyeTarget>,
390}
391
392#[derive(Debug)]
393pub(crate) struct PendingModuleControlRpc {
394    pub endpoint: ModuleEndpointId,
395    pub module_sink: FrameSink,
396    pub negotiated_ver: u8,
397    pub corr: u64,
398    pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
399}
400
401#[derive(Debug, Clone)]
402pub(crate) enum ModuleControlRpcOutcome {
403    Response(ModuleControlResponse),
404    Rejected(ErrorBody),
405    ModuleGone(String),
406    MalformedResponse(String),
407    UnexpectedOp { expected: String, actual: String },
408    DeadlineElapsed,
409}
410
411#[derive(Debug, Clone, PartialEq, Eq)]
412pub(crate) enum ModuleControlRpcCompletion {
413    Unknown,
414    Settled,
415    LateHealthAnswer {
416        module_id: String,
417        latency: Duration,
418    },
419}
420
421#[derive(Debug)]
422struct PendingModuleControlRpcEntry {
423    expected_op: String,
424    deadline: Instant,
425    health_probe_started_at: Option<Instant>,
426    sender: oneshot::Sender<ModuleControlRpcOutcome>,
427}
428
429#[derive(Debug)]
430struct HealthProbeTombstone {
431    expected_op: String,
432    module_id: String,
433    probe_started_at: Instant,
434    expires_at: Instant,
435}
436
437#[derive(Debug, Clone)]
438struct RouteReservation {
439    client_key: ClientRouteKey,
440    module_key: ModuleRouteKey,
441    client_epoch: u32,
442    module_epoch: u32,
443    project_root: Option<ProjectRootId>,
444}
445
446#[derive(Debug)]
447struct PendingRouteBindRelayEntry {
448    reservation: RouteReservation,
449    client_sink: FrameSink,
450    client_negotiated_ver: u8,
451    client_permit: crate::router::EgressPermit,
452    route_open_frame: Frame,
453    principal: Principal,
454    /// The scope admitted at `route.open`, with the tag captured then. Commit
455    /// compares it with the published tag.
456    scope: Option<BoundScope>,
457    deadline: Instant,
458    /// Set once `route.open` has finished putting the route.bind on the
459    /// module's egress queue. A path that drops this entry WITHOUT a module
460    /// reply (client teardown, module drain, the open's own abort) sends the
461    /// module a GOODBYE for the bind only when this is set, because only then
462    /// is the GOODBYE guaranteed to queue behind the bind. When it is not set,
463    /// the removal is recorded in `relays_released_before_enqueue` instead and
464    /// `route.open` sends that GOODBYE itself once its send has completed.
465    ///
466    /// A module reply does not consult it: an ack proves the module received
467    /// the bind, so the reply path decides about the GOODBYE on its own.
468    relay_enqueued: bool,
469    sender: oneshot::Sender<RouteBindRelayOutcome>,
470}
471
472/// What `mark_route_bind_relay_enqueued` found once `route.open` had put the
473/// route.bind on the module's queue.
474#[derive(Debug, Clone, Copy, PartialEq, Eq)]
475pub(crate) enum RouteBindRelayMark {
476    /// The relay is still pending; later removals handle their own GOODBYE.
477    Marked,
478    /// The module already answered and the reply path settled the relay:
479    /// it committed the route, or it refused it and sent any GOODBYE the
480    /// module needed. `route.open` must send nothing.
481    SettledByReply,
482    /// The relay was dropped without a module reply while the bind was still
483    /// being queued, so nothing has told the module to drop the binding it
484    /// is about to create. `route.open` must send that GOODBYE.
485    ReleasedWithoutReply,
486}
487
488#[derive(Debug, Clone)]
489pub(crate) enum RouteRelease {
490    Removed(GoodbyeTarget),
491    Stale,
492    Absent,
493}
494
495#[derive(Debug, Clone)]
496pub(crate) enum RoutePollSnapshot {
497    Bound {
498        module_id: String,
499        status: Option<String>,
500    },
501    Absent,
502}
503
504#[derive(Debug, Clone)]
505struct ModuleConnection {
506    endpoint: ModuleEndpointId,
507    sink: FrameSink,
508    negotiated_ver: u8,
509    concurrency: Concurrency,
510}
511
512#[derive(Debug, Default)]
513struct ForwardingInner {
514    operator_confirms: Arc<crate::operator_confirm::OperatorConfirms>,
515    daemon_draining: bool,
516    /// Whether forwarding reached quiescence during daemon shutdown, for each
517    /// active endpoint present when shutdown began. A pending wait is false
518    /// because it has not proved quiescence. Endpoint generations prevent a
519    /// result from describing a replacement connection with the same module id.
520    daemon_drain_results: HashMap<ModuleEndpointId, bool>,
521    /// The ACTIVE slot: the one endpoint per module id that routing resolves.
522    /// Every by-id lookup (relay reservation, drain-by-id, liveness, census,
523    /// live roots, module-control RPCs) reads this map and nothing else.
524    modules_by_id: HashMap<String, ModuleConnection>,
525    /// The CANDIDATE slot of a blue/green swap: a second process registered
526    /// under an id that already has an active endpoint. It has a full endpoint
527    /// identity (so its own connection can be looked up and torn down) but no
528    /// by-id lookup sees it, so nothing is routed to it until `cutover_candidate`
529    /// promotes it.
530    candidates_by_id: HashMap<String, ModuleConnection>,
531    /// Former active endpoints demoted by `cutover_candidate`, until their
532    /// connection is removed. Membership is what distinguishes an endpoint that
533    /// was SUPERSEDED by a promotion (still alive, still carrying bound routes,
534    /// about to be drained) from one that is merely STALE (replaced some other
535    /// way, which is treated as a fault on the acking connection).
536    superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
537    endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
538    module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
539    /// Endpoints mid-drain, keyed to the reason the drain was begun with. The
540    /// value serves the census ("draining because restart"); membership alone
541    /// still answers every admission-gate check.
542    draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
543    closing_connections: HashSet<ConnectionId>,
544    next_generation: u64,
545    reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
546    reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
547    next_client_channel: HashMap<ConnectionId, u16>,
548    next_module_channel: HashMap<ModuleEndpointId, u16>,
549    client_slot_epochs: HashMap<ClientRouteKey, u32>,
550    module_slot_epochs: HashMap<ModuleRouteKey, u32>,
551    last_published_epoch: HashMap<ClientRouteKey, u32>,
552    client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
553    module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
554    status: HashMap<(ClientRouteKey, u32), String>,
555    pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
556    /// Pending relays removed without a module reply before `route.open`
557    /// marked them enqueued. `route.open` consumes its key when it marks the
558    /// relay (and sends the GOODBYE the removal could not send safely) or when
559    /// it aborts, so an entry lives only while that open is mid-send.
560    relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
561    /// Each scope's current `(scope_epoch, version)`, keyed `(owner, ref)`,
562    /// published by a scope sync in the same step that changes the record. A
563    /// bind commit reads it here so it never takes the scope table's lock
564    /// while holding this one. Absent means the scope is not live.
565    scope_tags: HashMap<(String, String), ScopeTag>,
566    next_control_corr: HashMap<ModuleEndpointId, u64>,
567    pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
568    health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
569}
570
571#[derive(Debug, Clone)]
572pub(crate) struct CloseReason {
573    code: &'static str,
574    message: String,
575}
576
577impl CloseReason {
578    pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
579        Self {
580            code,
581            message: message.into(),
582        }
583    }
584}
585
586impl fmt::Display for CloseReason {
587    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
588        write!(f, "{}: {}", self.code, self.message)
589    }
590}
591
592pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
593
594/// Dynamic forwarding state shared by the control plane and data-plane router.
595#[derive(Debug, Default)]
596pub struct ForwardingTable {
597    inner: Arc<RwLock<ForwardingInner>>,
598    #[cfg(feature = "test-support")]
599    route_closing_hold: Mutex<Option<oneshot::Receiver<()>>>,
600    close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
601    counters: DaemonCounters,
602    /// Per-target-module bind-relay breaker state. It lives here, beside the
603    /// module connections it describes, because this is where a module
604    /// connection's identity is established and therefore where a stale
605    /// verdict has to be discarded.
606    route_bind_breakers: RouteBindBreakers,
607    /// Current route.bind relays keyed by target module. Admission is shared
608    /// across every client connection that points at the same endpoint.
609    route_bind_concurrency: RouteBindConcurrency,
610    /// Start and end of each module's `route.open` outage. Held here for the
611    /// same reason as the breakers: every handler built over this table must
612    /// see one outage per module, or each would log its own opening line.
613    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
614}
615
616impl ForwardingTable {
617    /// Install scripted operator confirmation for an in-process integration harness,
618    /// without bootstrapping a daemon process. Only available with `test-support`.
619    ///
620    /// The script has one response per line: `approve`, `decline`, `unavailable`,
621    /// `hang` (until withdrawn), or `approve when <path>` (wait for the path to
622    /// exist). Unreadable or exhausted scripts fail closed. If supplied, `events`
623    /// receives the provider's JSON-lines event log. `limits: None` uses
624    /// [`OperatorLimits::default`](crate::operator_confirm::OperatorLimits); use
625    /// `Some(OperatorLimits { queue_wait, prompt_timeout, stuck_grace })` to
626    /// override the queue wait, prompt timeout and stuck-provider grace periods.
627    ///
628    /// The current executable's file name **must start with `ckdev-`**. Otherwise
629    /// this logs a warning and keeps the OS provider, just like bootstrap.
630    /// `cortexkit-test-support`'s `ckdev_binary` copies a harness executable under
631    /// a qualifying name; run that copy to enable scripted confirmation. See
632    /// `crates/subc-client-rs/tests/real_daemon/operator_confirm.rs` for the
633    /// reference end-to-end harness.
634    ///
635    /// Configure the table shared by the harness's `ControlHandler` and `Router`
636    /// before serving any requests. This also configures privileged confirmations
637    /// that use the same table.
638    ///
639    /// # Panics
640    ///
641    /// Panics if operator-confirm processing has already started.
642    ///
643    /// # Example
644    ///
645    /// ```no_run
646    /// use std::{path::Path, sync::Arc};
647    /// use subc_daemon::{ControlHandler, ForwardingTable, Registry};
648    ///
649    /// let registry = Arc::new(Registry::default());
650    /// let forwarding = Arc::new(ForwardingTable::default());
651    /// forwarding.configure_operator_script(
652    ///     Path::new("operator.script"),
653    ///     Some(Path::new("operator.events")),
654    ///     None,
655    /// );
656    /// let handler = ControlHandler::with_forwarding(registry, forwarding);
657    /// ```
658    #[cfg(feature = "test-support")]
659    pub fn configure_operator_script(
660        &self,
661        script: &std::path::Path,
662        events: Option<&std::path::Path>,
663        limits: Option<crate::operator_confirm::OperatorLimits>,
664    ) {
665        self.operator_confirms()
666            .configure_script(script, events, limits);
667    }
668
669    /// Hold the next supervisor drain between enqueueing `module.draining` and
670    /// `route.closing`, so a test can forward a response in that window. Dropping
671    /// the returned sender also releases the hold, including on a test panic.
672    #[cfg(feature = "test-support")]
673    pub fn hold_route_closing_for_test(&self) -> oneshot::Sender<()> {
674        let (release, hold) = oneshot::channel();
675        *self.route_closing_hold.lock().unwrap() = Some(hold);
676        release
677    }
678
679    #[cfg(feature = "test-support")]
680    pub(crate) async fn wait_before_route_closing_for_test(&self) {
681        let hold = self.route_closing_hold.lock().unwrap().take();
682        if let Some(hold) = hold {
683            let _ = hold.await;
684        }
685    }
686
687    pub(crate) fn operator_confirms(&self) -> Arc<crate::operator_confirm::OperatorConfirms> {
688        Arc::clone(
689            &self
690                .inner
691                .read()
692                .unwrap_or_else(|p| p.into_inner())
693                .operator_confirms,
694        )
695    }
696
697    #[cfg(test)]
698    pub(crate) fn inject_operator_principal(&self, key: ModuleRouteKey, principal: Principal) {
699        let mut inner = self.write_inner().unwrap();
700        let old = inner.module_to_client.remove(&key).unwrap();
701        let client = ClientRouteKey {
702            connection_id: old.client_connection_id,
703            channel: old.client_channel,
704        };
705        inner.client_to_module.remove(&client);
706        let mut binding = Arc::try_unwrap(old).expect("test binding has no outstanding readers");
707        binding.principal = principal;
708        let binding = Arc::new(binding);
709        inner.client_to_module.insert(client, Arc::clone(&binding));
710        inner.module_to_client.insert(key, binding);
711    }
712
713    /// Keep the binding protected through confirmation admission. Route release
714    /// takes the write lock and then the same confirm-state lock, so it cannot
715    /// miss an admitted request or leave it waiting on a vanished route.
716    pub(crate) fn with_operator_route<T>(
717        &self,
718        connection_id: ConnectionId,
719        channel: u16,
720        epoch: u32,
721        admit: impl FnOnce(Option<&RouteBinding>) -> T,
722    ) -> Result<T, ForwardingError> {
723        let inner = self.read_inner()?;
724        let binding = inner
725            .endpoint_by_connection
726            .get(&connection_id)
727            .and_then(|endpoint| {
728                inner.module_to_client.get(&ModuleRouteKey {
729                    endpoint: *endpoint,
730                    channel,
731                })
732            })
733            .filter(|binding| binding.module_epoch == epoch);
734        Ok(admit(binding.map(Arc::as_ref)))
735    }
736
737    pub(crate) fn counters(&self) -> DaemonCounters {
738        self.counters.clone()
739    }
740
741    pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
742        self.route_bind_breakers.clone()
743    }
744
745    pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
746        self.route_bind_concurrency.clone()
747    }
748
749    pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
750        Arc::clone(&self.route_outages)
751    }
752
753    pub(crate) fn register_connection_close(
754        &self,
755        connection_id: ConnectionId,
756    ) -> ConnectionCloseReceiver {
757        let (sender, receiver) = oneshot::channel();
758        let replaced = self
759            .lock_close_registry()
760            .insert(connection_id, sender)
761            .is_some();
762        if replaced {
763            warn!(
764                connection_id = connection_id.get(),
765                "replaced existing connection close registration"
766            );
767        }
768        receiver
769    }
770
771    pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
772        self.lock_close_registry().remove(&connection_id);
773    }
774
775    /// Close every established connection, modules and clients alike, as the
776    /// last step of an announced daemon shutdown. By the time this runs each
777    /// registered module has already been sent a module GOODBYE (see
778    /// `Supervisor::end_children_for_daemon_shutdown`), so the EOF that follows
779    /// is a planned stop; EOF with no GOODBYE before it stays reserved for a
780    /// daemon that went away unannounced. Closing while the daemon is still
781    /// alive lets it wait for the modules' own teardowns before it ends
782    /// whatever is left.
783    #[cfg(unix)]
784    pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
785        let senders: Vec<_> = self.lock_close_registry().drain().collect();
786        let count = senders.len();
787        for (_, sender) in senders {
788            let _ = sender.send(reason.clone());
789        }
790        count
791    }
792
793    /// Every registered module connection, whichever slot it occupies (active,
794    /// swap candidate, or superseded incumbent), once each. Daemon shutdown
795    /// uses this to tell each of them it is a planned stop before closing it.
796    #[cfg(unix)]
797    pub(crate) fn module_connections(
798        &self,
799    ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
800        let inner = self.read_inner()?;
801        let mut seen = HashSet::new();
802        Ok(inner
803            .modules_by_id
804            .values()
805            .chain(inner.candidates_by_id.values())
806            .chain(inner.superseded_endpoints.values())
807            .filter(|module| seen.insert(module.endpoint))
808            .map(|module| ModuleConnectionTarget {
809                module_id: inner
810                    .module_id_by_endpoint
811                    .get(&module.endpoint)
812                    .cloned()
813                    .unwrap_or_default(),
814                endpoint: module.endpoint,
815                sink: module.sink.clone(),
816                negotiated_ver: module.negotiated_ver,
817            })
818            .collect())
819    }
820
821    /// Ask a registered connection to close. Returns true only for the request
822    /// that actually reached it; later requests for the same connection, and
823    /// requests for a connection that is not registered, return false.
824    pub(crate) fn request_connection_close(
825        &self,
826        connection_id: ConnectionId,
827        reason: CloseReason,
828    ) -> bool {
829        let sender = self.lock_close_registry().remove(&connection_id);
830        if let Some(sender) = sender {
831            debug!(
832                connection_id = connection_id.get(),
833                close_reason = %reason,
834                "requesting connection close"
835            );
836            let _ = sender.send(reason);
837            true
838        } else {
839            debug!(
840                connection_id = connection_id.get(),
841                close_reason = %reason,
842                "connection close request ignored for inactive connection"
843            );
844            false
845        }
846    }
847
848    pub fn register_module_connection(
849        &self,
850        connection_id: ConnectionId,
851        module_id: String,
852        negotiated_ver: u8,
853        concurrency: Concurrency,
854        sink: FrameSink,
855    ) -> Result<ModuleEndpointId, ForwardingError> {
856        self.register_module_connection_inner(
857            connection_id,
858            module_id,
859            negotiated_ver,
860            concurrency,
861            sink,
862            None,
863        )
864    }
865
866    /// Register a module connection and queue its HELLO_ACK as the first frame
867    /// on its sink, in the same write-lock critical section that makes the
868    /// endpoint visible.
869    ///
870    /// A module reads HELLO_ACK as the first frame after its HELLO and exits on
871    /// anything else. Once the endpoint is in `modules_by_id`, a `route.open`
872    /// on any other connection can queue a `route.bind` request onto this sink,
873    /// so the ack must already be queued by then. Every lookup that can queue a
874    /// frame for the module takes this lock, so nothing can get in ahead of it.
875    ///
876    /// If the ack cannot be queued (the sink is closed or full), the
877    /// registration fails with nothing inserted.
878    pub(crate) fn register_module_connection_acked(
879        &self,
880        connection_id: ConnectionId,
881        module_id: String,
882        negotiated_ver: u8,
883        concurrency: Concurrency,
884        sink: FrameSink,
885        hello_ack: Frame,
886    ) -> Result<ModuleEndpointId, ForwardingError> {
887        self.register_module_connection_inner(
888            connection_id,
889            module_id,
890            negotiated_ver,
891            concurrency,
892            sink,
893            Some(hello_ack),
894        )
895    }
896
897    fn register_module_connection_inner(
898        &self,
899        connection_id: ConnectionId,
900        module_id: String,
901        negotiated_ver: u8,
902        concurrency: Concurrency,
903        sink: FrameSink,
904        hello_ack: Option<Frame>,
905    ) -> Result<ModuleEndpointId, ForwardingError> {
906        let mut inner = self.write_inner()?;
907        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
908            return Err(ForwardingError::ConnectionClosing { connection_id });
909        }
910        check_module_connection_role_locked(&inner, connection_id)?;
911        // Every refusal check has passed and nothing has been mutated yet, so
912        // a failed enqueue leaves the table exactly as it was.
913        enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
914
915        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
916        let endpoint = ModuleEndpointId {
917            connection_id,
918            generation: inner.next_generation,
919        };
920        inner.endpoint_by_connection.insert(connection_id, endpoint);
921        inner
922            .module_id_by_endpoint
923            .insert(endpoint, module_id.clone());
924        inner.next_module_channel.insert(endpoint, 1);
925        inner.next_control_corr.insert(endpoint, 1);
926        inner.modules_by_id.insert(
927            module_id.clone(),
928            ModuleConnection {
929                endpoint,
930                sink,
931                negotiated_ver,
932                concurrency,
933            },
934        );
935        drop(inner);
936
937        // A new module connection has arrived under this id, so anything the
938        // bind-relay breaker learned was learned about a process that is no
939        // longer the one behind this name. See
940        // `RouteBindBreakers::reset_for_new_module_connection`.
941        //
942        // Keyed on ARRIVAL rather than on teardown deliberately: a connection
943        // going away is not evidence about anything, and a module whose
944        // connection drops without coming back should keep its verdict until
945        // something actually registers in its place. This also covers the
946        // unclean replacements -- a module killed mid-bind, or one whose
947        // connection was closing -- because registration is the single path by
948        // which any module connection becomes usable.
949        if let Some(discarded) = self
950            .route_bind_breakers
951            .reset_for_new_module_connection(&module_id)
952        {
953            info!(
954                module_id = %module_id,
955                discarded_consecutive_timeouts = discarded,
956                "route.bind breaker state discarded: a new module connection replaced the process it described"
957            );
958        }
959        Ok(endpoint)
960    }
961
962    /// Register a blue/green swap candidate for `module_id` into the candidate
963    /// slot, alongside whatever endpoint is active for the id.
964    ///
965    /// Unlike [`Self::register_module_connection`], this never touches the
966    /// active slot and never resets the bind-relay breaker: the incumbent is
967    /// still the process serving the id, so what the breaker learned about it
968    /// is still true. The breaker is reset at cutover instead, when the process
969    /// behind the name actually changes. A second candidate for the same id is
970    /// refused.
971    ///
972    /// Production registers candidates through
973    /// [`Self::register_candidate_module_connection_acked`]; this ack-less form
974    /// is for tests that build forwarding state directly.
975    #[cfg(test)]
976    pub(crate) fn register_candidate_module_connection(
977        &self,
978        connection_id: ConnectionId,
979        module_id: String,
980        negotiated_ver: u8,
981        concurrency: Concurrency,
982        sink: FrameSink,
983    ) -> Result<ModuleEndpointId, ForwardingError> {
984        self.register_candidate_module_connection_inner(
985            connection_id,
986            module_id,
987            negotiated_ver,
988            concurrency,
989            sink,
990            None,
991        )
992    }
993
994    /// The swap-candidate counterpart of
995    /// [`Self::register_module_connection_acked`]: the HELLO_ACK is queued on
996    /// the candidate's sink before its endpoint is inserted. No by-id lookup
997    /// sees a candidate, but its own connection's endpoint does become
998    /// resolvable here, and at cutover it becomes routable; the ack has to be
999    /// ahead of anything either can queue.
1000    pub(crate) fn register_candidate_module_connection_acked(
1001        &self,
1002        connection_id: ConnectionId,
1003        module_id: String,
1004        negotiated_ver: u8,
1005        concurrency: Concurrency,
1006        sink: FrameSink,
1007        hello_ack: Frame,
1008    ) -> Result<ModuleEndpointId, ForwardingError> {
1009        self.register_candidate_module_connection_inner(
1010            connection_id,
1011            module_id,
1012            negotiated_ver,
1013            concurrency,
1014            sink,
1015            Some(hello_ack),
1016        )
1017    }
1018
1019    fn register_candidate_module_connection_inner(
1020        &self,
1021        connection_id: ConnectionId,
1022        module_id: String,
1023        negotiated_ver: u8,
1024        concurrency: Concurrency,
1025        sink: FrameSink,
1026        hello_ack: Option<Frame>,
1027    ) -> Result<ModuleEndpointId, ForwardingError> {
1028        let mut inner = self.write_inner()?;
1029        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1030            return Err(ForwardingError::ConnectionClosing { connection_id });
1031        }
1032        check_module_connection_role_locked(&inner, connection_id)?;
1033        if inner.candidates_by_id.contains_key(&module_id) {
1034            return Err(ForwardingError::CandidateSlotOccupied { module_id });
1035        }
1036        // As in `register_module_connection_inner`: after every refusal check,
1037        // before any mutation.
1038        enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1039
1040        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1041        let endpoint = ModuleEndpointId {
1042            connection_id,
1043            generation: inner.next_generation,
1044        };
1045        inner.endpoint_by_connection.insert(connection_id, endpoint);
1046        inner
1047            .module_id_by_endpoint
1048            .insert(endpoint, module_id.clone());
1049        inner.next_module_channel.insert(endpoint, 1);
1050        inner.next_control_corr.insert(endpoint, 1);
1051        inner.candidates_by_id.insert(
1052            module_id,
1053            ModuleConnection {
1054                endpoint,
1055                sink,
1056                negotiated_ver,
1057                concurrency,
1058            },
1059        );
1060        Ok(endpoint)
1061    }
1062
1063    /// Promote `module_id`'s candidate to the active slot, in ONE forwarding
1064    /// write-lock critical section.
1065    ///
1066    /// That single section is the linearization point of a swap. Every relay
1067    /// reservation resolves the active slot under the same lock, so each one
1068    /// lands wholly before cutover (on the incumbent) or wholly after it (on the
1069    /// promoted candidate), never on a mix. A relay reserved on the incumbent
1070    /// before cutover can no longer commit: `commit_route_locked` requires the
1071    /// reservation's endpoint to be the active one, and an ack for it arriving
1072    /// later is answered by the superseded-endpoint arm of
1073    /// `complete_pending_relay` instead of ending the incumbent's connection.
1074    ///
1075    /// Both endpoints keep their identities: nothing is re-keyed, so the
1076    /// incumbent's bound routes and its pending correlation keys stay exactly
1077    /// where they are until it is drained with [`Self::begin_endpoint_drain`],
1078    /// using the incumbent endpoint this returns. Returns `Ok(None)` when there
1079    /// is no candidate for the id.
1080    pub(crate) fn cutover_candidate(
1081        &self,
1082        module_id: &str,
1083    ) -> Result<Option<ForwardingCutover>, ForwardingError> {
1084        let mut inner = self.write_inner()?;
1085        if inner.daemon_draining {
1086            return Err(ForwardingError::ModuleReloading {
1087                module_id: module_id.to_string(),
1088            });
1089        }
1090        let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
1091            return Ok(None);
1092        };
1093        let promoted = candidate.endpoint;
1094        let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
1095        let incumbent = incumbent.map(|incumbent| {
1096            let endpoint = incumbent.endpoint;
1097            inner.superseded_endpoints.insert(endpoint, incumbent);
1098            endpoint
1099        });
1100        drop(inner);
1101
1102        // A different process now answers for this id; see the matching reset
1103        // in `register_module_connection` for why a verdict about the old one
1104        // must not carry over.
1105        if let Some(discarded) = self
1106            .route_bind_breakers
1107            .reset_for_new_module_connection(module_id)
1108        {
1109            info!(
1110                module_id = %module_id,
1111                discarded_consecutive_timeouts = discarded,
1112                "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
1113            );
1114        }
1115        Ok(Some(ForwardingCutover {
1116            promoted,
1117            incumbent,
1118        }))
1119    }
1120
1121    #[allow(clippy::too_many_arguments)]
1122    pub(crate) async fn begin_route_bind_relay_for(
1123        &self,
1124        client_connection_id: ConnectionId,
1125        client_sink: FrameSink,
1126        client_negotiated_ver: u8,
1127        client_corr: u64,
1128        module_id: &str,
1129        principal: Principal,
1130        scope: Option<BoundScope>,
1131        project_root: Option<ProjectRootId>,
1132        deadline: Instant,
1133    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1134        // Reserve egress capacity before taking the forwarding lock. The permit is
1135        // held until the bind reaches one terminal state, so an accepted bind can
1136        // publish its table entry and RouteOpen response in one critical section.
1137        let client_permit =
1138            client_sink
1139                .reserve_owned()
1140                .await
1141                .map_err(|_| ForwardingError::ClientEgressClosed {
1142                    connection_id: client_connection_id,
1143                })?;
1144        self.begin_route_bind_relay_inner(
1145            client_connection_id,
1146            client_sink,
1147            client_negotiated_ver,
1148            client_corr,
1149            module_id,
1150            principal,
1151            scope,
1152            project_root,
1153            deadline,
1154            client_permit,
1155        )
1156    }
1157
1158    #[cfg(test)]
1159    pub(crate) fn begin_route_bind_relay_for_test(
1160        &self,
1161        client_connection_id: ConnectionId,
1162        client_sink: FrameSink,
1163        client_corr: u64,
1164        module_id: &str,
1165    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1166        let permit =
1167            client_sink
1168                .try_reserve_owned()
1169                .map_err(|_| ForwardingError::ClientEgressClosed {
1170                    connection_id: client_connection_id,
1171                })?;
1172        self.begin_route_bind_relay_inner(
1173            client_connection_id,
1174            client_sink,
1175            subc_protocol::PROTOCOL_VERSION,
1176            client_corr,
1177            module_id,
1178            Principal::Direct,
1179            None,
1180            None,
1181            Instant::now() + std::time::Duration::from_secs(60),
1182            permit,
1183        )
1184    }
1185
1186    pub(crate) fn begin_module_control_rpc_for(
1187        &self,
1188        module_id: &str,
1189        expected_op: &str,
1190        deadline: Instant,
1191    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1192        self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1193    }
1194
1195    pub(crate) fn begin_health_probe_rpc_for(
1196        &self,
1197        module_id: &str,
1198        expected_op: &str,
1199        probe_started_at: Instant,
1200        deadline: Instant,
1201    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1202        self.begin_module_control_rpc_inner(
1203            module_id,
1204            expected_op,
1205            deadline,
1206            Some(probe_started_at),
1207            false,
1208        )
1209    }
1210
1211    pub(crate) fn begin_drain_health_probe_rpc_for(
1212        &self,
1213        module_id: &str,
1214        expected_op: &str,
1215        probe_started_at: Instant,
1216        deadline: Instant,
1217    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1218        self.begin_module_control_rpc_inner(
1219            module_id,
1220            expected_op,
1221            deadline,
1222            Some(probe_started_at),
1223            true,
1224        )
1225    }
1226
1227    /// A health probe addressed to one endpoint, in whichever slot it is.
1228    ///
1229    /// Every other control RPC resolves the id's ACTIVE endpoint, which is how
1230    /// a swap candidate (not active until cutover) and a superseded incumbent
1231    /// (not active after it) are unreachable by them. A swap needs to probe
1232    /// exactly those two: the candidate before promoting it, and the incumbent
1233    /// for its busy gauges while it drains.
1234    pub(crate) fn begin_endpoint_health_probe_rpc_for(
1235        &self,
1236        endpoint: ModuleEndpointId,
1237        expected_op: &str,
1238        probe_started_at: Instant,
1239        deadline: Instant,
1240    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1241        let inner = self.write_inner()?;
1242        let module = module_connection_for_endpoint_locked(&inner, endpoint)
1243            .cloned()
1244            .ok_or(ForwardingError::NoModuleConnection)?;
1245        let module_id = inner
1246            .module_id_by_endpoint
1247            .get(&endpoint)
1248            .cloned()
1249            .unwrap_or_default();
1250        // A draining incumbent is exactly what this probes for busy gauges, so
1251        // draining is allowed, as it is for the by-id drain probe.
1252        self.begin_control_rpc_locked(
1253            inner,
1254            &module_id,
1255            module,
1256            expected_op,
1257            deadline,
1258            Some(probe_started_at),
1259            true,
1260        )
1261    }
1262
1263    fn begin_module_control_rpc_inner(
1264        &self,
1265        module_id: &str,
1266        expected_op: &str,
1267        deadline: Instant,
1268        health_probe_started_at: Option<Instant>,
1269        allow_draining: bool,
1270    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1271        let inner = self.write_inner()?;
1272        let module = inner
1273            .modules_by_id
1274            .get(module_id)
1275            .cloned()
1276            .ok_or(ForwardingError::NoModuleConnection)?;
1277        self.begin_control_rpc_locked(
1278            inner,
1279            module_id,
1280            module,
1281            expected_op,
1282            deadline,
1283            health_probe_started_at,
1284            allow_draining,
1285        )
1286    }
1287
1288    #[allow(clippy::too_many_arguments)]
1289    fn begin_control_rpc_locked(
1290        &self,
1291        mut inner: RwLockWriteGuard<'_, ForwardingInner>,
1292        module_id: &str,
1293        module: ModuleConnection,
1294        expected_op: &str,
1295        deadline: Instant,
1296        health_probe_started_at: Option<Instant>,
1297        allow_draining: bool,
1298    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1299        if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1300            return Err(ForwardingError::ModuleReloading {
1301                module_id: module_id.to_string(),
1302            });
1303        }
1304        if inner
1305            .closing_connections
1306            .contains(&module.endpoint.connection_id)
1307        {
1308            return Err(ForwardingError::ConnectionClosing {
1309                connection_id: module.endpoint.connection_id,
1310            });
1311        }
1312        if health_probe_started_at.is_some() {
1313            // Recurring health probes are serial per endpoint. Once the next one
1314            // starts, an older answer can no longer improve the current snapshot,
1315            // so retaining more than the newest unanswered probe has no value.
1316            inner
1317                .health_probe_tombstones
1318                .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1319        }
1320        let corr = match inner.allocate_control_corr(module.endpoint) {
1321            Ok(corr) => corr,
1322            Err(err) => {
1323                drop(inner);
1324                self.request_connection_close(
1325                    module.endpoint.connection_id,
1326                    CloseReason::new(
1327                        "control_correlation_exhausted",
1328                        "daemon-originated channel-0 correlation space exhausted",
1329                    ),
1330                );
1331                return Err(err);
1332            }
1333        };
1334        let (sender, receiver) = oneshot::channel();
1335        inner.pending_control_rpcs.insert(
1336            (module.endpoint, corr),
1337            PendingModuleControlRpcEntry {
1338                expected_op: expected_op.to_string(),
1339                deadline,
1340                health_probe_started_at,
1341                sender,
1342            },
1343        );
1344
1345        Ok(PendingModuleControlRpc {
1346            endpoint: module.endpoint,
1347            module_sink: module.sink,
1348            negotiated_ver: module.negotiated_ver,
1349            corr,
1350            receiver,
1351        })
1352    }
1353
1354    #[allow(clippy::too_many_arguments)]
1355    fn begin_route_bind_relay_inner(
1356        &self,
1357        client_connection_id: ConnectionId,
1358        client_sink: FrameSink,
1359        client_negotiated_ver: u8,
1360        client_corr: u64,
1361        expected_module_id: &str,
1362        principal: Principal,
1363        scope: Option<BoundScope>,
1364        project_root: Option<ProjectRootId>,
1365        deadline: Instant,
1366        client_permit: crate::router::EgressPermit,
1367    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1368        let mut inner = self.write_inner()?;
1369        // HELLO and route.open can run on different tasks. Decide the role under
1370        // the same lock that installs endpoints and reserves client channels so
1371        // neither task can turn one connection into both kinds of owner.
1372        if inner
1373            .endpoint_by_connection
1374            .contains_key(&client_connection_id)
1375        {
1376            return Err(ForwardingError::ConnectionRoleConflict {
1377                connection_id: client_connection_id,
1378            });
1379        }
1380        if inner.closing_connections.contains(&client_connection_id) {
1381            return Err(ForwardingError::ConnectionClosing {
1382                connection_id: client_connection_id,
1383            });
1384        }
1385        let module = inner
1386            .modules_by_id
1387            .get(expected_module_id)
1388            .cloned()
1389            .ok_or(ForwardingError::NoModuleConnection)?;
1390        if inner.draining_endpoints.contains_key(&module.endpoint) {
1391            return Err(ForwardingError::ModuleReloading {
1392                module_id: expected_module_id.to_string(),
1393            });
1394        }
1395        if inner
1396            .closing_connections
1397            .contains(&module.endpoint.connection_id)
1398        {
1399            return Err(ForwardingError::ConnectionClosing {
1400                connection_id: module.endpoint.connection_id,
1401            });
1402        }
1403
1404        let corr = match inner.allocate_control_corr(module.endpoint) {
1405            Ok(corr) => corr,
1406            Err(err) => {
1407                drop(inner);
1408                self.request_connection_close(
1409                    module.endpoint.connection_id,
1410                    CloseReason::new(
1411                        "control_correlation_exhausted",
1412                        "daemon-originated channel-0 correlation space exhausted",
1413                    ),
1414                );
1415                return Err(err);
1416            }
1417        };
1418        let (client_channel, client_epoch, module_channel, module_epoch) =
1419            inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1420        let client_key = ClientRouteKey {
1421            connection_id: client_connection_id,
1422            channel: client_channel,
1423        };
1424        let module_key = ModuleRouteKey {
1425            endpoint: module.endpoint,
1426            channel: module_channel,
1427        };
1428        let reservation = RouteReservation {
1429            client_key,
1430            module_key,
1431            client_epoch,
1432            module_epoch,
1433            project_root,
1434        };
1435        let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
1436            route_channel: client_channel,
1437            route_epoch: client_epoch,
1438        })
1439        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1440        let route_open_frame = Frame::build_with_version(
1441            client_negotiated_ver,
1442            FrameType::Response,
1443            Flags::new(false, Priority::Passive, false),
1444            0,
1445            0,
1446            client_corr,
1447            response_body,
1448        )
1449        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1450        let (sender, receiver) = oneshot::channel();
1451        inner.reserved_client.insert(client_key, module_key);
1452        inner.reserved_module.insert(module_key, client_key);
1453        inner.pending_relays.insert(
1454            (module.endpoint, corr),
1455            PendingRouteBindRelayEntry {
1456                reservation,
1457                client_sink,
1458                client_negotiated_ver,
1459                client_permit,
1460                route_open_frame,
1461                principal,
1462                scope,
1463                deadline,
1464                relay_enqueued: false,
1465                sender,
1466            },
1467        );
1468
1469        Ok(PendingRouteBindRelay {
1470            endpoint: module.endpoint,
1471            module_sink: module.sink,
1472            negotiated_ver: module.negotiated_ver,
1473            client_channel,
1474            client_epoch,
1475            module_channel,
1476            module_epoch,
1477            corr,
1478            receiver,
1479        })
1480    }
1481
1482    /// Called by `route.open` right after the route.bind entered the module's
1483    /// egress queue.
1484    ///
1485    /// The module reader runs concurrently with `route.open`, so a fast module
1486    /// can receive the bind, ack it, and have the reader commit the route
1487    /// before this runs. A missing entry therefore does not mean the bind was
1488    /// abandoned: only a removal recorded in `relays_released_before_enqueue`
1489    /// does. Telling the two apart is what stops `route.open` from sending a
1490    /// GOODBYE for a route that was just committed.
1491    pub(crate) fn mark_route_bind_relay_enqueued(
1492        &self,
1493        endpoint: ModuleEndpointId,
1494        corr: u64,
1495    ) -> Result<RouteBindRelayMark, ForwardingError> {
1496        let mut inner = self.write_inner()?;
1497        if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
1498            pending.relay_enqueued = true;
1499            return Ok(RouteBindRelayMark::Marked);
1500        }
1501        if inner
1502            .relays_released_before_enqueue
1503            .remove(&(endpoint, corr))
1504        {
1505            return Ok(RouteBindRelayMark::ReleasedWithoutReply);
1506        }
1507        Ok(RouteBindRelayMark::SettledByReply)
1508    }
1509
1510    pub(crate) fn release_client_route(
1511        &self,
1512        client_connection_id: ConnectionId,
1513        client_channel: u16,
1514        expected_epoch: u32,
1515    ) -> Result<RouteRelease, ForwardingError> {
1516        let mut inner = self.write_inner()?;
1517        let release = release_client_route_locked(
1518            &mut inner,
1519            ClientRouteKey {
1520                connection_id: client_connection_id,
1521                channel: client_channel,
1522            },
1523            expected_epoch,
1524        );
1525        self.record_route_release(&release);
1526        Ok(release)
1527    }
1528
1529    pub(crate) fn release_module_route(
1530        &self,
1531        module_connection_id: ConnectionId,
1532        module_channel: u16,
1533        expected_epoch: u32,
1534    ) -> Result<RouteRelease, ForwardingError> {
1535        let mut inner = self.write_inner()?;
1536        let Some(endpoint) = inner
1537            .endpoint_by_connection
1538            .get(&module_connection_id)
1539            .copied()
1540        else {
1541            return Ok(RouteRelease::Absent);
1542        };
1543        let release = release_module_route_locked(
1544            &mut inner,
1545            ModuleRouteKey {
1546                endpoint,
1547                channel: module_channel,
1548            },
1549            expected_epoch,
1550        );
1551        self.record_route_release(&release);
1552        Ok(release)
1553    }
1554
1555    pub(crate) fn abort_pending_relay(
1556        &self,
1557        endpoint: ModuleEndpointId,
1558        corr: u64,
1559        outcome: RouteBindRelayOutcome,
1560    ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
1561        let mut inner = self.write_inner()?;
1562        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1563            // If another path dropped the relay while `route.open` was still
1564            // sending the bind, and that open is now giving up without marking
1565            // it (its send failed or it was cancelled), forget the record so it
1566            // cannot outlive the open.
1567            inner
1568                .relays_released_before_enqueue
1569                .remove(&(endpoint, corr));
1570            return Ok(None);
1571        };
1572        release_reserved_route_locked(
1573            &mut inner,
1574            pending.reservation.client_key,
1575            pending.reservation.module_key,
1576        );
1577        let target = pending
1578            .relay_enqueued
1579            .then(|| abandoned_route_target(&inner, &pending.reservation));
1580        let _ = pending.sender.send(outcome);
1581        Ok(target.flatten())
1582    }
1583
1584    pub(crate) fn cancel_module_control_rpc(
1585        &self,
1586        endpoint: ModuleEndpointId,
1587        corr: u64,
1588    ) -> Result<(), ForwardingError> {
1589        self.write_inner()?
1590            .pending_control_rpcs
1591            .remove(&(endpoint, corr));
1592        Ok(())
1593    }
1594
1595    pub(crate) fn tombstone_health_probe_rpc(
1596        &self,
1597        endpoint: ModuleEndpointId,
1598        corr: u64,
1599    ) -> Result<bool, ForwardingError> {
1600        let key = (endpoint, corr);
1601        let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
1602        {
1603            let mut inner = self.write_inner()?;
1604            let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
1605                return Ok(false);
1606            };
1607            let Some(probe_started_at) = pending.health_probe_started_at else {
1608                inner.pending_control_rpcs.insert(key, pending);
1609                return Ok(false);
1610            };
1611            let module_id = inner
1612                .module_id_by_endpoint
1613                .get(&endpoint)
1614                .cloned()
1615                .unwrap_or_else(|| "unknown".to_string());
1616            inner.health_probe_tombstones.insert(
1617                key,
1618                HealthProbeTombstone {
1619                    expected_op: pending.expected_op,
1620                    module_id,
1621                    probe_started_at,
1622                    expires_at,
1623                },
1624            );
1625        }
1626        self.schedule_health_probe_tombstone_expiration(key, expires_at);
1627        Ok(true)
1628    }
1629
1630    fn schedule_health_probe_tombstone_expiration(
1631        &self,
1632        key: (ModuleEndpointId, u64),
1633        expires_at: Instant,
1634    ) {
1635        let inner = Arc::downgrade(&self.inner);
1636        tokio::spawn(async move {
1637            tokio::time::sleep_until(expires_at).await;
1638            let Some(inner) = inner.upgrade() else {
1639                return;
1640            };
1641            let Ok(mut inner) = inner.write() else {
1642                return;
1643            };
1644            let expired = inner
1645                .health_probe_tombstones
1646                .get(&key)
1647                .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
1648            if expired {
1649                inner.health_probe_tombstones.remove(&key);
1650            }
1651        });
1652    }
1653
1654    pub(crate) fn complete_pending_relay(
1655        &self,
1656        connection_id: ConnectionId,
1657        corr: u64,
1658        outcome: RouteBindRelayOutcome,
1659    ) -> Result<PendingRelayCompletion, ForwardingError> {
1660        let mut inner = self.write_inner()?;
1661        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1662            return Ok(PendingRelayCompletion {
1663                settled: false,
1664                abandoned: None,
1665            });
1666        };
1667        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1668            return Ok(PendingRelayCompletion {
1669                settled: false,
1670                abandoned: None,
1671            });
1672        };
1673
1674        if Instant::now() >= pending.deadline {
1675            release_reserved_route_locked(
1676                &mut inner,
1677                pending.reservation.client_key,
1678                pending.reservation.module_key,
1679            );
1680            let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
1681                .then(|| abandoned_route_target(&inner, &pending.reservation))
1682                .flatten();
1683            let _ = pending
1684                .sender
1685                .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody {
1686                    code: "module_timeout".to_string(),
1687                    message: "route.bind response arrived after its daemon deadline".to_string(),
1688                    detail: None,
1689                }));
1690            return Ok(PendingRelayCompletion {
1691                settled: true,
1692                abandoned,
1693            });
1694        }
1695
1696        match outcome {
1697            // Two shapes of "the client is not there to receive this route" that
1698            // must resolve identically: its egress is already closed, or it is
1699            // marked closing (its connection loop has been asked to end, but has
1700            // not drained yet, so the sink is still open).
1701            //
1702            // Only the first used to be caught here. The second fell through to
1703            // `commit_route_locked`, which refuses a closing client with
1704            // `ConnectionClosing` -- and this function is called from the MODULE
1705            // connection's frame handler, so that refusal ended the module's
1706            // connection instead of this one client's route. One dying client's
1707            // route.open then took down a connection carrying every other
1708            // client's routes to that module.
1709            //
1710            // The remedy for both is the same, which is why they share an arm:
1711            // give back the reserved handle pair, tell the waiting route.open the
1712            // route is gone, and report the module-side channel so the caller can
1713            // send a channel-scoped GOODBYE for the binding the module just
1714            // created. Nothing here touches the module connection.
1715            RouteBindRelayOutcome::Accepted
1716                if pending.client_sink.is_closed()
1717                    || inner
1718                        .closing_connections
1719                        .contains(&pending.reservation.client_key.connection_id) =>
1720            {
1721                let reason = if pending.client_sink.is_closed() {
1722                    "client egress closed before route publication"
1723                } else {
1724                    "client connection is closing before route publication"
1725                };
1726                release_reserved_route_locked(
1727                    &mut inner,
1728                    pending.reservation.client_key,
1729                    pending.reservation.module_key,
1730                );
1731                // Not gated on `relay_enqueued`: an ack proves the module
1732                // received the bind, and `route.open` sends nothing for a
1733                // relay a reply already settled, so this is the only GOODBYE.
1734                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1735                let _ = pending
1736                    .sender
1737                    .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
1738                return Ok(PendingRelayCompletion {
1739                    settled: true,
1740                    abandoned,
1741                });
1742            }
1743            // The endpoint that acked this bind is no longer accepting new
1744            // routes, for one of two reasons. Either it was the module's active
1745            // endpoint when this relay was reserved and a blue/green swap has
1746            // since promoted a candidate (the replacement process registered
1747            // beside it) into the active slot, superseding it; or a daemon
1748            // drain has gated it, and the ack arrived before the ordered module
1749            // drain settled pending relays. The module did nothing wrong: it
1750            // bound the route it was asked to bind, and the endpoint still
1751            // carries every other client's routes until it is drained. So the
1752            // ack is settled here, while the pending entry still holds the
1753            // client's sender and the reserved handle pair: release the pair,
1754            // refuse the waiting route.open with `module_reloading`, which
1755            // callers retry (after a swap the retry reserves on the promoted
1756            // endpoint), and hand back the module-side channel so the caller
1757            // sends one channel-scoped GOODBYE for the binding the module just
1758            // created. Returning an error instead would end the module
1759            // connection all those other routes share; nothing here touches it.
1760            //
1761            // An endpoint that stopped being active any other way is stale
1762            // (replaced, but not by a swap and not draining). It still falls
1763            // through to `commit_route_locked`, which refuses it with
1764            // `StaleModuleEndpoint` exactly as before swaps existed.
1765            RouteBindRelayOutcome::Accepted
1766                if inner.superseded_endpoints.contains_key(&endpoint)
1767                    || inner.draining_endpoints.contains_key(&endpoint) =>
1768            {
1769                release_reserved_route_locked(
1770                    &mut inner,
1771                    pending.reservation.client_key,
1772                    pending.reservation.module_key,
1773                );
1774                // Not gated on `relay_enqueued`: an ack proves the module
1775                // received the bind, whether or not the enqueue mark was set.
1776                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1777                let module_id = inner
1778                    .module_id_by_endpoint
1779                    .get(&endpoint)
1780                    .cloned()
1781                    .unwrap_or_else(|| "unknown".to_string());
1782                let _ = pending
1783                    .sender
1784                    .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
1785                        "module_reloading",
1786                        format!("module_id '{module_id}' is reloading"),
1787                    )));
1788                return Ok(PendingRelayCompletion {
1789                    settled: true,
1790                    abandoned,
1791                });
1792            }
1793            // The scope the open was admitted under ended or changed after
1794            // admission. The module bound a route under a stamp that is no
1795            // longer true, so it must not become routable. Settled here like
1796            // the superseded arm above, never as an error from
1797            // `commit_route_locked`, which would close the module's whole
1798            // connection and every other client's routes to it: release the
1799            // pair, answer the waiting route.open by name, and hand back the
1800            // module-side channel for one channel-scoped GOODBYE.
1801            RouteBindRelayOutcome::Accepted
1802                if pending
1803                    .scope
1804                    .as_ref()
1805                    .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
1806            {
1807                let refusal = pending
1808                    .scope
1809                    .as_ref()
1810                    .and_then(|scope| scope_refusal_locked(&inner, scope))
1811                    .expect("guard matched a refusal under the same lock");
1812                release_reserved_route_locked(
1813                    &mut inner,
1814                    pending.reservation.client_key,
1815                    pending.reservation.module_key,
1816                );
1817                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1818                let _ = pending
1819                    .sender
1820                    .send(RouteBindRelayOutcome::DaemonRejected(refusal));
1821                return Ok(PendingRelayCompletion {
1822                    settled: true,
1823                    abandoned,
1824                });
1825            }
1826            RouteBindRelayOutcome::Accepted => {
1827                let abandoned = commit_route_locked(&mut inner, pending)?;
1828                return Ok(PendingRelayCompletion {
1829                    settled: true,
1830                    abandoned,
1831                });
1832            }
1833            terminal => {
1834                release_reserved_route_locked(
1835                    &mut inner,
1836                    pending.reservation.client_key,
1837                    pending.reservation.module_key,
1838                );
1839                let _ = pending.sender.send(terminal);
1840            }
1841        }
1842        Ok(PendingRelayCompletion {
1843            settled: true,
1844            abandoned: None,
1845        })
1846    }
1847
1848    pub(crate) fn pending_module_control_op(
1849        &self,
1850        connection_id: ConnectionId,
1851        corr: u64,
1852    ) -> Result<Option<String>, ForwardingError> {
1853        let inner = self.read_inner()?;
1854        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1855            return Ok(None);
1856        };
1857        let key = (endpoint, corr);
1858        Ok(inner
1859            .pending_control_rpcs
1860            .get(&key)
1861            .map(|pending| pending.expected_op.clone())
1862            .or_else(|| {
1863                inner
1864                    .health_probe_tombstones
1865                    .get(&key)
1866                    .filter(|tombstone| tombstone.expires_at > Instant::now())
1867                    .map(|tombstone| tombstone.expected_op.clone())
1868            }))
1869    }
1870
1871    pub(crate) fn complete_module_control_rpc(
1872        &self,
1873        connection_id: ConnectionId,
1874        corr: u64,
1875        actual_op: Option<&str>,
1876        outcome: ModuleControlRpcOutcome,
1877    ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
1878        let now = Instant::now();
1879        let mut inner = self.write_inner()?;
1880        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1881            return Ok(ModuleControlRpcCompletion::Unknown);
1882        };
1883        let key = (endpoint, corr);
1884        if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
1885            if now >= pending.deadline {
1886                let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
1887                    ModuleControlRpcCompletion::LateHealthAnswer {
1888                        module_id: inner
1889                            .module_id_by_endpoint
1890                            .get(&endpoint)
1891                            .cloned()
1892                            .unwrap_or_else(|| "unknown".to_string()),
1893                        latency: now.saturating_duration_since(probe_started_at),
1894                    }
1895                });
1896                let _ = pending
1897                    .sender
1898                    .send(ModuleControlRpcOutcome::DeadlineElapsed);
1899                return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
1900            }
1901            let outcome = match actual_op {
1902                Some(actual) if actual != pending.expected_op => {
1903                    ModuleControlRpcOutcome::UnexpectedOp {
1904                        expected: pending.expected_op,
1905                        actual: actual.to_string(),
1906                    }
1907                }
1908                _ => outcome,
1909            };
1910            let _ = pending.sender.send(outcome);
1911            return Ok(ModuleControlRpcCompletion::Settled);
1912        }
1913
1914        let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
1915            return Ok(ModuleControlRpcCompletion::Unknown);
1916        };
1917        if tombstone.expires_at <= now {
1918            return Ok(ModuleControlRpcCompletion::Unknown);
1919        }
1920        Ok(ModuleControlRpcCompletion::LateHealthAnswer {
1921            module_id: tombstone.module_id,
1922            latency: now.saturating_duration_since(tombstone.probe_started_at),
1923        })
1924    }
1925
1926    #[cfg(test)]
1927    pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
1928        Ok(self.read_inner()?.health_probe_tombstones.len())
1929    }
1930
1931    #[cfg(test)]
1932    pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
1933        Ok(self.read_inner()?.closing_connections.len())
1934    }
1935
1936    /// Reserved-but-uncommitted route handles, counted on both index sides, so
1937    /// a test can assert a reservation pair was actually given back.
1938    #[cfg(test)]
1939    pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
1940        let inner = self.read_inner()?;
1941        Ok((inner.reserved_client.len(), inner.reserved_module.len()))
1942    }
1943
1944    pub(crate) fn module_endpoint_for_connection(
1945        &self,
1946        connection_id: ConnectionId,
1947    ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
1948        Ok(self
1949            .read_inner()?
1950            .endpoint_by_connection
1951            .get(&connection_id)
1952            .copied())
1953    }
1954
1955    /// Looks up the module registered on a data-plane connection so route-drop
1956    /// diagnostics name the emitter instead of only reporting a daemon total.
1957    pub(crate) fn module_id_for_connection(
1958        &self,
1959        connection_id: ConnectionId,
1960    ) -> Result<Option<String>, ForwardingError> {
1961        let inner = self.read_inner()?;
1962        Ok(inner
1963            .endpoint_by_connection
1964            .get(&connection_id)
1965            .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
1966            .cloned())
1967    }
1968
1969    /// Whether the daemon ever allocated `(channel, epoch)` on this module
1970    /// connection. Epochs on a module channel are handed out as 1, 2, 3, ...
1971    /// and the last one handed out is remembered until the connection ends, so
1972    /// every epoch from 1 up to that one was a real route at some point. Used
1973    /// only on the drop path, to tell a module still sending on a route the
1974    /// daemon released from one sending on a route that never existed.
1975    pub(crate) fn module_route_epoch_was_allocated(
1976        &self,
1977        connection_id: ConnectionId,
1978        channel: u16,
1979        epoch: u32,
1980    ) -> Result<bool, ForwardingError> {
1981        let inner = self.read_inner()?;
1982        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1983            return Ok(false);
1984        };
1985        Ok(inner
1986            .module_slot_epochs
1987            .get(&ModuleRouteKey { endpoint, channel })
1988            .is_some_and(|last| epoch != 0 && epoch <= *last))
1989    }
1990
1991    pub(crate) fn has_live_module_connection(
1992        &self,
1993        module_id: &str,
1994    ) -> Result<bool, ForwardingError> {
1995        Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
1996    }
1997
1998    pub(crate) fn lookup_data_route(
1999        &self,
2000        connection_id: ConnectionId,
2001        channel: u16,
2002        epoch: u32,
2003    ) -> Result<DataRoute, ForwardingError> {
2004        let inner = self.read_inner()?;
2005        let state = if let Some(endpoint) =
2006            inner.endpoint_by_connection.get(&connection_id).copied()
2007        {
2008            let key = ModuleRouteKey { endpoint, channel };
2009            match inner.module_to_client.get(&key) {
2010                Some(route) if route.module_epoch == epoch => {
2011                    DataRouteState::Bound(Arc::clone(route))
2012                }
2013                Some(_) => DataRouteState::EpochMismatch,
2014                None if inner.reserved_module.contains_key(&key)
2015                    && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
2016                {
2017                    DataRouteState::Reserved
2018                }
2019                None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
2020                None => DataRouteState::Absent,
2021            }
2022        } else {
2023            let key = ClientRouteKey {
2024                connection_id,
2025                channel,
2026            };
2027            match inner.client_to_module.get(&key) {
2028                Some(route) if route.client_epoch == epoch => {
2029                    DataRouteState::Bound(Arc::clone(route))
2030                }
2031                Some(_) => DataRouteState::EpochMismatch,
2032                None if inner.reserved_client.contains_key(&key)
2033                    && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
2034                {
2035                    DataRouteState::Reserved
2036                }
2037                None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
2038                None => DataRouteState::Absent,
2039            }
2040        };
2041        Ok(
2042            if inner.endpoint_by_connection.contains_key(&connection_id) {
2043                DataRoute::Module(state)
2044            } else {
2045                DataRoute::Client(state)
2046            },
2047        )
2048    }
2049
2050    #[cfg(test)]
2051    pub(crate) fn inject_client_slot_epoch(
2052        &self,
2053        connection_id: ConnectionId,
2054        channel: u16,
2055        last_epoch: u32,
2056    ) {
2057        let mut inner = self.write_inner().expect("forwarding lock");
2058        inner.client_slot_epochs.insert(
2059            ClientRouteKey {
2060                connection_id,
2061                channel,
2062            },
2063            last_epoch,
2064        );
2065        inner.next_client_channel.insert(connection_id, channel);
2066    }
2067
2068    #[cfg(test)]
2069    pub(crate) fn inject_module_slot_epoch(
2070        &self,
2071        endpoint: ModuleEndpointId,
2072        channel: u16,
2073        last_epoch: u32,
2074    ) {
2075        let mut inner = self.write_inner().expect("forwarding lock");
2076        inner
2077            .module_slot_epochs
2078            .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
2079        inner.next_module_channel.insert(endpoint, channel);
2080    }
2081
2082    #[cfg(test)]
2083    pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
2084        self.write_inner()
2085            .expect("forwarding lock")
2086            .next_control_corr
2087            .insert(endpoint, next_corr);
2088    }
2089
2090    pub(crate) fn cache_status(
2091        &self,
2092        endpoint: ModuleEndpointId,
2093        module_channel: u16,
2094        module_epoch: u32,
2095        status: String,
2096    ) -> Result<bool, ForwardingError> {
2097        let mut inner = self.write_inner()?;
2098        if !inner.module_id_by_endpoint.contains_key(&endpoint) {
2099            return Err(ForwardingError::StaleModuleEndpoint);
2100        }
2101
2102        let module_key = ModuleRouteKey {
2103            endpoint,
2104            channel: module_channel,
2105        };
2106        let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
2107            (route.module_epoch == module_epoch).then_some((
2108                ClientRouteKey {
2109                    connection_id: route.client_connection_id,
2110                    channel: route.client_channel,
2111                },
2112                route.client_epoch,
2113            ))
2114        } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
2115            (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
2116                client_key,
2117                inner
2118                    .client_slot_epochs
2119                    .get(&client_key)
2120                    .copied()
2121                    .unwrap_or(0),
2122            ))
2123        } else {
2124            None
2125        };
2126
2127        if let Some(handle) = handle {
2128            inner.status.insert(handle, status);
2129            Ok(true)
2130        } else {
2131            debug!(
2132                module_channel,
2133                module_epoch,
2134                generation = endpoint.generation,
2135                connection_id = endpoint.connection_id.get(),
2136                "dropping stale status update for module route handle"
2137            );
2138            Ok(false)
2139        }
2140    }
2141
2142    pub(crate) fn route_poll_snapshot(
2143        &self,
2144        client_connection_id: ConnectionId,
2145        client_channel: u16,
2146        client_epoch: u32,
2147    ) -> Result<RoutePollSnapshot, ForwardingError> {
2148        let inner = self.read_inner()?;
2149        let client_key = ClientRouteKey {
2150            connection_id: client_connection_id,
2151            channel: client_channel,
2152        };
2153        let Some(route) = inner.client_to_module.get(&client_key) else {
2154            return Ok(RoutePollSnapshot::Absent);
2155        };
2156        if route.client_epoch != client_epoch
2157            || !inner
2158                .module_id_by_endpoint
2159                .contains_key(&route.module_endpoint)
2160        {
2161            return Ok(RoutePollSnapshot::Absent);
2162        }
2163        Ok(RoutePollSnapshot::Bound {
2164            module_id: route.module_id.clone(),
2165            status: inner.status.get(&(client_key, client_epoch)).cloned(),
2166        })
2167    }
2168
2169    pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2170        Ok(self.read_inner()?.client_to_module.len())
2171    }
2172
2173    /// How many distinct client connections hold at least one committed route,
2174    /// alongside the largest number of routes any single connection holds.
2175    ///
2176    /// `connected_clients` alone cannot distinguish many clients with a route
2177    /// each from one client accumulating hundreds, and those have opposite
2178    /// causes. Reading it required an out-of-band `lsof` during a live
2179    /// investigation, and the count of connections was mistaken for a count of
2180    /// client processes — which sent two of us after cleanup paths that were
2181    /// working correctly.
2182    pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2183        let inner = self.read_inner()?;
2184        let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2185        for key in inner.client_to_module.keys() {
2186            *per_connection.entry(key.connection_id).or_insert(0) += 1;
2187        }
2188        let max = per_connection.values().copied().max().unwrap_or(0);
2189        Ok((per_connection.len(), max))
2190    }
2191
2192    pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2193        let inner = self.read_inner()?;
2194        Ok(inner
2195            .client_to_module
2196            .keys()
2197            .any(|key| key.channel == route_channel))
2198    }
2199
2200    /// Whether daemon-wide shutdown has begun draining all providers.
2201    /// `None` means normal operation; during daemon shutdown, return the
2202    /// endpoint's measured result, or false if no completed measurement exists.
2203    pub(crate) fn daemon_drain_result_for_connection(
2204        &self,
2205        connection_id: ConnectionId,
2206    ) -> Result<Option<bool>, ForwardingError> {
2207        let inner = self.read_inner()?;
2208        Ok(inner.daemon_draining.then(|| {
2209            inner
2210                .endpoint_by_connection
2211                .get(&connection_id)
2212                .and_then(|endpoint| inner.daemon_drain_results.get(endpoint))
2213                .copied()
2214                .unwrap_or(false)
2215        }))
2216    }
2217
2218    #[cfg(unix)]
2219    pub(crate) fn record_daemon_drain_result(
2220        &self,
2221        endpoint: ModuleEndpointId,
2222        drained: bool,
2223    ) -> Result<(), ForwardingError> {
2224        let mut inner = self.write_inner()?;
2225        // Only update an endpoint captured at the start of daemon shutdown.
2226        // Teardown removes its entry, so a late wait cannot recreate the result
2227        // or attribute it to a replacement connection.
2228        if let Some(result) = inner.daemon_drain_results.get_mut(&endpoint) {
2229            *result = drained;
2230        }
2231        Ok(())
2232    }
2233
2234    /// Gate every provider atomically, including registrations racing shutdown.
2235    /// No supervisor lock is held while taking the forwarding lock.
2236    #[cfg(unix)]
2237    pub(crate) fn begin_daemon_drain(
2238        &self,
2239    ) -> Result<Vec<(String, ModuleEndpointId)>, ForwardingError> {
2240        let mut inner = self.write_inner()?;
2241        if inner.daemon_draining {
2242            return Ok(Vec::new());
2243        }
2244        inner.daemon_draining = true;
2245        let modules = inner
2246            .modules_by_id
2247            .iter()
2248            .map(|(id, module)| (id.clone(), module.endpoint))
2249            .collect::<Vec<_>>();
2250        for (_, endpoint) in &modules {
2251            inner.daemon_drain_results.insert(*endpoint, false);
2252            inner
2253                .draining_endpoints
2254                .insert(*endpoint, RouteCloseReason::Restart);
2255        }
2256        // Swap candidates and superseded incumbents are not routable, but they
2257        // are live endpoints that could still be sent module-control work, so
2258        // the daemon-wide gate covers them too. Only the active endpoints are
2259        // returned to the supervisor for measurement, one per module id.
2260        let off_slot_endpoints = inner
2261            .candidates_by_id
2262            .values()
2263            .map(|module| module.endpoint)
2264            .chain(inner.superseded_endpoints.keys().copied())
2265            .collect::<Vec<_>>();
2266        for endpoint in off_slot_endpoints {
2267            inner
2268                .draining_endpoints
2269                .insert(endpoint, RouteCloseReason::Restart);
2270        }
2271        Ok(modules)
2272    }
2273
2274    /// Begin draining whatever endpoint is ACTIVE for `module_id`.
2275    ///
2276    /// After a swap's cutover the active endpoint is the promoted candidate, so
2277    /// this must not be used to drain the incumbent; use
2278    /// [`Self::begin_endpoint_drain`] with the endpoint `cutover_candidate`
2279    /// returned.
2280    pub(crate) fn begin_module_drain(
2281        &self,
2282        module_id: &str,
2283        reason: RouteCloseReason,
2284    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2285        let mut inner = self.write_inner()?;
2286        let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2287            return Ok(None);
2288        };
2289        Ok(Some(begin_drain_locked(
2290            &mut inner, module_id, module, reason,
2291        )))
2292    }
2293
2294    /// Begin draining one specific endpoint, whichever slot it is in.
2295    ///
2296    /// This is how a swap drains its incumbent after cutover: the incumbent's
2297    /// endpoint is captured by `cutover_candidate`, and resolving it by module id
2298    /// instead would find the promoted candidate and leave neither process
2299    /// routable. Returns `Ok(None)` when the endpoint is no longer registered.
2300    pub(crate) fn begin_endpoint_drain(
2301        &self,
2302        endpoint: ModuleEndpointId,
2303        reason: RouteCloseReason,
2304    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2305        let mut inner = self.write_inner()?;
2306        let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2307            return Ok(None);
2308        };
2309        let module_id = inner
2310            .module_id_by_endpoint
2311            .get(&endpoint)
2312            .cloned()
2313            .expect("an endpoint resolved to a module connection has a module id");
2314        Ok(Some(begin_drain_locked(
2315            &mut inner, &module_id, module, reason,
2316        )))
2317    }
2318}
2319
2320/// Mark `module`'s endpoint draining and settle everything still pending on it.
2321/// Shared by the by-id and by-endpoint drain entry points, which differ only in
2322/// how they find the endpoint.
2323fn begin_drain_locked(
2324    inner: &mut ForwardingInner,
2325    module_id: &str,
2326    module: ModuleConnection,
2327    reason: RouteCloseReason,
2328) -> ModuleDrainTarget {
2329    {
2330        let endpoint = module.endpoint;
2331        inner.draining_endpoints.insert(endpoint, reason);
2332
2333        let flows = inner
2334            .client_to_module
2335            .values()
2336            .filter(|route| route.module_endpoint == endpoint)
2337            .map(|route| Arc::clone(&route.flow))
2338            .collect::<Vec<_>>();
2339        let excluded_subscriptions = flows
2340            .into_iter()
2341            .map(|flow| flow.begin_drain())
2342            .fold(0u32, u32::saturating_add);
2343
2344        let pending_keys = inner
2345            .pending_relays
2346            .keys()
2347            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2348            .copied()
2349            .collect::<Vec<_>>();
2350        let mut abandoned_bindings = Vec::new();
2351        for key in pending_keys {
2352            let Some(pending) = inner.pending_relays.remove(&key) else {
2353                continue;
2354            };
2355            release_reserved_route_locked(
2356                inner,
2357                pending.reservation.client_key,
2358                pending.reservation.module_key,
2359            );
2360            if pending.relay_enqueued {
2361                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2362                    abandoned_bindings.push(target);
2363                }
2364            } else {
2365                // The bind may still be on its way into the module's queue;
2366                // a GOODBYE sent now could overtake it. `route.open` sends
2367                // it once its send completes.
2368                inner.relays_released_before_enqueue.insert(key);
2369            }
2370            let _ = pending
2371                .sender
2372                .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
2373                    "module_reloading",
2374                    format!("module_id '{module_id}' is reloading"),
2375                )));
2376        }
2377
2378        let pending_control_keys = inner
2379            .pending_control_rpcs
2380            .keys()
2381            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2382            .copied()
2383            .collect::<Vec<_>>();
2384        for key in pending_control_keys {
2385            if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2386                let _ = pending
2387                    .sender
2388                    .send(ModuleControlRpcOutcome::ModuleGone(format!(
2389                        "module '{module_id}' began draining during module-control RPC"
2390                    )));
2391            }
2392        }
2393
2394        ModuleDrainTarget {
2395            endpoint,
2396            sink: module.sink,
2397            negotiated_ver: module.negotiated_ver,
2398            abandoned_bindings,
2399            excluded_subscriptions,
2400        }
2401    }
2402}
2403
2404/// What a drain that timed out was still waiting on, for the log line that
2405/// reports the timeout. Per-request ages are not tracked.
2406#[derive(Debug, Default, PartialEq, Eq)]
2407pub(crate) struct DrainHoldouts {
2408    /// Requests still counted against the drain (subscriptions flagged as
2409    /// such are excluded from the drain and not counted here).
2410    pub(crate) requests: usize,
2411    /// Routes holding at least one of those requests.
2412    pub(crate) routes: usize,
2413    /// Every route on the endpoint, for scale.
2414    pub(crate) total_routes: usize,
2415    /// The client connections holding the most requests, largest first, at
2416    /// most three: enough to name the consumer without listing every route.
2417    pub(crate) top_connections: Vec<(u64, usize)>,
2418    /// The held requests themselves, as the module saw them, so its own log
2419    /// can say what each one was: `(module channel, corr)`, ordered by channel
2420    /// then corr, at most [`DRAIN_HELD_REQUESTS_LISTED`]. The daemon never reads
2421    /// request bodies, so it cannot name a request's method; the module can,
2422    /// from the channel and corr. A request is released only when the module
2423    /// sends a terminal frame (Response, Error or StreamEnd) with its corr on
2424    /// its route, so every pair listed here is one the module ended, if at all,
2425    /// without sending that frame.
2426    pub(crate) held: Vec<(u16, u64)>,
2427}
2428
2429/// How many held requests the drain-timeout line lists by channel and corr.
2430pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
2431
2432impl ForwardingTable {
2433    /// Summarise the requests one endpoint's drain is still waiting on.
2434    pub(crate) fn endpoint_drain_holdouts(
2435        &self,
2436        endpoint: ModuleEndpointId,
2437    ) -> Result<DrainHoldouts, ForwardingError> {
2438        let inner = self.read_inner()?;
2439        let mut holdouts = DrainHoldouts::default();
2440        let mut by_connection: HashMap<u64, usize> = HashMap::new();
2441        for (key, route) in &inner.client_to_module {
2442            if route.module_endpoint != endpoint {
2443                continue;
2444            }
2445            holdouts.total_routes += 1;
2446            let held = route.flow.drain_in_flight();
2447            if held == 0 {
2448                continue;
2449            }
2450            holdouts.requests += held;
2451            holdouts.routes += 1;
2452            *by_connection.entry(key.connection_id.get()).or_default() += held;
2453            holdouts.held.extend(
2454                route
2455                    .flow
2456                    .drain_held_corrs()
2457                    .into_iter()
2458                    .map(|corr| (route.module_channel, corr)),
2459            );
2460        }
2461        holdouts.held.sort_unstable();
2462        holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
2463        let mut connections = by_connection.into_iter().collect::<Vec<_>>();
2464        connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
2465        connections.truncate(3);
2466        holdouts.top_connections = connections;
2467        Ok(holdouts)
2468    }
2469
2470    pub(crate) fn endpoint_in_flight_count(
2471        &self,
2472        endpoint: ModuleEndpointId,
2473    ) -> Result<usize, ForwardingError> {
2474        let inner = self.read_inner()?;
2475        Ok(inner
2476            .client_to_module
2477            .values()
2478            .filter(|route| route.module_endpoint == endpoint)
2479            .map(|route| route.flow.drain_in_flight())
2480            .sum())
2481    }
2482
2483    pub(crate) fn endpoint_is_draining(
2484        &self,
2485        endpoint: ModuleEndpointId,
2486    ) -> Result<bool, ForwardingError> {
2487        Ok(self
2488            .read_inner()?
2489            .draining_endpoints
2490            .contains_key(&endpoint))
2491    }
2492
2493    pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
2494        let inner = self.read_inner()?;
2495        Ok(inner
2496            .modules_by_id
2497            .get(module_id)
2498            .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
2499    }
2500
2501    pub(crate) fn release_module_endpoint_routes(
2502        &self,
2503        endpoint: ModuleEndpointId,
2504    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2505        let mut inner = self.write_inner()?;
2506        let routes = inner
2507            .module_to_client
2508            .iter()
2509            .filter(|(module_key, _)| module_key.endpoint == endpoint)
2510            .map(|(module_key, route)| (*module_key, route.module_epoch))
2511            .collect::<Vec<_>>();
2512        let mut released = Vec::with_capacity(routes.len());
2513        for (module_key, epoch) in routes {
2514            if let RouteRelease::Removed(target) =
2515                release_module_route_locked(&mut inner, module_key, epoch)
2516            {
2517                released.push(target);
2518            }
2519        }
2520        Ok(released)
2521    }
2522
2523    /// Enumerate one endpoint's current routes without contacting the module.
2524    ///
2525    /// The read lock makes this safe while the endpoint drains: the returned
2526    /// `draining` marker describes the same table state that owns the route,
2527    /// rather than inferring liveness from a module that may be stopping.
2528    pub(crate) fn endpoint_routes(
2529        &self,
2530        endpoint: ModuleEndpointId,
2531    ) -> Result<Vec<EndpointRoute>, ForwardingError> {
2532        let inner = self.read_inner()?;
2533        Ok(endpoint_routes_locked(&inner, endpoint))
2534    }
2535
2536    /// Snapshot all live endpoint route sets under one forwarding-table read lock.
2537    pub(crate) fn route_census(
2538        &self,
2539        module_id: Option<&str>,
2540    ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
2541        let inner = self.read_inner()?;
2542        let mut endpoints = inner
2543            .modules_by_id
2544            .iter()
2545            .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
2546            .map(|(id, module)| (id.clone(), module.endpoint))
2547            .collect::<Vec<_>>();
2548        endpoints.sort_by(|left, right| left.0.cmp(&right.0));
2549        Ok(endpoints
2550            .into_iter()
2551            .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
2552            .collect())
2553    }
2554
2555    /// Snapshot only the endpoint currently routable under this module id.
2556    pub(crate) fn live_roots(
2557        &self,
2558        module_id: &str,
2559    ) -> Result<ModuleControlResponseToModule, ForwardingError> {
2560        let inner = self.read_inner()?;
2561        let endpoint = inner
2562            .modules_by_id
2563            .get(module_id)
2564            .map(|module| module.endpoint);
2565        let mut roots = BTreeMap::new();
2566        let mut unknown_root_bindings = 0;
2567        let mut total_bindings = 0;
2568        if let Some(endpoint) = endpoint {
2569            for binding in inner
2570                .module_to_client
2571                .values()
2572                .filter(|binding| binding.module_endpoint == endpoint)
2573            {
2574                total_bindings += 1;
2575                let workdir = binding
2576                    .scope
2577                    .as_ref()
2578                    .and_then(|scope| scope.workdir.as_deref())
2579                    .map(std::path::Path::new);
2580                if let Some(root) = &binding.project_root {
2581                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2582                    entry.0 += 1;
2583                } else if workdir.is_none() {
2584                    unknown_root_bindings += 1;
2585                }
2586                if let Some(workdir) = workdir.filter(|workdir| {
2587                    binding.project_root.as_ref().map(ProjectRootId::as_path) != Some(*workdir)
2588                }) {
2589                    roots.entry(workdir.to_path_buf()).or_insert((0, 0)).0 += 1;
2590                }
2591            }
2592            for pending in inner
2593                .pending_relays
2594                .values()
2595                .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
2596            {
2597                total_bindings += 1;
2598                if let Some(root) = &pending.reservation.project_root {
2599                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2600                    entry.1 += 1;
2601                } else {
2602                    unknown_root_bindings += 1;
2603                }
2604            }
2605        }
2606        Ok(ModuleControlResponseToModule::LiveRoots {
2607            roots: roots
2608                .into_iter()
2609                .map(|(project_root, (bound, pending))| LiveRoot {
2610                    project_root,
2611                    bound,
2612                    pending,
2613                })
2614                .collect(),
2615            unknown_root_bindings,
2616            total_bindings,
2617        })
2618    }
2619
2620    /// True if this connection already owns committed or reserved CLIENT routes.
2621    /// A module registers (HELLO) before serving and never opens client routes, so
2622    /// a connection that has client routes must not also become a module endpoint
2623    /// — otherwise one connection holds both client and module state and cleanup
2624    /// only releases one side.
2625    pub(crate) fn connection_has_client_routes(
2626        &self,
2627        connection_id: ConnectionId,
2628    ) -> Result<bool, ForwardingError> {
2629        let inner = self.read_inner()?;
2630        Ok(connection_has_client_routes_locked(&inner, connection_id))
2631    }
2632
2633    pub(crate) fn cleanup_connection(
2634        &self,
2635        connection_id: ConnectionId,
2636    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2637        self.cleanup_connection_counted(connection_id)
2638            .map(|cleanup| cleanup.released)
2639    }
2640
2641    /// [`Self::cleanup_connection`], also reporting how many pending
2642    /// route.bind relays to the closed module were aborted. That count is the
2643    /// `abandoned` figure of the `route.closed` push sent for a lost module
2644    /// connection; it is always zero for a client connection.
2645    pub(crate) fn cleanup_connection_counted(
2646        &self,
2647        connection_id: ConnectionId,
2648    ) -> Result<ConnectionCleanup, ForwardingError> {
2649        let mut inner = self.write_inner()?;
2650        inner.closing_connections.insert(connection_id);
2651        let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
2652            remove_module_connection_locked(&mut inner, endpoint)
2653        } else {
2654            ConnectionCleanup {
2655                released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
2656                abandoned_relays: 0,
2657            }
2658        };
2659        // The closing mark refuses new work for a connection whose teardown is
2660        // still pending. This is the latest point at which lifting it is safe:
2661        // teardown has just removed every per-connection entry above, under
2662        // this same write lock, so no lookup can still find live state for the
2663        // id; and ids come from a monotonic counter that never reissues one,
2664        // so the id can never name a future connection either. Keeping the
2665        // mark past this point only grew the set by one entry per connection
2666        // for the life of the daemon.
2667        inner.closing_connections.remove(&connection_id);
2668        Ok(cleanup)
2669    }
2670
2671    fn cleanup_client_connection_locked(
2672        inner: &mut ForwardingInner,
2673        connection_id: ConnectionId,
2674    ) -> Vec<GoodbyeTarget> {
2675        let routes = inner
2676            .client_to_module
2677            .iter()
2678            .filter(|(key, _)| key.connection_id == connection_id)
2679            .map(|(key, route)| (*key, route.client_epoch))
2680            .collect::<Vec<_>>();
2681        let mut released = Vec::with_capacity(routes.len());
2682        for (client_key, epoch) in routes {
2683            if let RouteRelease::Removed(target) =
2684                release_client_route_locked(inner, client_key, epoch)
2685            {
2686                released.push(target);
2687            }
2688        }
2689
2690        let pending_keys = inner
2691            .pending_relays
2692            .iter()
2693            .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
2694            .map(|(key, _)| *key)
2695            .collect::<Vec<_>>();
2696        for key in pending_keys {
2697            let Some(pending) = inner.pending_relays.remove(&key) else {
2698                continue;
2699            };
2700            release_reserved_route_locked(
2701                inner,
2702                pending.reservation.client_key,
2703                pending.reservation.module_key,
2704            );
2705            if pending.relay_enqueued {
2706                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2707                    released.push(target);
2708                }
2709            } else {
2710                // The bind may still be on its way into the module's queue;
2711                // a GOODBYE sent now could overtake it. `route.open` sends
2712                // it once its send completes.
2713                inner.relays_released_before_enqueue.insert(key);
2714            }
2715            let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
2716                "client connection closed during route.bind relay".to_string(),
2717            ));
2718        }
2719
2720        let orphaned = inner
2721            .reserved_client
2722            .iter()
2723            .filter(|(key, _)| key.connection_id == connection_id)
2724            .map(|(client, module)| (*client, *module))
2725            .collect::<Vec<_>>();
2726        for (client_key, module_key) in orphaned {
2727            release_reserved_route_locked(inner, client_key, module_key);
2728        }
2729        inner.next_client_channel.remove(&connection_id);
2730        inner
2731            .client_slot_epochs
2732            .retain(|key, _| key.connection_id != connection_id);
2733        inner
2734            .last_published_epoch
2735            .retain(|key, _| key.connection_id != connection_id);
2736        inner
2737            .status
2738            .retain(|(key, _), _| key.connection_id != connection_id);
2739
2740        released
2741    }
2742
2743    /// Close a client connection whose egress queue refused a frame for the
2744    /// route `(connection_id, channel)` at `expected_epoch`. The whole
2745    /// connection closes because its queue is shared by every route on it.
2746    ///
2747    /// The first request that actually closes the connection logs one WARN with
2748    /// the diagnosis: which principals the connection's routes belonged to,
2749    /// which module's frame did not fit and on which client channel, and what
2750    /// the queue held at that moment.
2751    pub(crate) fn escalate_client_delivery_failure(
2752        &self,
2753        connection_id: ConnectionId,
2754        channel: u16,
2755        expected_epoch: u32,
2756        reason: CloseReason,
2757        undelivered: UndeliveredFrame<'_>,
2758    ) -> Result<bool, ForwardingError> {
2759        self.escalate_client_delivery_failure_diagnosed(
2760            connection_id,
2761            channel,
2762            expected_epoch,
2763            reason,
2764            undelivered,
2765            None,
2766        )
2767    }
2768
2769    pub(crate) fn escalate_client_delivery_failure_diagnosed(
2770        &self,
2771        connection_id: ConnectionId,
2772        channel: u16,
2773        expected_epoch: u32,
2774        reason: CloseReason,
2775        undelivered: UndeliveredFrame<'_>,
2776        diagnosis: Option<&crate::router::ClientDeliveryDiagnosis>,
2777    ) -> Result<bool, ForwardingError> {
2778        let principals = {
2779            let mut inner = self.write_inner()?;
2780            let key = ClientRouteKey {
2781                connection_id,
2782                channel,
2783            };
2784            if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
2785                None
2786            } else {
2787                inner.closing_connections.insert(connection_id);
2788                Some(connection_principals_locked(&inner, connection_id))
2789            }
2790        };
2791        let Some(principals) = principals else {
2792            return Ok(false);
2793        };
2794        let backlog = undelivered.sink.backlog();
2795        let close_reason = reason.to_string();
2796        if self.request_connection_close(connection_id, reason) {
2797            warn!(
2798                connection_id = connection_id.get(),
2799                principals = %principals,
2800                module_id = undelivered.module_id.unwrap_or("unknown"),
2801                client_channel = channel,
2802                queued_bytes = backlog.queued_bytes,
2803                queued_frames = backlog.queued_frames,
2804                frame_type = ?diagnosis.map(|failure| failure.frame_type),
2805                frame_flags = ?diagnosis.map(|failure| failure.flags.0),
2806                close_cause = diagnosis.map(|failure| failure.cause).unwrap_or("delivery_failed"),
2807                oldest_queued_ms = backlog
2808                    .oldest_age
2809                    .map(|age| age.as_millis() as u64)
2810                    .unwrap_or(0),
2811                close_reason = %close_reason,
2812                "closing client connection: its egress queue could not take a frame"
2813            );
2814        }
2815        Ok(true)
2816    }
2817
2818    /// Publish the tags a scope sync changed and close the live routes its
2819    /// drain table selects, in one critical section.
2820    ///
2821    /// The caller holds the scope table's write lock (scope table, then this
2822    /// table, always). Publishing and selecting under one lock is what closes
2823    /// the race with a bind commit: a commit before this sees the old tag and
2824    /// its route is selected here like any other; a commit after it sees the
2825    /// new tag and is refused.
2826    ///
2827    /// Every client route index entry is scanned, which includes routes on a
2828    /// swap's superseded endpoint (the old process of a module being replaced
2829    /// blue/green, which keeps its existing routes until they drain): those
2830    /// stay indexed until drained, so ending a scope reaches them too.
2831    pub(crate) fn publish_scope_changes(
2832        &self,
2833        changes: &[ScopeTagChange],
2834    ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
2835        let mut inner = self.write_inner()?;
2836        let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
2837        for change in changes {
2838            let key = (change.owner.clone(), change.scope_ref.clone());
2839            match change.after {
2840                Some(tag) => {
2841                    inner.scope_tags.insert(key, tag);
2842                }
2843                None => {
2844                    inner.scope_tags.remove(&key);
2845                }
2846            }
2847            by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
2848        }
2849        let mut selected = Vec::new();
2850        for (client_key, route) in &inner.client_to_module {
2851            let Some(scope) = &route.scope else {
2852                continue;
2853            };
2854            let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
2855            else {
2856                continue;
2857            };
2858            if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
2859                continue;
2860            }
2861            let reason = match &change.drain {
2862                ScopeDrain::Nothing => continue,
2863                ScopeDrain::All(reason) => *reason,
2864                ScopeDrain::Carriers(narrowed) => {
2865                    let owner = Principal::Reserved {
2866                        module_id: scope.owner.clone(),
2867                    };
2868                    let hit = route.principal != owner
2869                        && narrowed.iter().any(|(principal, allowed)| {
2870                            *principal == route.principal
2871                                && allowed
2872                                    .as_ref()
2873                                    .is_none_or(|targets| !targets.contains(&route.module_id))
2874                        });
2875                    if !hit {
2876                        continue;
2877                    }
2878                    RouteCloseReason::ScopeCarrierRemoved
2879                }
2880            };
2881            selected.push((*client_key, route.client_epoch, reason));
2882        }
2883        let mut drained = Vec::new();
2884        for (client_key, client_epoch, reason) in selected {
2885            let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
2886                continue;
2887            };
2888            let release = release_client_route_locked(&mut inner, client_key, client_epoch);
2889            self.record_route_release(&release);
2890            if let RouteRelease::Removed(module) = release {
2891                drained.push(ScopeDrainedRoute {
2892                    reason,
2893                    scope: route
2894                        .scope
2895                        .clone()
2896                        .expect("selected scope drain has a bound scope"),
2897                    module_id: route.module_id.clone(),
2898                    client: GoodbyeTarget {
2899                        connection_id: route.client_connection_id,
2900                        sink: route.client_sink.clone(),
2901                        negotiated_ver: route.client_negotiated_ver,
2902                        channel: route.client_channel,
2903                        epoch: route.client_epoch,
2904                        kind: GoodbyeTargetKind::Client,
2905                        module_id: Some(route.module_id.clone()),
2906                    },
2907                    module,
2908                });
2909            }
2910        }
2911        Ok(drained)
2912    }
2913
2914    /// The published tag of one scope, for tests of the commit re-check.
2915    #[cfg(test)]
2916    pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
2917        self.read_inner()
2918            .ok()?
2919            .scope_tags
2920            .get(&(owner.to_string(), scope_ref.to_string()))
2921            .copied()
2922    }
2923
2924    fn record_route_release(&self, release: &RouteRelease) {
2925        match release {
2926            RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
2927            RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
2928            RouteRelease::Absent => {}
2929        }
2930    }
2931
2932    fn read_inner(&self) -> Result<RwLockReadGuard<'_, ForwardingInner>, ForwardingError> {
2933        self.inner.read().map_err(|_| ForwardingError::Poisoned)
2934    }
2935
2936    fn write_inner(&self) -> Result<RwLockWriteGuard<'_, ForwardingInner>, ForwardingError> {
2937        self.inner.write().map_err(|_| ForwardingError::Poisoned)
2938    }
2939
2940    fn lock_close_registry(
2941        &self,
2942    ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
2943        self.close_registry
2944            .lock()
2945            .unwrap_or_else(|poisoned| poisoned.into_inner())
2946    }
2947}
2948
2949impl ForwardingInner {
2950    fn allocate_route_slots(
2951        &mut self,
2952        connection_id: ConnectionId,
2953        endpoint: ModuleEndpointId,
2954    ) -> Result<(u16, u32, u16, u32), ForwardingError> {
2955        let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
2956        let mut client_channel = client_start;
2957        let client_channel = loop {
2958            let key = ClientRouteKey {
2959                connection_id,
2960                channel: client_channel,
2961            };
2962            let eligible = !self.client_to_module.contains_key(&key)
2963                && !self.reserved_client.contains_key(&key)
2964                && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2965            if eligible {
2966                break client_channel;
2967            }
2968            client_channel = next_channel(client_channel);
2969            if client_channel == client_start {
2970                return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
2971            }
2972        };
2973
2974        let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
2975        let mut module_channel = module_start;
2976        let module_channel = loop {
2977            let key = ModuleRouteKey {
2978                endpoint,
2979                channel: module_channel,
2980            };
2981            let eligible = !self.module_to_client.contains_key(&key)
2982                && !self.reserved_module.contains_key(&key)
2983                && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2984            if eligible {
2985                break module_channel;
2986            }
2987            module_channel = next_channel(module_channel);
2988            if module_channel == module_start {
2989                return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
2990            }
2991        };
2992
2993        let client_key = ClientRouteKey {
2994            connection_id,
2995            channel: client_channel,
2996        };
2997        let module_key = ModuleRouteKey {
2998            endpoint,
2999            channel: module_channel,
3000        };
3001        let client_epoch = self
3002            .client_slot_epochs
3003            .get(&client_key)
3004            .copied()
3005            .unwrap_or(0)
3006            + 1;
3007        let module_epoch = self
3008            .module_slot_epochs
3009            .get(&module_key)
3010            .copied()
3011            .unwrap_or(0)
3012            + 1;
3013        self.client_slot_epochs.insert(client_key, client_epoch);
3014        self.module_slot_epochs.insert(module_key, module_epoch);
3015        self.next_client_channel
3016            .insert(connection_id, next_channel(client_channel));
3017        self.next_module_channel
3018            .insert(endpoint, next_channel(module_channel));
3019        Ok((client_channel, client_epoch, module_channel, module_epoch))
3020    }
3021
3022    fn allocate_control_corr(
3023        &mut self,
3024        endpoint: ModuleEndpointId,
3025    ) -> Result<u64, ForwardingError> {
3026        let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
3027        if candidate == 0 {
3028            self.closing_connections.insert(endpoint.connection_id);
3029            return Err(ForwardingError::RelayCorrelationExhausted);
3030        }
3031        self.next_control_corr.insert(
3032            endpoint,
3033            if candidate == u64::MAX {
3034                0
3035            } else {
3036                candidate + 1
3037            },
3038        );
3039        Ok(candidate)
3040    }
3041}
3042
3043fn connection_has_client_routes_locked(
3044    inner: &ForwardingInner,
3045    connection_id: ConnectionId,
3046) -> bool {
3047    inner
3048        .client_to_module
3049        .keys()
3050        .any(|key| key.connection_id == connection_id)
3051        || inner
3052            .reserved_client
3053            .keys()
3054            .any(|key| key.connection_id == connection_id)
3055}
3056
3057fn check_module_connection_role_locked(
3058    inner: &ForwardingInner,
3059    connection_id: ConnectionId,
3060) -> Result<(), ForwardingError> {
3061    if inner.endpoint_by_connection.contains_key(&connection_id)
3062        || connection_has_client_routes_locked(inner, connection_id)
3063    {
3064        return Err(ForwardingError::ConnectionRoleConflict { connection_id });
3065    }
3066    Ok(())
3067}
3068
3069fn next_channel(channel: u16) -> u16 {
3070    let next = channel.wrapping_add(1);
3071    if next == 0 {
3072        1
3073    } else {
3074        next
3075    }
3076}
3077
3078fn endpoint_routes_locked(
3079    inner: &ForwardingInner,
3080    endpoint: ModuleEndpointId,
3081) -> Vec<EndpointRoute> {
3082    let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
3083    let draining = drain_reason.is_some();
3084    let mut routes = inner
3085        .module_to_client
3086        .iter()
3087        .filter(|(module_key, _)| module_key.endpoint == endpoint)
3088        .map(|(_, route)| EndpointRoute {
3089            goodbye_target: GoodbyeTarget {
3090                connection_id: route.client_connection_id,
3091                sink: route.client_sink.clone(),
3092                negotiated_ver: route.client_negotiated_ver,
3093                channel: route.client_channel,
3094                epoch: route.client_epoch,
3095                kind: GoodbyeTargetKind::Client,
3096                module_id: Some(route.module_id.clone()),
3097            },
3098            principal: route.principal.clone(),
3099            bound_at: route.bound_at,
3100            draining,
3101            drain_reason,
3102        })
3103        .collect::<Vec<_>>();
3104    routes.sort_by_key(|route| {
3105        (
3106            route.goodbye_target.connection_id.get(),
3107            route.goodbye_target.channel,
3108            route.goodbye_target.epoch,
3109        )
3110    });
3111    routes
3112}
3113
3114/// Why a bind admitted under `scope` may no longer commit, read from the
3115/// published tags: `scope_ended` when the scope is gone or at another epoch,
3116/// `scope_changed` (retryable) when only its content moved.
3117fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
3118    let key = (scope.owner.clone(), scope.scope_ref.clone());
3119    match inner.scope_tags.get(&key) {
3120        Some(current) if *current == scope.tag => None,
3121        Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
3122            subc_protocol::error_codes::SCOPE_CHANGED,
3123            format!(
3124                "scope '{}' of {} changed while the route was being bound; re-open it",
3125                scope.scope_ref, scope.owner
3126            ),
3127        )),
3128        _ => Some(ErrorBody::new(
3129            subc_protocol::error_codes::SCOPE_ENDED,
3130            format!(
3131                "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
3132                scope.scope_ref, scope.owner, scope.tag.scope_epoch
3133            ),
3134        )),
3135    }
3136}
3137
3138fn release_reserved_route_locked(
3139    inner: &mut ForwardingInner,
3140    client_key: ClientRouteKey,
3141    module_key: ModuleRouteKey,
3142) {
3143    if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
3144        inner.reserved_client.remove(&client_key);
3145    }
3146    if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
3147        inner.reserved_module.remove(&module_key);
3148    }
3149    inner.status.retain(|(key, _), _| *key != client_key);
3150}
3151
3152fn release_client_route_locked(
3153    inner: &mut ForwardingInner,
3154    client_key: ClientRouteKey,
3155    expected_epoch: u32,
3156) -> RouteRelease {
3157    let Some(route) = inner.client_to_module.get(&client_key) else {
3158        return RouteRelease::Absent;
3159    };
3160    if route.client_epoch != expected_epoch {
3161        return RouteRelease::Stale;
3162    }
3163    let route = inner
3164        .client_to_module
3165        .remove(&client_key)
3166        .expect("route checked under the same forwarding lock");
3167    route.flow.close();
3168    if let Some(scope) = &route.scope {
3169        scope.release_bind();
3170    }
3171    inner.module_to_client.remove(&ModuleRouteKey {
3172        endpoint: route.module_endpoint,
3173        channel: route.module_channel,
3174    });
3175    inner.operator_confirms.route_closed(ModuleRouteKey {
3176        endpoint: route.module_endpoint,
3177        channel: route.module_channel,
3178    });
3179    inner.status.remove(&(client_key, expected_epoch));
3180    RouteRelease::Removed(GoodbyeTarget {
3181        connection_id: route.module_endpoint.connection_id,
3182        sink: route.module_sink.clone(),
3183        negotiated_ver: route.module_negotiated_ver,
3184        channel: route.module_channel,
3185        epoch: route.module_epoch,
3186        kind: GoodbyeTargetKind::Module,
3187        module_id: Some(route.module_id.clone()),
3188    })
3189}
3190
3191fn release_module_route_locked(
3192    inner: &mut ForwardingInner,
3193    module_key: ModuleRouteKey,
3194    expected_epoch: u32,
3195) -> RouteRelease {
3196    let Some(route) = inner.module_to_client.get(&module_key) else {
3197        return RouteRelease::Absent;
3198    };
3199    if route.module_epoch != expected_epoch {
3200        return RouteRelease::Stale;
3201    }
3202    let route = inner
3203        .module_to_client
3204        .remove(&module_key)
3205        .expect("route checked under the same forwarding lock");
3206    inner.operator_confirms.route_closed(module_key);
3207    route.flow.close();
3208    if let Some(scope) = &route.scope {
3209        scope.release_bind();
3210    }
3211    let client_key = ClientRouteKey {
3212        connection_id: route.client_connection_id,
3213        channel: route.client_channel,
3214    };
3215    inner.client_to_module.remove(&client_key);
3216    inner.status.remove(&(client_key, route.client_epoch));
3217    RouteRelease::Removed(GoodbyeTarget {
3218        connection_id: route.client_connection_id,
3219        sink: route.client_sink.clone(),
3220        negotiated_ver: route.client_negotiated_ver,
3221        channel: route.client_channel,
3222        epoch: route.client_epoch,
3223        kind: GoodbyeTargetKind::Client,
3224        module_id: Some(route.module_id.clone()),
3225    })
3226}
3227
3228fn commit_route_locked(
3229    inner: &mut ForwardingInner,
3230    pending: PendingRouteBindRelayEntry,
3231) -> Result<Option<GoodbyeTarget>, ForwardingError> {
3232    let reservation = pending.reservation;
3233    if inner
3234        .closing_connections
3235        .contains(&reservation.client_key.connection_id)
3236    {
3237        return Err(ForwardingError::ConnectionClosing {
3238            connection_id: reservation.client_key.connection_id,
3239        });
3240    }
3241    let module_id = inner
3242        .module_id_by_endpoint
3243        .get(&reservation.module_key.endpoint)
3244        .cloned()
3245        .ok_or(ForwardingError::StaleModuleEndpoint)?;
3246    if inner
3247        .draining_endpoints
3248        .contains_key(&reservation.module_key.endpoint)
3249    {
3250        return Err(ForwardingError::ModuleReloading { module_id });
3251    }
3252    if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
3253        || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
3254    {
3255        return Err(ForwardingError::UnknownReservation {
3256            client_channel: reservation.client_key.channel,
3257            module_channel: reservation.module_key.channel,
3258        });
3259    }
3260    let module = inner
3261        .modules_by_id
3262        .get(&module_id)
3263        .filter(|module| module.endpoint == reservation.module_key.endpoint)
3264        .cloned()
3265        .ok_or(ForwardingError::StaleModuleEndpoint)?;
3266    let binding = Arc::new(RouteBinding {
3267        client_connection_id: reservation.client_key.connection_id,
3268        client_sink: pending.client_sink,
3269        client_negotiated_ver: pending.client_negotiated_ver,
3270        client_channel: reservation.client_key.channel,
3271        client_epoch: reservation.client_epoch,
3272        module_id,
3273        module_endpoint: reservation.module_key.endpoint,
3274        module_sink: module.sink,
3275        module_negotiated_ver: module.negotiated_ver,
3276        module_channel: reservation.module_key.channel,
3277        module_epoch: reservation.module_epoch,
3278        principal: pending.principal,
3279        project_root: reservation.project_root.clone(),
3280        bound_at: Instant::now(),
3281        flow: Arc::new(ChannelFlow::new(window_for(&module.concurrency))),
3282        scope: pending.scope,
3283    });
3284    inner
3285        .client_to_module
3286        .insert(reservation.client_key, Arc::clone(&binding));
3287    inner
3288        .module_to_client
3289        .insert(reservation.module_key, Arc::clone(&binding));
3290    let previous_published = inner
3291        .last_published_epoch
3292        .insert(reservation.client_key, reservation.client_epoch);
3293
3294    // This point is reachable only through an accepted route.bind acknowledgement.
3295    // Publish history before notifying the opener, so a subsequent
3296    // `scope.served_by` query sees the acknowledged bind.
3297    if let Some(scope) = &binding.scope {
3298        scope.record_served(&binding.module_id);
3299    }
3300
3301    // Sending through the reserved slot cannot fail, but it reports a receiver
3302    // that closed after reservation and before this locked publication point.
3303    // The frame is stamped and charged to the queued-byte count here, when it
3304    // actually enters the queue, not when the slot was reserved.
3305    let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
3306    if client_writer_closed {
3307        // Not gated on `relay_enqueued`: only a module ack reaches this
3308        // commit, which proves the module holds the binding, and `route.open`
3309        // sends nothing for a relay a reply already settled.
3310        let abandoned = abandoned_route_target(inner, &reservation);
3311        if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
3312            route.flow.close();
3313        }
3314        inner.module_to_client.remove(&reservation.module_key);
3315        inner
3316            .status
3317            .remove(&(reservation.client_key, reservation.client_epoch));
3318        match previous_published {
3319            Some(epoch) => {
3320                inner
3321                    .last_published_epoch
3322                    .insert(reservation.client_key, epoch);
3323            }
3324            None => {
3325                inner.last_published_epoch.remove(&reservation.client_key);
3326            }
3327        }
3328        let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3329            "client egress closed during route publication".to_string(),
3330        ));
3331        return Ok(abandoned);
3332    }
3333
3334    if let Some(scope) = &binding.scope {
3335        scope.commit_presence();
3336    }
3337    let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
3338    Ok(None)
3339}
3340
3341/// The live module connection registered under exactly `endpoint`, whichever
3342/// slot holds it: active, swap candidate, or superseded incumbent.
3343///
3344/// Resolving by endpoint rather than by module id is what keeps an incumbent
3345/// addressable after a swap promoted a candidate over its id. An endpoint that
3346/// is in none of the three (a stale one, replaced without a promotion) resolves
3347/// to nothing, as it always has.
3348fn module_connection_for_endpoint_locked(
3349    inner: &ForwardingInner,
3350    endpoint: ModuleEndpointId,
3351) -> Option<&ModuleConnection> {
3352    let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
3353    inner
3354        .modules_by_id
3355        .get(module_id)
3356        .filter(|module| module.endpoint == endpoint)
3357        .or_else(|| {
3358            inner
3359                .candidates_by_id
3360                .get(module_id)
3361                .filter(|module| module.endpoint == endpoint)
3362        })
3363        .or_else(|| inner.superseded_endpoints.get(&endpoint))
3364}
3365
3366fn abandoned_route_target(
3367    inner: &ForwardingInner,
3368    reservation: &RouteReservation,
3369) -> Option<GoodbyeTarget> {
3370    let module_id = inner
3371        .module_id_by_endpoint
3372        .get(&reservation.module_key.endpoint)?;
3373    let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
3374    (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
3375        connection_id: module.endpoint.connection_id,
3376        sink: module.sink.clone(),
3377        negotiated_ver: module.negotiated_ver,
3378        channel: reservation.module_key.channel,
3379        epoch: reservation.module_epoch,
3380        kind: GoodbyeTargetKind::Module,
3381        module_id: Some(module_id.clone()),
3382    })
3383}
3384
3385/// Queue a registering module's HELLO_ACK on its sink. Called with the
3386/// forwarding write lock held, before the endpoint is inserted, which is what
3387/// puts the ack ahead of any `route.bind` or control RPC routed to the module.
3388/// `try_send` never waits, so holding the lock across it is safe.
3389fn enqueue_hello_ack_locked(
3390    sink: &FrameSink,
3391    connection_id: ConnectionId,
3392    hello_ack: Option<Frame>,
3393) -> Result<(), ForwardingError> {
3394    let Some(hello_ack) = hello_ack else {
3395        return Ok(());
3396    };
3397    sink.try_send(hello_ack)
3398        .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
3399}
3400
3401fn remove_module_connection_locked(
3402    inner: &mut ForwardingInner,
3403    endpoint: ModuleEndpointId,
3404) -> ConnectionCleanup {
3405    // Commit the disconnect before its routes are released, so their teardown
3406    // cannot replace module_closed with route_closed.
3407    inner.operator_confirms.module_closed(endpoint);
3408    inner.draining_endpoints.remove(&endpoint);
3409    inner.daemon_drain_results.remove(&endpoint);
3410    let module_id = inner.module_id_by_endpoint.remove(&endpoint);
3411    if let Some(module_id) = module_id.as_ref() {
3412        if inner
3413            .modules_by_id
3414            .get(module_id)
3415            .is_some_and(|module| module.endpoint == endpoint)
3416        {
3417            inner.modules_by_id.remove(module_id);
3418        }
3419        if inner
3420            .candidates_by_id
3421            .get(module_id)
3422            .is_some_and(|module| module.endpoint == endpoint)
3423        {
3424            inner.candidates_by_id.remove(module_id);
3425        }
3426    }
3427    inner.superseded_endpoints.remove(&endpoint);
3428    inner.endpoint_by_connection.remove(&endpoint.connection_id);
3429    inner.next_module_channel.remove(&endpoint);
3430    inner.next_control_corr.remove(&endpoint);
3431    inner
3432        .health_probe_tombstones
3433        .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
3434    inner
3435        .module_slot_epochs
3436        .retain(|key, _| key.endpoint != endpoint);
3437    let reserved_module_keys: Vec<ModuleRouteKey> = inner
3438        .reserved_module
3439        .keys()
3440        .filter(|module_key| module_key.endpoint == endpoint)
3441        .copied()
3442        .collect();
3443    for module_key in reserved_module_keys {
3444        if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
3445            release_reserved_route_locked(inner, client_key, module_key);
3446        }
3447    }
3448
3449    let pending_keys: Vec<_> = inner
3450        .pending_relays
3451        .keys()
3452        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3453        .copied()
3454        .collect();
3455    let pending: Vec<_> = pending_keys
3456        .into_iter()
3457        .filter_map(|key| {
3458            inner
3459                .pending_relays
3460                .remove(&key)
3461                .map(|pending| (key, pending))
3462        })
3463        .collect();
3464    let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
3465    for (key, pending) in pending {
3466        // Recorded like every other removal without a reply, so a `route.open`
3467        // still sending the bind learns the relay was released rather than
3468        // settled by the module.
3469        if !pending.relay_enqueued {
3470            inner.relays_released_before_enqueue.insert(key);
3471        }
3472        let module_label = module_id.as_deref().unwrap_or("unknown");
3473        let _ = pending
3474            .sender
3475            .send(RouteBindRelayOutcome::ModuleGone(format!(
3476                "module '{module_label}' connection closed during route.bind relay"
3477            )));
3478    }
3479
3480    let pending_control_keys: Vec<_> = inner
3481        .pending_control_rpcs
3482        .keys()
3483        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3484        .copied()
3485        .collect();
3486    let pending_control: Vec<_> = pending_control_keys
3487        .into_iter()
3488        .filter_map(|key| inner.pending_control_rpcs.remove(&key))
3489        .collect();
3490    for pending in pending_control {
3491        let module_label = module_id.as_deref().unwrap_or("unknown");
3492        let _ = pending
3493            .sender
3494            .send(ModuleControlRpcOutcome::ModuleGone(format!(
3495                "module '{module_label}' connection closed during module-control RPC"
3496            )));
3497    }
3498
3499    let module_routes = inner
3500        .module_to_client
3501        .iter()
3502        .filter(|(module_key, _)| module_key.endpoint == endpoint)
3503        .map(|(module_key, route)| (*module_key, route.module_epoch))
3504        .collect::<Vec<_>>();
3505    let mut released = Vec::with_capacity(module_routes.len());
3506    for (module_key, epoch) in module_routes {
3507        if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
3508        {
3509            released.push(target);
3510        }
3511    }
3512    ConnectionCleanup {
3513        released,
3514        abandoned_relays,
3515    }
3516}
3517
3518#[derive(Debug, Clone, Copy)]
3519struct RequestCredit {
3520    subscription: bool,
3521    excluded_from_drain: bool,
3522}
3523
3524#[derive(Debug, Default)]
3525struct CreditLedger {
3526    by_corr: HashMap<u64, Vec<RequestCredit>>,
3527}
3528
3529impl CreditLedger {
3530    fn acquire(&mut self, corr: u64, subscription: bool) {
3531        self.by_corr.entry(corr).or_default().push(RequestCredit {
3532            subscription,
3533            excluded_from_drain: false,
3534        });
3535    }
3536
3537    fn release(&mut self, corr: u64) -> bool {
3538        let Some(credits) = self.by_corr.get_mut(&corr) else {
3539            return false;
3540        };
3541        let released = credits.pop().is_some();
3542        if credits.is_empty() {
3543            self.by_corr.remove(&corr);
3544        }
3545        released
3546    }
3547
3548    fn capture_subscription_exclusions(&mut self) -> u32 {
3549        let mut excluded = 0u32;
3550        for credit in self.by_corr.values_mut().flatten() {
3551            if credit.subscription && !credit.excluded_from_drain {
3552                credit.excluded_from_drain = true;
3553                excluded = excluded.saturating_add(1);
3554            }
3555        }
3556        excluded
3557    }
3558
3559    #[cfg(test)]
3560    fn in_flight(&self) -> usize {
3561        self.by_corr.values().map(Vec::len).sum()
3562    }
3563
3564    fn drain_in_flight(&self) -> usize {
3565        self.by_corr
3566            .values()
3567            .flatten()
3568            .filter(|credit| !credit.excluded_from_drain)
3569            .count()
3570    }
3571
3572    /// The correlation ids of the requests a drain still waits on, one entry
3573    /// per held credit, in ascending order.
3574    fn drain_held_corrs(&self) -> Vec<u64> {
3575        let mut corrs = self
3576            .by_corr
3577            .iter()
3578            .flat_map(|(corr, credits)| {
3579                credits
3580                    .iter()
3581                    .filter(|credit| !credit.excluded_from_drain)
3582                    .map(move |_| *corr)
3583            })
3584            .collect::<Vec<_>>();
3585        corrs.sort_unstable();
3586        corrs
3587    }
3588}
3589
3590#[derive(Debug, Default)]
3591struct ChannelFlowState {
3592    closed: bool,
3593    credits: CreditLedger,
3594}
3595
3596/// Per-channel request-credit accounting shared by the client and module route halves.
3597#[derive(Debug)]
3598pub(crate) struct ChannelFlow {
3599    sem: Semaphore,
3600    window: usize,
3601    state: Mutex<ChannelFlowState>,
3602}
3603
3604impl ChannelFlow {
3605    pub(crate) fn new(window: usize) -> Self {
3606        debug_assert!(window > 0, "flow-control window must be non-zero");
3607        Self {
3608            sem: Semaphore::new(window),
3609            window,
3610            state: Mutex::new(ChannelFlowState::default()),
3611        }
3612    }
3613
3614    #[cfg(test)]
3615    pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
3616        self.acquire_tagged(0, false).await
3617    }
3618
3619    pub(crate) async fn acquire_tagged(
3620        &self,
3621        corr: u64,
3622        subscription: bool,
3623    ) -> Result<(), ChannelFlowClosed> {
3624        let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
3625        let mut state = self
3626            .state
3627            .lock()
3628            .unwrap_or_else(|poisoned| poisoned.into_inner());
3629        if state.closed {
3630            return Err(ChannelFlowClosed);
3631        }
3632        state.credits.acquire(corr, subscription);
3633        permit.forget();
3634        Ok(())
3635    }
3636
3637    #[cfg(test)]
3638    pub(crate) fn release(&self) {
3639        self.release_corr(0);
3640    }
3641
3642    pub(crate) fn release_corr(&self, corr: u64) {
3643        let released = self
3644            .state
3645            .lock()
3646            .unwrap_or_else(|poisoned| poisoned.into_inner())
3647            .credits
3648            .release(corr);
3649        if !released {
3650            // Protocol-conforming modules emit exactly one terminal per request.
3651            // This guard is a best-effort safety net against window growth, not a
3652            // security boundary against malicious peers.
3653            warn!(
3654                window = self.window,
3655                available = self.sem.available_permits(),
3656                "flow-control over-release ignored"
3657            );
3658            return;
3659        }
3660        if !self.sem.is_closed() {
3661            self.sem.add_permits(1);
3662        }
3663    }
3664
3665    #[cfg(test)]
3666    pub(crate) fn in_flight(&self) -> usize {
3667        self.state
3668            .lock()
3669            .unwrap_or_else(|poisoned| poisoned.into_inner())
3670            .credits
3671            .in_flight()
3672    }
3673
3674    pub(crate) fn drain_in_flight(&self) -> usize {
3675        self.state
3676            .lock()
3677            .unwrap_or_else(|poisoned| poisoned.into_inner())
3678            .credits
3679            .drain_in_flight()
3680    }
3681
3682    pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
3683        self.state
3684            .lock()
3685            .unwrap_or_else(|poisoned| poisoned.into_inner())
3686            .credits
3687            .drain_held_corrs()
3688    }
3689
3690    #[cfg(test)]
3691    pub(crate) fn available_permits(&self) -> usize {
3692        self.sem.available_permits()
3693    }
3694
3695    pub(crate) fn begin_drain(&self) -> u32 {
3696        let mut state = self
3697            .state
3698            .lock()
3699            .unwrap_or_else(|poisoned| poisoned.into_inner());
3700        state.closed = true;
3701        self.sem.close();
3702        state.credits.capture_subscription_exclusions()
3703    }
3704
3705    pub(crate) fn close(&self) {
3706        self.state
3707            .lock()
3708            .unwrap_or_else(|poisoned| poisoned.into_inner())
3709            .closed = true;
3710        self.sem.close();
3711    }
3712}
3713
3714#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3715pub(crate) struct ChannelFlowClosed;
3716
3717impl fmt::Display for ChannelFlowClosed {
3718    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3719        write!(f, "flow-control window closed")
3720    }
3721}
3722
3723impl Error for ChannelFlowClosed {}
3724
3725fn window_for(concurrency: &Concurrency) -> usize {
3726    match concurrency {
3727        Concurrency::Serial => 1,
3728        Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
3729        Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
3730    }
3731}
3732
3733#[derive(Debug, Clone, PartialEq, Eq)]
3734pub enum ForwardingError {
3735    /// A connection cannot hold both client routes and a module endpoint, or
3736    /// replace a module identity while that endpoint is still registered.
3737    ConnectionRoleConflict {
3738        connection_id: ConnectionId,
3739    },
3740    NoModuleConnection,
3741    ModuleReloading {
3742        module_id: String,
3743    },
3744    StaleModuleEndpoint,
3745    UnknownReservation {
3746        client_channel: u16,
3747        module_channel: u16,
3748    },
3749    ClientRouteChannelExhausted {
3750        connection_id: ConnectionId,
3751    },
3752    ModuleRouteChannelExhausted {
3753        endpoint: ModuleEndpointId,
3754    },
3755    RelayCorrelationExhausted,
3756    ConnectionClosing {
3757        connection_id: ConnectionId,
3758    },
3759    ClientEgressClosed {
3760        connection_id: ConnectionId,
3761    },
3762    RouteOpenBuild(String),
3763    /// A swap candidate is already registered for this module id.
3764    CandidateSlotOccupied {
3765        module_id: String,
3766    },
3767    /// A registering module's HELLO_ACK could not be queued because its
3768    /// outbound queue is closed or full.
3769    ModuleEgressUnavailable {
3770        connection_id: ConnectionId,
3771    },
3772    Poisoned,
3773}
3774
3775impl fmt::Display for ForwardingError {
3776    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3777        match self {
3778            Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
3779            Self::NoModuleConnection => write!(f, "no module connection is registered"),
3780            Self::ModuleReloading { module_id } => {
3781                write!(f, "module_id '{module_id}' is reloading")
3782            }
3783            Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
3784            Self::UnknownReservation {
3785                client_channel,
3786                module_channel,
3787            } => write!(
3788                f,
3789                "route reservation client channel {client_channel} / module channel {module_channel} was not found"
3790            ),
3791            Self::ClientRouteChannelExhausted { connection_id } => write!(
3792                f,
3793                "no client route channels are available for connection {}",
3794                connection_id.get()
3795            ),
3796            Self::ModuleRouteChannelExhausted { endpoint } => write!(
3797                f,
3798                "no module route channels are available for endpoint generation {} on connection {}",
3799                endpoint.generation,
3800                endpoint.connection_id.get()
3801            ),
3802            Self::RelayCorrelationExhausted => {
3803                write!(f, "module control correlation ids are exhausted")
3804            }
3805            Self::ConnectionClosing { connection_id } => write!(
3806                f,
3807                "connection {} is closing and cannot accept route allocation",
3808                connection_id.get()
3809            ),
3810            Self::ClientEgressClosed { connection_id } => write!(
3811                f,
3812                "client connection {} egress is closed",
3813                connection_id.get()
3814            ),
3815            Self::RouteOpenBuild(message) => {
3816                write!(f, "failed to prebuild route.open response: {message}")
3817            }
3818            Self::CandidateSlotOccupied { module_id } => write!(
3819                f,
3820                "module_id '{module_id}' already has a swap candidate registered"
3821            ),
3822            Self::ModuleEgressUnavailable { connection_id } => write!(
3823                f,
3824                "module connection {} egress is unavailable; HELLO_ACK could not be queued",
3825                connection_id.get()
3826            ),
3827            Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
3828        }
3829    }
3830}
3831
3832impl Error for ForwardingError {}
3833
3834#[cfg(test)]
3835mod tests {
3836    use std::time::Duration;
3837
3838    use super::*;
3839    use tokio::sync::mpsc;
3840
3841    #[test]
3842    fn ordinary_long_running_request_is_not_excluded_from_drain() {
3843        let mut ledger = CreditLedger::default();
3844        ledger.acquire(1, false);
3845
3846        assert_eq!(ledger.capture_subscription_exclusions(), 0);
3847        assert_eq!(ledger.drain_in_flight(), 1);
3848    }
3849
3850    #[test]
3851    fn bit_set_subscription_is_excluded_and_counted() {
3852        let mut ledger = CreditLedger::default();
3853        ledger.acquire(1, true);
3854
3855        assert_eq!(ledger.capture_subscription_exclusions(), 1);
3856        assert_eq!(ledger.drain_in_flight(), 0);
3857    }
3858
3859    #[test]
3860    fn subscription_opened_after_drain_snapshot_is_not_excluded() {
3861        let mut ledger = CreditLedger::default();
3862        ledger.acquire(1, true);
3863        assert_eq!(ledger.capture_subscription_exclusions(), 1);
3864
3865        ledger.acquire(2, true);
3866
3867        assert_eq!(ledger.drain_in_flight(), 1);
3868    }
3869
3870    #[test]
3871    fn drain_with_no_subscriptions_reports_zero_excluded() {
3872        let mut ledger = CreditLedger::default();
3873        assert_eq!(ledger.capture_subscription_exclusions(), 0);
3874    }
3875
3876    fn test_hello_ack(corr: u64) -> Frame {
3877        Frame::build(
3878            FrameType::HelloAck,
3879            Flags::new(false, Priority::Passive, false),
3880            0,
3881            0,
3882            corr,
3883            Vec::new(),
3884        )
3885        .unwrap()
3886    }
3887
3888    /// An acked registration whose HELLO_ACK cannot be queued must leave no
3889    /// endpoint behind: a routable module that never got its ack would read a
3890    /// route.bind first and exit. A closed queue and a full one both refuse.
3891    #[test]
3892    fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
3893        let forwarding = ForwardingTable::default();
3894
3895        let (closed_tx, closed_rx) = mpsc::channel(8);
3896        drop(closed_rx);
3897        let closed = ConnectionId::new(1);
3898        assert_eq!(
3899            forwarding.register_module_connection_acked(
3900                closed,
3901                "closed".to_string(),
3902                2,
3903                Concurrency::ModuleManaged,
3904                FrameSink::new(closed_tx),
3905                test_hello_ack(1),
3906            ),
3907            Err(ForwardingError::ModuleEgressUnavailable {
3908                connection_id: closed
3909            })
3910        );
3911
3912        let (full_tx, _full_rx) = mpsc::channel(1);
3913        let full_sink = FrameSink::new(full_tx);
3914        full_sink.try_send(test_hello_ack(99)).unwrap();
3915        let full = ConnectionId::new(2);
3916        assert_eq!(
3917            forwarding.register_module_connection_acked(
3918                full,
3919                "full".to_string(),
3920                2,
3921                Concurrency::ModuleManaged,
3922                full_sink.clone(),
3923                test_hello_ack(2),
3924            ),
3925            Err(ForwardingError::ModuleEgressUnavailable {
3926                connection_id: full
3927            })
3928        );
3929        assert_eq!(
3930            forwarding.register_candidate_module_connection_acked(
3931                full,
3932                "full".to_string(),
3933                2,
3934                Concurrency::ModuleManaged,
3935                full_sink,
3936                test_hello_ack(3),
3937            ),
3938            Err(ForwardingError::ModuleEgressUnavailable {
3939                connection_id: full
3940            })
3941        );
3942
3943        for (connection, module_id) in [(closed, "closed"), (full, "full")] {
3944            assert_eq!(
3945                forwarding
3946                    .module_endpoint_for_connection(connection)
3947                    .unwrap(),
3948                None
3949            );
3950            let (client_tx, _client_rx) = mpsc::channel(8);
3951            assert_eq!(
3952                forwarding
3953                    .begin_route_bind_relay_for_test(
3954                        ConnectionId::new(50),
3955                        FrameSink::new(client_tx),
3956                        1,
3957                        module_id,
3958                    )
3959                    .err(),
3960                Some(ForwardingError::NoModuleConnection)
3961            );
3962        }
3963        assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
3964    }
3965
3966    /// Both acked registration forms put the HELLO_ACK on the module's queue
3967    /// by the time the endpoint can be resolved.
3968    #[test]
3969    fn acked_registration_queues_the_hello_ack_first() {
3970        let forwarding = ForwardingTable::default();
3971        let (active_tx, mut active_rx) = mpsc::channel(8);
3972        forwarding
3973            .register_module_connection_acked(
3974                ConnectionId::new(1),
3975                "acked".to_string(),
3976                2,
3977                Concurrency::ModuleManaged,
3978                FrameSink::new(active_tx),
3979                test_hello_ack(11),
3980            )
3981            .unwrap();
3982        let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
3983        forwarding
3984            .register_candidate_module_connection_acked(
3985                ConnectionId::new(2),
3986                "acked".to_string(),
3987                2,
3988                Concurrency::ModuleManaged,
3989                FrameSink::new(candidate_tx),
3990                test_hello_ack(12),
3991            )
3992            .unwrap();
3993
3994        let active_first = active_rx.try_recv().unwrap().frame;
3995        assert_eq!(active_first.header.ty, FrameType::HelloAck);
3996        assert_eq!(active_first.header.corr, 11);
3997        let candidate_first = candidate_rx.try_recv().unwrap().frame;
3998        assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
3999        assert_eq!(candidate_first.header.corr, 12);
4000    }
4001
4002    #[test]
4003    fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
4004        let forwarding = ForwardingTable::default();
4005        let module_connection = ConnectionId::new(10);
4006        let exhausted_client = ConnectionId::new(20);
4007        let second_client = ConnectionId::new(30);
4008        let (module_tx, _module_rx) = mpsc::channel(1);
4009        let endpoint = forwarding
4010            .register_module_connection(
4011                module_connection,
4012                "route-limit-provider".to_string(),
4013                1,
4014                Concurrency::ModuleManaged,
4015                FrameSink::new(module_tx),
4016            )
4017            .unwrap();
4018
4019        {
4020            let mut inner = forwarding.inner.write().unwrap();
4021            for channel in 1..=u16::MAX {
4022                inner.reserved_client.insert(
4023                    ClientRouteKey {
4024                        connection_id: exhausted_client,
4025                        channel,
4026                    },
4027                    ModuleRouteKey {
4028                        endpoint,
4029                        channel: 1,
4030                    },
4031                );
4032            }
4033        }
4034
4035        let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
4036        let err = forwarding
4037            .begin_route_bind_relay_for_test(
4038                exhausted_client,
4039                FrameSink::new(exhausted_tx),
4040                1,
4041                "route-limit-provider",
4042            )
4043            .unwrap_err();
4044        assert!(matches!(
4045            err,
4046            ForwardingError::ClientRouteChannelExhausted { connection_id }
4047                if connection_id == exhausted_client
4048        ));
4049
4050        let (second_tx, _second_rx) = mpsc::channel(1);
4051        let pending = forwarding
4052            .begin_route_bind_relay_for_test(
4053                second_client,
4054                FrameSink::new(second_tx),
4055                2,
4056                "route-limit-provider",
4057            )
4058            .unwrap();
4059        assert_eq!(pending.client_channel, 1);
4060    }
4061
4062    #[test]
4063    fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
4064        let forwarding = ForwardingTable::default();
4065        let module_connection = ConnectionId::new(40);
4066        let client = ConnectionId::new(50);
4067        let (module_tx, _module_rx) = mpsc::channel(1);
4068        forwarding
4069            .register_module_connection(
4070                module_connection,
4071                "slot-reuse-provider".to_string(),
4072                1,
4073                Concurrency::ModuleManaged,
4074                FrameSink::new(module_tx),
4075            )
4076            .unwrap();
4077
4078        let (client_tx, _client_rx) = mpsc::channel(1);
4079        let client_sink = FrameSink::new(client_tx);
4080        let mut wrapped_channel = None;
4081        for index in 0..=usize::from(u16::MAX) {
4082            let pending = forwarding
4083                .begin_route_bind_relay_for_test(
4084                    client,
4085                    client_sink.clone(),
4086                    index as u64 + 1,
4087                    "slot-reuse-provider",
4088                )
4089                .unwrap();
4090            if index == usize::from(u16::MAX) {
4091                wrapped_channel = Some(pending.module_channel);
4092            }
4093            forwarding
4094                .abort_pending_relay(
4095                    pending.endpoint,
4096                    pending.corr,
4097                    RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
4098                )
4099                .unwrap();
4100        }
4101
4102        assert_eq!(wrapped_channel, Some(1));
4103    }
4104
4105    #[test]
4106    fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
4107        let forwarding = ForwardingTable::default();
4108        let client = ConnectionId::new(60);
4109        forwarding
4110            .inner
4111            .write()
4112            .unwrap()
4113            .next_client_channel
4114            .insert(client, 41);
4115
4116        let released = forwarding.cleanup_connection(client).unwrap();
4117
4118        assert!(released.is_empty());
4119        assert!(!forwarding
4120            .inner
4121            .read()
4122            .unwrap()
4123            .next_client_channel
4124            .contains_key(&client));
4125    }
4126
4127    #[test]
4128    fn stale_module_cleanup_preserves_fast_reconnect_successor() {
4129        let forwarding = ForwardingTable::default();
4130        let module_id = "fast-reconnect-provider";
4131        let first_connection = ConnectionId::new(70);
4132        let second_connection = ConnectionId::new(80);
4133        let (first_tx, _first_rx) = mpsc::channel(1);
4134        let first_endpoint = forwarding
4135            .register_module_connection(
4136                first_connection,
4137                module_id.to_string(),
4138                1,
4139                Concurrency::ModuleManaged,
4140                FrameSink::new(first_tx),
4141            )
4142            .unwrap();
4143        let (second_tx, _second_rx) = mpsc::channel(1);
4144        let second_endpoint = forwarding
4145            .register_module_connection(
4146                second_connection,
4147                module_id.to_string(),
4148                1,
4149                Concurrency::ModuleManaged,
4150                FrameSink::new(second_tx),
4151            )
4152            .unwrap();
4153        assert_ne!(first_endpoint, second_endpoint);
4154
4155        let released = forwarding.cleanup_connection(first_connection).unwrap();
4156
4157        assert!(released.is_empty());
4158        assert_eq!(
4159            forwarding
4160                .inner
4161                .read()
4162                .unwrap()
4163                .modules_by_id
4164                .get(module_id)
4165                .map(|module| module.endpoint),
4166            Some(second_endpoint)
4167        );
4168        assert!(forwarding.has_live_module_connection(module_id).unwrap());
4169        let control_rpc = forwarding
4170            .begin_module_control_rpc_for(
4171                module_id,
4172                "health.check",
4173                Instant::now() + Duration::from_secs(1),
4174            )
4175            .unwrap();
4176        assert_eq!(control_rpc.endpoint, second_endpoint);
4177    }
4178
4179    fn route_fixture(
4180        module_id: &str,
4181    ) -> (
4182        ForwardingTable,
4183        ConnectionId,
4184        ModuleEndpointId,
4185        ConnectionId,
4186        FrameSink,
4187        mpsc::Receiver<crate::router::OutboundFrame>,
4188    ) {
4189        let forwarding = ForwardingTable::default();
4190        let module_connection = ConnectionId::new(100);
4191        let client_connection = ConnectionId::new(200);
4192        let (module_tx, _module_rx) = mpsc::channel(8);
4193        let endpoint = forwarding
4194            .register_module_connection(
4195                module_connection,
4196                module_id.to_string(),
4197                2,
4198                Concurrency::ModuleManaged,
4199                FrameSink::new(module_tx),
4200            )
4201            .unwrap();
4202        let (client_tx, client_rx) = mpsc::channel(8);
4203        (
4204            forwarding,
4205            module_connection,
4206            endpoint,
4207            client_connection,
4208            FrameSink::new(client_tx),
4209            client_rx,
4210        )
4211    }
4212
4213    #[test]
4214    #[cfg(unix)]
4215    fn daemon_drain_gates_current_and_racing_provider_registrations() {
4216        let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
4217        assert_eq!(
4218            forwarding.begin_daemon_drain().unwrap(),
4219            [("provider".to_string(), endpoint)]
4220        );
4221        assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
4222        assert!(matches!(
4223            forwarding.register_module_connection(
4224                ConnectionId::new(300),
4225                "late-provider".into(),
4226                2,
4227                Concurrency::ModuleManaged,
4228                sink,
4229            ),
4230            Err(ForwardingError::ConnectionClosing { .. })
4231        ));
4232    }
4233
4234    #[test]
4235    #[cfg(unix)]
4236    fn daemon_drain_results_are_endpoint_fenced_and_removed_on_cleanup() {
4237        let (forwarding, connection, endpoint, _, _, _) = route_fixture("provider");
4238        assert_eq!(
4239            forwarding
4240                .daemon_drain_result_for_connection(connection)
4241                .unwrap(),
4242            None
4243        );
4244        forwarding
4245            .record_daemon_drain_result(endpoint, true)
4246            .unwrap();
4247        forwarding.begin_daemon_drain().unwrap();
4248        assert_eq!(
4249            forwarding
4250                .daemon_drain_result_for_connection(connection)
4251                .unwrap(),
4252            Some(false)
4253        );
4254        let replacement = ModuleEndpointId {
4255            generation: endpoint.generation + 1,
4256            ..endpoint
4257        };
4258        forwarding
4259            .record_daemon_drain_result(replacement, true)
4260            .unwrap();
4261        assert_eq!(
4262            forwarding
4263                .daemon_drain_result_for_connection(connection)
4264                .unwrap(),
4265            Some(false)
4266        );
4267        forwarding
4268            .record_daemon_drain_result(endpoint, true)
4269            .unwrap();
4270        assert_eq!(
4271            forwarding
4272                .daemon_drain_result_for_connection(connection)
4273                .unwrap(),
4274            Some(true)
4275        );
4276        assert_eq!(
4277            forwarding
4278                .daemon_drain_result_for_connection(ConnectionId::new(300))
4279                .unwrap(),
4280            Some(false)
4281        );
4282        forwarding.cleanup_connection(connection).unwrap();
4283        forwarding
4284            .record_daemon_drain_result(endpoint, true)
4285            .unwrap();
4286        assert_eq!(
4287            forwarding
4288                .daemon_drain_result_for_connection(connection)
4289                .unwrap(),
4290            Some(false)
4291        );
4292        assert!(forwarding
4293            .read_inner()
4294            .unwrap()
4295            .daemon_drain_results
4296            .is_empty());
4297    }
4298
4299    #[test]
4300    #[cfg(unix)]
4301    fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
4302        let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
4303            route_fixture("provider");
4304        let mut pending =
4305            begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
4306        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
4307        forwarding.begin_daemon_drain().unwrap();
4308        let completion = forwarding
4309            .complete_pending_relay(
4310                module_connection,
4311                pending.corr,
4312                RouteBindRelayOutcome::Accepted,
4313            )
4314            .expect("draining admission is not a fatal module error");
4315        assert!(completion.settled);
4316        let abandoned = completion
4317            .abandoned
4318            .expect("late accepted bind needs a route GOODBYE");
4319        assert_eq!(abandoned.channel, pending.module_channel);
4320        assert!(matches!(pending.receiver.try_recv().unwrap(),
4321            RouteBindRelayOutcome::DaemonRejected(error) if error.code == "module_reloading"));
4322        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4323        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4324        assert!(
4325            rx.try_recv().is_err(),
4326            "no route.open may be published during drain"
4327        );
4328        assert_eq!(
4329            forwarding
4330                .module_endpoint_for_connection(module_connection)
4331                .unwrap(),
4332            Some(endpoint)
4333        );
4334        assert!(
4335            !forwarding
4336                .complete_pending_relay(
4337                    module_connection,
4338                    pending.corr,
4339                    RouteBindRelayOutcome::Accepted,
4340                )
4341                .unwrap()
4342                .settled
4343        );
4344    }
4345
4346    fn test_ping(corr: u64) -> Frame {
4347        Frame::build(
4348            FrameType::Ping,
4349            Flags::new(false, Priority::Passive, false),
4350            0,
4351            0,
4352            corr,
4353            Vec::new(),
4354        )
4355        .unwrap()
4356    }
4357
4358    fn begin_test_route(
4359        forwarding: &ForwardingTable,
4360        client_connection: ConnectionId,
4361        client_sink: FrameSink,
4362        corr: u64,
4363        module_id: &str,
4364    ) -> PendingRouteBindRelay {
4365        forwarding
4366            .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
4367            .unwrap()
4368    }
4369
4370    #[test]
4371    fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
4372        for candidate in [false, true] {
4373            for register_first in [false, true] {
4374                let (forwarding, module_connection, _, client_connection, sink, _rx) =
4375                    route_fixture("target");
4376                let register = || {
4377                    if candidate {
4378                        forwarding.register_candidate_module_connection(
4379                            client_connection,
4380                            "source".into(),
4381                            2,
4382                            Concurrency::ModuleManaged,
4383                            sink.clone(),
4384                        )
4385                    } else {
4386                        forwarding.register_module_connection(
4387                            client_connection,
4388                            "source".into(),
4389                            2,
4390                            Concurrency::ModuleManaged,
4391                            sink.clone(),
4392                        )
4393                    }
4394                };
4395                if register_first {
4396                    register().unwrap();
4397                    assert!(
4398                        forwarding
4399                            .begin_route_bind_relay_for_test(
4400                                client_connection,
4401                                sink.clone(),
4402                                1,
4403                                "target",
4404                            )
4405                            .is_err(),
4406                        "a deferred route.open cannot reserve after HELLO"
4407                    );
4408                    assert!(forwarding
4409                        .cleanup_connection(client_connection)
4410                        .unwrap()
4411                        .is_empty());
4412                } else {
4413                    let pending =
4414                        begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
4415                    assert!(
4416                        register().is_err(),
4417                        "HELLO cannot register after a route reservation"
4418                    );
4419                    forwarding
4420                        .complete_pending_relay(
4421                            module_connection,
4422                            pending.corr,
4423                            RouteBindRelayOutcome::Accepted,
4424                        )
4425                        .unwrap();
4426                    assert_eq!(
4427                        forwarding
4428                            .cleanup_connection(client_connection)
4429                            .unwrap()
4430                            .len(),
4431                        1
4432                    );
4433                }
4434                assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4435                assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4436            }
4437        }
4438    }
4439
4440    /// A pending route.open reserves its queue slot with `reserve_owned` and
4441    /// sends its prebuilt response through that slot when the module accepts.
4442    /// A queue already holding many data frames (far more than the old 64-frame
4443    /// queue) must not stop the open from reserving or completing, and the
4444    /// response must be charged to the queue's byte count when it is sent.
4445    #[tokio::test]
4446    async fn pending_route_open_completes_behind_queued_data_frames() {
4447        assert_eq!(
4448            crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
4449            8,
4450            "the per-connection pending route.open limit is its own constant"
4451        );
4452        let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
4453            route_fixture("open-behind-data");
4454        let (sink, mut client_rx) = crate::server::connection_egress();
4455        const DATA_FRAMES: usize = 1_000;
4456        let data = |corr: u64| {
4457            Frame::build(
4458                FrameType::StreamData,
4459                Flags::new(false, Priority::Interactive, false),
4460                9,
4461                1,
4462                corr,
4463                vec![b'x'; 200],
4464            )
4465            .unwrap()
4466        };
4467        for corr in 0..DATA_FRAMES as u64 {
4468            sink.try_send(data(corr)).unwrap();
4469        }
4470        let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
4471        assert_eq!(sink.backlog().queued_bytes, data_bytes);
4472
4473        let pending = tokio::time::timeout(
4474            Duration::from_secs(5),
4475            forwarding.begin_route_bind_relay_for(
4476                client,
4477                sink.clone(),
4478                subc_protocol::PROTOCOL_VERSION,
4479                4_242,
4480                "open-behind-data",
4481                Principal::Direct,
4482                None,
4483                None,
4484                Instant::now() + Duration::from_secs(60),
4485            ),
4486        )
4487        .await
4488        .expect("reserving the route.open slot must not wait behind data frames")
4489        .unwrap();
4490        forwarding
4491            .complete_pending_relay(
4492                module_connection,
4493                pending.corr,
4494                RouteBindRelayOutcome::Accepted,
4495            )
4496            .unwrap();
4497
4498        let backlog = sink.backlog();
4499        assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
4500        assert!(
4501            backlog.queued_bytes > data_bytes,
4502            "the route.open response must be counted in queued bytes: {backlog:?}"
4503        );
4504        for corr in 0..DATA_FRAMES as u64 {
4505            assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
4506        }
4507        let open = client_rx.recv().await.unwrap();
4508        assert_eq!(open.header.corr, 4_242);
4509        assert_eq!(open.header.ty, FrameType::Response);
4510        drop(open);
4511        assert_eq!(sink.backlog().queued_bytes, 0);
4512        assert_eq!(sink.backlog().queued_frames, 0);
4513    }
4514
4515    /// The drain-timeout line reports these numbers, so they must count the
4516    /// requests the drain is waiting on and no others: a route with nothing in
4517    /// flight is not a holdout, and a flagged subscription is excluded from the
4518    /// drain and so from the count.
4519    #[tokio::test]
4520    async fn drain_holdouts_count_held_requests_and_name_the_connection() {
4521        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4522            route_fixture("holdouts");
4523        let mut bound = |corr| {
4524            let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
4525            forwarding
4526                .complete_pending_relay(
4527                    module_connection,
4528                    route.corr,
4529                    RouteBindRelayOutcome::Accepted,
4530                )
4531                .unwrap();
4532            client_rx.try_recv().unwrap();
4533            match forwarding
4534                .lookup_data_route(client, route.client_channel, route.client_epoch)
4535                .unwrap()
4536            {
4537                DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4538                other => panic!("expected live route, got {other:?}"),
4539            }
4540        };
4541        let holding = bound(61);
4542        let _idle = bound(62);
4543        holding.flow.acquire_tagged(7, false).await.unwrap();
4544        holding.flow.acquire_tagged(2, false).await.unwrap();
4545        holding.flow.acquire_tagged(3, true).await.unwrap();
4546        forwarding
4547            .begin_module_drain("holdouts", RouteCloseReason::Restart)
4548            .unwrap();
4549
4550        let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
4551        assert_eq!(
4552            holdouts,
4553            DrainHoldouts {
4554                requests: 2,
4555                routes: 1,
4556                total_routes: 2,
4557                top_connections: vec![(client.get(), 2)],
4558                // The held requests by the module's channel and corr, ascending,
4559                // without the excluded subscription (corr 3).
4560                held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
4561            }
4562        );
4563    }
4564
4565    #[test]
4566    fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
4567        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4568            route_fixture("census");
4569        let pending = begin_test_route(&forwarding, client, sink, 1, "census");
4570        forwarding
4571            .complete_pending_relay(
4572                module_connection,
4573                pending.corr,
4574                RouteBindRelayOutcome::Accepted,
4575            )
4576            .unwrap();
4577
4578        let routes = forwarding.endpoint_routes(endpoint).unwrap();
4579        assert_eq!(routes.len(), 1);
4580        assert!(matches!(routes[0].principal, Principal::Direct));
4581        assert_eq!(routes[0].goodbye_target.connection_id, client);
4582        assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
4583        assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
4584        assert!(!routes[0].draining);
4585
4586        forwarding
4587            .begin_module_drain("census", RouteCloseReason::Restart)
4588            .unwrap();
4589        let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
4590        assert_eq!(draining_routes.len(), 1);
4591        assert!(draining_routes[0].draining);
4592    }
4593
4594    #[test]
4595    fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
4596        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
4597        let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
4598        assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
4599        forwarding
4600            .abort_pending_relay(
4601                first.endpoint,
4602                first.corr,
4603                RouteBindRelayOutcome::ModuleGone("abort".into()),
4604            )
4605            .unwrap();
4606        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4607        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4608
4609        let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
4610        assert_eq!(second.client_channel, first.client_channel);
4611        assert_eq!(second.module_channel, first.module_channel);
4612        assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
4613    }
4614
4615    #[test]
4616    fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
4617        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4618            route_fixture("epoch-release");
4619        let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
4620        forwarding
4621            .complete_pending_relay(
4622                module_connection,
4623                first.corr,
4624                RouteBindRelayOutcome::Accepted,
4625            )
4626            .unwrap();
4627        assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
4628        assert!(matches!(
4629            forwarding
4630                .release_client_route(client, first.client_channel, first.client_epoch)
4631                .unwrap(),
4632            RouteRelease::Removed(_)
4633        ));
4634        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4635        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4636
4637        let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
4638        forwarding
4639            .complete_pending_relay(
4640                module_connection,
4641                second.corr,
4642                RouteBindRelayOutcome::Accepted,
4643            )
4644            .unwrap();
4645        assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
4646        assert!(matches!(
4647            forwarding
4648                .release_client_route(client, second.client_channel, first.client_epoch)
4649                .unwrap(),
4650            RouteRelease::Stale
4651        ));
4652        assert!(!forwarding
4653            .cache_status(
4654                endpoint,
4655                second.module_channel,
4656                first.module_epoch,
4657                "stale".into(),
4658            )
4659            .unwrap());
4660        assert!(forwarding
4661            .cache_status(
4662                endpoint,
4663                second.module_channel,
4664                second.module_epoch,
4665                "current".into(),
4666            )
4667            .unwrap());
4668        match forwarding
4669            .route_poll_snapshot(client, second.client_channel, second.client_epoch)
4670            .unwrap()
4671        {
4672            RoutePollSnapshot::Bound { status, .. } => {
4673                assert_eq!(status.as_deref(), Some("current"));
4674            }
4675            RoutePollSnapshot::Absent => panic!("successor binding was removed"),
4676        }
4677        let counters = forwarding.counters().snapshot();
4678        assert_eq!(counters["route_released_epoch_fenced"], 1);
4679        assert_eq!(counters["route_release_stale_skipped"], 1);
4680    }
4681
4682    #[test]
4683    fn max_epoch_reservation_retires_only_that_slot() {
4684        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
4685        forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
4686        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
4687        let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
4688        assert_eq!(
4689            (final_use.client_channel, final_use.client_epoch),
4690            (7, u32::MAX)
4691        );
4692        assert_eq!(
4693            (final_use.module_channel, final_use.module_epoch),
4694            (9, u32::MAX)
4695        );
4696        forwarding
4697            .abort_pending_relay(
4698                endpoint,
4699                final_use.corr,
4700                RouteBindRelayOutcome::ModuleGone("abort".into()),
4701            )
4702            .unwrap();
4703        forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
4704        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
4705        let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
4706        assert_ne!(next.client_channel, 7);
4707        assert_ne!(next.module_channel, 9);
4708        assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
4709    }
4710
4711    #[test]
4712    fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
4713        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4714            route_fixture("corr-shared");
4715        let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
4716        assert_eq!(bind.corr, 1);
4717        forwarding
4718            .abort_pending_relay(
4719                endpoint,
4720                bind.corr,
4721                RouteBindRelayOutcome::ModuleGone("abort".into()),
4722            )
4723            .unwrap();
4724        let rpc = forwarding
4725            .begin_module_control_rpc_for(
4726                "corr-shared",
4727                "health.check",
4728                Instant::now() - Duration::from_millis(1),
4729            )
4730            .unwrap();
4731        assert_eq!(rpc.corr, 2);
4732        assert_eq!(
4733            forwarding
4734                .complete_module_control_rpc(
4735                    module_connection,
4736                    rpc.corr,
4737                    Some("health.check"),
4738                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
4739                        status: subc_protocol::session::HealthStatus::Ok,
4740                        detail: None,
4741                        metrics: None,
4742                    }),
4743                )
4744                .unwrap(),
4745            ModuleControlRpcCompletion::Settled
4746        );
4747        assert!(matches!(
4748            rpc.receiver.blocking_recv().unwrap(),
4749            ModuleControlRpcOutcome::DeadlineElapsed
4750        ));
4751    }
4752
4753    #[tokio::test(start_paused = true)]
4754    async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
4755        let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
4756        let probe_started_at = Instant::now();
4757        let rpc = forwarding
4758            .begin_health_probe_rpc_for(
4759                "tombstone-ttl",
4760                "health.check",
4761                probe_started_at,
4762                probe_started_at + Duration::from_secs(5),
4763            )
4764            .unwrap();
4765        assert!(forwarding
4766            .tombstone_health_probe_rpc(endpoint, rpc.corr)
4767            .unwrap());
4768        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
4769
4770        tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
4771        tokio::task::yield_now().await;
4772
4773        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
4774    }
4775
4776    #[test]
4777    fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
4778        let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
4779        let mut close = forwarding.register_connection_close(endpoint.connection_id);
4780        forwarding.inject_control_corr(endpoint, u64::MAX);
4781        let final_rpc = forwarding
4782            .begin_module_control_rpc_for(
4783                "corr-max",
4784                "health.check",
4785                Instant::now() + Duration::from_secs(1),
4786            )
4787            .unwrap();
4788        assert_eq!(final_rpc.corr, u64::MAX);
4789        forwarding
4790            .cancel_module_control_rpc(endpoint, final_rpc.corr)
4791            .unwrap();
4792        assert!(matches!(
4793            forwarding.begin_module_control_rpc_for(
4794                "corr-max",
4795                "health.check",
4796                Instant::now() + Duration::from_secs(1),
4797            ),
4798            Err(ForwardingError::RelayCorrelationExhausted)
4799        ));
4800        assert!(close.try_recv().is_ok());
4801    }
4802
4803    #[test]
4804    fn publication_epoch_controls_delivery_failure_escalation() {
4805        fn setup_successor(
4806            commit_successor: Option<bool>,
4807        ) -> (ForwardingTable, ConnectionId, u16, u32) {
4808            let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4809                route_fixture("escalation");
4810            let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
4811            forwarding
4812                .complete_pending_relay(
4813                    module_connection,
4814                    first.corr,
4815                    RouteBindRelayOutcome::Accepted,
4816                )
4817                .unwrap();
4818            client_rx.try_recv().unwrap();
4819            assert!(matches!(
4820                forwarding
4821                    .release_client_route(client, first.client_channel, first.client_epoch)
4822                    .unwrap(),
4823                RouteRelease::Removed(_)
4824            ));
4825            if let Some(commit_successor) = commit_successor {
4826                forwarding.inject_client_slot_epoch(
4827                    client,
4828                    first.client_channel,
4829                    first.client_epoch,
4830                );
4831                forwarding.inject_module_slot_epoch(
4832                    endpoint,
4833                    first.module_channel,
4834                    first.module_epoch,
4835                );
4836                let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
4837                if commit_successor {
4838                    forwarding
4839                        .complete_pending_relay(
4840                            module_connection,
4841                            successor.corr,
4842                            RouteBindRelayOutcome::Accepted,
4843                        )
4844                        .unwrap();
4845                    client_rx.try_recv().unwrap();
4846                } else {
4847                    forwarding
4848                        .abort_pending_relay(
4849                            endpoint,
4850                            successor.corr,
4851                            RouteBindRelayOutcome::ModuleGone("abort".into()),
4852                        )
4853                        .unwrap();
4854                }
4855            }
4856            (forwarding, client, first.client_channel, first.client_epoch)
4857        }
4858
4859        let probe_sink = FrameSink::new(mpsc::channel(1).0);
4860        let (no_successor, client, channel, epoch) = setup_successor(None);
4861        let mut close = no_successor.register_connection_close(client);
4862        assert!(no_successor
4863            .escalate_client_delivery_failure(
4864                client,
4865                channel,
4866                epoch,
4867                CloseReason::new("delivery", "failed"),
4868                UndeliveredFrame {
4869                    module_id: None,
4870                    sink: &probe_sink,
4871                },
4872            )
4873            .unwrap());
4874        assert!(close.try_recv().is_ok());
4875
4876        let (aborted, client, channel, epoch) = setup_successor(Some(false));
4877        let mut close = aborted.register_connection_close(client);
4878        assert!(aborted
4879            .escalate_client_delivery_failure(
4880                client,
4881                channel,
4882                epoch,
4883                CloseReason::new("delivery", "failed"),
4884                UndeliveredFrame {
4885                    module_id: None,
4886                    sink: &probe_sink,
4887                },
4888            )
4889            .unwrap());
4890        assert!(close.try_recv().is_ok());
4891
4892        let (published, client, channel, epoch) = setup_successor(Some(true));
4893        let mut close = published.register_connection_close(client);
4894        assert!(!published
4895            .escalate_client_delivery_failure(
4896                client,
4897                channel,
4898                epoch,
4899                CloseReason::new("delivery", "stale failure"),
4900                UndeliveredFrame {
4901                    module_id: None,
4902                    sink: &probe_sink,
4903                },
4904            )
4905            .unwrap());
4906        assert!(close.try_recv().is_err());
4907    }
4908
4909    #[test]
4910    fn route_concentration_separates_client_count_from_routes_per_client() {
4911        // The distinction this asserts is the one a bare connection count cannot
4912        // make: two connections holding one route each and one connection
4913        // holding two are the same total, and have opposite causes.
4914        let (forwarding, module_connection, _, client, sink, _client_rx) =
4915            route_fixture("concentration");
4916        assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
4917
4918        for corr in [70_u64, 71] {
4919            let pending =
4920                begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
4921            forwarding
4922                .complete_pending_relay(
4923                    module_connection,
4924                    pending.corr,
4925                    RouteBindRelayOutcome::Accepted,
4926                )
4927                .unwrap();
4928        }
4929
4930        // One connection, two routes — not two connections with a route each.
4931        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
4932        assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
4933    }
4934
4935    #[test]
4936    fn cleanup_and_accepted_resolution_have_one_lock_winner() {
4937        let (forwarding, module_connection, _, client, sink, mut client_rx) =
4938            route_fixture("cleanup-race");
4939        let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
4940        forwarding
4941            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4942            .unwrap();
4943        let released = forwarding.cleanup_connection(client).unwrap();
4944        assert_eq!(released.len(), 1);
4945        let completion = forwarding
4946            .complete_pending_relay(
4947                module_connection,
4948                pending.corr,
4949                RouteBindRelayOutcome::Accepted,
4950            )
4951            .unwrap();
4952        assert!(!completion.settled);
4953        assert!(client_rx.try_recv().is_err());
4954        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4955
4956        let (forwarding, module_connection, _, client, sink, mut client_rx) =
4957            route_fixture("accepted-race");
4958        let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
4959        forwarding
4960            .complete_pending_relay(
4961                module_connection,
4962                pending.corr,
4963                RouteBindRelayOutcome::Accepted,
4964            )
4965            .unwrap();
4966        assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
4967        let released = forwarding.cleanup_connection(client).unwrap();
4968        assert_eq!(released.len(), 1);
4969        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4970    }
4971
4972    #[test]
4973    fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
4974        let (forwarding, module_connection, _, client, sink, mut client_rx) =
4975            route_fixture("drain-gap");
4976        let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
4977        forwarding
4978            .complete_pending_relay(
4979                module_connection,
4980                live.corr,
4981                RouteBindRelayOutcome::Accepted,
4982            )
4983            .unwrap();
4984        client_rx.try_recv().unwrap();
4985        let binding = match forwarding
4986            .lookup_data_route(client, live.client_channel, live.client_epoch)
4987            .unwrap()
4988        {
4989            DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4990            other => panic!("expected live route, got {other:?}"),
4991        };
4992
4993        let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
4994        forwarding
4995            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4996            .unwrap();
4997        let control_rpc = forwarding
4998            .begin_module_control_rpc_for(
4999                "drain-gap",
5000                "health.check",
5001                Instant::now() + Duration::from_secs(1),
5002            )
5003            .unwrap();
5004        let target = forwarding
5005            .begin_module_drain("drain-gap", RouteCloseReason::Reload)
5006            .unwrap()
5007            .unwrap();
5008        assert!(matches!(
5009            control_rpc.receiver.blocking_recv().unwrap(),
5010            ModuleControlRpcOutcome::ModuleGone(_)
5011        ));
5012        assert_eq!(target.abandoned_bindings.len(), 1);
5013        assert!(binding.flow.sem.is_closed());
5014        assert!(
5015            !forwarding
5016                .complete_pending_relay(
5017                    module_connection,
5018                    pending.corr,
5019                    RouteBindRelayOutcome::Accepted,
5020                )
5021                .unwrap()
5022                .settled
5023        );
5024        assert!(matches!(
5025            forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
5026            Err(ForwardingError::ModuleReloading { .. })
5027        ));
5028        let released = forwarding
5029            .release_module_endpoint_routes(target.endpoint)
5030            .unwrap();
5031        assert_eq!(released.len(), 1);
5032        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5033    }
5034
5035    /// A client can be marked closing while its egress is still open: the daemon
5036    /// asks a connection to close (here through the production path, a module
5037    /// frame that its egress refused) and the connection loop tears down a moment
5038    /// later. A route.bind ack that lands inside that window is answered on the
5039    /// MODULE connection's frame handler, so resolving it must not produce an
5040    /// error -- an error there ends the module connection, and that connection
5041    /// carries every other client's routes to the module.
5042    #[test]
5043    fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
5044        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5045            route_fixture("closing-client");
5046
5047        // A published route on this client: escalate_client_delivery_failure only
5048        // marks a connection closing for a route it has already published.
5049        let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
5050        forwarding
5051            .complete_pending_relay(
5052                module_connection,
5053                live.corr,
5054                RouteBindRelayOutcome::Accepted,
5055            )
5056            .unwrap();
5057        client_rx.try_recv().unwrap();
5058
5059        // A second route.open from the same client, relayed and awaiting its ack.
5060        let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
5061        forwarding
5062            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5063            .unwrap();
5064
5065        // The window: closing, but the sink is still open.
5066        assert!(forwarding
5067            .escalate_client_delivery_failure(
5068                client,
5069                live.client_channel,
5070                live.client_epoch,
5071                CloseReason::new(
5072                    "module_to_client_delivery_failed",
5073                    "client egress refused a module frame",
5074                ),
5075                UndeliveredFrame {
5076                    module_id: None,
5077                    sink: &sink,
5078                },
5079            )
5080            .unwrap());
5081        assert!(!sink.is_closed());
5082
5083        let completion = forwarding
5084            .complete_pending_relay(
5085                module_connection,
5086                pending.corr,
5087                RouteBindRelayOutcome::Accepted,
5088            )
5089            .expect("a closing client must not turn a module's ack into an error");
5090
5091        assert!(completion.settled);
5092        let abandoned = completion
5093            .abandoned
5094            .expect("the module must be told to drop the binding it just created");
5095        assert_eq!(abandoned.connection_id, module_connection);
5096        assert_eq!(abandoned.channel, pending.module_channel);
5097        assert_eq!(abandoned.epoch, pending.module_epoch);
5098        assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
5099        assert!(matches!(
5100            pending.receiver.blocking_recv().unwrap(),
5101            RouteBindRelayOutcome::ModuleGone(_)
5102        ));
5103        // No route was published to a client that is on its way out, and the
5104        // reserved handle pair went back.
5105        assert!(client_rx.try_recv().is_err());
5106        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
5107
5108        // The module endpoint is untouched: still live, and still able to take a
5109        // route from another client.
5110        assert!(forwarding
5111            .has_live_module_connection("closing-client")
5112            .unwrap());
5113        let cotenant = ConnectionId::new(201);
5114        let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
5115        let cotenant_route = begin_test_route(
5116            &forwarding,
5117            cotenant,
5118            FrameSink::new(cotenant_tx),
5119            62,
5120            "closing-client",
5121        );
5122        assert_eq!(cotenant_route.endpoint, endpoint);
5123        forwarding
5124            .complete_pending_relay(
5125                module_connection,
5126                cotenant_route.corr,
5127                RouteBindRelayOutcome::Accepted,
5128            )
5129            .unwrap();
5130        assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
5131        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5132    }
5133
5134    #[test]
5135    fn pending_route_permit_is_released_on_rejection_and_abort() {
5136        let forwarding = ForwardingTable::default();
5137        let module_connection = ConnectionId::new(300);
5138        let client = ConnectionId::new(301);
5139        let (module_tx, _module_rx) = mpsc::channel(1);
5140        let endpoint = forwarding
5141            .register_module_connection(
5142                module_connection,
5143                "permit".into(),
5144                2,
5145                Concurrency::ModuleManaged,
5146                FrameSink::new(module_tx),
5147            )
5148            .unwrap();
5149        let (client_tx, mut client_rx) = mpsc::channel(1);
5150        let sink = FrameSink::new(client_tx);
5151        let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
5152        assert!(sink.try_send(test_ping(999)).is_err());
5153        forwarding
5154            .complete_pending_relay(
5155                module_connection,
5156                rejected.corr,
5157                RouteBindRelayOutcome::Rejected(ErrorBody {
5158                    code: "no".into(),
5159                    message: "rejected".into(),
5160                    detail: None,
5161                }),
5162            )
5163            .unwrap();
5164        sink.try_send(test_ping(1000)).unwrap();
5165        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
5166
5167        let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
5168        assert!(sink.try_send(test_ping(1001)).is_err());
5169        forwarding
5170            .abort_pending_relay(
5171                endpoint,
5172                aborted.corr,
5173                RouteBindRelayOutcome::ModuleGone("abort".into()),
5174            )
5175            .unwrap();
5176        sink.try_send(test_ping(1002)).unwrap();
5177        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
5178
5179        let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
5180        forwarding
5181            .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
5182            .unwrap();
5183        drop(client_rx);
5184        let completion = forwarding
5185            .complete_pending_relay(
5186                module_connection,
5187                receiver_closed.corr,
5188                RouteBindRelayOutcome::Accepted,
5189            )
5190            .unwrap();
5191        assert!(completion.abandoned.is_some());
5192        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5193    }
5194
5195    /// Every connection teardown passes through `cleanup_connection`, and
5196    /// connection ids come from a monotonic counter that never hands an id out
5197    /// twice. If the closing mark survives teardown, the set grows by one entry
5198    /// per connection for the life of the daemon -- the self-watchdog alone
5199    /// reconnects once a minute.
5200    #[test]
5201    fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
5202        let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
5203            route_fixture("closing-set-leak");
5204
5205        const CONNECTIONS: u64 = 32;
5206        for index in 0..CONNECTIONS {
5207            let client = ConnectionId::new(1000 + index);
5208            let (client_tx, _client_rx) = mpsc::channel(8);
5209            let route = begin_test_route(
5210                &forwarding,
5211                client,
5212                FrameSink::new(client_tx),
5213                index + 1,
5214                "closing-set-leak",
5215            );
5216            forwarding
5217                .complete_pending_relay(
5218                    module_connection,
5219                    route.corr,
5220                    RouteBindRelayOutcome::Accepted,
5221                )
5222                .unwrap();
5223            forwarding.cleanup_connection(client).unwrap();
5224        }
5225        forwarding.cleanup_connection(module_connection).unwrap();
5226
5227        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5228    }
5229
5230    /// The closing mark exists to refuse new work for a connection that is on
5231    /// its way out but whose teardown has not run yet: the daemon asks the
5232    /// connection loop to end, and only when the loop reacts does
5233    /// `cleanup_connection` strip the connection's state. Inside that window an
5234    /// operation for the dying connection must still be refused; only after
5235    /// cleanup completes may the mark go.
5236    #[test]
5237    fn closing_connection_is_refused_new_work_until_cleanup_completes() {
5238        let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
5239            route_fixture("closing-gate");
5240
5241        // A published route: escalate_client_delivery_failure only marks a
5242        // connection closing for a route it has already published.
5243        let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
5244        forwarding
5245            .complete_pending_relay(
5246                module_connection,
5247                live.corr,
5248                RouteBindRelayOutcome::Accepted,
5249            )
5250            .unwrap();
5251        client_rx.try_recv().unwrap();
5252
5253        // Mark the connection closing through the production path without
5254        // running teardown, pinning the window open.
5255        assert!(forwarding
5256            .escalate_client_delivery_failure(
5257                client,
5258                live.client_channel,
5259                live.client_epoch,
5260                CloseReason::new(
5261                    "module_to_client_delivery_failed",
5262                    "client egress refused a module frame",
5263                ),
5264                UndeliveredFrame {
5265                    module_id: None,
5266                    sink: &sink,
5267                },
5268            )
5269            .unwrap());
5270        assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
5271
5272        // A late route.open for the closing client is refused ...
5273        assert!(matches!(
5274            forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
5275            Err(ForwardingError::ConnectionClosing { connection_id })
5276                if connection_id == client
5277        ));
5278        // ... and so is a late attempt to register the connection as a module.
5279        let (late_tx, _late_rx) = mpsc::channel(1);
5280        assert!(matches!(
5281            forwarding.register_module_connection(
5282                client,
5283                "late-module".into(),
5284                2,
5285                Concurrency::ModuleManaged,
5286                FrameSink::new(late_tx),
5287            ),
5288            Err(ForwardingError::ConnectionClosing { connection_id })
5289                if connection_id == client
5290        ));
5291
5292        // Teardown is the point that lifts the mark: it has just removed every
5293        // per-connection entry under the same lock, so the gate has nothing
5294        // left to protect for this id.
5295        forwarding.cleanup_connection(client).unwrap();
5296        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5297    }
5298}
5299
5300/// Blue/green swap slots: a candidate registered beside the active endpoint,
5301/// promoted by `cutover_candidate`, with the old incumbent drained by endpoint.
5302#[cfg(test)]
5303mod swap_slot_tests {
5304    use std::time::Duration;
5305
5306    use super::*;
5307    use tokio::sync::mpsc;
5308
5309    const MODULE_ID: &str = "swapped";
5310
5311    struct SwapFixture {
5312        forwarding: ForwardingTable,
5313        incumbent_connection: ConnectionId,
5314        incumbent: ModuleEndpointId,
5315        candidate_connection: ConnectionId,
5316        candidate: ModuleEndpointId,
5317        _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
5318    }
5319
5320    fn swap_fixture() -> SwapFixture {
5321        let forwarding = ForwardingTable::default();
5322        let incumbent_connection = ConnectionId::new(100);
5323        let candidate_connection = ConnectionId::new(110);
5324        let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
5325        let incumbent = forwarding
5326            .register_module_connection(
5327                incumbent_connection,
5328                MODULE_ID.to_string(),
5329                2,
5330                Concurrency::ModuleManaged,
5331                FrameSink::new(incumbent_tx),
5332            )
5333            .unwrap();
5334        let (candidate_tx, candidate_rx) = mpsc::channel(8);
5335        let candidate = forwarding
5336            .register_candidate_module_connection(
5337                candidate_connection,
5338                MODULE_ID.to_string(),
5339                2,
5340                Concurrency::ModuleManaged,
5341                FrameSink::new(candidate_tx),
5342            )
5343            .unwrap();
5344        SwapFixture {
5345            forwarding,
5346            incumbent_connection,
5347            incumbent,
5348            candidate_connection,
5349            candidate,
5350            _module_rxs: vec![incumbent_rx, candidate_rx],
5351        }
5352    }
5353
5354    fn client(
5355        raw: u64,
5356    ) -> (
5357        ConnectionId,
5358        FrameSink,
5359        mpsc::Receiver<crate::router::OutboundFrame>,
5360    ) {
5361        let (tx, rx) = mpsc::channel(8);
5362        (ConnectionId::new(raw), FrameSink::new(tx), rx)
5363    }
5364
5365    fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
5366        forwarding
5367            .read_inner()
5368            .unwrap()
5369            .client_to_module
5370            .values()
5371            .map(|route| route.module_endpoint)
5372            .collect()
5373    }
5374
5375    #[test]
5376    fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
5377        let fixture = swap_fixture();
5378        let forwarding = &fixture.forwarding;
5379        assert_ne!(fixture.incumbent, fixture.candidate);
5380
5381        // Every by-id consumer still resolves the incumbent.
5382        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5383        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5384        let (client_connection, client_sink, _client_rx) = client(200);
5385        let pending = forwarding
5386            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5387            .unwrap();
5388        assert_eq!(pending.endpoint, fixture.incumbent);
5389        let rpc = forwarding
5390            .begin_module_control_rpc_for(
5391                MODULE_ID,
5392                "health.check",
5393                Instant::now() + Duration::from_secs(1),
5394            )
5395            .unwrap();
5396        assert_eq!(rpc.endpoint, fixture.incumbent);
5397        let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
5398        assert_eq!(census.len(), 1, "the census lists one endpoint per id");
5399
5400        // Connection-keyed lookups see the candidate, so its own frames resolve.
5401        assert_eq!(
5402            forwarding
5403                .module_endpoint_for_connection(fixture.candidate_connection)
5404                .unwrap(),
5405            Some(fixture.candidate)
5406        );
5407        assert_eq!(
5408            forwarding
5409                .module_id_for_connection(fixture.candidate_connection)
5410                .unwrap()
5411                .as_deref(),
5412            Some(MODULE_ID)
5413        );
5414
5415        // One candidate per id.
5416        let (other_tx, _other_rx) = mpsc::channel(1);
5417        assert_eq!(
5418            forwarding.register_candidate_module_connection(
5419                ConnectionId::new(120),
5420                MODULE_ID.to_string(),
5421                2,
5422                Concurrency::ModuleManaged,
5423                FrameSink::new(other_tx),
5424            ),
5425            Err(ForwardingError::CandidateSlotOccupied {
5426                module_id: MODULE_ID.to_string()
5427            })
5428        );
5429    }
5430
5431    /// The cutover linearization point. A relay reserved on the incumbent before
5432    /// cutover must never become a route on the incumbent, and every relay
5433    /// reserved after it must land on the promoted candidate.
5434    #[test]
5435    fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
5436        let fixture = swap_fixture();
5437        let forwarding = &fixture.forwarding;
5438        let (early_client, early_sink, _early_rx) = client(200);
5439        let mut early = forwarding
5440            .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
5441            .unwrap();
5442        assert_eq!(early.endpoint, fixture.incumbent);
5443        assert_eq!(
5444            forwarding
5445                .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
5446                .unwrap(),
5447            RouteBindRelayMark::Marked
5448        );
5449
5450        let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5451        assert_eq!(
5452            cutover,
5453            ForwardingCutover {
5454                promoted: fixture.candidate,
5455                incumbent: Some(fixture.incumbent),
5456            }
5457        );
5458
5459        // A route.open reserved after cutover goes to the promoted candidate.
5460        let (late_client, late_sink, _late_rx) = client(201);
5461        let late = forwarding
5462            .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
5463            .unwrap();
5464        assert_eq!(
5465            late.endpoint, fixture.candidate,
5466            "a route.open after cutover was reserved on the incumbent"
5467        );
5468
5469        // The incumbent acks the early relay after cutover.
5470        let completion = forwarding
5471            .complete_pending_relay(
5472                fixture.incumbent_connection,
5473                early.corr,
5474                RouteBindRelayOutcome::Accepted,
5475            )
5476            .expect("a superseded endpoint's ack is not an error on its connection");
5477        assert!(completion.settled);
5478        assert!(
5479            !committed_endpoints(forwarding).contains(&fixture.incumbent),
5480            "a relay reserved before cutover committed a route on the incumbent"
5481        );
5482        let goodbye = completion
5483            .abandoned
5484            .expect("the incumbent is told to drop the binding it just created");
5485        assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
5486        assert_eq!(goodbye.channel, early.module_channel);
5487        assert_eq!(goodbye.epoch, early.module_epoch);
5488        assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
5489        match early.receiver.try_recv() {
5490            Ok(RouteBindRelayOutcome::DaemonRejected(body)) => {
5491                assert_eq!(body.code, "module_reloading")
5492            }
5493            other => panic!("expected a retryable module_reloading answer, got {other:?}"),
5494        }
5495        assert!(matches!(
5496            forwarding
5497                .lookup_data_route(early_client, early.client_channel, early.client_epoch)
5498                .unwrap(),
5499            DataRoute::Client(DataRouteState::Absent)
5500        ));
5501
5502        // The late relay commits on the candidate; only its pair stays reserved
5503        // until then.
5504        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
5505        forwarding
5506            .complete_pending_relay(
5507                fixture.candidate_connection,
5508                late.corr,
5509                RouteBindRelayOutcome::Accepted,
5510            )
5511            .unwrap();
5512        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5513        assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
5514    }
5515
5516    #[test]
5517    fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
5518        let fixture = swap_fixture();
5519        let forwarding = &fixture.forwarding;
5520        // One bound route and one in-flight relay on the incumbent.
5521        let (bound_client, bound_sink, _bound_rx) = client(200);
5522        let bound = forwarding
5523            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5524            .unwrap();
5525        forwarding
5526            .complete_pending_relay(
5527                fixture.incumbent_connection,
5528                bound.corr,
5529                RouteBindRelayOutcome::Accepted,
5530            )
5531            .unwrap();
5532        let (pending_client, pending_sink, _pending_rx) = client(201);
5533        let mut in_flight = forwarding
5534            .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
5535            .unwrap();
5536        forwarding
5537            .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
5538            .unwrap();
5539
5540        let incumbent = forwarding
5541            .cutover_candidate(MODULE_ID)
5542            .unwrap()
5543            .unwrap()
5544            .incumbent
5545            .unwrap();
5546        let target = forwarding
5547            .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
5548            .unwrap()
5549            .expect("the superseded incumbent is still registered");
5550
5551        assert_eq!(target.endpoint, fixture.incumbent);
5552        assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
5553        assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
5554        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5555        assert_eq!(target.abandoned_bindings.len(), 1);
5556        assert_eq!(
5557            target.abandoned_bindings[0].channel,
5558            in_flight.module_channel
5559        );
5560        assert!(matches!(
5561            in_flight.receiver.try_recv(),
5562            Ok(RouteBindRelayOutcome::DaemonRejected(body)) if body.code == "module_reloading"
5563        ));
5564        assert_eq!(
5565            forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
5566            1,
5567            "the incumbent's bound route stays until its drain finishes"
5568        );
5569
5570        let (next_client, next_sink, _next_rx) = client(202);
5571        let next = forwarding
5572            .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
5573            .expect("the promoted candidate keeps accepting routes");
5574        assert_eq!(next.endpoint, fixture.candidate);
5575    }
5576
5577    /// An endpoint replaced WITHOUT a promotion (a successor registered over it
5578    /// as an ordinary active HELLO) is stale, not superseded, and its ack still
5579    /// fails the acking connection exactly as it did before swap slots existed:
5580    /// `StaleModuleEndpoint`, the reservation indexes already stripped, and the
5581    /// waiting client's sender dropped unanswered.
5582    #[test]
5583    fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
5584        let forwarding = ForwardingTable::default();
5585        let first_connection = ConnectionId::new(70);
5586        let (first_tx, _first_rx) = mpsc::channel(8);
5587        forwarding
5588            .register_module_connection(
5589                first_connection,
5590                MODULE_ID.to_string(),
5591                2,
5592                Concurrency::ModuleManaged,
5593                FrameSink::new(first_tx),
5594            )
5595            .unwrap();
5596        let (client_connection, client_sink, _client_rx) = client(200);
5597        let mut pending = forwarding
5598            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5599            .unwrap();
5600        let (second_tx, _second_rx) = mpsc::channel(8);
5601        forwarding
5602            .register_module_connection(
5603                ConnectionId::new(80),
5604                MODULE_ID.to_string(),
5605                2,
5606                Concurrency::ModuleManaged,
5607                FrameSink::new(second_tx),
5608            )
5609            .unwrap();
5610
5611        assert_eq!(
5612            forwarding
5613                .complete_pending_relay(
5614                    first_connection,
5615                    pending.corr,
5616                    RouteBindRelayOutcome::Accepted
5617                )
5618                .unwrap_err(),
5619            ForwardingError::StaleModuleEndpoint
5620        );
5621        assert!(committed_endpoints(&forwarding).is_empty());
5622        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5623        assert!(matches!(
5624            pending.receiver.try_recv(),
5625            Err(oneshot::error::TryRecvError::Closed)
5626        ));
5627    }
5628
5629    #[test]
5630    fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
5631        // A candidate whose connection drops leaves the incumbent routable.
5632        let fixture = swap_fixture();
5633        let forwarding = &fixture.forwarding;
5634        assert!(forwarding
5635            .cleanup_connection(fixture.candidate_connection)
5636            .unwrap()
5637            .is_empty());
5638        assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
5639        let (client_connection, client_sink, _client_rx) = client(200);
5640        assert_eq!(
5641            forwarding
5642                .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5643                .unwrap()
5644                .endpoint,
5645            fixture.incumbent
5646        );
5647
5648        // After a cutover, the incumbent's teardown releases its own routes and
5649        // leaves the promoted candidate in place.
5650        let fixture = swap_fixture();
5651        let forwarding = &fixture.forwarding;
5652        let (bound_client, bound_sink, _bound_rx) = client(200);
5653        let bound = forwarding
5654            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5655            .unwrap();
5656        forwarding
5657            .complete_pending_relay(
5658                fixture.incumbent_connection,
5659                bound.corr,
5660                RouteBindRelayOutcome::Accepted,
5661            )
5662            .unwrap();
5663        forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5664        let released = forwarding
5665            .cleanup_connection(fixture.incumbent_connection)
5666            .unwrap();
5667        assert_eq!(released.len(), 1);
5668        assert_eq!(released[0].connection_id, bound_client);
5669        assert!(forwarding
5670            .read_inner()
5671            .unwrap()
5672            .superseded_endpoints
5673            .is_empty());
5674        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5675        let (next_client, next_sink, _next_rx) = client(201);
5676        assert_eq!(
5677            forwarding
5678                .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
5679                .unwrap()
5680                .endpoint,
5681            fixture.candidate
5682        );
5683    }
5684}