Skip to main content

subc_daemon/
forwarding.rs

1use std::{
2    collections::{BTreeMap, HashMap, HashSet},
3    error::Error,
4    fmt,
5    sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
6    time::Duration,
7};
8
9use subc_control::{ClientControlResponse, RouteCloseReason};
10use subc_protocol::{
11    manifest::Concurrency,
12    session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
13    ErrorBody, Flags, FrameType, Principal, Priority,
14};
15use tokio::sync::{oneshot, Semaphore};
16use tokio::time::Instant;
17use tracing::{debug, info, warn};
18
19use crate::{
20    control::{RouteBindBreakers, RouteBindConcurrency},
21    observability::DaemonCounters,
22    registry::ConnectionId,
23    router::FrameSink,
24    scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
25    Frame, ProjectRootId,
26};
27
28/// Default per-channel request-credit window for modules that schedule internally.
29const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
30
31/// High per-channel cap for stateless modules; this is an OOM guard, not scheduling policy.
32const STATELESS_PARALLEL_WINDOW: usize = 1024;
33
34/// A stopped probe cycle cannot retain its last unanswered correlation forever.
35/// Active endpoints replace the tombstone on their next serial health probe;
36/// this backstop covers endpoints that stop probing altogether.
37const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
38
39/// Module connection identity used in forwarding keys.
40///
41/// The generation is bumped every time a module connection is registered so a future restart cannot
42/// accidentally reuse a stale `(connection_id, route_channel)` binding.
43#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
44pub struct ModuleEndpointId {
45    pub connection_id: ConnectionId,
46    pub generation: u64,
47}
48
49/// Client-local route key. A route channel is unique only within one client connection.
50#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
51pub(crate) struct ClientRouteKey {
52    pub connection_id: ConnectionId,
53    pub channel: u16,
54}
55
56/// Module-local route key. A route channel is unique only within one live module endpoint.
57#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
58pub(crate) struct ModuleRouteKey {
59    pub endpoint: ModuleEndpointId,
60    pub channel: u16,
61}
62
63#[derive(Debug)]
64pub(crate) struct RouteBinding {
65    pub client_connection_id: ConnectionId,
66    pub client_sink: FrameSink,
67    pub client_negotiated_ver: u8,
68    pub client_channel: u16,
69    pub client_epoch: u32,
70    pub module_id: String,
71    pub module_endpoint: ModuleEndpointId,
72    pub module_sink: FrameSink,
73    pub module_negotiated_ver: u8,
74    pub module_channel: u16,
75    pub module_epoch: u32,
76    pub principal: Principal,
77    pub project_root: Option<ProjectRootId>,
78    pub bound_at: Instant,
79    pub flow: Arc<ChannelFlow>,
80    /// The scope the route was admitted under, with the tag its bind was
81    /// stamped at. A scope change finds the routes to close by it.
82    pub scope: Option<BoundScope>,
83}
84
85#[derive(Debug, Clone)]
86pub(crate) enum DataRoute {
87    Client(DataRouteState),
88    Module(DataRouteState),
89}
90
91#[derive(Debug, Clone)]
92pub(crate) enum DataRouteState {
93    Bound(Arc<RouteBinding>),
94    Reserved,
95    EpochMismatch,
96    Absent,
97}
98
99/// Which kind of peer a route GOODBYE is being delivered to. This decides what
100/// happens when the GOODBYE cannot be enqueued (egress full/closed):
101/// - `Client`: escalate to closing that client connection (a socket close is a
102///   stronger teardown signal, and a full client egress means it is the slow
103///   client we would drop anyway).
104/// - `Module`: never close; deliver late instead (see
105///   [`send_module_route_goodbye`]), and drop only if the module still has no
106///   room after [`LATE_MODULE_GOODBYE_DEADLINE`]. A client-disconnect notifies the
107///   SHARED module that one client's route is gone; closing the module on its
108///   egress backpressure would tear down every co-tenant client (the exact
109///   cross-tenant blast radius this never-close rule exists to prevent — observed when a
110///   flooding dead client filled BOTH its own and the module's egress, so its
111///   route-gone GOODBYE to the module failed and closed the shared connection).
112///   subc has already removed the route from its forwarding state and drops
113///   stale module frames for the released channel (see router.rs), so subc's
114///   own routing is correct. The residual: under SUSTAINED module-egress
115///   backpressure a module-targeted route-gone notification can be lost, which a
116///   module using it for client-refcounting (e.g. AFT's session accounting)
117///   would miss. This is INTENTIONALLY ACCEPTED, not a gap. A consuming module
118///   must bound stale bindings with its own idle-activity reaper (last-touched
119///   TTL) independent of route-gone signals — AFT does exactly this, so a lost
120///   GOODBYE degrades to "the
121///   binding stays warm until its idle TTL" (bounded wasted resources), never an
122///   unbounded leak; disk-durable replay is unaffected. A dedicated reliable
123///   module control lane was evaluated and deliberately NOT
124///   built: it would add starvation-avoidance machinery to the thin core for a
125///   bounded warm-resource window that is not a correctness issue. never-close
126///   is the invariant that matters here.
127#[derive(Debug, Clone, Copy, PartialEq, Eq)]
128pub(crate) enum GoodbyeTargetKind {
129    Client,
130    Module,
131}
132
133#[derive(Debug, Clone)]
134pub(crate) struct GoodbyeTarget {
135    pub connection_id: ConnectionId,
136    pub sink: FrameSink,
137    pub negotiated_ver: u8,
138    pub channel: u16,
139    pub epoch: u32,
140    pub kind: GoodbyeTargetKind,
141    /// The module on the other end of the route: for a module-targeted relay
142    /// the receiving module (attributing a dropped relay), for a client target
143    /// the module whose route is going away (naming it when an undeliverable
144    /// relay closes the client).
145    pub module_id: Option<String>,
146}
147
148/// The frame a client connection's egress queue refused, for the diagnosis
149/// logged when that refusal closes the connection.
150#[derive(Debug, Clone, Copy)]
151pub(crate) struct UndeliveredFrame<'a> {
152    /// The module on the other end of the route the frame belonged to, when known.
153    pub module_id: Option<&'a str>,
154    /// The refusing connection's sink, read for what its queue held.
155    pub sink: &'a FrameSink,
156}
157
158/// How a route's principal appears in logs: `direct`, or `reserved:<module>`.
159pub(crate) fn principal_label(principal: &Principal) -> String {
160    match principal {
161        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
162        Principal::Direct => "direct".to_string(),
163        other => format!("{other:?}"),
164    }
165}
166
167/// The distinct principals of the routes bound on one client connection,
168/// sorted and comma-joined, or `none` when it has no bound route.
169fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
170    let labels = inner
171        .client_to_module
172        .iter()
173        .filter(|(key, _)| key.connection_id == connection_id)
174        .map(|(_, route)| principal_label(&route.principal))
175        .collect::<std::collections::BTreeSet<_>>();
176    if labels.is_empty() {
177        "none".to_string()
178    } else {
179        labels.into_iter().collect::<Vec<_>>().join(",")
180    }
181}
182
183impl GoodbyeTarget {
184    /// True only when an undeliverable GOODBYE should escalate to closing the
185    /// target connection. Never escalate for module recipients.
186    pub(crate) fn close_on_delivery_failure(&self) -> bool {
187        matches!(self.kind, GoodbyeTargetKind::Client)
188    }
189}
190
191/// How long a route GOODBYE that a module's egress queue refused keeps waiting
192/// for room before it is given up and counted as dropped.
193///
194/// A module that stops reading for a moment (a GC pause, a long synchronous
195/// handler, a reconnect herd it is still working through) should still learn
196/// that the route is gone, because a module that never hears the GOODBYE keeps
197/// the route and keeps sending on it for as long as its connection lives. The
198/// value is the default module drain timeout: the daemon already treats that as
199/// the longest it is reasonable to wait on a module that is busy but alive, and
200/// a module that cannot free 21 bytes of egress in that time is wedged rather
201/// than slow. Resolving each module's own configured drain timeout here would
202/// need a registry lookup on a path that holds only the module's sink.
203pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
204
205/// Send a route GOODBYE to a module without ever closing the module's shared
206/// connection, which would sever every other route it serves.
207///
208/// The frame is enqueued at once when the module's egress queue has room. When
209/// it does not, a detached task waits for room (bounded by
210/// [`LATE_MODULE_GOODBYE_DEADLINE`]) and sends it then. Arriving late is safe:
211/// the daemon has already released the route, and module SDKs tear a route
212/// down only when both the channel AND the epoch match the route installed on
213/// that channel, so a late GOODBYE for a channel since reused at a newer epoch
214/// is ignored. The GOODBYE is counted in `goodbye_relay_module_dropped` only
215/// when the connection is closed or the deadline passes.
216pub(crate) fn send_module_route_goodbye(
217    counters: &DaemonCounters,
218    sink: &FrameSink,
219    frame: Frame,
220    module_id: Option<&str>,
221    context: &'static str,
222) {
223    let channel = frame.header.channel;
224    let epoch = frame.header.epoch;
225    let Err(err) = sink.try_send(frame.clone()) else {
226        return;
227    };
228    // Waiting is pointless once the module's writer is gone, and impossible
229    // outside a Tokio runtime (cleanup that runs from a destructor at shutdown).
230    let runtime = match tokio::runtime::Handle::try_current() {
231        Ok(runtime) if !sink.is_closed() => runtime,
232        _ => {
233            counters.increment_goodbye_relay_module_dropped(module_id);
234            warn!(
235            module_id = module_id.unwrap_or("unknown"),
236            route_channel = channel,
237            route_epoch = epoch,
238            error = %err,
239            context,
240            "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
241            );
242            return;
243        }
244    };
245    debug!(
246        module_id = module_id.unwrap_or("unknown"),
247        route_channel = channel,
248        route_epoch = epoch,
249        error = %err,
250        context,
251        "module egress queue refused route GOODBYE; delivering it once the module frees room"
252    );
253    let counters = counters.clone();
254    let sink = sink.clone();
255    let module_id = module_id.map(str::to_string);
256    runtime.spawn(async move {
257        let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
258        let why = match outcome {
259            Ok(Ok(())) => {
260                debug!(
261                    module_id = module_id.as_deref().unwrap_or("unknown"),
262                    route_channel = channel,
263                    route_epoch = epoch,
264                    context,
265                    "late route GOODBYE delivered to module"
266                );
267                return;
268            }
269            Ok(Err(err)) => err.to_string(),
270            Err(_) => format!(
271                "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
272            ),
273        };
274        counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
275        warn!(
276            module_id = module_id.as_deref().unwrap_or("unknown"),
277            route_channel = channel,
278            route_epoch = epoch,
279            error = %why,
280            context,
281            "route GOODBYE to module dropped under backpressure; not closing shared module connection"
282        );
283    });
284}
285
286/// One route currently served by a module endpoint.
287///
288/// `goodbye_target` is deliberately retained alongside the census projection so
289/// a draining caller can address exactly the same route set that this read
290/// reports, without a second forwarding-table pass.
291#[derive(Debug, Clone)]
292pub(crate) struct EndpointRoute {
293    pub goodbye_target: GoodbyeTarget,
294    pub principal: Principal,
295    pub bound_at: Instant,
296    pub draining: bool,
297    /// WHY the endpoint is draining, when it is. Carried per-route so the
298    /// census can answer "closing because of what" without a second lookup;
299    /// `None` exactly when `draining` is false (one source: the drain map).
300    pub drain_reason: Option<RouteCloseReason>,
301}
302
303/// One live route a scope change closed: both ends to send GOODBYE to, and the
304/// reason for the client's `route.closed`.
305#[derive(Debug, Clone)]
306pub(crate) struct ScopeDrainedRoute {
307    pub reason: RouteCloseReason,
308    pub scope: BoundScope,
309    pub module_id: String,
310    pub client: GoodbyeTarget,
311    pub module: GoodbyeTarget,
312}
313
314#[derive(Debug)]
315pub(crate) struct PendingRouteBindRelay {
316    pub endpoint: ModuleEndpointId,
317    pub module_sink: FrameSink,
318    pub negotiated_ver: u8,
319    pub client_channel: u16,
320    pub client_epoch: u32,
321    pub module_channel: u16,
322    pub module_epoch: u32,
323    pub corr: u64,
324    pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
325}
326
327#[derive(Debug, Clone)]
328pub(crate) struct ModuleDrainTarget {
329    pub endpoint: ModuleEndpointId,
330    pub sink: FrameSink,
331    pub negotiated_ver: u8,
332    pub abandoned_bindings: Vec<GoodbyeTarget>,
333    pub excluded_subscriptions: u32,
334}
335
336/// One registered module connection, as [`ForwardingTable::module_connections`]
337/// reports it: enough to send it a frame and then close it.
338#[cfg(unix)]
339#[derive(Debug, Clone)]
340pub(crate) struct ModuleConnectionTarget {
341    pub module_id: String,
342    pub endpoint: ModuleEndpointId,
343    pub sink: FrameSink,
344    pub negotiated_ver: u8,
345}
346
347#[derive(Debug, Clone)]
348pub(crate) enum RouteBindRelayOutcome {
349    Accepted,
350    Rejected(ErrorBody),
351    ModuleGone(String),
352}
353
354/// What [`ForwardingTable::cutover_candidate`] did.
355#[derive(Debug, Clone, Copy, PartialEq, Eq)]
356pub(crate) struct ForwardingCutover {
357    /// The endpoint now in the active slot (the former candidate).
358    pub promoted: ModuleEndpointId,
359    /// The endpoint demoted out of the active slot, to be drained by endpoint.
360    /// `None` when the incumbent's connection was already gone.
361    pub incumbent: Option<ModuleEndpointId>,
362}
363
364/// What tearing down one connection released.
365#[derive(Debug)]
366pub(crate) struct ConnectionCleanup {
367    /// Routes whose other end must be sent GOODBYE.
368    pub released: Vec<GoodbyeTarget>,
369    /// Pending route.bind relays to the closed module that were aborted: opens
370    /// that had not been bound yet. Zero when a client connection closed.
371    pub abandoned_relays: u32,
372}
373
374#[derive(Debug, Clone)]
375pub(crate) struct PendingRelayCompletion {
376    pub settled: bool,
377    pub abandoned: Option<GoodbyeTarget>,
378}
379
380#[derive(Debug)]
381pub(crate) struct PendingModuleControlRpc {
382    pub endpoint: ModuleEndpointId,
383    pub module_sink: FrameSink,
384    pub negotiated_ver: u8,
385    pub corr: u64,
386    pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
387}
388
389#[derive(Debug, Clone)]
390pub(crate) enum ModuleControlRpcOutcome {
391    Response(ModuleControlResponse),
392    Rejected(ErrorBody),
393    ModuleGone(String),
394    MalformedResponse(String),
395    UnexpectedOp { expected: String, actual: String },
396    DeadlineElapsed,
397}
398
399#[derive(Debug, Clone, PartialEq, Eq)]
400pub(crate) enum ModuleControlRpcCompletion {
401    Unknown,
402    Settled,
403    LateHealthAnswer {
404        module_id: String,
405        latency: Duration,
406    },
407}
408
409#[derive(Debug)]
410struct PendingModuleControlRpcEntry {
411    expected_op: String,
412    deadline: Instant,
413    health_probe_started_at: Option<Instant>,
414    sender: oneshot::Sender<ModuleControlRpcOutcome>,
415}
416
417#[derive(Debug)]
418struct HealthProbeTombstone {
419    expected_op: String,
420    module_id: String,
421    probe_started_at: Instant,
422    expires_at: Instant,
423}
424
425#[derive(Debug, Clone)]
426struct RouteReservation {
427    client_key: ClientRouteKey,
428    module_key: ModuleRouteKey,
429    client_epoch: u32,
430    module_epoch: u32,
431    project_root: Option<ProjectRootId>,
432}
433
434#[derive(Debug)]
435struct PendingRouteBindRelayEntry {
436    reservation: RouteReservation,
437    client_sink: FrameSink,
438    client_negotiated_ver: u8,
439    client_permit: crate::router::EgressPermit,
440    route_open_frame: Frame,
441    principal: Principal,
442    /// The scope admitted at `route.open`, with the tag captured then. Commit
443    /// compares it with the published tag.
444    scope: Option<BoundScope>,
445    deadline: Instant,
446    /// Set once `route.open` has finished putting the route.bind on the
447    /// module's egress queue. A path that drops this entry WITHOUT a module
448    /// reply (client teardown, module drain, the open's own abort) sends the
449    /// module a GOODBYE for the bind only when this is set, because only then
450    /// is the GOODBYE guaranteed to queue behind the bind. When it is not set,
451    /// the removal is recorded in `relays_released_before_enqueue` instead and
452    /// `route.open` sends that GOODBYE itself once its send has completed.
453    ///
454    /// A module reply does not consult it: an ack proves the module received
455    /// the bind, so the reply path decides about the GOODBYE on its own.
456    relay_enqueued: bool,
457    sender: oneshot::Sender<RouteBindRelayOutcome>,
458}
459
460/// What `mark_route_bind_relay_enqueued` found once `route.open` had put the
461/// route.bind on the module's queue.
462#[derive(Debug, Clone, Copy, PartialEq, Eq)]
463pub(crate) enum RouteBindRelayMark {
464    /// The relay is still pending; later removals handle their own GOODBYE.
465    Marked,
466    /// The module already answered and the reply path settled the relay:
467    /// it committed the route, or it refused it and sent any GOODBYE the
468    /// module needed. `route.open` must send nothing.
469    SettledByReply,
470    /// The relay was dropped without a module reply while the bind was still
471    /// being queued, so nothing has told the module to drop the binding it
472    /// is about to create. `route.open` must send that GOODBYE.
473    ReleasedWithoutReply,
474}
475
476#[derive(Debug, Clone)]
477pub(crate) enum RouteRelease {
478    Removed(GoodbyeTarget),
479    Stale,
480    Absent,
481}
482
483#[derive(Debug, Clone)]
484pub(crate) enum RoutePollSnapshot {
485    Bound {
486        module_id: String,
487        status: Option<String>,
488    },
489    Absent,
490}
491
492#[derive(Debug, Clone)]
493struct ModuleConnection {
494    endpoint: ModuleEndpointId,
495    sink: FrameSink,
496    negotiated_ver: u8,
497    concurrency: Concurrency,
498}
499
500#[derive(Debug, Default)]
501struct ForwardingInner {
502    operator_confirms: Arc<crate::operator_confirm::OperatorConfirms>,
503    daemon_draining: bool,
504    /// Whether forwarding reached quiescence during daemon shutdown, for each
505    /// active endpoint present when shutdown began. A pending wait is false
506    /// because it has not proved quiescence. Endpoint generations prevent a
507    /// result from describing a replacement connection with the same module id.
508    daemon_drain_results: HashMap<ModuleEndpointId, bool>,
509    /// The ACTIVE slot: the one endpoint per module id that routing resolves.
510    /// Every by-id lookup (relay reservation, drain-by-id, liveness, census,
511    /// live roots, module-control RPCs) reads this map and nothing else.
512    modules_by_id: HashMap<String, ModuleConnection>,
513    /// The CANDIDATE slot of a blue/green swap: a second process registered
514    /// under an id that already has an active endpoint. It has a full endpoint
515    /// identity (so its own connection can be looked up and torn down) but no
516    /// by-id lookup sees it, so nothing is routed to it until `cutover_candidate`
517    /// promotes it.
518    candidates_by_id: HashMap<String, ModuleConnection>,
519    /// Former active endpoints demoted by `cutover_candidate`, until their
520    /// connection is removed. Membership is what distinguishes an endpoint that
521    /// was SUPERSEDED by a promotion (still alive, still carrying bound routes,
522    /// about to be drained) from one that is merely STALE (replaced some other
523    /// way, which is treated as a fault on the acking connection).
524    superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
525    endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
526    module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
527    /// Endpoints mid-drain, keyed to the reason the drain was begun with. The
528    /// value serves the census ("draining because restart"); membership alone
529    /// still answers every admission-gate check.
530    draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
531    closing_connections: HashSet<ConnectionId>,
532    next_generation: u64,
533    reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
534    reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
535    next_client_channel: HashMap<ConnectionId, u16>,
536    next_module_channel: HashMap<ModuleEndpointId, u16>,
537    client_slot_epochs: HashMap<ClientRouteKey, u32>,
538    module_slot_epochs: HashMap<ModuleRouteKey, u32>,
539    last_published_epoch: HashMap<ClientRouteKey, u32>,
540    client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
541    module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
542    status: HashMap<(ClientRouteKey, u32), String>,
543    pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
544    /// Pending relays removed without a module reply before `route.open`
545    /// marked them enqueued. `route.open` consumes its key when it marks the
546    /// relay (and sends the GOODBYE the removal could not send safely) or when
547    /// it aborts, so an entry lives only while that open is mid-send.
548    relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
549    /// Each scope's current `(scope_epoch, version)`, keyed `(owner, ref)`,
550    /// published by a scope sync in the same step that changes the record. A
551    /// bind commit reads it here so it never takes the scope table's lock
552    /// while holding this one. Absent means the scope is not live.
553    scope_tags: HashMap<(String, String), ScopeTag>,
554    next_control_corr: HashMap<ModuleEndpointId, u64>,
555    pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
556    health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
557}
558
559#[derive(Debug, Clone)]
560pub(crate) struct CloseReason {
561    code: &'static str,
562    message: String,
563}
564
565impl CloseReason {
566    pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
567        Self {
568            code,
569            message: message.into(),
570        }
571    }
572}
573
574impl fmt::Display for CloseReason {
575    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
576        write!(f, "{}: {}", self.code, self.message)
577    }
578}
579
580pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
581
582/// Dynamic forwarding state shared by the control plane and data-plane router.
583#[derive(Debug, Default)]
584pub struct ForwardingTable {
585    inner: Arc<RwLock<ForwardingInner>>,
586    #[cfg(feature = "test-support")]
587    route_closing_hold: Mutex<Option<oneshot::Receiver<()>>>,
588    close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
589    counters: DaemonCounters,
590    /// Per-target-module bind-relay breaker state. It lives here, beside the
591    /// module connections it describes, because this is where a module
592    /// connection's identity is established and therefore where a stale
593    /// verdict has to be discarded.
594    route_bind_breakers: RouteBindBreakers,
595    /// Current route.bind relays keyed by target module. Admission is shared
596    /// across every client connection that points at the same endpoint.
597    route_bind_concurrency: RouteBindConcurrency,
598    /// Start and end of each module's `route.open` outage. Held here for the
599    /// same reason as the breakers: every handler built over this table must
600    /// see one outage per module, or each would log its own opening line.
601    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
602}
603
604impl ForwardingTable {
605    /// Install scripted operator confirmation for an in-process integration harness,
606    /// without bootstrapping a daemon process. Only available with `test-support`.
607    ///
608    /// The script has one response per line: `approve`, `decline`, `unavailable`,
609    /// `hang` (until withdrawn), or `approve when <path>` (wait for the path to
610    /// exist). Unreadable or exhausted scripts fail closed. If supplied, `events`
611    /// receives the provider's JSON-lines event log. `limits: None` uses
612    /// [`OperatorLimits::default`](crate::operator_confirm::OperatorLimits); use
613    /// `Some(OperatorLimits { queue_wait, prompt_timeout, stuck_grace })` to
614    /// override the queue wait, prompt timeout and stuck-provider grace periods.
615    ///
616    /// The current executable's file name **must start with `ckdev-`**. Otherwise
617    /// this logs a warning and keeps the OS provider, just like bootstrap.
618    /// `cortexkit-test-support`'s `ckdev_binary` copies a harness executable under
619    /// a qualifying name; run that copy to enable scripted confirmation. See
620    /// `crates/subc-client-rs/tests/real_daemon/operator_confirm.rs` for the
621    /// reference end-to-end harness.
622    ///
623    /// Configure the table shared by the harness's `ControlHandler` and `Router`
624    /// before serving any requests. This also configures privileged confirmations
625    /// that use the same table.
626    ///
627    /// # Panics
628    ///
629    /// Panics if operator-confirm processing has already started.
630    ///
631    /// # Example
632    ///
633    /// ```no_run
634    /// use std::{path::Path, sync::Arc};
635    /// use subc_daemon::{ControlHandler, ForwardingTable, Registry};
636    ///
637    /// let registry = Arc::new(Registry::default());
638    /// let forwarding = Arc::new(ForwardingTable::default());
639    /// forwarding.configure_operator_script(
640    ///     Path::new("operator.script"),
641    ///     Some(Path::new("operator.events")),
642    ///     None,
643    /// );
644    /// let handler = ControlHandler::with_forwarding(registry, forwarding);
645    /// ```
646    #[cfg(feature = "test-support")]
647    pub fn configure_operator_script(
648        &self,
649        script: &std::path::Path,
650        events: Option<&std::path::Path>,
651        limits: Option<crate::operator_confirm::OperatorLimits>,
652    ) {
653        self.operator_confirms()
654            .configure_script(script, events, limits);
655    }
656
657    /// Hold the next supervisor drain between enqueueing `module.draining` and
658    /// `route.closing`, so a test can forward a response in that window. Dropping
659    /// the returned sender also releases the hold, including on a test panic.
660    #[cfg(feature = "test-support")]
661    pub fn hold_route_closing_for_test(&self) -> oneshot::Sender<()> {
662        let (release, hold) = oneshot::channel();
663        *self.route_closing_hold.lock().unwrap() = Some(hold);
664        release
665    }
666
667    #[cfg(feature = "test-support")]
668    pub(crate) async fn wait_before_route_closing_for_test(&self) {
669        let hold = self.route_closing_hold.lock().unwrap().take();
670        if let Some(hold) = hold {
671            let _ = hold.await;
672        }
673    }
674
675    pub(crate) fn operator_confirms(&self) -> Arc<crate::operator_confirm::OperatorConfirms> {
676        Arc::clone(
677            &self
678                .inner
679                .read()
680                .unwrap_or_else(|p| p.into_inner())
681                .operator_confirms,
682        )
683    }
684
685    #[cfg(test)]
686    pub(crate) fn inject_operator_principal(&self, key: ModuleRouteKey, principal: Principal) {
687        let mut inner = self.write_inner().unwrap();
688        let old = inner.module_to_client.remove(&key).unwrap();
689        let client = ClientRouteKey {
690            connection_id: old.client_connection_id,
691            channel: old.client_channel,
692        };
693        inner.client_to_module.remove(&client);
694        let mut binding = Arc::try_unwrap(old).expect("test binding has no outstanding readers");
695        binding.principal = principal;
696        let binding = Arc::new(binding);
697        inner.client_to_module.insert(client, Arc::clone(&binding));
698        inner.module_to_client.insert(key, binding);
699    }
700
701    /// Keep the binding protected through confirmation admission. Route release
702    /// takes the write lock and then the same confirm-state lock, so it cannot
703    /// miss an admitted request or leave it waiting on a vanished route.
704    pub(crate) fn with_operator_route<T>(
705        &self,
706        connection_id: ConnectionId,
707        channel: u16,
708        epoch: u32,
709        admit: impl FnOnce(Option<&RouteBinding>) -> T,
710    ) -> Result<T, ForwardingError> {
711        let inner = self.read_inner()?;
712        let binding = inner
713            .endpoint_by_connection
714            .get(&connection_id)
715            .and_then(|endpoint| {
716                inner.module_to_client.get(&ModuleRouteKey {
717                    endpoint: *endpoint,
718                    channel,
719                })
720            })
721            .filter(|binding| binding.module_epoch == epoch);
722        Ok(admit(binding.map(Arc::as_ref)))
723    }
724
725    pub(crate) fn counters(&self) -> DaemonCounters {
726        self.counters.clone()
727    }
728
729    pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
730        self.route_bind_breakers.clone()
731    }
732
733    pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
734        self.route_bind_concurrency.clone()
735    }
736
737    pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
738        Arc::clone(&self.route_outages)
739    }
740
741    pub(crate) fn register_connection_close(
742        &self,
743        connection_id: ConnectionId,
744    ) -> ConnectionCloseReceiver {
745        let (sender, receiver) = oneshot::channel();
746        let replaced = self
747            .lock_close_registry()
748            .insert(connection_id, sender)
749            .is_some();
750        if replaced {
751            warn!(
752                connection_id = connection_id.get(),
753                "replaced existing connection close registration"
754            );
755        }
756        receiver
757    }
758
759    pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
760        self.lock_close_registry().remove(&connection_id);
761    }
762
763    /// Close every established connection, modules and clients alike, as the
764    /// last step of an announced daemon shutdown. By the time this runs each
765    /// registered module has already been sent a module GOODBYE (see
766    /// `Supervisor::end_children_for_daemon_shutdown`), so the EOF that follows
767    /// is a planned stop; EOF with no GOODBYE before it stays reserved for a
768    /// daemon that went away unannounced. Closing while the daemon is still
769    /// alive lets it wait for the modules' own teardowns before it ends
770    /// whatever is left.
771    #[cfg(unix)]
772    pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
773        let senders: Vec<_> = self.lock_close_registry().drain().collect();
774        let count = senders.len();
775        for (_, sender) in senders {
776            let _ = sender.send(reason.clone());
777        }
778        count
779    }
780
781    /// Every registered module connection, whichever slot it occupies (active,
782    /// swap candidate, or superseded incumbent), once each. Daemon shutdown
783    /// uses this to tell each of them it is a planned stop before closing it.
784    #[cfg(unix)]
785    pub(crate) fn module_connections(
786        &self,
787    ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
788        let inner = self.read_inner()?;
789        let mut seen = HashSet::new();
790        Ok(inner
791            .modules_by_id
792            .values()
793            .chain(inner.candidates_by_id.values())
794            .chain(inner.superseded_endpoints.values())
795            .filter(|module| seen.insert(module.endpoint))
796            .map(|module| ModuleConnectionTarget {
797                module_id: inner
798                    .module_id_by_endpoint
799                    .get(&module.endpoint)
800                    .cloned()
801                    .unwrap_or_default(),
802                endpoint: module.endpoint,
803                sink: module.sink.clone(),
804                negotiated_ver: module.negotiated_ver,
805            })
806            .collect())
807    }
808
809    /// Ask a registered connection to close. Returns true only for the request
810    /// that actually reached it; later requests for the same connection, and
811    /// requests for a connection that is not registered, return false.
812    pub(crate) fn request_connection_close(
813        &self,
814        connection_id: ConnectionId,
815        reason: CloseReason,
816    ) -> bool {
817        let sender = self.lock_close_registry().remove(&connection_id);
818        if let Some(sender) = sender {
819            debug!(
820                connection_id = connection_id.get(),
821                close_reason = %reason,
822                "requesting connection close"
823            );
824            let _ = sender.send(reason);
825            true
826        } else {
827            debug!(
828                connection_id = connection_id.get(),
829                close_reason = %reason,
830                "connection close request ignored for inactive connection"
831            );
832            false
833        }
834    }
835
836    pub fn register_module_connection(
837        &self,
838        connection_id: ConnectionId,
839        module_id: String,
840        negotiated_ver: u8,
841        concurrency: Concurrency,
842        sink: FrameSink,
843    ) -> Result<ModuleEndpointId, ForwardingError> {
844        self.register_module_connection_inner(
845            connection_id,
846            module_id,
847            negotiated_ver,
848            concurrency,
849            sink,
850            None,
851        )
852    }
853
854    /// Register a module connection and queue its HELLO_ACK as the first frame
855    /// on its sink, in the same write-lock critical section that makes the
856    /// endpoint visible.
857    ///
858    /// A module reads HELLO_ACK as the first frame after its HELLO and exits on
859    /// anything else. Once the endpoint is in `modules_by_id`, a `route.open`
860    /// on any other connection can queue a `route.bind` request onto this sink,
861    /// so the ack must already be queued by then. Every lookup that can queue a
862    /// frame for the module takes this lock, so nothing can get in ahead of it.
863    ///
864    /// If the ack cannot be queued (the sink is closed or full), the
865    /// registration fails with nothing inserted.
866    pub(crate) fn register_module_connection_acked(
867        &self,
868        connection_id: ConnectionId,
869        module_id: String,
870        negotiated_ver: u8,
871        concurrency: Concurrency,
872        sink: FrameSink,
873        hello_ack: Frame,
874    ) -> Result<ModuleEndpointId, ForwardingError> {
875        self.register_module_connection_inner(
876            connection_id,
877            module_id,
878            negotiated_ver,
879            concurrency,
880            sink,
881            Some(hello_ack),
882        )
883    }
884
885    fn register_module_connection_inner(
886        &self,
887        connection_id: ConnectionId,
888        module_id: String,
889        negotiated_ver: u8,
890        concurrency: Concurrency,
891        sink: FrameSink,
892        hello_ack: Option<Frame>,
893    ) -> Result<ModuleEndpointId, ForwardingError> {
894        let mut inner = self.write_inner()?;
895        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
896            return Err(ForwardingError::ConnectionClosing { connection_id });
897        }
898        check_module_connection_role_locked(&inner, connection_id)?;
899        // Every refusal check has passed and nothing has been mutated yet, so
900        // a failed enqueue leaves the table exactly as it was.
901        enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
902
903        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
904        let endpoint = ModuleEndpointId {
905            connection_id,
906            generation: inner.next_generation,
907        };
908        inner.endpoint_by_connection.insert(connection_id, endpoint);
909        inner
910            .module_id_by_endpoint
911            .insert(endpoint, module_id.clone());
912        inner.next_module_channel.insert(endpoint, 1);
913        inner.next_control_corr.insert(endpoint, 1);
914        inner.modules_by_id.insert(
915            module_id.clone(),
916            ModuleConnection {
917                endpoint,
918                sink,
919                negotiated_ver,
920                concurrency,
921            },
922        );
923        drop(inner);
924
925        // A new module connection has arrived under this id, so anything the
926        // bind-relay breaker learned was learned about a process that is no
927        // longer the one behind this name. See
928        // `RouteBindBreakers::reset_for_new_module_connection`.
929        //
930        // Keyed on ARRIVAL rather than on teardown deliberately: a connection
931        // going away is not evidence about anything, and a module whose
932        // connection drops without coming back should keep its verdict until
933        // something actually registers in its place. This also covers the
934        // unclean replacements -- a module killed mid-bind, or one whose
935        // connection was closing -- because registration is the single path by
936        // which any module connection becomes usable.
937        if let Some(discarded) = self
938            .route_bind_breakers
939            .reset_for_new_module_connection(&module_id)
940        {
941            info!(
942                module_id = %module_id,
943                discarded_consecutive_timeouts = discarded,
944                "route.bind breaker state discarded: a new module connection replaced the process it described"
945            );
946        }
947        Ok(endpoint)
948    }
949
950    /// Register a blue/green swap candidate for `module_id` into the candidate
951    /// slot, alongside whatever endpoint is active for the id.
952    ///
953    /// Unlike [`Self::register_module_connection`], this never touches the
954    /// active slot and never resets the bind-relay breaker: the incumbent is
955    /// still the process serving the id, so what the breaker learned about it
956    /// is still true. The breaker is reset at cutover instead, when the process
957    /// behind the name actually changes. A second candidate for the same id is
958    /// refused.
959    ///
960    /// Production registers candidates through
961    /// [`Self::register_candidate_module_connection_acked`]; this ack-less form
962    /// is for tests that build forwarding state directly.
963    #[cfg(test)]
964    pub(crate) fn register_candidate_module_connection(
965        &self,
966        connection_id: ConnectionId,
967        module_id: String,
968        negotiated_ver: u8,
969        concurrency: Concurrency,
970        sink: FrameSink,
971    ) -> Result<ModuleEndpointId, ForwardingError> {
972        self.register_candidate_module_connection_inner(
973            connection_id,
974            module_id,
975            negotiated_ver,
976            concurrency,
977            sink,
978            None,
979        )
980    }
981
982    /// The swap-candidate counterpart of
983    /// [`Self::register_module_connection_acked`]: the HELLO_ACK is queued on
984    /// the candidate's sink before its endpoint is inserted. No by-id lookup
985    /// sees a candidate, but its own connection's endpoint does become
986    /// resolvable here, and at cutover it becomes routable; the ack has to be
987    /// ahead of anything either can queue.
988    pub(crate) fn register_candidate_module_connection_acked(
989        &self,
990        connection_id: ConnectionId,
991        module_id: String,
992        negotiated_ver: u8,
993        concurrency: Concurrency,
994        sink: FrameSink,
995        hello_ack: Frame,
996    ) -> Result<ModuleEndpointId, ForwardingError> {
997        self.register_candidate_module_connection_inner(
998            connection_id,
999            module_id,
1000            negotiated_ver,
1001            concurrency,
1002            sink,
1003            Some(hello_ack),
1004        )
1005    }
1006
1007    fn register_candidate_module_connection_inner(
1008        &self,
1009        connection_id: ConnectionId,
1010        module_id: String,
1011        negotiated_ver: u8,
1012        concurrency: Concurrency,
1013        sink: FrameSink,
1014        hello_ack: Option<Frame>,
1015    ) -> Result<ModuleEndpointId, ForwardingError> {
1016        let mut inner = self.write_inner()?;
1017        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1018            return Err(ForwardingError::ConnectionClosing { connection_id });
1019        }
1020        check_module_connection_role_locked(&inner, connection_id)?;
1021        if inner.candidates_by_id.contains_key(&module_id) {
1022            return Err(ForwardingError::CandidateSlotOccupied { module_id });
1023        }
1024        // As in `register_module_connection_inner`: after every refusal check,
1025        // before any mutation.
1026        enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1027
1028        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1029        let endpoint = ModuleEndpointId {
1030            connection_id,
1031            generation: inner.next_generation,
1032        };
1033        inner.endpoint_by_connection.insert(connection_id, endpoint);
1034        inner
1035            .module_id_by_endpoint
1036            .insert(endpoint, module_id.clone());
1037        inner.next_module_channel.insert(endpoint, 1);
1038        inner.next_control_corr.insert(endpoint, 1);
1039        inner.candidates_by_id.insert(
1040            module_id,
1041            ModuleConnection {
1042                endpoint,
1043                sink,
1044                negotiated_ver,
1045                concurrency,
1046            },
1047        );
1048        Ok(endpoint)
1049    }
1050
1051    /// Promote `module_id`'s candidate to the active slot, in ONE forwarding
1052    /// write-lock critical section.
1053    ///
1054    /// That single section is the linearization point of a swap. Every relay
1055    /// reservation resolves the active slot under the same lock, so each one
1056    /// lands wholly before cutover (on the incumbent) or wholly after it (on the
1057    /// promoted candidate), never on a mix. A relay reserved on the incumbent
1058    /// before cutover can no longer commit: `commit_route_locked` requires the
1059    /// reservation's endpoint to be the active one, and an ack for it arriving
1060    /// later is answered by the superseded-endpoint arm of
1061    /// `complete_pending_relay` instead of ending the incumbent's connection.
1062    ///
1063    /// Both endpoints keep their identities: nothing is re-keyed, so the
1064    /// incumbent's bound routes and its pending correlation keys stay exactly
1065    /// where they are until it is drained with [`Self::begin_endpoint_drain`],
1066    /// using the incumbent endpoint this returns. Returns `Ok(None)` when there
1067    /// is no candidate for the id.
1068    pub(crate) fn cutover_candidate(
1069        &self,
1070        module_id: &str,
1071    ) -> Result<Option<ForwardingCutover>, ForwardingError> {
1072        let mut inner = self.write_inner()?;
1073        if inner.daemon_draining {
1074            return Err(ForwardingError::ModuleReloading {
1075                module_id: module_id.to_string(),
1076            });
1077        }
1078        let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
1079            return Ok(None);
1080        };
1081        let promoted = candidate.endpoint;
1082        let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
1083        let incumbent = incumbent.map(|incumbent| {
1084            let endpoint = incumbent.endpoint;
1085            inner.superseded_endpoints.insert(endpoint, incumbent);
1086            endpoint
1087        });
1088        drop(inner);
1089
1090        // A different process now answers for this id; see the matching reset
1091        // in `register_module_connection` for why a verdict about the old one
1092        // must not carry over.
1093        if let Some(discarded) = self
1094            .route_bind_breakers
1095            .reset_for_new_module_connection(module_id)
1096        {
1097            info!(
1098                module_id = %module_id,
1099                discarded_consecutive_timeouts = discarded,
1100                "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
1101            );
1102        }
1103        Ok(Some(ForwardingCutover {
1104            promoted,
1105            incumbent,
1106        }))
1107    }
1108
1109    #[allow(clippy::too_many_arguments)]
1110    pub(crate) async fn begin_route_bind_relay_for(
1111        &self,
1112        client_connection_id: ConnectionId,
1113        client_sink: FrameSink,
1114        client_negotiated_ver: u8,
1115        client_corr: u64,
1116        module_id: &str,
1117        principal: Principal,
1118        scope: Option<BoundScope>,
1119        project_root: Option<ProjectRootId>,
1120        deadline: Instant,
1121    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1122        // Reserve egress capacity before taking the forwarding lock. The permit is
1123        // held until the bind reaches one terminal state, so an accepted bind can
1124        // publish its table entry and RouteOpen response in one critical section.
1125        let client_permit =
1126            client_sink
1127                .reserve_owned()
1128                .await
1129                .map_err(|_| ForwardingError::ClientEgressClosed {
1130                    connection_id: client_connection_id,
1131                })?;
1132        self.begin_route_bind_relay_inner(
1133            client_connection_id,
1134            client_sink,
1135            client_negotiated_ver,
1136            client_corr,
1137            module_id,
1138            principal,
1139            scope,
1140            project_root,
1141            deadline,
1142            client_permit,
1143        )
1144    }
1145
1146    #[cfg(test)]
1147    pub(crate) fn begin_route_bind_relay_for_test(
1148        &self,
1149        client_connection_id: ConnectionId,
1150        client_sink: FrameSink,
1151        client_corr: u64,
1152        module_id: &str,
1153    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1154        let permit =
1155            client_sink
1156                .try_reserve_owned()
1157                .map_err(|_| ForwardingError::ClientEgressClosed {
1158                    connection_id: client_connection_id,
1159                })?;
1160        self.begin_route_bind_relay_inner(
1161            client_connection_id,
1162            client_sink,
1163            subc_protocol::PROTOCOL_VERSION,
1164            client_corr,
1165            module_id,
1166            Principal::Direct,
1167            None,
1168            None,
1169            Instant::now() + std::time::Duration::from_secs(60),
1170            permit,
1171        )
1172    }
1173
1174    pub(crate) fn begin_module_control_rpc_for(
1175        &self,
1176        module_id: &str,
1177        expected_op: &str,
1178        deadline: Instant,
1179    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1180        self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1181    }
1182
1183    pub(crate) fn begin_health_probe_rpc_for(
1184        &self,
1185        module_id: &str,
1186        expected_op: &str,
1187        probe_started_at: Instant,
1188        deadline: Instant,
1189    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1190        self.begin_module_control_rpc_inner(
1191            module_id,
1192            expected_op,
1193            deadline,
1194            Some(probe_started_at),
1195            false,
1196        )
1197    }
1198
1199    pub(crate) fn begin_drain_health_probe_rpc_for(
1200        &self,
1201        module_id: &str,
1202        expected_op: &str,
1203        probe_started_at: Instant,
1204        deadline: Instant,
1205    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1206        self.begin_module_control_rpc_inner(
1207            module_id,
1208            expected_op,
1209            deadline,
1210            Some(probe_started_at),
1211            true,
1212        )
1213    }
1214
1215    /// A health probe addressed to one endpoint, in whichever slot it is.
1216    ///
1217    /// Every other control RPC resolves the id's ACTIVE endpoint, which is how
1218    /// a swap candidate (not active until cutover) and a superseded incumbent
1219    /// (not active after it) are unreachable by them. A swap needs to probe
1220    /// exactly those two: the candidate before promoting it, and the incumbent
1221    /// for its busy gauges while it drains.
1222    pub(crate) fn begin_endpoint_health_probe_rpc_for(
1223        &self,
1224        endpoint: ModuleEndpointId,
1225        expected_op: &str,
1226        probe_started_at: Instant,
1227        deadline: Instant,
1228    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1229        let inner = self.write_inner()?;
1230        let module = module_connection_for_endpoint_locked(&inner, endpoint)
1231            .cloned()
1232            .ok_or(ForwardingError::NoModuleConnection)?;
1233        let module_id = inner
1234            .module_id_by_endpoint
1235            .get(&endpoint)
1236            .cloned()
1237            .unwrap_or_default();
1238        // A draining incumbent is exactly what this probes for busy gauges, so
1239        // draining is allowed, as it is for the by-id drain probe.
1240        self.begin_control_rpc_locked(
1241            inner,
1242            &module_id,
1243            module,
1244            expected_op,
1245            deadline,
1246            Some(probe_started_at),
1247            true,
1248        )
1249    }
1250
1251    fn begin_module_control_rpc_inner(
1252        &self,
1253        module_id: &str,
1254        expected_op: &str,
1255        deadline: Instant,
1256        health_probe_started_at: Option<Instant>,
1257        allow_draining: bool,
1258    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1259        let inner = self.write_inner()?;
1260        let module = inner
1261            .modules_by_id
1262            .get(module_id)
1263            .cloned()
1264            .ok_or(ForwardingError::NoModuleConnection)?;
1265        self.begin_control_rpc_locked(
1266            inner,
1267            module_id,
1268            module,
1269            expected_op,
1270            deadline,
1271            health_probe_started_at,
1272            allow_draining,
1273        )
1274    }
1275
1276    #[allow(clippy::too_many_arguments)]
1277    fn begin_control_rpc_locked(
1278        &self,
1279        mut inner: RwLockWriteGuard<'_, ForwardingInner>,
1280        module_id: &str,
1281        module: ModuleConnection,
1282        expected_op: &str,
1283        deadline: Instant,
1284        health_probe_started_at: Option<Instant>,
1285        allow_draining: bool,
1286    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1287        if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1288            return Err(ForwardingError::ModuleReloading {
1289                module_id: module_id.to_string(),
1290            });
1291        }
1292        if inner
1293            .closing_connections
1294            .contains(&module.endpoint.connection_id)
1295        {
1296            return Err(ForwardingError::ConnectionClosing {
1297                connection_id: module.endpoint.connection_id,
1298            });
1299        }
1300        if health_probe_started_at.is_some() {
1301            // Recurring health probes are serial per endpoint. Once the next one
1302            // starts, an older answer can no longer improve the current snapshot,
1303            // so retaining more than the newest unanswered probe has no value.
1304            inner
1305                .health_probe_tombstones
1306                .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1307        }
1308        let corr = match inner.allocate_control_corr(module.endpoint) {
1309            Ok(corr) => corr,
1310            Err(err) => {
1311                drop(inner);
1312                self.request_connection_close(
1313                    module.endpoint.connection_id,
1314                    CloseReason::new(
1315                        "control_correlation_exhausted",
1316                        "daemon-originated channel-0 correlation space exhausted",
1317                    ),
1318                );
1319                return Err(err);
1320            }
1321        };
1322        let (sender, receiver) = oneshot::channel();
1323        inner.pending_control_rpcs.insert(
1324            (module.endpoint, corr),
1325            PendingModuleControlRpcEntry {
1326                expected_op: expected_op.to_string(),
1327                deadline,
1328                health_probe_started_at,
1329                sender,
1330            },
1331        );
1332
1333        Ok(PendingModuleControlRpc {
1334            endpoint: module.endpoint,
1335            module_sink: module.sink,
1336            negotiated_ver: module.negotiated_ver,
1337            corr,
1338            receiver,
1339        })
1340    }
1341
1342    #[allow(clippy::too_many_arguments)]
1343    fn begin_route_bind_relay_inner(
1344        &self,
1345        client_connection_id: ConnectionId,
1346        client_sink: FrameSink,
1347        client_negotiated_ver: u8,
1348        client_corr: u64,
1349        expected_module_id: &str,
1350        principal: Principal,
1351        scope: Option<BoundScope>,
1352        project_root: Option<ProjectRootId>,
1353        deadline: Instant,
1354        client_permit: crate::router::EgressPermit,
1355    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1356        let mut inner = self.write_inner()?;
1357        // HELLO and route.open can run on different tasks. Decide the role under
1358        // the same lock that installs endpoints and reserves client channels so
1359        // neither task can turn one connection into both kinds of owner.
1360        if inner
1361            .endpoint_by_connection
1362            .contains_key(&client_connection_id)
1363        {
1364            return Err(ForwardingError::ConnectionRoleConflict {
1365                connection_id: client_connection_id,
1366            });
1367        }
1368        if inner.closing_connections.contains(&client_connection_id) {
1369            return Err(ForwardingError::ConnectionClosing {
1370                connection_id: client_connection_id,
1371            });
1372        }
1373        let module = inner
1374            .modules_by_id
1375            .get(expected_module_id)
1376            .cloned()
1377            .ok_or(ForwardingError::NoModuleConnection)?;
1378        if inner.draining_endpoints.contains_key(&module.endpoint) {
1379            return Err(ForwardingError::ModuleReloading {
1380                module_id: expected_module_id.to_string(),
1381            });
1382        }
1383        if inner
1384            .closing_connections
1385            .contains(&module.endpoint.connection_id)
1386        {
1387            return Err(ForwardingError::ConnectionClosing {
1388                connection_id: module.endpoint.connection_id,
1389            });
1390        }
1391
1392        let corr = match inner.allocate_control_corr(module.endpoint) {
1393            Ok(corr) => corr,
1394            Err(err) => {
1395                drop(inner);
1396                self.request_connection_close(
1397                    module.endpoint.connection_id,
1398                    CloseReason::new(
1399                        "control_correlation_exhausted",
1400                        "daemon-originated channel-0 correlation space exhausted",
1401                    ),
1402                );
1403                return Err(err);
1404            }
1405        };
1406        let (client_channel, client_epoch, module_channel, module_epoch) =
1407            inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1408        let client_key = ClientRouteKey {
1409            connection_id: client_connection_id,
1410            channel: client_channel,
1411        };
1412        let module_key = ModuleRouteKey {
1413            endpoint: module.endpoint,
1414            channel: module_channel,
1415        };
1416        let reservation = RouteReservation {
1417            client_key,
1418            module_key,
1419            client_epoch,
1420            module_epoch,
1421            project_root,
1422        };
1423        let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
1424            route_channel: client_channel,
1425            route_epoch: client_epoch,
1426        })
1427        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1428        let route_open_frame = Frame::build_with_version(
1429            client_negotiated_ver,
1430            FrameType::Response,
1431            Flags::new(false, Priority::Passive, false),
1432            0,
1433            0,
1434            client_corr,
1435            response_body,
1436        )
1437        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1438        let (sender, receiver) = oneshot::channel();
1439        inner.reserved_client.insert(client_key, module_key);
1440        inner.reserved_module.insert(module_key, client_key);
1441        inner.pending_relays.insert(
1442            (module.endpoint, corr),
1443            PendingRouteBindRelayEntry {
1444                reservation,
1445                client_sink,
1446                client_negotiated_ver,
1447                client_permit,
1448                route_open_frame,
1449                principal,
1450                scope,
1451                deadline,
1452                relay_enqueued: false,
1453                sender,
1454            },
1455        );
1456
1457        Ok(PendingRouteBindRelay {
1458            endpoint: module.endpoint,
1459            module_sink: module.sink,
1460            negotiated_ver: module.negotiated_ver,
1461            client_channel,
1462            client_epoch,
1463            module_channel,
1464            module_epoch,
1465            corr,
1466            receiver,
1467        })
1468    }
1469
1470    /// Called by `route.open` right after the route.bind entered the module's
1471    /// egress queue.
1472    ///
1473    /// The module reader runs concurrently with `route.open`, so a fast module
1474    /// can receive the bind, ack it, and have the reader commit the route
1475    /// before this runs. A missing entry therefore does not mean the bind was
1476    /// abandoned: only a removal recorded in `relays_released_before_enqueue`
1477    /// does. Telling the two apart is what stops `route.open` from sending a
1478    /// GOODBYE for a route that was just committed.
1479    pub(crate) fn mark_route_bind_relay_enqueued(
1480        &self,
1481        endpoint: ModuleEndpointId,
1482        corr: u64,
1483    ) -> Result<RouteBindRelayMark, ForwardingError> {
1484        let mut inner = self.write_inner()?;
1485        if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
1486            pending.relay_enqueued = true;
1487            return Ok(RouteBindRelayMark::Marked);
1488        }
1489        if inner
1490            .relays_released_before_enqueue
1491            .remove(&(endpoint, corr))
1492        {
1493            return Ok(RouteBindRelayMark::ReleasedWithoutReply);
1494        }
1495        Ok(RouteBindRelayMark::SettledByReply)
1496    }
1497
1498    pub(crate) fn release_client_route(
1499        &self,
1500        client_connection_id: ConnectionId,
1501        client_channel: u16,
1502        expected_epoch: u32,
1503    ) -> Result<RouteRelease, ForwardingError> {
1504        let mut inner = self.write_inner()?;
1505        let release = release_client_route_locked(
1506            &mut inner,
1507            ClientRouteKey {
1508                connection_id: client_connection_id,
1509                channel: client_channel,
1510            },
1511            expected_epoch,
1512        );
1513        self.record_route_release(&release);
1514        Ok(release)
1515    }
1516
1517    pub(crate) fn release_module_route(
1518        &self,
1519        module_connection_id: ConnectionId,
1520        module_channel: u16,
1521        expected_epoch: u32,
1522    ) -> Result<RouteRelease, ForwardingError> {
1523        let mut inner = self.write_inner()?;
1524        let Some(endpoint) = inner
1525            .endpoint_by_connection
1526            .get(&module_connection_id)
1527            .copied()
1528        else {
1529            return Ok(RouteRelease::Absent);
1530        };
1531        let release = release_module_route_locked(
1532            &mut inner,
1533            ModuleRouteKey {
1534                endpoint,
1535                channel: module_channel,
1536            },
1537            expected_epoch,
1538        );
1539        self.record_route_release(&release);
1540        Ok(release)
1541    }
1542
1543    pub(crate) fn abort_pending_relay(
1544        &self,
1545        endpoint: ModuleEndpointId,
1546        corr: u64,
1547        outcome: RouteBindRelayOutcome,
1548    ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
1549        let mut inner = self.write_inner()?;
1550        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1551            // If another path dropped the relay while `route.open` was still
1552            // sending the bind, and that open is now giving up without marking
1553            // it (its send failed or it was cancelled), forget the record so it
1554            // cannot outlive the open.
1555            inner
1556                .relays_released_before_enqueue
1557                .remove(&(endpoint, corr));
1558            return Ok(None);
1559        };
1560        release_reserved_route_locked(
1561            &mut inner,
1562            pending.reservation.client_key,
1563            pending.reservation.module_key,
1564        );
1565        let target = pending
1566            .relay_enqueued
1567            .then(|| abandoned_route_target(&inner, &pending.reservation));
1568        let _ = pending.sender.send(outcome);
1569        Ok(target.flatten())
1570    }
1571
1572    pub(crate) fn cancel_module_control_rpc(
1573        &self,
1574        endpoint: ModuleEndpointId,
1575        corr: u64,
1576    ) -> Result<(), ForwardingError> {
1577        self.write_inner()?
1578            .pending_control_rpcs
1579            .remove(&(endpoint, corr));
1580        Ok(())
1581    }
1582
1583    pub(crate) fn tombstone_health_probe_rpc(
1584        &self,
1585        endpoint: ModuleEndpointId,
1586        corr: u64,
1587    ) -> Result<bool, ForwardingError> {
1588        let key = (endpoint, corr);
1589        let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
1590        {
1591            let mut inner = self.write_inner()?;
1592            let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
1593                return Ok(false);
1594            };
1595            let Some(probe_started_at) = pending.health_probe_started_at else {
1596                inner.pending_control_rpcs.insert(key, pending);
1597                return Ok(false);
1598            };
1599            let module_id = inner
1600                .module_id_by_endpoint
1601                .get(&endpoint)
1602                .cloned()
1603                .unwrap_or_else(|| "unknown".to_string());
1604            inner.health_probe_tombstones.insert(
1605                key,
1606                HealthProbeTombstone {
1607                    expected_op: pending.expected_op,
1608                    module_id,
1609                    probe_started_at,
1610                    expires_at,
1611                },
1612            );
1613        }
1614        self.schedule_health_probe_tombstone_expiration(key, expires_at);
1615        Ok(true)
1616    }
1617
1618    fn schedule_health_probe_tombstone_expiration(
1619        &self,
1620        key: (ModuleEndpointId, u64),
1621        expires_at: Instant,
1622    ) {
1623        let inner = Arc::downgrade(&self.inner);
1624        tokio::spawn(async move {
1625            tokio::time::sleep_until(expires_at).await;
1626            let Some(inner) = inner.upgrade() else {
1627                return;
1628            };
1629            let Ok(mut inner) = inner.write() else {
1630                return;
1631            };
1632            let expired = inner
1633                .health_probe_tombstones
1634                .get(&key)
1635                .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
1636            if expired {
1637                inner.health_probe_tombstones.remove(&key);
1638            }
1639        });
1640    }
1641
1642    pub(crate) fn complete_pending_relay(
1643        &self,
1644        connection_id: ConnectionId,
1645        corr: u64,
1646        outcome: RouteBindRelayOutcome,
1647    ) -> Result<PendingRelayCompletion, ForwardingError> {
1648        let mut inner = self.write_inner()?;
1649        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1650            return Ok(PendingRelayCompletion {
1651                settled: false,
1652                abandoned: None,
1653            });
1654        };
1655        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1656            return Ok(PendingRelayCompletion {
1657                settled: false,
1658                abandoned: None,
1659            });
1660        };
1661
1662        if Instant::now() >= pending.deadline {
1663            release_reserved_route_locked(
1664                &mut inner,
1665                pending.reservation.client_key,
1666                pending.reservation.module_key,
1667            );
1668            let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
1669                .then(|| abandoned_route_target(&inner, &pending.reservation))
1670                .flatten();
1671            let _ = pending
1672                .sender
1673                .send(RouteBindRelayOutcome::Rejected(ErrorBody {
1674                    code: "module_timeout".to_string(),
1675                    message: "route.bind response arrived after its daemon deadline".to_string(),
1676                    detail: None,
1677                }));
1678            return Ok(PendingRelayCompletion {
1679                settled: true,
1680                abandoned,
1681            });
1682        }
1683
1684        match outcome {
1685            // Two shapes of "the client is not there to receive this route" that
1686            // must resolve identically: its egress is already closed, or it is
1687            // marked closing (its connection loop has been asked to end, but has
1688            // not drained yet, so the sink is still open).
1689            //
1690            // Only the first used to be caught here. The second fell through to
1691            // `commit_route_locked`, which refuses a closing client with
1692            // `ConnectionClosing` -- and this function is called from the MODULE
1693            // connection's frame handler, so that refusal ended the module's
1694            // connection instead of this one client's route. One dying client's
1695            // route.open then took down a connection carrying every other
1696            // client's routes to that module.
1697            //
1698            // The remedy for both is the same, which is why they share an arm:
1699            // give back the reserved handle pair, tell the waiting route.open the
1700            // route is gone, and report the module-side channel so the caller can
1701            // send a channel-scoped GOODBYE for the binding the module just
1702            // created. Nothing here touches the module connection.
1703            RouteBindRelayOutcome::Accepted
1704                if pending.client_sink.is_closed()
1705                    || inner
1706                        .closing_connections
1707                        .contains(&pending.reservation.client_key.connection_id) =>
1708            {
1709                let reason = if pending.client_sink.is_closed() {
1710                    "client egress closed before route publication"
1711                } else {
1712                    "client connection is closing before route publication"
1713                };
1714                release_reserved_route_locked(
1715                    &mut inner,
1716                    pending.reservation.client_key,
1717                    pending.reservation.module_key,
1718                );
1719                // Not gated on `relay_enqueued`: an ack proves the module
1720                // received the bind, and `route.open` sends nothing for a
1721                // relay a reply already settled, so this is the only GOODBYE.
1722                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1723                let _ = pending
1724                    .sender
1725                    .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
1726                return Ok(PendingRelayCompletion {
1727                    settled: true,
1728                    abandoned,
1729                });
1730            }
1731            // The endpoint that acked this bind is no longer accepting new
1732            // routes, for one of two reasons. Either it was the module's active
1733            // endpoint when this relay was reserved and a blue/green swap has
1734            // since promoted a candidate (the replacement process registered
1735            // beside it) into the active slot, superseding it; or a daemon
1736            // drain has gated it, and the ack arrived before the ordered module
1737            // drain settled pending relays. The module did nothing wrong: it
1738            // bound the route it was asked to bind, and the endpoint still
1739            // carries every other client's routes until it is drained. So the
1740            // ack is settled here, while the pending entry still holds the
1741            // client's sender and the reserved handle pair: release the pair,
1742            // refuse the waiting route.open with `module_reloading`, which
1743            // callers retry (after a swap the retry reserves on the promoted
1744            // endpoint), and hand back the module-side channel so the caller
1745            // sends one channel-scoped GOODBYE for the binding the module just
1746            // created. Returning an error instead would end the module
1747            // connection all those other routes share; nothing here touches it.
1748            //
1749            // An endpoint that stopped being active any other way is stale
1750            // (replaced, but not by a swap and not draining). It still falls
1751            // through to `commit_route_locked`, which refuses it with
1752            // `StaleModuleEndpoint` exactly as before swaps existed.
1753            RouteBindRelayOutcome::Accepted
1754                if inner.superseded_endpoints.contains_key(&endpoint)
1755                    || inner.draining_endpoints.contains_key(&endpoint) =>
1756            {
1757                release_reserved_route_locked(
1758                    &mut inner,
1759                    pending.reservation.client_key,
1760                    pending.reservation.module_key,
1761                );
1762                // Not gated on `relay_enqueued`: an ack proves the module
1763                // received the bind, whether or not the enqueue mark was set.
1764                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1765                let module_id = inner
1766                    .module_id_by_endpoint
1767                    .get(&endpoint)
1768                    .cloned()
1769                    .unwrap_or_else(|| "unknown".to_string());
1770                let _ = pending
1771                    .sender
1772                    .send(RouteBindRelayOutcome::Rejected(ErrorBody::new(
1773                        "module_reloading",
1774                        format!("module_id '{module_id}' is reloading"),
1775                    )));
1776                return Ok(PendingRelayCompletion {
1777                    settled: true,
1778                    abandoned,
1779                });
1780            }
1781            // The scope the open was admitted under ended or changed after
1782            // admission. The module bound a route under a stamp that is no
1783            // longer true, so it must not become routable. Settled here like
1784            // the superseded arm above, never as an error from
1785            // `commit_route_locked`, which would close the module's whole
1786            // connection and every other client's routes to it: release the
1787            // pair, answer the waiting route.open by name, and hand back the
1788            // module-side channel for one channel-scoped GOODBYE.
1789            RouteBindRelayOutcome::Accepted
1790                if pending
1791                    .scope
1792                    .as_ref()
1793                    .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
1794            {
1795                let refusal = pending
1796                    .scope
1797                    .as_ref()
1798                    .and_then(|scope| scope_refusal_locked(&inner, scope))
1799                    .expect("guard matched a refusal under the same lock");
1800                release_reserved_route_locked(
1801                    &mut inner,
1802                    pending.reservation.client_key,
1803                    pending.reservation.module_key,
1804                );
1805                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1806                let _ = pending
1807                    .sender
1808                    .send(RouteBindRelayOutcome::Rejected(refusal));
1809                return Ok(PendingRelayCompletion {
1810                    settled: true,
1811                    abandoned,
1812                });
1813            }
1814            RouteBindRelayOutcome::Accepted => {
1815                let abandoned = commit_route_locked(&mut inner, pending)?;
1816                return Ok(PendingRelayCompletion {
1817                    settled: true,
1818                    abandoned,
1819                });
1820            }
1821            terminal => {
1822                release_reserved_route_locked(
1823                    &mut inner,
1824                    pending.reservation.client_key,
1825                    pending.reservation.module_key,
1826                );
1827                let _ = pending.sender.send(terminal);
1828            }
1829        }
1830        Ok(PendingRelayCompletion {
1831            settled: true,
1832            abandoned: None,
1833        })
1834    }
1835
1836    pub(crate) fn pending_module_control_op(
1837        &self,
1838        connection_id: ConnectionId,
1839        corr: u64,
1840    ) -> Result<Option<String>, ForwardingError> {
1841        let inner = self.read_inner()?;
1842        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1843            return Ok(None);
1844        };
1845        let key = (endpoint, corr);
1846        Ok(inner
1847            .pending_control_rpcs
1848            .get(&key)
1849            .map(|pending| pending.expected_op.clone())
1850            .or_else(|| {
1851                inner
1852                    .health_probe_tombstones
1853                    .get(&key)
1854                    .filter(|tombstone| tombstone.expires_at > Instant::now())
1855                    .map(|tombstone| tombstone.expected_op.clone())
1856            }))
1857    }
1858
1859    pub(crate) fn complete_module_control_rpc(
1860        &self,
1861        connection_id: ConnectionId,
1862        corr: u64,
1863        actual_op: Option<&str>,
1864        outcome: ModuleControlRpcOutcome,
1865    ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
1866        let now = Instant::now();
1867        let mut inner = self.write_inner()?;
1868        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1869            return Ok(ModuleControlRpcCompletion::Unknown);
1870        };
1871        let key = (endpoint, corr);
1872        if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
1873            if now >= pending.deadline {
1874                let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
1875                    ModuleControlRpcCompletion::LateHealthAnswer {
1876                        module_id: inner
1877                            .module_id_by_endpoint
1878                            .get(&endpoint)
1879                            .cloned()
1880                            .unwrap_or_else(|| "unknown".to_string()),
1881                        latency: now.saturating_duration_since(probe_started_at),
1882                    }
1883                });
1884                let _ = pending
1885                    .sender
1886                    .send(ModuleControlRpcOutcome::DeadlineElapsed);
1887                return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
1888            }
1889            let outcome = match actual_op {
1890                Some(actual) if actual != pending.expected_op => {
1891                    ModuleControlRpcOutcome::UnexpectedOp {
1892                        expected: pending.expected_op,
1893                        actual: actual.to_string(),
1894                    }
1895                }
1896                _ => outcome,
1897            };
1898            let _ = pending.sender.send(outcome);
1899            return Ok(ModuleControlRpcCompletion::Settled);
1900        }
1901
1902        let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
1903            return Ok(ModuleControlRpcCompletion::Unknown);
1904        };
1905        if tombstone.expires_at <= now {
1906            return Ok(ModuleControlRpcCompletion::Unknown);
1907        }
1908        Ok(ModuleControlRpcCompletion::LateHealthAnswer {
1909            module_id: tombstone.module_id,
1910            latency: now.saturating_duration_since(tombstone.probe_started_at),
1911        })
1912    }
1913
1914    #[cfg(test)]
1915    pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
1916        Ok(self.read_inner()?.health_probe_tombstones.len())
1917    }
1918
1919    #[cfg(test)]
1920    pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
1921        Ok(self.read_inner()?.closing_connections.len())
1922    }
1923
1924    /// Reserved-but-uncommitted route handles, counted on both index sides, so
1925    /// a test can assert a reservation pair was actually given back.
1926    #[cfg(test)]
1927    pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
1928        let inner = self.read_inner()?;
1929        Ok((inner.reserved_client.len(), inner.reserved_module.len()))
1930    }
1931
1932    pub(crate) fn module_endpoint_for_connection(
1933        &self,
1934        connection_id: ConnectionId,
1935    ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
1936        Ok(self
1937            .read_inner()?
1938            .endpoint_by_connection
1939            .get(&connection_id)
1940            .copied())
1941    }
1942
1943    /// Looks up the module registered on a data-plane connection so route-drop
1944    /// diagnostics name the emitter instead of only reporting a daemon total.
1945    pub(crate) fn module_id_for_connection(
1946        &self,
1947        connection_id: ConnectionId,
1948    ) -> Result<Option<String>, ForwardingError> {
1949        let inner = self.read_inner()?;
1950        Ok(inner
1951            .endpoint_by_connection
1952            .get(&connection_id)
1953            .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
1954            .cloned())
1955    }
1956
1957    /// Whether the daemon ever allocated `(channel, epoch)` on this module
1958    /// connection. Epochs on a module channel are handed out as 1, 2, 3, ...
1959    /// and the last one handed out is remembered until the connection ends, so
1960    /// every epoch from 1 up to that one was a real route at some point. Used
1961    /// only on the drop path, to tell a module still sending on a route the
1962    /// daemon released from one sending on a route that never existed.
1963    pub(crate) fn module_route_epoch_was_allocated(
1964        &self,
1965        connection_id: ConnectionId,
1966        channel: u16,
1967        epoch: u32,
1968    ) -> Result<bool, ForwardingError> {
1969        let inner = self.read_inner()?;
1970        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1971            return Ok(false);
1972        };
1973        Ok(inner
1974            .module_slot_epochs
1975            .get(&ModuleRouteKey { endpoint, channel })
1976            .is_some_and(|last| epoch != 0 && epoch <= *last))
1977    }
1978
1979    pub(crate) fn has_live_module_connection(
1980        &self,
1981        module_id: &str,
1982    ) -> Result<bool, ForwardingError> {
1983        Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
1984    }
1985
1986    pub(crate) fn lookup_data_route(
1987        &self,
1988        connection_id: ConnectionId,
1989        channel: u16,
1990        epoch: u32,
1991    ) -> Result<DataRoute, ForwardingError> {
1992        let inner = self.read_inner()?;
1993        let state = if let Some(endpoint) =
1994            inner.endpoint_by_connection.get(&connection_id).copied()
1995        {
1996            let key = ModuleRouteKey { endpoint, channel };
1997            match inner.module_to_client.get(&key) {
1998                Some(route) if route.module_epoch == epoch => {
1999                    DataRouteState::Bound(Arc::clone(route))
2000                }
2001                Some(_) => DataRouteState::EpochMismatch,
2002                None if inner.reserved_module.contains_key(&key)
2003                    && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
2004                {
2005                    DataRouteState::Reserved
2006                }
2007                None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
2008                None => DataRouteState::Absent,
2009            }
2010        } else {
2011            let key = ClientRouteKey {
2012                connection_id,
2013                channel,
2014            };
2015            match inner.client_to_module.get(&key) {
2016                Some(route) if route.client_epoch == epoch => {
2017                    DataRouteState::Bound(Arc::clone(route))
2018                }
2019                Some(_) => DataRouteState::EpochMismatch,
2020                None if inner.reserved_client.contains_key(&key)
2021                    && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
2022                {
2023                    DataRouteState::Reserved
2024                }
2025                None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
2026                None => DataRouteState::Absent,
2027            }
2028        };
2029        Ok(
2030            if inner.endpoint_by_connection.contains_key(&connection_id) {
2031                DataRoute::Module(state)
2032            } else {
2033                DataRoute::Client(state)
2034            },
2035        )
2036    }
2037
2038    #[cfg(test)]
2039    pub(crate) fn inject_client_slot_epoch(
2040        &self,
2041        connection_id: ConnectionId,
2042        channel: u16,
2043        last_epoch: u32,
2044    ) {
2045        let mut inner = self.write_inner().expect("forwarding lock");
2046        inner.client_slot_epochs.insert(
2047            ClientRouteKey {
2048                connection_id,
2049                channel,
2050            },
2051            last_epoch,
2052        );
2053        inner.next_client_channel.insert(connection_id, channel);
2054    }
2055
2056    #[cfg(test)]
2057    pub(crate) fn inject_module_slot_epoch(
2058        &self,
2059        endpoint: ModuleEndpointId,
2060        channel: u16,
2061        last_epoch: u32,
2062    ) {
2063        let mut inner = self.write_inner().expect("forwarding lock");
2064        inner
2065            .module_slot_epochs
2066            .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
2067        inner.next_module_channel.insert(endpoint, channel);
2068    }
2069
2070    #[cfg(test)]
2071    pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
2072        self.write_inner()
2073            .expect("forwarding lock")
2074            .next_control_corr
2075            .insert(endpoint, next_corr);
2076    }
2077
2078    pub(crate) fn cache_status(
2079        &self,
2080        endpoint: ModuleEndpointId,
2081        module_channel: u16,
2082        module_epoch: u32,
2083        status: String,
2084    ) -> Result<bool, ForwardingError> {
2085        let mut inner = self.write_inner()?;
2086        if !inner.module_id_by_endpoint.contains_key(&endpoint) {
2087            return Err(ForwardingError::StaleModuleEndpoint);
2088        }
2089
2090        let module_key = ModuleRouteKey {
2091            endpoint,
2092            channel: module_channel,
2093        };
2094        let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
2095            (route.module_epoch == module_epoch).then_some((
2096                ClientRouteKey {
2097                    connection_id: route.client_connection_id,
2098                    channel: route.client_channel,
2099                },
2100                route.client_epoch,
2101            ))
2102        } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
2103            (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
2104                client_key,
2105                inner
2106                    .client_slot_epochs
2107                    .get(&client_key)
2108                    .copied()
2109                    .unwrap_or(0),
2110            ))
2111        } else {
2112            None
2113        };
2114
2115        if let Some(handle) = handle {
2116            inner.status.insert(handle, status);
2117            Ok(true)
2118        } else {
2119            debug!(
2120                module_channel,
2121                module_epoch,
2122                generation = endpoint.generation,
2123                connection_id = endpoint.connection_id.get(),
2124                "dropping stale status update for module route handle"
2125            );
2126            Ok(false)
2127        }
2128    }
2129
2130    pub(crate) fn route_poll_snapshot(
2131        &self,
2132        client_connection_id: ConnectionId,
2133        client_channel: u16,
2134        client_epoch: u32,
2135    ) -> Result<RoutePollSnapshot, ForwardingError> {
2136        let inner = self.read_inner()?;
2137        let client_key = ClientRouteKey {
2138            connection_id: client_connection_id,
2139            channel: client_channel,
2140        };
2141        let Some(route) = inner.client_to_module.get(&client_key) else {
2142            return Ok(RoutePollSnapshot::Absent);
2143        };
2144        if route.client_epoch != client_epoch
2145            || !inner
2146                .module_id_by_endpoint
2147                .contains_key(&route.module_endpoint)
2148        {
2149            return Ok(RoutePollSnapshot::Absent);
2150        }
2151        Ok(RoutePollSnapshot::Bound {
2152            module_id: route.module_id.clone(),
2153            status: inner.status.get(&(client_key, client_epoch)).cloned(),
2154        })
2155    }
2156
2157    pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2158        Ok(self.read_inner()?.client_to_module.len())
2159    }
2160
2161    /// How many distinct client connections hold at least one committed route,
2162    /// alongside the largest number of routes any single connection holds.
2163    ///
2164    /// `connected_clients` alone cannot distinguish many clients with a route
2165    /// each from one client accumulating hundreds, and those have opposite
2166    /// causes. Reading it required an out-of-band `lsof` during a live
2167    /// investigation, and the count of connections was mistaken for a count of
2168    /// client processes — which sent two of us after cleanup paths that were
2169    /// working correctly.
2170    pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2171        let inner = self.read_inner()?;
2172        let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2173        for key in inner.client_to_module.keys() {
2174            *per_connection.entry(key.connection_id).or_insert(0) += 1;
2175        }
2176        let max = per_connection.values().copied().max().unwrap_or(0);
2177        Ok((per_connection.len(), max))
2178    }
2179
2180    pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2181        let inner = self.read_inner()?;
2182        Ok(inner
2183            .client_to_module
2184            .keys()
2185            .any(|key| key.channel == route_channel))
2186    }
2187
2188    /// Whether daemon-wide shutdown has begun draining all providers.
2189    /// `None` means normal operation; during daemon shutdown, return the
2190    /// endpoint's measured result, or false if no completed measurement exists.
2191    pub(crate) fn daemon_drain_result_for_connection(
2192        &self,
2193        connection_id: ConnectionId,
2194    ) -> Result<Option<bool>, ForwardingError> {
2195        let inner = self.read_inner()?;
2196        Ok(inner.daemon_draining.then(|| {
2197            inner
2198                .endpoint_by_connection
2199                .get(&connection_id)
2200                .and_then(|endpoint| inner.daemon_drain_results.get(endpoint))
2201                .copied()
2202                .unwrap_or(false)
2203        }))
2204    }
2205
2206    #[cfg(unix)]
2207    pub(crate) fn record_daemon_drain_result(
2208        &self,
2209        endpoint: ModuleEndpointId,
2210        drained: bool,
2211    ) -> Result<(), ForwardingError> {
2212        let mut inner = self.write_inner()?;
2213        // Only update an endpoint captured at the start of daemon shutdown.
2214        // Teardown removes its entry, so a late wait cannot recreate the result
2215        // or attribute it to a replacement connection.
2216        if let Some(result) = inner.daemon_drain_results.get_mut(&endpoint) {
2217            *result = drained;
2218        }
2219        Ok(())
2220    }
2221
2222    /// Gate every provider atomically, including registrations racing shutdown.
2223    /// No supervisor lock is held while taking the forwarding lock.
2224    #[cfg(unix)]
2225    pub(crate) fn begin_daemon_drain(
2226        &self,
2227    ) -> Result<Vec<(String, ModuleEndpointId)>, ForwardingError> {
2228        let mut inner = self.write_inner()?;
2229        if inner.daemon_draining {
2230            return Ok(Vec::new());
2231        }
2232        inner.daemon_draining = true;
2233        let modules = inner
2234            .modules_by_id
2235            .iter()
2236            .map(|(id, module)| (id.clone(), module.endpoint))
2237            .collect::<Vec<_>>();
2238        for (_, endpoint) in &modules {
2239            inner.daemon_drain_results.insert(*endpoint, false);
2240            inner
2241                .draining_endpoints
2242                .insert(*endpoint, RouteCloseReason::Restart);
2243        }
2244        // Swap candidates and superseded incumbents are not routable, but they
2245        // are live endpoints that could still be sent module-control work, so
2246        // the daemon-wide gate covers them too. Only the active endpoints are
2247        // returned to the supervisor for measurement, one per module id.
2248        let off_slot_endpoints = inner
2249            .candidates_by_id
2250            .values()
2251            .map(|module| module.endpoint)
2252            .chain(inner.superseded_endpoints.keys().copied())
2253            .collect::<Vec<_>>();
2254        for endpoint in off_slot_endpoints {
2255            inner
2256                .draining_endpoints
2257                .insert(endpoint, RouteCloseReason::Restart);
2258        }
2259        Ok(modules)
2260    }
2261
2262    /// Begin draining whatever endpoint is ACTIVE for `module_id`.
2263    ///
2264    /// After a swap's cutover the active endpoint is the promoted candidate, so
2265    /// this must not be used to drain the incumbent; use
2266    /// [`Self::begin_endpoint_drain`] with the endpoint `cutover_candidate`
2267    /// returned.
2268    pub(crate) fn begin_module_drain(
2269        &self,
2270        module_id: &str,
2271        reason: RouteCloseReason,
2272    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2273        let mut inner = self.write_inner()?;
2274        let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2275            return Ok(None);
2276        };
2277        Ok(Some(begin_drain_locked(
2278            &mut inner, module_id, module, reason,
2279        )))
2280    }
2281
2282    /// Begin draining one specific endpoint, whichever slot it is in.
2283    ///
2284    /// This is how a swap drains its incumbent after cutover: the incumbent's
2285    /// endpoint is captured by `cutover_candidate`, and resolving it by module id
2286    /// instead would find the promoted candidate and leave neither process
2287    /// routable. Returns `Ok(None)` when the endpoint is no longer registered.
2288    pub(crate) fn begin_endpoint_drain(
2289        &self,
2290        endpoint: ModuleEndpointId,
2291        reason: RouteCloseReason,
2292    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2293        let mut inner = self.write_inner()?;
2294        let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2295            return Ok(None);
2296        };
2297        let module_id = inner
2298            .module_id_by_endpoint
2299            .get(&endpoint)
2300            .cloned()
2301            .expect("an endpoint resolved to a module connection has a module id");
2302        Ok(Some(begin_drain_locked(
2303            &mut inner, &module_id, module, reason,
2304        )))
2305    }
2306}
2307
2308/// Mark `module`'s endpoint draining and settle everything still pending on it.
2309/// Shared by the by-id and by-endpoint drain entry points, which differ only in
2310/// how they find the endpoint.
2311fn begin_drain_locked(
2312    inner: &mut ForwardingInner,
2313    module_id: &str,
2314    module: ModuleConnection,
2315    reason: RouteCloseReason,
2316) -> ModuleDrainTarget {
2317    {
2318        let endpoint = module.endpoint;
2319        inner.draining_endpoints.insert(endpoint, reason);
2320
2321        let flows = inner
2322            .client_to_module
2323            .values()
2324            .filter(|route| route.module_endpoint == endpoint)
2325            .map(|route| Arc::clone(&route.flow))
2326            .collect::<Vec<_>>();
2327        let excluded_subscriptions = flows
2328            .into_iter()
2329            .map(|flow| flow.begin_drain())
2330            .fold(0u32, u32::saturating_add);
2331
2332        let pending_keys = inner
2333            .pending_relays
2334            .keys()
2335            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2336            .copied()
2337            .collect::<Vec<_>>();
2338        let mut abandoned_bindings = Vec::new();
2339        for key in pending_keys {
2340            let Some(pending) = inner.pending_relays.remove(&key) else {
2341                continue;
2342            };
2343            release_reserved_route_locked(
2344                inner,
2345                pending.reservation.client_key,
2346                pending.reservation.module_key,
2347            );
2348            if pending.relay_enqueued {
2349                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2350                    abandoned_bindings.push(target);
2351                }
2352            } else {
2353                // The bind may still be on its way into the module's queue;
2354                // a GOODBYE sent now could overtake it. `route.open` sends
2355                // it once its send completes.
2356                inner.relays_released_before_enqueue.insert(key);
2357            }
2358            let _ = pending
2359                .sender
2360                .send(RouteBindRelayOutcome::Rejected(ErrorBody::new(
2361                    "module_reloading",
2362                    format!("module_id '{module_id}' is reloading"),
2363                )));
2364        }
2365
2366        let pending_control_keys = inner
2367            .pending_control_rpcs
2368            .keys()
2369            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2370            .copied()
2371            .collect::<Vec<_>>();
2372        for key in pending_control_keys {
2373            if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2374                let _ = pending
2375                    .sender
2376                    .send(ModuleControlRpcOutcome::ModuleGone(format!(
2377                        "module '{module_id}' began draining during module-control RPC"
2378                    )));
2379            }
2380        }
2381
2382        ModuleDrainTarget {
2383            endpoint,
2384            sink: module.sink,
2385            negotiated_ver: module.negotiated_ver,
2386            abandoned_bindings,
2387            excluded_subscriptions,
2388        }
2389    }
2390}
2391
2392/// What a drain that timed out was still waiting on, for the log line that
2393/// reports the timeout. Per-request ages are not tracked.
2394#[derive(Debug, Default, PartialEq, Eq)]
2395pub(crate) struct DrainHoldouts {
2396    /// Requests still counted against the drain (subscriptions flagged as
2397    /// such are excluded from the drain and not counted here).
2398    pub(crate) requests: usize,
2399    /// Routes holding at least one of those requests.
2400    pub(crate) routes: usize,
2401    /// Every route on the endpoint, for scale.
2402    pub(crate) total_routes: usize,
2403    /// The client connections holding the most requests, largest first, at
2404    /// most three: enough to name the consumer without listing every route.
2405    pub(crate) top_connections: Vec<(u64, usize)>,
2406    /// The held requests themselves, as the module saw them, so its own log
2407    /// can say what each one was: `(module channel, corr)`, ordered by channel
2408    /// then corr, at most [`DRAIN_HELD_REQUESTS_LISTED`]. The daemon never reads
2409    /// request bodies, so it cannot name a request's method; the module can,
2410    /// from the channel and corr. A request is released only when the module
2411    /// sends a terminal frame (Response, Error or StreamEnd) with its corr on
2412    /// its route, so every pair listed here is one the module ended, if at all,
2413    /// without sending that frame.
2414    pub(crate) held: Vec<(u16, u64)>,
2415}
2416
2417/// How many held requests the drain-timeout line lists by channel and corr.
2418pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
2419
2420impl ForwardingTable {
2421    /// Summarise the requests one endpoint's drain is still waiting on.
2422    pub(crate) fn endpoint_drain_holdouts(
2423        &self,
2424        endpoint: ModuleEndpointId,
2425    ) -> Result<DrainHoldouts, ForwardingError> {
2426        let inner = self.read_inner()?;
2427        let mut holdouts = DrainHoldouts::default();
2428        let mut by_connection: HashMap<u64, usize> = HashMap::new();
2429        for (key, route) in &inner.client_to_module {
2430            if route.module_endpoint != endpoint {
2431                continue;
2432            }
2433            holdouts.total_routes += 1;
2434            let held = route.flow.drain_in_flight();
2435            if held == 0 {
2436                continue;
2437            }
2438            holdouts.requests += held;
2439            holdouts.routes += 1;
2440            *by_connection.entry(key.connection_id.get()).or_default() += held;
2441            holdouts.held.extend(
2442                route
2443                    .flow
2444                    .drain_held_corrs()
2445                    .into_iter()
2446                    .map(|corr| (route.module_channel, corr)),
2447            );
2448        }
2449        holdouts.held.sort_unstable();
2450        holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
2451        let mut connections = by_connection.into_iter().collect::<Vec<_>>();
2452        connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
2453        connections.truncate(3);
2454        holdouts.top_connections = connections;
2455        Ok(holdouts)
2456    }
2457
2458    pub(crate) fn endpoint_in_flight_count(
2459        &self,
2460        endpoint: ModuleEndpointId,
2461    ) -> Result<usize, ForwardingError> {
2462        let inner = self.read_inner()?;
2463        Ok(inner
2464            .client_to_module
2465            .values()
2466            .filter(|route| route.module_endpoint == endpoint)
2467            .map(|route| route.flow.drain_in_flight())
2468            .sum())
2469    }
2470
2471    pub(crate) fn endpoint_is_draining(
2472        &self,
2473        endpoint: ModuleEndpointId,
2474    ) -> Result<bool, ForwardingError> {
2475        Ok(self
2476            .read_inner()?
2477            .draining_endpoints
2478            .contains_key(&endpoint))
2479    }
2480
2481    pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
2482        let inner = self.read_inner()?;
2483        Ok(inner
2484            .modules_by_id
2485            .get(module_id)
2486            .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
2487    }
2488
2489    pub(crate) fn release_module_endpoint_routes(
2490        &self,
2491        endpoint: ModuleEndpointId,
2492    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2493        let mut inner = self.write_inner()?;
2494        let routes = inner
2495            .module_to_client
2496            .iter()
2497            .filter(|(module_key, _)| module_key.endpoint == endpoint)
2498            .map(|(module_key, route)| (*module_key, route.module_epoch))
2499            .collect::<Vec<_>>();
2500        let mut released = Vec::with_capacity(routes.len());
2501        for (module_key, epoch) in routes {
2502            if let RouteRelease::Removed(target) =
2503                release_module_route_locked(&mut inner, module_key, epoch)
2504            {
2505                released.push(target);
2506            }
2507        }
2508        Ok(released)
2509    }
2510
2511    /// Enumerate one endpoint's current routes without contacting the module.
2512    ///
2513    /// The read lock makes this safe while the endpoint drains: the returned
2514    /// `draining` marker describes the same table state that owns the route,
2515    /// rather than inferring liveness from a module that may be stopping.
2516    pub(crate) fn endpoint_routes(
2517        &self,
2518        endpoint: ModuleEndpointId,
2519    ) -> Result<Vec<EndpointRoute>, ForwardingError> {
2520        let inner = self.read_inner()?;
2521        Ok(endpoint_routes_locked(&inner, endpoint))
2522    }
2523
2524    /// Snapshot all live endpoint route sets under one forwarding-table read lock.
2525    pub(crate) fn route_census(
2526        &self,
2527        module_id: Option<&str>,
2528    ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
2529        let inner = self.read_inner()?;
2530        let mut endpoints = inner
2531            .modules_by_id
2532            .iter()
2533            .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
2534            .map(|(id, module)| (id.clone(), module.endpoint))
2535            .collect::<Vec<_>>();
2536        endpoints.sort_by(|left, right| left.0.cmp(&right.0));
2537        Ok(endpoints
2538            .into_iter()
2539            .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
2540            .collect())
2541    }
2542
2543    /// Snapshot only the endpoint currently routable under this module id.
2544    pub(crate) fn live_roots(
2545        &self,
2546        module_id: &str,
2547    ) -> Result<ModuleControlResponseToModule, ForwardingError> {
2548        let inner = self.read_inner()?;
2549        let endpoint = inner
2550            .modules_by_id
2551            .get(module_id)
2552            .map(|module| module.endpoint);
2553        let mut roots = BTreeMap::new();
2554        let mut unknown_root_bindings = 0;
2555        let mut total_bindings = 0;
2556        if let Some(endpoint) = endpoint {
2557            for binding in inner
2558                .module_to_client
2559                .values()
2560                .filter(|binding| binding.module_endpoint == endpoint)
2561            {
2562                total_bindings += 1;
2563                if let Some(root) = &binding.project_root {
2564                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2565                    entry.0 += 1;
2566                } else {
2567                    unknown_root_bindings += 1;
2568                }
2569            }
2570            for pending in inner
2571                .pending_relays
2572                .values()
2573                .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
2574            {
2575                total_bindings += 1;
2576                if let Some(root) = &pending.reservation.project_root {
2577                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2578                    entry.1 += 1;
2579                } else {
2580                    unknown_root_bindings += 1;
2581                }
2582            }
2583        }
2584        Ok(ModuleControlResponseToModule::LiveRoots {
2585            roots: roots
2586                .into_iter()
2587                .map(|(project_root, (bound, pending))| LiveRoot {
2588                    project_root,
2589                    bound,
2590                    pending,
2591                })
2592                .collect(),
2593            unknown_root_bindings,
2594            total_bindings,
2595        })
2596    }
2597
2598    /// True if this connection already owns committed or reserved CLIENT routes.
2599    /// A module registers (HELLO) before serving and never opens client routes, so
2600    /// a connection that has client routes must not also become a module endpoint
2601    /// — otherwise one connection holds both client and module state and cleanup
2602    /// only releases one side.
2603    pub(crate) fn connection_has_client_routes(
2604        &self,
2605        connection_id: ConnectionId,
2606    ) -> Result<bool, ForwardingError> {
2607        let inner = self.read_inner()?;
2608        Ok(connection_has_client_routes_locked(&inner, connection_id))
2609    }
2610
2611    pub(crate) fn cleanup_connection(
2612        &self,
2613        connection_id: ConnectionId,
2614    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2615        self.cleanup_connection_counted(connection_id)
2616            .map(|cleanup| cleanup.released)
2617    }
2618
2619    /// [`Self::cleanup_connection`], also reporting how many pending
2620    /// route.bind relays to the closed module were aborted. That count is the
2621    /// `abandoned` figure of the `route.closed` push sent for a lost module
2622    /// connection; it is always zero for a client connection.
2623    pub(crate) fn cleanup_connection_counted(
2624        &self,
2625        connection_id: ConnectionId,
2626    ) -> Result<ConnectionCleanup, ForwardingError> {
2627        let mut inner = self.write_inner()?;
2628        inner.closing_connections.insert(connection_id);
2629        let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
2630            remove_module_connection_locked(&mut inner, endpoint)
2631        } else {
2632            ConnectionCleanup {
2633                released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
2634                abandoned_relays: 0,
2635            }
2636        };
2637        // The closing mark refuses new work for a connection whose teardown is
2638        // still pending. This is the latest point at which lifting it is safe:
2639        // teardown has just removed every per-connection entry above, under
2640        // this same write lock, so no lookup can still find live state for the
2641        // id; and ids come from a monotonic counter that never reissues one,
2642        // so the id can never name a future connection either. Keeping the
2643        // mark past this point only grew the set by one entry per connection
2644        // for the life of the daemon.
2645        inner.closing_connections.remove(&connection_id);
2646        Ok(cleanup)
2647    }
2648
2649    fn cleanup_client_connection_locked(
2650        inner: &mut ForwardingInner,
2651        connection_id: ConnectionId,
2652    ) -> Vec<GoodbyeTarget> {
2653        let routes = inner
2654            .client_to_module
2655            .iter()
2656            .filter(|(key, _)| key.connection_id == connection_id)
2657            .map(|(key, route)| (*key, route.client_epoch))
2658            .collect::<Vec<_>>();
2659        let mut released = Vec::with_capacity(routes.len());
2660        for (client_key, epoch) in routes {
2661            if let RouteRelease::Removed(target) =
2662                release_client_route_locked(inner, client_key, epoch)
2663            {
2664                released.push(target);
2665            }
2666        }
2667
2668        let pending_keys = inner
2669            .pending_relays
2670            .iter()
2671            .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
2672            .map(|(key, _)| *key)
2673            .collect::<Vec<_>>();
2674        for key in pending_keys {
2675            let Some(pending) = inner.pending_relays.remove(&key) else {
2676                continue;
2677            };
2678            release_reserved_route_locked(
2679                inner,
2680                pending.reservation.client_key,
2681                pending.reservation.module_key,
2682            );
2683            if pending.relay_enqueued {
2684                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2685                    released.push(target);
2686                }
2687            } else {
2688                // The bind may still be on its way into the module's queue;
2689                // a GOODBYE sent now could overtake it. `route.open` sends
2690                // it once its send completes.
2691                inner.relays_released_before_enqueue.insert(key);
2692            }
2693            let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
2694                "client connection closed during route.bind relay".to_string(),
2695            ));
2696        }
2697
2698        let orphaned = inner
2699            .reserved_client
2700            .iter()
2701            .filter(|(key, _)| key.connection_id == connection_id)
2702            .map(|(client, module)| (*client, *module))
2703            .collect::<Vec<_>>();
2704        for (client_key, module_key) in orphaned {
2705            release_reserved_route_locked(inner, client_key, module_key);
2706        }
2707        inner.next_client_channel.remove(&connection_id);
2708        inner
2709            .client_slot_epochs
2710            .retain(|key, _| key.connection_id != connection_id);
2711        inner
2712            .last_published_epoch
2713            .retain(|key, _| key.connection_id != connection_id);
2714        inner
2715            .status
2716            .retain(|(key, _), _| key.connection_id != connection_id);
2717
2718        released
2719    }
2720
2721    /// Close a client connection whose egress queue refused a frame for the
2722    /// route `(connection_id, channel)` at `expected_epoch`. The whole
2723    /// connection closes because its queue is shared by every route on it.
2724    ///
2725    /// The first request that actually closes the connection logs one WARN with
2726    /// the diagnosis: which principals the connection's routes belonged to,
2727    /// which module's frame did not fit and on which client channel, and what
2728    /// the queue held at that moment.
2729    pub(crate) fn escalate_client_delivery_failure(
2730        &self,
2731        connection_id: ConnectionId,
2732        channel: u16,
2733        expected_epoch: u32,
2734        reason: CloseReason,
2735        undelivered: UndeliveredFrame<'_>,
2736    ) -> Result<bool, ForwardingError> {
2737        self.escalate_client_delivery_failure_diagnosed(
2738            connection_id,
2739            channel,
2740            expected_epoch,
2741            reason,
2742            undelivered,
2743            None,
2744        )
2745    }
2746
2747    pub(crate) fn escalate_client_delivery_failure_diagnosed(
2748        &self,
2749        connection_id: ConnectionId,
2750        channel: u16,
2751        expected_epoch: u32,
2752        reason: CloseReason,
2753        undelivered: UndeliveredFrame<'_>,
2754        diagnosis: Option<&crate::router::ClientDeliveryDiagnosis>,
2755    ) -> Result<bool, ForwardingError> {
2756        let principals = {
2757            let mut inner = self.write_inner()?;
2758            let key = ClientRouteKey {
2759                connection_id,
2760                channel,
2761            };
2762            if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
2763                None
2764            } else {
2765                inner.closing_connections.insert(connection_id);
2766                Some(connection_principals_locked(&inner, connection_id))
2767            }
2768        };
2769        let Some(principals) = principals else {
2770            return Ok(false);
2771        };
2772        let backlog = undelivered.sink.backlog();
2773        let close_reason = reason.to_string();
2774        if self.request_connection_close(connection_id, reason) {
2775            warn!(
2776                connection_id = connection_id.get(),
2777                principals = %principals,
2778                module_id = undelivered.module_id.unwrap_or("unknown"),
2779                client_channel = channel,
2780                queued_bytes = backlog.queued_bytes,
2781                queued_frames = backlog.queued_frames,
2782                frame_type = ?diagnosis.map(|failure| failure.frame_type),
2783                frame_flags = ?diagnosis.map(|failure| failure.flags.0),
2784                close_cause = diagnosis.map(|failure| failure.cause).unwrap_or("delivery_failed"),
2785                oldest_queued_ms = backlog
2786                    .oldest_age
2787                    .map(|age| age.as_millis() as u64)
2788                    .unwrap_or(0),
2789                close_reason = %close_reason,
2790                "closing client connection: its egress queue could not take a frame"
2791            );
2792        }
2793        Ok(true)
2794    }
2795
2796    /// Publish the tags a scope sync changed and close the live routes its
2797    /// drain table selects, in one critical section.
2798    ///
2799    /// The caller holds the scope table's write lock (scope table, then this
2800    /// table, always). Publishing and selecting under one lock is what closes
2801    /// the race with a bind commit: a commit before this sees the old tag and
2802    /// its route is selected here like any other; a commit after it sees the
2803    /// new tag and is refused.
2804    ///
2805    /// Every client route index entry is scanned, which includes routes on a
2806    /// swap's superseded endpoint (the old process of a module being replaced
2807    /// blue/green, which keeps its existing routes until they drain): those
2808    /// stay indexed until drained, so ending a scope reaches them too.
2809    pub(crate) fn publish_scope_changes(
2810        &self,
2811        changes: &[ScopeTagChange],
2812    ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
2813        let mut inner = self.write_inner()?;
2814        let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
2815        for change in changes {
2816            let key = (change.owner.clone(), change.scope_ref.clone());
2817            match change.after {
2818                Some(tag) => {
2819                    inner.scope_tags.insert(key, tag);
2820                }
2821                None => {
2822                    inner.scope_tags.remove(&key);
2823                }
2824            }
2825            by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
2826        }
2827        let mut selected = Vec::new();
2828        for (client_key, route) in &inner.client_to_module {
2829            let Some(scope) = &route.scope else {
2830                continue;
2831            };
2832            let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
2833            else {
2834                continue;
2835            };
2836            if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
2837                continue;
2838            }
2839            let reason = match &change.drain {
2840                ScopeDrain::Nothing => continue,
2841                ScopeDrain::All(reason) => *reason,
2842                ScopeDrain::Carriers(narrowed) => {
2843                    let owner = Principal::Reserved {
2844                        module_id: scope.owner.clone(),
2845                    };
2846                    let hit = route.principal != owner
2847                        && narrowed.iter().any(|(principal, allowed)| {
2848                            *principal == route.principal
2849                                && allowed
2850                                    .as_ref()
2851                                    .is_none_or(|targets| !targets.contains(&route.module_id))
2852                        });
2853                    if !hit {
2854                        continue;
2855                    }
2856                    RouteCloseReason::ScopeCarrierRemoved
2857                }
2858            };
2859            selected.push((*client_key, route.client_epoch, reason));
2860        }
2861        let mut drained = Vec::new();
2862        for (client_key, client_epoch, reason) in selected {
2863            let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
2864                continue;
2865            };
2866            let release = release_client_route_locked(&mut inner, client_key, client_epoch);
2867            self.record_route_release(&release);
2868            if let RouteRelease::Removed(module) = release {
2869                drained.push(ScopeDrainedRoute {
2870                    reason,
2871                    scope: route
2872                        .scope
2873                        .clone()
2874                        .expect("selected scope drain has a bound scope"),
2875                    module_id: route.module_id.clone(),
2876                    client: GoodbyeTarget {
2877                        connection_id: route.client_connection_id,
2878                        sink: route.client_sink.clone(),
2879                        negotiated_ver: route.client_negotiated_ver,
2880                        channel: route.client_channel,
2881                        epoch: route.client_epoch,
2882                        kind: GoodbyeTargetKind::Client,
2883                        module_id: Some(route.module_id.clone()),
2884                    },
2885                    module,
2886                });
2887            }
2888        }
2889        Ok(drained)
2890    }
2891
2892    /// The published tag of one scope, for tests of the commit re-check.
2893    #[cfg(test)]
2894    pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
2895        self.read_inner()
2896            .ok()?
2897            .scope_tags
2898            .get(&(owner.to_string(), scope_ref.to_string()))
2899            .copied()
2900    }
2901
2902    fn record_route_release(&self, release: &RouteRelease) {
2903        match release {
2904            RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
2905            RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
2906            RouteRelease::Absent => {}
2907        }
2908    }
2909
2910    fn read_inner(&self) -> Result<RwLockReadGuard<'_, ForwardingInner>, ForwardingError> {
2911        self.inner.read().map_err(|_| ForwardingError::Poisoned)
2912    }
2913
2914    fn write_inner(&self) -> Result<RwLockWriteGuard<'_, ForwardingInner>, ForwardingError> {
2915        self.inner.write().map_err(|_| ForwardingError::Poisoned)
2916    }
2917
2918    fn lock_close_registry(
2919        &self,
2920    ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
2921        self.close_registry
2922            .lock()
2923            .unwrap_or_else(|poisoned| poisoned.into_inner())
2924    }
2925}
2926
2927impl ForwardingInner {
2928    fn allocate_route_slots(
2929        &mut self,
2930        connection_id: ConnectionId,
2931        endpoint: ModuleEndpointId,
2932    ) -> Result<(u16, u32, u16, u32), ForwardingError> {
2933        let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
2934        let mut client_channel = client_start;
2935        let client_channel = loop {
2936            let key = ClientRouteKey {
2937                connection_id,
2938                channel: client_channel,
2939            };
2940            let eligible = !self.client_to_module.contains_key(&key)
2941                && !self.reserved_client.contains_key(&key)
2942                && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2943            if eligible {
2944                break client_channel;
2945            }
2946            client_channel = next_channel(client_channel);
2947            if client_channel == client_start {
2948                return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
2949            }
2950        };
2951
2952        let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
2953        let mut module_channel = module_start;
2954        let module_channel = loop {
2955            let key = ModuleRouteKey {
2956                endpoint,
2957                channel: module_channel,
2958            };
2959            let eligible = !self.module_to_client.contains_key(&key)
2960                && !self.reserved_module.contains_key(&key)
2961                && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2962            if eligible {
2963                break module_channel;
2964            }
2965            module_channel = next_channel(module_channel);
2966            if module_channel == module_start {
2967                return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
2968            }
2969        };
2970
2971        let client_key = ClientRouteKey {
2972            connection_id,
2973            channel: client_channel,
2974        };
2975        let module_key = ModuleRouteKey {
2976            endpoint,
2977            channel: module_channel,
2978        };
2979        let client_epoch = self
2980            .client_slot_epochs
2981            .get(&client_key)
2982            .copied()
2983            .unwrap_or(0)
2984            + 1;
2985        let module_epoch = self
2986            .module_slot_epochs
2987            .get(&module_key)
2988            .copied()
2989            .unwrap_or(0)
2990            + 1;
2991        self.client_slot_epochs.insert(client_key, client_epoch);
2992        self.module_slot_epochs.insert(module_key, module_epoch);
2993        self.next_client_channel
2994            .insert(connection_id, next_channel(client_channel));
2995        self.next_module_channel
2996            .insert(endpoint, next_channel(module_channel));
2997        Ok((client_channel, client_epoch, module_channel, module_epoch))
2998    }
2999
3000    fn allocate_control_corr(
3001        &mut self,
3002        endpoint: ModuleEndpointId,
3003    ) -> Result<u64, ForwardingError> {
3004        let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
3005        if candidate == 0 {
3006            self.closing_connections.insert(endpoint.connection_id);
3007            return Err(ForwardingError::RelayCorrelationExhausted);
3008        }
3009        self.next_control_corr.insert(
3010            endpoint,
3011            if candidate == u64::MAX {
3012                0
3013            } else {
3014                candidate + 1
3015            },
3016        );
3017        Ok(candidate)
3018    }
3019}
3020
3021fn connection_has_client_routes_locked(
3022    inner: &ForwardingInner,
3023    connection_id: ConnectionId,
3024) -> bool {
3025    inner
3026        .client_to_module
3027        .keys()
3028        .any(|key| key.connection_id == connection_id)
3029        || inner
3030            .reserved_client
3031            .keys()
3032            .any(|key| key.connection_id == connection_id)
3033}
3034
3035fn check_module_connection_role_locked(
3036    inner: &ForwardingInner,
3037    connection_id: ConnectionId,
3038) -> Result<(), ForwardingError> {
3039    if inner.endpoint_by_connection.contains_key(&connection_id)
3040        || connection_has_client_routes_locked(inner, connection_id)
3041    {
3042        return Err(ForwardingError::ConnectionRoleConflict { connection_id });
3043    }
3044    Ok(())
3045}
3046
3047fn next_channel(channel: u16) -> u16 {
3048    let next = channel.wrapping_add(1);
3049    if next == 0 {
3050        1
3051    } else {
3052        next
3053    }
3054}
3055
3056fn endpoint_routes_locked(
3057    inner: &ForwardingInner,
3058    endpoint: ModuleEndpointId,
3059) -> Vec<EndpointRoute> {
3060    let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
3061    let draining = drain_reason.is_some();
3062    let mut routes = inner
3063        .module_to_client
3064        .iter()
3065        .filter(|(module_key, _)| module_key.endpoint == endpoint)
3066        .map(|(_, route)| EndpointRoute {
3067            goodbye_target: GoodbyeTarget {
3068                connection_id: route.client_connection_id,
3069                sink: route.client_sink.clone(),
3070                negotiated_ver: route.client_negotiated_ver,
3071                channel: route.client_channel,
3072                epoch: route.client_epoch,
3073                kind: GoodbyeTargetKind::Client,
3074                module_id: Some(route.module_id.clone()),
3075            },
3076            principal: route.principal.clone(),
3077            bound_at: route.bound_at,
3078            draining,
3079            drain_reason,
3080        })
3081        .collect::<Vec<_>>();
3082    routes.sort_by_key(|route| {
3083        (
3084            route.goodbye_target.connection_id.get(),
3085            route.goodbye_target.channel,
3086            route.goodbye_target.epoch,
3087        )
3088    });
3089    routes
3090}
3091
3092/// Why a bind admitted under `scope` may no longer commit, read from the
3093/// published tags: `scope_ended` when the scope is gone or at another epoch,
3094/// `scope_changed` (retryable) when only its content moved.
3095fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
3096    let key = (scope.owner.clone(), scope.scope_ref.clone());
3097    match inner.scope_tags.get(&key) {
3098        Some(current) if *current == scope.tag => None,
3099        Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
3100            subc_protocol::error_codes::SCOPE_CHANGED,
3101            format!(
3102                "scope '{}' of {} changed while the route was being bound; re-open it",
3103                scope.scope_ref, scope.owner
3104            ),
3105        )),
3106        _ => Some(ErrorBody::new(
3107            subc_protocol::error_codes::SCOPE_ENDED,
3108            format!(
3109                "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
3110                scope.scope_ref, scope.owner, scope.tag.scope_epoch
3111            ),
3112        )),
3113    }
3114}
3115
3116fn release_reserved_route_locked(
3117    inner: &mut ForwardingInner,
3118    client_key: ClientRouteKey,
3119    module_key: ModuleRouteKey,
3120) {
3121    if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
3122        inner.reserved_client.remove(&client_key);
3123    }
3124    if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
3125        inner.reserved_module.remove(&module_key);
3126    }
3127    inner.status.retain(|(key, _), _| *key != client_key);
3128}
3129
3130fn release_client_route_locked(
3131    inner: &mut ForwardingInner,
3132    client_key: ClientRouteKey,
3133    expected_epoch: u32,
3134) -> RouteRelease {
3135    let Some(route) = inner.client_to_module.get(&client_key) else {
3136        return RouteRelease::Absent;
3137    };
3138    if route.client_epoch != expected_epoch {
3139        return RouteRelease::Stale;
3140    }
3141    let route = inner
3142        .client_to_module
3143        .remove(&client_key)
3144        .expect("route checked under the same forwarding lock");
3145    route.flow.close();
3146    if let Some(scope) = &route.scope {
3147        scope.release_bind();
3148    }
3149    inner.module_to_client.remove(&ModuleRouteKey {
3150        endpoint: route.module_endpoint,
3151        channel: route.module_channel,
3152    });
3153    inner.operator_confirms.route_closed(ModuleRouteKey {
3154        endpoint: route.module_endpoint,
3155        channel: route.module_channel,
3156    });
3157    inner.status.remove(&(client_key, expected_epoch));
3158    RouteRelease::Removed(GoodbyeTarget {
3159        connection_id: route.module_endpoint.connection_id,
3160        sink: route.module_sink.clone(),
3161        negotiated_ver: route.module_negotiated_ver,
3162        channel: route.module_channel,
3163        epoch: route.module_epoch,
3164        kind: GoodbyeTargetKind::Module,
3165        module_id: Some(route.module_id.clone()),
3166    })
3167}
3168
3169fn release_module_route_locked(
3170    inner: &mut ForwardingInner,
3171    module_key: ModuleRouteKey,
3172    expected_epoch: u32,
3173) -> RouteRelease {
3174    let Some(route) = inner.module_to_client.get(&module_key) else {
3175        return RouteRelease::Absent;
3176    };
3177    if route.module_epoch != expected_epoch {
3178        return RouteRelease::Stale;
3179    }
3180    let route = inner
3181        .module_to_client
3182        .remove(&module_key)
3183        .expect("route checked under the same forwarding lock");
3184    inner.operator_confirms.route_closed(module_key);
3185    route.flow.close();
3186    if let Some(scope) = &route.scope {
3187        scope.release_bind();
3188    }
3189    let client_key = ClientRouteKey {
3190        connection_id: route.client_connection_id,
3191        channel: route.client_channel,
3192    };
3193    inner.client_to_module.remove(&client_key);
3194    inner.status.remove(&(client_key, route.client_epoch));
3195    RouteRelease::Removed(GoodbyeTarget {
3196        connection_id: route.client_connection_id,
3197        sink: route.client_sink.clone(),
3198        negotiated_ver: route.client_negotiated_ver,
3199        channel: route.client_channel,
3200        epoch: route.client_epoch,
3201        kind: GoodbyeTargetKind::Client,
3202        module_id: Some(route.module_id.clone()),
3203    })
3204}
3205
3206fn commit_route_locked(
3207    inner: &mut ForwardingInner,
3208    pending: PendingRouteBindRelayEntry,
3209) -> Result<Option<GoodbyeTarget>, ForwardingError> {
3210    let reservation = pending.reservation;
3211    if inner
3212        .closing_connections
3213        .contains(&reservation.client_key.connection_id)
3214    {
3215        return Err(ForwardingError::ConnectionClosing {
3216            connection_id: reservation.client_key.connection_id,
3217        });
3218    }
3219    let module_id = inner
3220        .module_id_by_endpoint
3221        .get(&reservation.module_key.endpoint)
3222        .cloned()
3223        .ok_or(ForwardingError::StaleModuleEndpoint)?;
3224    if inner
3225        .draining_endpoints
3226        .contains_key(&reservation.module_key.endpoint)
3227    {
3228        return Err(ForwardingError::ModuleReloading { module_id });
3229    }
3230    if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
3231        || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
3232    {
3233        return Err(ForwardingError::UnknownReservation {
3234            client_channel: reservation.client_key.channel,
3235            module_channel: reservation.module_key.channel,
3236        });
3237    }
3238    let module = inner
3239        .modules_by_id
3240        .get(&module_id)
3241        .filter(|module| module.endpoint == reservation.module_key.endpoint)
3242        .cloned()
3243        .ok_or(ForwardingError::StaleModuleEndpoint)?;
3244    let binding = Arc::new(RouteBinding {
3245        client_connection_id: reservation.client_key.connection_id,
3246        client_sink: pending.client_sink,
3247        client_negotiated_ver: pending.client_negotiated_ver,
3248        client_channel: reservation.client_key.channel,
3249        client_epoch: reservation.client_epoch,
3250        module_id,
3251        module_endpoint: reservation.module_key.endpoint,
3252        module_sink: module.sink,
3253        module_negotiated_ver: module.negotiated_ver,
3254        module_channel: reservation.module_key.channel,
3255        module_epoch: reservation.module_epoch,
3256        principal: pending.principal,
3257        project_root: reservation.project_root.clone(),
3258        bound_at: Instant::now(),
3259        flow: Arc::new(ChannelFlow::new(window_for(&module.concurrency))),
3260        scope: pending.scope,
3261    });
3262    inner
3263        .client_to_module
3264        .insert(reservation.client_key, Arc::clone(&binding));
3265    inner
3266        .module_to_client
3267        .insert(reservation.module_key, Arc::clone(&binding));
3268    let previous_published = inner
3269        .last_published_epoch
3270        .insert(reservation.client_key, reservation.client_epoch);
3271
3272    // This point is reachable only through an accepted route.bind acknowledgement.
3273    // Publish history before notifying the opener, so a subsequent
3274    // `scope.served_by` query sees the acknowledged bind.
3275    if let Some(scope) = &binding.scope {
3276        scope.record_served(&binding.module_id);
3277    }
3278
3279    // Sending through the reserved slot cannot fail, but it reports a receiver
3280    // that closed after reservation and before this locked publication point.
3281    // The frame is stamped and charged to the queued-byte count here, when it
3282    // actually enters the queue, not when the slot was reserved.
3283    let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
3284    if client_writer_closed {
3285        // Not gated on `relay_enqueued`: only a module ack reaches this
3286        // commit, which proves the module holds the binding, and `route.open`
3287        // sends nothing for a relay a reply already settled.
3288        let abandoned = abandoned_route_target(inner, &reservation);
3289        if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
3290            route.flow.close();
3291        }
3292        inner.module_to_client.remove(&reservation.module_key);
3293        inner
3294            .status
3295            .remove(&(reservation.client_key, reservation.client_epoch));
3296        match previous_published {
3297            Some(epoch) => {
3298                inner
3299                    .last_published_epoch
3300                    .insert(reservation.client_key, epoch);
3301            }
3302            None => {
3303                inner.last_published_epoch.remove(&reservation.client_key);
3304            }
3305        }
3306        let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3307            "client egress closed during route publication".to_string(),
3308        ));
3309        return Ok(abandoned);
3310    }
3311
3312    if let Some(scope) = &binding.scope {
3313        scope.commit_presence();
3314    }
3315    let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
3316    Ok(None)
3317}
3318
3319/// The live module connection registered under exactly `endpoint`, whichever
3320/// slot holds it: active, swap candidate, or superseded incumbent.
3321///
3322/// Resolving by endpoint rather than by module id is what keeps an incumbent
3323/// addressable after a swap promoted a candidate over its id. An endpoint that
3324/// is in none of the three (a stale one, replaced without a promotion) resolves
3325/// to nothing, as it always has.
3326fn module_connection_for_endpoint_locked(
3327    inner: &ForwardingInner,
3328    endpoint: ModuleEndpointId,
3329) -> Option<&ModuleConnection> {
3330    let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
3331    inner
3332        .modules_by_id
3333        .get(module_id)
3334        .filter(|module| module.endpoint == endpoint)
3335        .or_else(|| {
3336            inner
3337                .candidates_by_id
3338                .get(module_id)
3339                .filter(|module| module.endpoint == endpoint)
3340        })
3341        .or_else(|| inner.superseded_endpoints.get(&endpoint))
3342}
3343
3344fn abandoned_route_target(
3345    inner: &ForwardingInner,
3346    reservation: &RouteReservation,
3347) -> Option<GoodbyeTarget> {
3348    let module_id = inner
3349        .module_id_by_endpoint
3350        .get(&reservation.module_key.endpoint)?;
3351    let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
3352    (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
3353        connection_id: module.endpoint.connection_id,
3354        sink: module.sink.clone(),
3355        negotiated_ver: module.negotiated_ver,
3356        channel: reservation.module_key.channel,
3357        epoch: reservation.module_epoch,
3358        kind: GoodbyeTargetKind::Module,
3359        module_id: Some(module_id.clone()),
3360    })
3361}
3362
3363/// Queue a registering module's HELLO_ACK on its sink. Called with the
3364/// forwarding write lock held, before the endpoint is inserted, which is what
3365/// puts the ack ahead of any `route.bind` or control RPC routed to the module.
3366/// `try_send` never waits, so holding the lock across it is safe.
3367fn enqueue_hello_ack_locked(
3368    sink: &FrameSink,
3369    connection_id: ConnectionId,
3370    hello_ack: Option<Frame>,
3371) -> Result<(), ForwardingError> {
3372    let Some(hello_ack) = hello_ack else {
3373        return Ok(());
3374    };
3375    sink.try_send(hello_ack)
3376        .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
3377}
3378
3379fn remove_module_connection_locked(
3380    inner: &mut ForwardingInner,
3381    endpoint: ModuleEndpointId,
3382) -> ConnectionCleanup {
3383    // Commit the disconnect before its routes are released, so their teardown
3384    // cannot replace module_closed with route_closed.
3385    inner.operator_confirms.module_closed(endpoint);
3386    inner.draining_endpoints.remove(&endpoint);
3387    inner.daemon_drain_results.remove(&endpoint);
3388    let module_id = inner.module_id_by_endpoint.remove(&endpoint);
3389    if let Some(module_id) = module_id.as_ref() {
3390        if inner
3391            .modules_by_id
3392            .get(module_id)
3393            .is_some_and(|module| module.endpoint == endpoint)
3394        {
3395            inner.modules_by_id.remove(module_id);
3396        }
3397        if inner
3398            .candidates_by_id
3399            .get(module_id)
3400            .is_some_and(|module| module.endpoint == endpoint)
3401        {
3402            inner.candidates_by_id.remove(module_id);
3403        }
3404    }
3405    inner.superseded_endpoints.remove(&endpoint);
3406    inner.endpoint_by_connection.remove(&endpoint.connection_id);
3407    inner.next_module_channel.remove(&endpoint);
3408    inner.next_control_corr.remove(&endpoint);
3409    inner
3410        .health_probe_tombstones
3411        .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
3412    inner
3413        .module_slot_epochs
3414        .retain(|key, _| key.endpoint != endpoint);
3415    let reserved_module_keys: Vec<ModuleRouteKey> = inner
3416        .reserved_module
3417        .keys()
3418        .filter(|module_key| module_key.endpoint == endpoint)
3419        .copied()
3420        .collect();
3421    for module_key in reserved_module_keys {
3422        if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
3423            release_reserved_route_locked(inner, client_key, module_key);
3424        }
3425    }
3426
3427    let pending_keys: Vec<_> = inner
3428        .pending_relays
3429        .keys()
3430        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3431        .copied()
3432        .collect();
3433    let pending: Vec<_> = pending_keys
3434        .into_iter()
3435        .filter_map(|key| {
3436            inner
3437                .pending_relays
3438                .remove(&key)
3439                .map(|pending| (key, pending))
3440        })
3441        .collect();
3442    let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
3443    for (key, pending) in pending {
3444        // Recorded like every other removal without a reply, so a `route.open`
3445        // still sending the bind learns the relay was released rather than
3446        // settled by the module.
3447        if !pending.relay_enqueued {
3448            inner.relays_released_before_enqueue.insert(key);
3449        }
3450        let module_label = module_id.as_deref().unwrap_or("unknown");
3451        let _ = pending
3452            .sender
3453            .send(RouteBindRelayOutcome::ModuleGone(format!(
3454                "module '{module_label}' connection closed during route.bind relay"
3455            )));
3456    }
3457
3458    let pending_control_keys: Vec<_> = inner
3459        .pending_control_rpcs
3460        .keys()
3461        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3462        .copied()
3463        .collect();
3464    let pending_control: Vec<_> = pending_control_keys
3465        .into_iter()
3466        .filter_map(|key| inner.pending_control_rpcs.remove(&key))
3467        .collect();
3468    for pending in pending_control {
3469        let module_label = module_id.as_deref().unwrap_or("unknown");
3470        let _ = pending
3471            .sender
3472            .send(ModuleControlRpcOutcome::ModuleGone(format!(
3473                "module '{module_label}' connection closed during module-control RPC"
3474            )));
3475    }
3476
3477    let module_routes = inner
3478        .module_to_client
3479        .iter()
3480        .filter(|(module_key, _)| module_key.endpoint == endpoint)
3481        .map(|(module_key, route)| (*module_key, route.module_epoch))
3482        .collect::<Vec<_>>();
3483    let mut released = Vec::with_capacity(module_routes.len());
3484    for (module_key, epoch) in module_routes {
3485        if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
3486        {
3487            released.push(target);
3488        }
3489    }
3490    ConnectionCleanup {
3491        released,
3492        abandoned_relays,
3493    }
3494}
3495
3496#[derive(Debug, Clone, Copy)]
3497struct RequestCredit {
3498    subscription: bool,
3499    excluded_from_drain: bool,
3500}
3501
3502#[derive(Debug, Default)]
3503struct CreditLedger {
3504    by_corr: HashMap<u64, Vec<RequestCredit>>,
3505}
3506
3507impl CreditLedger {
3508    fn acquire(&mut self, corr: u64, subscription: bool) {
3509        self.by_corr.entry(corr).or_default().push(RequestCredit {
3510            subscription,
3511            excluded_from_drain: false,
3512        });
3513    }
3514
3515    fn release(&mut self, corr: u64) -> bool {
3516        let Some(credits) = self.by_corr.get_mut(&corr) else {
3517            return false;
3518        };
3519        let released = credits.pop().is_some();
3520        if credits.is_empty() {
3521            self.by_corr.remove(&corr);
3522        }
3523        released
3524    }
3525
3526    fn capture_subscription_exclusions(&mut self) -> u32 {
3527        let mut excluded = 0u32;
3528        for credit in self.by_corr.values_mut().flatten() {
3529            if credit.subscription && !credit.excluded_from_drain {
3530                credit.excluded_from_drain = true;
3531                excluded = excluded.saturating_add(1);
3532            }
3533        }
3534        excluded
3535    }
3536
3537    #[cfg(test)]
3538    fn in_flight(&self) -> usize {
3539        self.by_corr.values().map(Vec::len).sum()
3540    }
3541
3542    fn drain_in_flight(&self) -> usize {
3543        self.by_corr
3544            .values()
3545            .flatten()
3546            .filter(|credit| !credit.excluded_from_drain)
3547            .count()
3548    }
3549
3550    /// The correlation ids of the requests a drain still waits on, one entry
3551    /// per held credit, in ascending order.
3552    fn drain_held_corrs(&self) -> Vec<u64> {
3553        let mut corrs = self
3554            .by_corr
3555            .iter()
3556            .flat_map(|(corr, credits)| {
3557                credits
3558                    .iter()
3559                    .filter(|credit| !credit.excluded_from_drain)
3560                    .map(move |_| *corr)
3561            })
3562            .collect::<Vec<_>>();
3563        corrs.sort_unstable();
3564        corrs
3565    }
3566}
3567
3568#[derive(Debug, Default)]
3569struct ChannelFlowState {
3570    closed: bool,
3571    credits: CreditLedger,
3572}
3573
3574/// Per-channel request-credit accounting shared by the client and module route halves.
3575#[derive(Debug)]
3576pub(crate) struct ChannelFlow {
3577    sem: Semaphore,
3578    window: usize,
3579    state: Mutex<ChannelFlowState>,
3580}
3581
3582impl ChannelFlow {
3583    pub(crate) fn new(window: usize) -> Self {
3584        debug_assert!(window > 0, "flow-control window must be non-zero");
3585        Self {
3586            sem: Semaphore::new(window),
3587            window,
3588            state: Mutex::new(ChannelFlowState::default()),
3589        }
3590    }
3591
3592    #[cfg(test)]
3593    pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
3594        self.acquire_tagged(0, false).await
3595    }
3596
3597    pub(crate) async fn acquire_tagged(
3598        &self,
3599        corr: u64,
3600        subscription: bool,
3601    ) -> Result<(), ChannelFlowClosed> {
3602        let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
3603        let mut state = self
3604            .state
3605            .lock()
3606            .unwrap_or_else(|poisoned| poisoned.into_inner());
3607        if state.closed {
3608            return Err(ChannelFlowClosed);
3609        }
3610        state.credits.acquire(corr, subscription);
3611        permit.forget();
3612        Ok(())
3613    }
3614
3615    #[cfg(test)]
3616    pub(crate) fn release(&self) {
3617        self.release_corr(0);
3618    }
3619
3620    pub(crate) fn release_corr(&self, corr: u64) {
3621        let released = self
3622            .state
3623            .lock()
3624            .unwrap_or_else(|poisoned| poisoned.into_inner())
3625            .credits
3626            .release(corr);
3627        if !released {
3628            // Protocol-conforming modules emit exactly one terminal per request.
3629            // This guard is a best-effort safety net against window growth, not a
3630            // security boundary against malicious peers.
3631            warn!(
3632                window = self.window,
3633                available = self.sem.available_permits(),
3634                "flow-control over-release ignored"
3635            );
3636            return;
3637        }
3638        if !self.sem.is_closed() {
3639            self.sem.add_permits(1);
3640        }
3641    }
3642
3643    #[cfg(test)]
3644    pub(crate) fn in_flight(&self) -> usize {
3645        self.state
3646            .lock()
3647            .unwrap_or_else(|poisoned| poisoned.into_inner())
3648            .credits
3649            .in_flight()
3650    }
3651
3652    pub(crate) fn drain_in_flight(&self) -> usize {
3653        self.state
3654            .lock()
3655            .unwrap_or_else(|poisoned| poisoned.into_inner())
3656            .credits
3657            .drain_in_flight()
3658    }
3659
3660    pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
3661        self.state
3662            .lock()
3663            .unwrap_or_else(|poisoned| poisoned.into_inner())
3664            .credits
3665            .drain_held_corrs()
3666    }
3667
3668    #[cfg(test)]
3669    pub(crate) fn available_permits(&self) -> usize {
3670        self.sem.available_permits()
3671    }
3672
3673    pub(crate) fn begin_drain(&self) -> u32 {
3674        let mut state = self
3675            .state
3676            .lock()
3677            .unwrap_or_else(|poisoned| poisoned.into_inner());
3678        state.closed = true;
3679        self.sem.close();
3680        state.credits.capture_subscription_exclusions()
3681    }
3682
3683    pub(crate) fn close(&self) {
3684        self.state
3685            .lock()
3686            .unwrap_or_else(|poisoned| poisoned.into_inner())
3687            .closed = true;
3688        self.sem.close();
3689    }
3690}
3691
3692#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3693pub(crate) struct ChannelFlowClosed;
3694
3695impl fmt::Display for ChannelFlowClosed {
3696    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3697        write!(f, "flow-control window closed")
3698    }
3699}
3700
3701impl Error for ChannelFlowClosed {}
3702
3703fn window_for(concurrency: &Concurrency) -> usize {
3704    match concurrency {
3705        Concurrency::Serial => 1,
3706        Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
3707        Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
3708    }
3709}
3710
3711#[derive(Debug, Clone, PartialEq, Eq)]
3712pub enum ForwardingError {
3713    /// A connection cannot hold both client routes and a module endpoint, or
3714    /// replace a module identity while that endpoint is still registered.
3715    ConnectionRoleConflict {
3716        connection_id: ConnectionId,
3717    },
3718    NoModuleConnection,
3719    ModuleReloading {
3720        module_id: String,
3721    },
3722    StaleModuleEndpoint,
3723    UnknownReservation {
3724        client_channel: u16,
3725        module_channel: u16,
3726    },
3727    ClientRouteChannelExhausted {
3728        connection_id: ConnectionId,
3729    },
3730    ModuleRouteChannelExhausted {
3731        endpoint: ModuleEndpointId,
3732    },
3733    RelayCorrelationExhausted,
3734    ConnectionClosing {
3735        connection_id: ConnectionId,
3736    },
3737    ClientEgressClosed {
3738        connection_id: ConnectionId,
3739    },
3740    RouteOpenBuild(String),
3741    /// A swap candidate is already registered for this module id.
3742    CandidateSlotOccupied {
3743        module_id: String,
3744    },
3745    /// A registering module's HELLO_ACK could not be queued because its
3746    /// outbound queue is closed or full.
3747    ModuleEgressUnavailable {
3748        connection_id: ConnectionId,
3749    },
3750    Poisoned,
3751}
3752
3753impl fmt::Display for ForwardingError {
3754    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3755        match self {
3756            Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
3757            Self::NoModuleConnection => write!(f, "no module connection is registered"),
3758            Self::ModuleReloading { module_id } => {
3759                write!(f, "module_id '{module_id}' is reloading")
3760            }
3761            Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
3762            Self::UnknownReservation {
3763                client_channel,
3764                module_channel,
3765            } => write!(
3766                f,
3767                "route reservation client channel {client_channel} / module channel {module_channel} was not found"
3768            ),
3769            Self::ClientRouteChannelExhausted { connection_id } => write!(
3770                f,
3771                "no client route channels are available for connection {}",
3772                connection_id.get()
3773            ),
3774            Self::ModuleRouteChannelExhausted { endpoint } => write!(
3775                f,
3776                "no module route channels are available for endpoint generation {} on connection {}",
3777                endpoint.generation,
3778                endpoint.connection_id.get()
3779            ),
3780            Self::RelayCorrelationExhausted => {
3781                write!(f, "module control correlation ids are exhausted")
3782            }
3783            Self::ConnectionClosing { connection_id } => write!(
3784                f,
3785                "connection {} is closing and cannot accept route allocation",
3786                connection_id.get()
3787            ),
3788            Self::ClientEgressClosed { connection_id } => write!(
3789                f,
3790                "client connection {} egress is closed",
3791                connection_id.get()
3792            ),
3793            Self::RouteOpenBuild(message) => {
3794                write!(f, "failed to prebuild route.open response: {message}")
3795            }
3796            Self::CandidateSlotOccupied { module_id } => write!(
3797                f,
3798                "module_id '{module_id}' already has a swap candidate registered"
3799            ),
3800            Self::ModuleEgressUnavailable { connection_id } => write!(
3801                f,
3802                "module connection {} egress is unavailable; HELLO_ACK could not be queued",
3803                connection_id.get()
3804            ),
3805            Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
3806        }
3807    }
3808}
3809
3810impl Error for ForwardingError {}
3811
3812#[cfg(test)]
3813mod tests {
3814    use std::time::Duration;
3815
3816    use super::*;
3817    use tokio::sync::mpsc;
3818
3819    #[test]
3820    fn ordinary_long_running_request_is_not_excluded_from_drain() {
3821        let mut ledger = CreditLedger::default();
3822        ledger.acquire(1, false);
3823
3824        assert_eq!(ledger.capture_subscription_exclusions(), 0);
3825        assert_eq!(ledger.drain_in_flight(), 1);
3826    }
3827
3828    #[test]
3829    fn bit_set_subscription_is_excluded_and_counted() {
3830        let mut ledger = CreditLedger::default();
3831        ledger.acquire(1, true);
3832
3833        assert_eq!(ledger.capture_subscription_exclusions(), 1);
3834        assert_eq!(ledger.drain_in_flight(), 0);
3835    }
3836
3837    #[test]
3838    fn subscription_opened_after_drain_snapshot_is_not_excluded() {
3839        let mut ledger = CreditLedger::default();
3840        ledger.acquire(1, true);
3841        assert_eq!(ledger.capture_subscription_exclusions(), 1);
3842
3843        ledger.acquire(2, true);
3844
3845        assert_eq!(ledger.drain_in_flight(), 1);
3846    }
3847
3848    #[test]
3849    fn drain_with_no_subscriptions_reports_zero_excluded() {
3850        let mut ledger = CreditLedger::default();
3851        assert_eq!(ledger.capture_subscription_exclusions(), 0);
3852    }
3853
3854    fn test_hello_ack(corr: u64) -> Frame {
3855        Frame::build(
3856            FrameType::HelloAck,
3857            Flags::new(false, Priority::Passive, false),
3858            0,
3859            0,
3860            corr,
3861            Vec::new(),
3862        )
3863        .unwrap()
3864    }
3865
3866    /// An acked registration whose HELLO_ACK cannot be queued must leave no
3867    /// endpoint behind: a routable module that never got its ack would read a
3868    /// route.bind first and exit. A closed queue and a full one both refuse.
3869    #[test]
3870    fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
3871        let forwarding = ForwardingTable::default();
3872
3873        let (closed_tx, closed_rx) = mpsc::channel(8);
3874        drop(closed_rx);
3875        let closed = ConnectionId::new(1);
3876        assert_eq!(
3877            forwarding.register_module_connection_acked(
3878                closed,
3879                "closed".to_string(),
3880                2,
3881                Concurrency::ModuleManaged,
3882                FrameSink::new(closed_tx),
3883                test_hello_ack(1),
3884            ),
3885            Err(ForwardingError::ModuleEgressUnavailable {
3886                connection_id: closed
3887            })
3888        );
3889
3890        let (full_tx, _full_rx) = mpsc::channel(1);
3891        let full_sink = FrameSink::new(full_tx);
3892        full_sink.try_send(test_hello_ack(99)).unwrap();
3893        let full = ConnectionId::new(2);
3894        assert_eq!(
3895            forwarding.register_module_connection_acked(
3896                full,
3897                "full".to_string(),
3898                2,
3899                Concurrency::ModuleManaged,
3900                full_sink.clone(),
3901                test_hello_ack(2),
3902            ),
3903            Err(ForwardingError::ModuleEgressUnavailable {
3904                connection_id: full
3905            })
3906        );
3907        assert_eq!(
3908            forwarding.register_candidate_module_connection_acked(
3909                full,
3910                "full".to_string(),
3911                2,
3912                Concurrency::ModuleManaged,
3913                full_sink,
3914                test_hello_ack(3),
3915            ),
3916            Err(ForwardingError::ModuleEgressUnavailable {
3917                connection_id: full
3918            })
3919        );
3920
3921        for (connection, module_id) in [(closed, "closed"), (full, "full")] {
3922            assert_eq!(
3923                forwarding
3924                    .module_endpoint_for_connection(connection)
3925                    .unwrap(),
3926                None
3927            );
3928            let (client_tx, _client_rx) = mpsc::channel(8);
3929            assert_eq!(
3930                forwarding
3931                    .begin_route_bind_relay_for_test(
3932                        ConnectionId::new(50),
3933                        FrameSink::new(client_tx),
3934                        1,
3935                        module_id,
3936                    )
3937                    .err(),
3938                Some(ForwardingError::NoModuleConnection)
3939            );
3940        }
3941        assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
3942    }
3943
3944    /// Both acked registration forms put the HELLO_ACK on the module's queue
3945    /// by the time the endpoint can be resolved.
3946    #[test]
3947    fn acked_registration_queues_the_hello_ack_first() {
3948        let forwarding = ForwardingTable::default();
3949        let (active_tx, mut active_rx) = mpsc::channel(8);
3950        forwarding
3951            .register_module_connection_acked(
3952                ConnectionId::new(1),
3953                "acked".to_string(),
3954                2,
3955                Concurrency::ModuleManaged,
3956                FrameSink::new(active_tx),
3957                test_hello_ack(11),
3958            )
3959            .unwrap();
3960        let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
3961        forwarding
3962            .register_candidate_module_connection_acked(
3963                ConnectionId::new(2),
3964                "acked".to_string(),
3965                2,
3966                Concurrency::ModuleManaged,
3967                FrameSink::new(candidate_tx),
3968                test_hello_ack(12),
3969            )
3970            .unwrap();
3971
3972        let active_first = active_rx.try_recv().unwrap().frame;
3973        assert_eq!(active_first.header.ty, FrameType::HelloAck);
3974        assert_eq!(active_first.header.corr, 11);
3975        let candidate_first = candidate_rx.try_recv().unwrap().frame;
3976        assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
3977        assert_eq!(candidate_first.header.corr, 12);
3978    }
3979
3980    #[test]
3981    fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
3982        let forwarding = ForwardingTable::default();
3983        let module_connection = ConnectionId::new(10);
3984        let exhausted_client = ConnectionId::new(20);
3985        let second_client = ConnectionId::new(30);
3986        let (module_tx, _module_rx) = mpsc::channel(1);
3987        let endpoint = forwarding
3988            .register_module_connection(
3989                module_connection,
3990                "route-limit-provider".to_string(),
3991                1,
3992                Concurrency::ModuleManaged,
3993                FrameSink::new(module_tx),
3994            )
3995            .unwrap();
3996
3997        {
3998            let mut inner = forwarding.inner.write().unwrap();
3999            for channel in 1..=u16::MAX {
4000                inner.reserved_client.insert(
4001                    ClientRouteKey {
4002                        connection_id: exhausted_client,
4003                        channel,
4004                    },
4005                    ModuleRouteKey {
4006                        endpoint,
4007                        channel: 1,
4008                    },
4009                );
4010            }
4011        }
4012
4013        let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
4014        let err = forwarding
4015            .begin_route_bind_relay_for_test(
4016                exhausted_client,
4017                FrameSink::new(exhausted_tx),
4018                1,
4019                "route-limit-provider",
4020            )
4021            .unwrap_err();
4022        assert!(matches!(
4023            err,
4024            ForwardingError::ClientRouteChannelExhausted { connection_id }
4025                if connection_id == exhausted_client
4026        ));
4027
4028        let (second_tx, _second_rx) = mpsc::channel(1);
4029        let pending = forwarding
4030            .begin_route_bind_relay_for_test(
4031                second_client,
4032                FrameSink::new(second_tx),
4033                2,
4034                "route-limit-provider",
4035            )
4036            .unwrap();
4037        assert_eq!(pending.client_channel, 1);
4038    }
4039
4040    #[test]
4041    fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
4042        let forwarding = ForwardingTable::default();
4043        let module_connection = ConnectionId::new(40);
4044        let client = ConnectionId::new(50);
4045        let (module_tx, _module_rx) = mpsc::channel(1);
4046        forwarding
4047            .register_module_connection(
4048                module_connection,
4049                "slot-reuse-provider".to_string(),
4050                1,
4051                Concurrency::ModuleManaged,
4052                FrameSink::new(module_tx),
4053            )
4054            .unwrap();
4055
4056        let (client_tx, _client_rx) = mpsc::channel(1);
4057        let client_sink = FrameSink::new(client_tx);
4058        let mut wrapped_channel = None;
4059        for index in 0..=usize::from(u16::MAX) {
4060            let pending = forwarding
4061                .begin_route_bind_relay_for_test(
4062                    client,
4063                    client_sink.clone(),
4064                    index as u64 + 1,
4065                    "slot-reuse-provider",
4066                )
4067                .unwrap();
4068            if index == usize::from(u16::MAX) {
4069                wrapped_channel = Some(pending.module_channel);
4070            }
4071            forwarding
4072                .abort_pending_relay(
4073                    pending.endpoint,
4074                    pending.corr,
4075                    RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
4076                )
4077                .unwrap();
4078        }
4079
4080        assert_eq!(wrapped_channel, Some(1));
4081    }
4082
4083    #[test]
4084    fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
4085        let forwarding = ForwardingTable::default();
4086        let client = ConnectionId::new(60);
4087        forwarding
4088            .inner
4089            .write()
4090            .unwrap()
4091            .next_client_channel
4092            .insert(client, 41);
4093
4094        let released = forwarding.cleanup_connection(client).unwrap();
4095
4096        assert!(released.is_empty());
4097        assert!(!forwarding
4098            .inner
4099            .read()
4100            .unwrap()
4101            .next_client_channel
4102            .contains_key(&client));
4103    }
4104
4105    #[test]
4106    fn stale_module_cleanup_preserves_fast_reconnect_successor() {
4107        let forwarding = ForwardingTable::default();
4108        let module_id = "fast-reconnect-provider";
4109        let first_connection = ConnectionId::new(70);
4110        let second_connection = ConnectionId::new(80);
4111        let (first_tx, _first_rx) = mpsc::channel(1);
4112        let first_endpoint = forwarding
4113            .register_module_connection(
4114                first_connection,
4115                module_id.to_string(),
4116                1,
4117                Concurrency::ModuleManaged,
4118                FrameSink::new(first_tx),
4119            )
4120            .unwrap();
4121        let (second_tx, _second_rx) = mpsc::channel(1);
4122        let second_endpoint = forwarding
4123            .register_module_connection(
4124                second_connection,
4125                module_id.to_string(),
4126                1,
4127                Concurrency::ModuleManaged,
4128                FrameSink::new(second_tx),
4129            )
4130            .unwrap();
4131        assert_ne!(first_endpoint, second_endpoint);
4132
4133        let released = forwarding.cleanup_connection(first_connection).unwrap();
4134
4135        assert!(released.is_empty());
4136        assert_eq!(
4137            forwarding
4138                .inner
4139                .read()
4140                .unwrap()
4141                .modules_by_id
4142                .get(module_id)
4143                .map(|module| module.endpoint),
4144            Some(second_endpoint)
4145        );
4146        assert!(forwarding.has_live_module_connection(module_id).unwrap());
4147        let control_rpc = forwarding
4148            .begin_module_control_rpc_for(
4149                module_id,
4150                "health.check",
4151                Instant::now() + Duration::from_secs(1),
4152            )
4153            .unwrap();
4154        assert_eq!(control_rpc.endpoint, second_endpoint);
4155    }
4156
4157    fn route_fixture(
4158        module_id: &str,
4159    ) -> (
4160        ForwardingTable,
4161        ConnectionId,
4162        ModuleEndpointId,
4163        ConnectionId,
4164        FrameSink,
4165        mpsc::Receiver<crate::router::OutboundFrame>,
4166    ) {
4167        let forwarding = ForwardingTable::default();
4168        let module_connection = ConnectionId::new(100);
4169        let client_connection = ConnectionId::new(200);
4170        let (module_tx, _module_rx) = mpsc::channel(8);
4171        let endpoint = forwarding
4172            .register_module_connection(
4173                module_connection,
4174                module_id.to_string(),
4175                2,
4176                Concurrency::ModuleManaged,
4177                FrameSink::new(module_tx),
4178            )
4179            .unwrap();
4180        let (client_tx, client_rx) = mpsc::channel(8);
4181        (
4182            forwarding,
4183            module_connection,
4184            endpoint,
4185            client_connection,
4186            FrameSink::new(client_tx),
4187            client_rx,
4188        )
4189    }
4190
4191    #[test]
4192    #[cfg(unix)]
4193    fn daemon_drain_gates_current_and_racing_provider_registrations() {
4194        let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
4195        assert_eq!(
4196            forwarding.begin_daemon_drain().unwrap(),
4197            [("provider".to_string(), endpoint)]
4198        );
4199        assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
4200        assert!(matches!(
4201            forwarding.register_module_connection(
4202                ConnectionId::new(300),
4203                "late-provider".into(),
4204                2,
4205                Concurrency::ModuleManaged,
4206                sink,
4207            ),
4208            Err(ForwardingError::ConnectionClosing { .. })
4209        ));
4210    }
4211
4212    #[test]
4213    #[cfg(unix)]
4214    fn daemon_drain_results_are_endpoint_fenced_and_removed_on_cleanup() {
4215        let (forwarding, connection, endpoint, _, _, _) = route_fixture("provider");
4216        assert_eq!(
4217            forwarding
4218                .daemon_drain_result_for_connection(connection)
4219                .unwrap(),
4220            None
4221        );
4222        forwarding
4223            .record_daemon_drain_result(endpoint, true)
4224            .unwrap();
4225        forwarding.begin_daemon_drain().unwrap();
4226        assert_eq!(
4227            forwarding
4228                .daemon_drain_result_for_connection(connection)
4229                .unwrap(),
4230            Some(false)
4231        );
4232        let replacement = ModuleEndpointId {
4233            generation: endpoint.generation + 1,
4234            ..endpoint
4235        };
4236        forwarding
4237            .record_daemon_drain_result(replacement, true)
4238            .unwrap();
4239        assert_eq!(
4240            forwarding
4241                .daemon_drain_result_for_connection(connection)
4242                .unwrap(),
4243            Some(false)
4244        );
4245        forwarding
4246            .record_daemon_drain_result(endpoint, true)
4247            .unwrap();
4248        assert_eq!(
4249            forwarding
4250                .daemon_drain_result_for_connection(connection)
4251                .unwrap(),
4252            Some(true)
4253        );
4254        assert_eq!(
4255            forwarding
4256                .daemon_drain_result_for_connection(ConnectionId::new(300))
4257                .unwrap(),
4258            Some(false)
4259        );
4260        forwarding.cleanup_connection(connection).unwrap();
4261        forwarding
4262            .record_daemon_drain_result(endpoint, true)
4263            .unwrap();
4264        assert_eq!(
4265            forwarding
4266                .daemon_drain_result_for_connection(connection)
4267                .unwrap(),
4268            Some(false)
4269        );
4270        assert!(forwarding
4271            .read_inner()
4272            .unwrap()
4273            .daemon_drain_results
4274            .is_empty());
4275    }
4276
4277    #[test]
4278    #[cfg(unix)]
4279    fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
4280        let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
4281            route_fixture("provider");
4282        let mut pending =
4283            begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
4284        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
4285        forwarding.begin_daemon_drain().unwrap();
4286        let completion = forwarding
4287            .complete_pending_relay(
4288                module_connection,
4289                pending.corr,
4290                RouteBindRelayOutcome::Accepted,
4291            )
4292            .expect("draining admission is not a fatal module error");
4293        assert!(completion.settled);
4294        let abandoned = completion
4295            .abandoned
4296            .expect("late accepted bind needs a route GOODBYE");
4297        assert_eq!(abandoned.channel, pending.module_channel);
4298        assert!(matches!(pending.receiver.try_recv().unwrap(),
4299            RouteBindRelayOutcome::Rejected(error) if error.code == "module_reloading"));
4300        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4301        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4302        assert!(
4303            rx.try_recv().is_err(),
4304            "no route.open may be published during drain"
4305        );
4306        assert_eq!(
4307            forwarding
4308                .module_endpoint_for_connection(module_connection)
4309                .unwrap(),
4310            Some(endpoint)
4311        );
4312        assert!(
4313            !forwarding
4314                .complete_pending_relay(
4315                    module_connection,
4316                    pending.corr,
4317                    RouteBindRelayOutcome::Accepted,
4318                )
4319                .unwrap()
4320                .settled
4321        );
4322    }
4323
4324    fn test_ping(corr: u64) -> Frame {
4325        Frame::build(
4326            FrameType::Ping,
4327            Flags::new(false, Priority::Passive, false),
4328            0,
4329            0,
4330            corr,
4331            Vec::new(),
4332        )
4333        .unwrap()
4334    }
4335
4336    fn begin_test_route(
4337        forwarding: &ForwardingTable,
4338        client_connection: ConnectionId,
4339        client_sink: FrameSink,
4340        corr: u64,
4341        module_id: &str,
4342    ) -> PendingRouteBindRelay {
4343        forwarding
4344            .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
4345            .unwrap()
4346    }
4347
4348    #[test]
4349    fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
4350        for candidate in [false, true] {
4351            for register_first in [false, true] {
4352                let (forwarding, module_connection, _, client_connection, sink, _rx) =
4353                    route_fixture("target");
4354                let register = || {
4355                    if candidate {
4356                        forwarding.register_candidate_module_connection(
4357                            client_connection,
4358                            "source".into(),
4359                            2,
4360                            Concurrency::ModuleManaged,
4361                            sink.clone(),
4362                        )
4363                    } else {
4364                        forwarding.register_module_connection(
4365                            client_connection,
4366                            "source".into(),
4367                            2,
4368                            Concurrency::ModuleManaged,
4369                            sink.clone(),
4370                        )
4371                    }
4372                };
4373                if register_first {
4374                    register().unwrap();
4375                    assert!(
4376                        forwarding
4377                            .begin_route_bind_relay_for_test(
4378                                client_connection,
4379                                sink.clone(),
4380                                1,
4381                                "target",
4382                            )
4383                            .is_err(),
4384                        "a deferred route.open cannot reserve after HELLO"
4385                    );
4386                    assert!(forwarding
4387                        .cleanup_connection(client_connection)
4388                        .unwrap()
4389                        .is_empty());
4390                } else {
4391                    let pending =
4392                        begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
4393                    assert!(
4394                        register().is_err(),
4395                        "HELLO cannot register after a route reservation"
4396                    );
4397                    forwarding
4398                        .complete_pending_relay(
4399                            module_connection,
4400                            pending.corr,
4401                            RouteBindRelayOutcome::Accepted,
4402                        )
4403                        .unwrap();
4404                    assert_eq!(
4405                        forwarding
4406                            .cleanup_connection(client_connection)
4407                            .unwrap()
4408                            .len(),
4409                        1
4410                    );
4411                }
4412                assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4413                assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4414            }
4415        }
4416    }
4417
4418    /// A pending route.open reserves its queue slot with `reserve_owned` and
4419    /// sends its prebuilt response through that slot when the module accepts.
4420    /// A queue already holding many data frames (far more than the old 64-frame
4421    /// queue) must not stop the open from reserving or completing, and the
4422    /// response must be charged to the queue's byte count when it is sent.
4423    #[tokio::test]
4424    async fn pending_route_open_completes_behind_queued_data_frames() {
4425        assert_eq!(
4426            crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
4427            8,
4428            "the per-connection pending route.open limit is its own constant"
4429        );
4430        let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
4431            route_fixture("open-behind-data");
4432        let (sink, mut client_rx) = crate::server::connection_egress();
4433        const DATA_FRAMES: usize = 1_000;
4434        let data = |corr: u64| {
4435            Frame::build(
4436                FrameType::StreamData,
4437                Flags::new(false, Priority::Interactive, false),
4438                9,
4439                1,
4440                corr,
4441                vec![b'x'; 200],
4442            )
4443            .unwrap()
4444        };
4445        for corr in 0..DATA_FRAMES as u64 {
4446            sink.try_send(data(corr)).unwrap();
4447        }
4448        let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
4449        assert_eq!(sink.backlog().queued_bytes, data_bytes);
4450
4451        let pending = tokio::time::timeout(
4452            Duration::from_secs(5),
4453            forwarding.begin_route_bind_relay_for(
4454                client,
4455                sink.clone(),
4456                subc_protocol::PROTOCOL_VERSION,
4457                4_242,
4458                "open-behind-data",
4459                Principal::Direct,
4460                None,
4461                None,
4462                Instant::now() + Duration::from_secs(60),
4463            ),
4464        )
4465        .await
4466        .expect("reserving the route.open slot must not wait behind data frames")
4467        .unwrap();
4468        forwarding
4469            .complete_pending_relay(
4470                module_connection,
4471                pending.corr,
4472                RouteBindRelayOutcome::Accepted,
4473            )
4474            .unwrap();
4475
4476        let backlog = sink.backlog();
4477        assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
4478        assert!(
4479            backlog.queued_bytes > data_bytes,
4480            "the route.open response must be counted in queued bytes: {backlog:?}"
4481        );
4482        for corr in 0..DATA_FRAMES as u64 {
4483            assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
4484        }
4485        let open = client_rx.recv().await.unwrap();
4486        assert_eq!(open.header.corr, 4_242);
4487        assert_eq!(open.header.ty, FrameType::Response);
4488        drop(open);
4489        assert_eq!(sink.backlog().queued_bytes, 0);
4490        assert_eq!(sink.backlog().queued_frames, 0);
4491    }
4492
4493    /// The drain-timeout line reports these numbers, so they must count the
4494    /// requests the drain is waiting on and no others: a route with nothing in
4495    /// flight is not a holdout, and a flagged subscription is excluded from the
4496    /// drain and so from the count.
4497    #[tokio::test]
4498    async fn drain_holdouts_count_held_requests_and_name_the_connection() {
4499        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4500            route_fixture("holdouts");
4501        let mut bound = |corr| {
4502            let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
4503            forwarding
4504                .complete_pending_relay(
4505                    module_connection,
4506                    route.corr,
4507                    RouteBindRelayOutcome::Accepted,
4508                )
4509                .unwrap();
4510            client_rx.try_recv().unwrap();
4511            match forwarding
4512                .lookup_data_route(client, route.client_channel, route.client_epoch)
4513                .unwrap()
4514            {
4515                DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4516                other => panic!("expected live route, got {other:?}"),
4517            }
4518        };
4519        let holding = bound(61);
4520        let _idle = bound(62);
4521        holding.flow.acquire_tagged(7, false).await.unwrap();
4522        holding.flow.acquire_tagged(2, false).await.unwrap();
4523        holding.flow.acquire_tagged(3, true).await.unwrap();
4524        forwarding
4525            .begin_module_drain("holdouts", RouteCloseReason::Restart)
4526            .unwrap();
4527
4528        let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
4529        assert_eq!(
4530            holdouts,
4531            DrainHoldouts {
4532                requests: 2,
4533                routes: 1,
4534                total_routes: 2,
4535                top_connections: vec![(client.get(), 2)],
4536                // The held requests by the module's channel and corr, ascending,
4537                // without the excluded subscription (corr 3).
4538                held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
4539            }
4540        );
4541    }
4542
4543    #[test]
4544    fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
4545        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4546            route_fixture("census");
4547        let pending = begin_test_route(&forwarding, client, sink, 1, "census");
4548        forwarding
4549            .complete_pending_relay(
4550                module_connection,
4551                pending.corr,
4552                RouteBindRelayOutcome::Accepted,
4553            )
4554            .unwrap();
4555
4556        let routes = forwarding.endpoint_routes(endpoint).unwrap();
4557        assert_eq!(routes.len(), 1);
4558        assert!(matches!(routes[0].principal, Principal::Direct));
4559        assert_eq!(routes[0].goodbye_target.connection_id, client);
4560        assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
4561        assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
4562        assert!(!routes[0].draining);
4563
4564        forwarding
4565            .begin_module_drain("census", RouteCloseReason::Restart)
4566            .unwrap();
4567        let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
4568        assert_eq!(draining_routes.len(), 1);
4569        assert!(draining_routes[0].draining);
4570    }
4571
4572    #[test]
4573    fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
4574        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
4575        let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
4576        assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
4577        forwarding
4578            .abort_pending_relay(
4579                first.endpoint,
4580                first.corr,
4581                RouteBindRelayOutcome::ModuleGone("abort".into()),
4582            )
4583            .unwrap();
4584        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4585        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4586
4587        let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
4588        assert_eq!(second.client_channel, first.client_channel);
4589        assert_eq!(second.module_channel, first.module_channel);
4590        assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
4591    }
4592
4593    #[test]
4594    fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
4595        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4596            route_fixture("epoch-release");
4597        let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
4598        forwarding
4599            .complete_pending_relay(
4600                module_connection,
4601                first.corr,
4602                RouteBindRelayOutcome::Accepted,
4603            )
4604            .unwrap();
4605        assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
4606        assert!(matches!(
4607            forwarding
4608                .release_client_route(client, first.client_channel, first.client_epoch)
4609                .unwrap(),
4610            RouteRelease::Removed(_)
4611        ));
4612        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4613        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4614
4615        let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
4616        forwarding
4617            .complete_pending_relay(
4618                module_connection,
4619                second.corr,
4620                RouteBindRelayOutcome::Accepted,
4621            )
4622            .unwrap();
4623        assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
4624        assert!(matches!(
4625            forwarding
4626                .release_client_route(client, second.client_channel, first.client_epoch)
4627                .unwrap(),
4628            RouteRelease::Stale
4629        ));
4630        assert!(!forwarding
4631            .cache_status(
4632                endpoint,
4633                second.module_channel,
4634                first.module_epoch,
4635                "stale".into(),
4636            )
4637            .unwrap());
4638        assert!(forwarding
4639            .cache_status(
4640                endpoint,
4641                second.module_channel,
4642                second.module_epoch,
4643                "current".into(),
4644            )
4645            .unwrap());
4646        match forwarding
4647            .route_poll_snapshot(client, second.client_channel, second.client_epoch)
4648            .unwrap()
4649        {
4650            RoutePollSnapshot::Bound { status, .. } => {
4651                assert_eq!(status.as_deref(), Some("current"));
4652            }
4653            RoutePollSnapshot::Absent => panic!("successor binding was removed"),
4654        }
4655        let counters = forwarding.counters().snapshot();
4656        assert_eq!(counters["route_released_epoch_fenced"], 1);
4657        assert_eq!(counters["route_release_stale_skipped"], 1);
4658    }
4659
4660    #[test]
4661    fn max_epoch_reservation_retires_only_that_slot() {
4662        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
4663        forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
4664        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
4665        let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
4666        assert_eq!(
4667            (final_use.client_channel, final_use.client_epoch),
4668            (7, u32::MAX)
4669        );
4670        assert_eq!(
4671            (final_use.module_channel, final_use.module_epoch),
4672            (9, u32::MAX)
4673        );
4674        forwarding
4675            .abort_pending_relay(
4676                endpoint,
4677                final_use.corr,
4678                RouteBindRelayOutcome::ModuleGone("abort".into()),
4679            )
4680            .unwrap();
4681        forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
4682        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
4683        let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
4684        assert_ne!(next.client_channel, 7);
4685        assert_ne!(next.module_channel, 9);
4686        assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
4687    }
4688
4689    #[test]
4690    fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
4691        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4692            route_fixture("corr-shared");
4693        let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
4694        assert_eq!(bind.corr, 1);
4695        forwarding
4696            .abort_pending_relay(
4697                endpoint,
4698                bind.corr,
4699                RouteBindRelayOutcome::ModuleGone("abort".into()),
4700            )
4701            .unwrap();
4702        let rpc = forwarding
4703            .begin_module_control_rpc_for(
4704                "corr-shared",
4705                "health.check",
4706                Instant::now() - Duration::from_millis(1),
4707            )
4708            .unwrap();
4709        assert_eq!(rpc.corr, 2);
4710        assert_eq!(
4711            forwarding
4712                .complete_module_control_rpc(
4713                    module_connection,
4714                    rpc.corr,
4715                    Some("health.check"),
4716                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
4717                        status: subc_protocol::session::HealthStatus::Ok,
4718                        detail: None,
4719                        metrics: None,
4720                    }),
4721                )
4722                .unwrap(),
4723            ModuleControlRpcCompletion::Settled
4724        );
4725        assert!(matches!(
4726            rpc.receiver.blocking_recv().unwrap(),
4727            ModuleControlRpcOutcome::DeadlineElapsed
4728        ));
4729    }
4730
4731    #[tokio::test(start_paused = true)]
4732    async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
4733        let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
4734        let probe_started_at = Instant::now();
4735        let rpc = forwarding
4736            .begin_health_probe_rpc_for(
4737                "tombstone-ttl",
4738                "health.check",
4739                probe_started_at,
4740                probe_started_at + Duration::from_secs(5),
4741            )
4742            .unwrap();
4743        assert!(forwarding
4744            .tombstone_health_probe_rpc(endpoint, rpc.corr)
4745            .unwrap());
4746        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
4747
4748        tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
4749        tokio::task::yield_now().await;
4750
4751        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
4752    }
4753
4754    #[test]
4755    fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
4756        let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
4757        let mut close = forwarding.register_connection_close(endpoint.connection_id);
4758        forwarding.inject_control_corr(endpoint, u64::MAX);
4759        let final_rpc = forwarding
4760            .begin_module_control_rpc_for(
4761                "corr-max",
4762                "health.check",
4763                Instant::now() + Duration::from_secs(1),
4764            )
4765            .unwrap();
4766        assert_eq!(final_rpc.corr, u64::MAX);
4767        forwarding
4768            .cancel_module_control_rpc(endpoint, final_rpc.corr)
4769            .unwrap();
4770        assert!(matches!(
4771            forwarding.begin_module_control_rpc_for(
4772                "corr-max",
4773                "health.check",
4774                Instant::now() + Duration::from_secs(1),
4775            ),
4776            Err(ForwardingError::RelayCorrelationExhausted)
4777        ));
4778        assert!(close.try_recv().is_ok());
4779    }
4780
4781    #[test]
4782    fn publication_epoch_controls_delivery_failure_escalation() {
4783        fn setup_successor(
4784            commit_successor: Option<bool>,
4785        ) -> (ForwardingTable, ConnectionId, u16, u32) {
4786            let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4787                route_fixture("escalation");
4788            let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
4789            forwarding
4790                .complete_pending_relay(
4791                    module_connection,
4792                    first.corr,
4793                    RouteBindRelayOutcome::Accepted,
4794                )
4795                .unwrap();
4796            client_rx.try_recv().unwrap();
4797            assert!(matches!(
4798                forwarding
4799                    .release_client_route(client, first.client_channel, first.client_epoch)
4800                    .unwrap(),
4801                RouteRelease::Removed(_)
4802            ));
4803            if let Some(commit_successor) = commit_successor {
4804                forwarding.inject_client_slot_epoch(
4805                    client,
4806                    first.client_channel,
4807                    first.client_epoch,
4808                );
4809                forwarding.inject_module_slot_epoch(
4810                    endpoint,
4811                    first.module_channel,
4812                    first.module_epoch,
4813                );
4814                let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
4815                if commit_successor {
4816                    forwarding
4817                        .complete_pending_relay(
4818                            module_connection,
4819                            successor.corr,
4820                            RouteBindRelayOutcome::Accepted,
4821                        )
4822                        .unwrap();
4823                    client_rx.try_recv().unwrap();
4824                } else {
4825                    forwarding
4826                        .abort_pending_relay(
4827                            endpoint,
4828                            successor.corr,
4829                            RouteBindRelayOutcome::ModuleGone("abort".into()),
4830                        )
4831                        .unwrap();
4832                }
4833            }
4834            (forwarding, client, first.client_channel, first.client_epoch)
4835        }
4836
4837        let probe_sink = FrameSink::new(mpsc::channel(1).0);
4838        let (no_successor, client, channel, epoch) = setup_successor(None);
4839        let mut close = no_successor.register_connection_close(client);
4840        assert!(no_successor
4841            .escalate_client_delivery_failure(
4842                client,
4843                channel,
4844                epoch,
4845                CloseReason::new("delivery", "failed"),
4846                UndeliveredFrame {
4847                    module_id: None,
4848                    sink: &probe_sink,
4849                },
4850            )
4851            .unwrap());
4852        assert!(close.try_recv().is_ok());
4853
4854        let (aborted, client, channel, epoch) = setup_successor(Some(false));
4855        let mut close = aborted.register_connection_close(client);
4856        assert!(aborted
4857            .escalate_client_delivery_failure(
4858                client,
4859                channel,
4860                epoch,
4861                CloseReason::new("delivery", "failed"),
4862                UndeliveredFrame {
4863                    module_id: None,
4864                    sink: &probe_sink,
4865                },
4866            )
4867            .unwrap());
4868        assert!(close.try_recv().is_ok());
4869
4870        let (published, client, channel, epoch) = setup_successor(Some(true));
4871        let mut close = published.register_connection_close(client);
4872        assert!(!published
4873            .escalate_client_delivery_failure(
4874                client,
4875                channel,
4876                epoch,
4877                CloseReason::new("delivery", "stale failure"),
4878                UndeliveredFrame {
4879                    module_id: None,
4880                    sink: &probe_sink,
4881                },
4882            )
4883            .unwrap());
4884        assert!(close.try_recv().is_err());
4885    }
4886
4887    #[test]
4888    fn route_concentration_separates_client_count_from_routes_per_client() {
4889        // The distinction this asserts is the one a bare connection count cannot
4890        // make: two connections holding one route each and one connection
4891        // holding two are the same total, and have opposite causes.
4892        let (forwarding, module_connection, _, client, sink, _client_rx) =
4893            route_fixture("concentration");
4894        assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
4895
4896        for corr in [70_u64, 71] {
4897            let pending =
4898                begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
4899            forwarding
4900                .complete_pending_relay(
4901                    module_connection,
4902                    pending.corr,
4903                    RouteBindRelayOutcome::Accepted,
4904                )
4905                .unwrap();
4906        }
4907
4908        // One connection, two routes — not two connections with a route each.
4909        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
4910        assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
4911    }
4912
4913    #[test]
4914    fn cleanup_and_accepted_resolution_have_one_lock_winner() {
4915        let (forwarding, module_connection, _, client, sink, mut client_rx) =
4916            route_fixture("cleanup-race");
4917        let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
4918        forwarding
4919            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4920            .unwrap();
4921        let released = forwarding.cleanup_connection(client).unwrap();
4922        assert_eq!(released.len(), 1);
4923        let completion = forwarding
4924            .complete_pending_relay(
4925                module_connection,
4926                pending.corr,
4927                RouteBindRelayOutcome::Accepted,
4928            )
4929            .unwrap();
4930        assert!(!completion.settled);
4931        assert!(client_rx.try_recv().is_err());
4932        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4933
4934        let (forwarding, module_connection, _, client, sink, mut client_rx) =
4935            route_fixture("accepted-race");
4936        let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
4937        forwarding
4938            .complete_pending_relay(
4939                module_connection,
4940                pending.corr,
4941                RouteBindRelayOutcome::Accepted,
4942            )
4943            .unwrap();
4944        assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
4945        let released = forwarding.cleanup_connection(client).unwrap();
4946        assert_eq!(released.len(), 1);
4947        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4948    }
4949
4950    #[test]
4951    fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
4952        let (forwarding, module_connection, _, client, sink, mut client_rx) =
4953            route_fixture("drain-gap");
4954        let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
4955        forwarding
4956            .complete_pending_relay(
4957                module_connection,
4958                live.corr,
4959                RouteBindRelayOutcome::Accepted,
4960            )
4961            .unwrap();
4962        client_rx.try_recv().unwrap();
4963        let binding = match forwarding
4964            .lookup_data_route(client, live.client_channel, live.client_epoch)
4965            .unwrap()
4966        {
4967            DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4968            other => panic!("expected live route, got {other:?}"),
4969        };
4970
4971        let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
4972        forwarding
4973            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4974            .unwrap();
4975        let control_rpc = forwarding
4976            .begin_module_control_rpc_for(
4977                "drain-gap",
4978                "health.check",
4979                Instant::now() + Duration::from_secs(1),
4980            )
4981            .unwrap();
4982        let target = forwarding
4983            .begin_module_drain("drain-gap", RouteCloseReason::Reload)
4984            .unwrap()
4985            .unwrap();
4986        assert!(matches!(
4987            control_rpc.receiver.blocking_recv().unwrap(),
4988            ModuleControlRpcOutcome::ModuleGone(_)
4989        ));
4990        assert_eq!(target.abandoned_bindings.len(), 1);
4991        assert!(binding.flow.sem.is_closed());
4992        assert!(
4993            !forwarding
4994                .complete_pending_relay(
4995                    module_connection,
4996                    pending.corr,
4997                    RouteBindRelayOutcome::Accepted,
4998                )
4999                .unwrap()
5000                .settled
5001        );
5002        assert!(matches!(
5003            forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
5004            Err(ForwardingError::ModuleReloading { .. })
5005        ));
5006        let released = forwarding
5007            .release_module_endpoint_routes(target.endpoint)
5008            .unwrap();
5009        assert_eq!(released.len(), 1);
5010        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5011    }
5012
5013    /// A client can be marked closing while its egress is still open: the daemon
5014    /// asks a connection to close (here through the production path, a module
5015    /// frame that its egress refused) and the connection loop tears down a moment
5016    /// later. A route.bind ack that lands inside that window is answered on the
5017    /// MODULE connection's frame handler, so resolving it must not produce an
5018    /// error -- an error there ends the module connection, and that connection
5019    /// carries every other client's routes to the module.
5020    #[test]
5021    fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
5022        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5023            route_fixture("closing-client");
5024
5025        // A published route on this client: escalate_client_delivery_failure only
5026        // marks a connection closing for a route it has already published.
5027        let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
5028        forwarding
5029            .complete_pending_relay(
5030                module_connection,
5031                live.corr,
5032                RouteBindRelayOutcome::Accepted,
5033            )
5034            .unwrap();
5035        client_rx.try_recv().unwrap();
5036
5037        // A second route.open from the same client, relayed and awaiting its ack.
5038        let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
5039        forwarding
5040            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5041            .unwrap();
5042
5043        // The window: closing, but the sink is still open.
5044        assert!(forwarding
5045            .escalate_client_delivery_failure(
5046                client,
5047                live.client_channel,
5048                live.client_epoch,
5049                CloseReason::new(
5050                    "module_to_client_delivery_failed",
5051                    "client egress refused a module frame",
5052                ),
5053                UndeliveredFrame {
5054                    module_id: None,
5055                    sink: &sink,
5056                },
5057            )
5058            .unwrap());
5059        assert!(!sink.is_closed());
5060
5061        let completion = forwarding
5062            .complete_pending_relay(
5063                module_connection,
5064                pending.corr,
5065                RouteBindRelayOutcome::Accepted,
5066            )
5067            .expect("a closing client must not turn a module's ack into an error");
5068
5069        assert!(completion.settled);
5070        let abandoned = completion
5071            .abandoned
5072            .expect("the module must be told to drop the binding it just created");
5073        assert_eq!(abandoned.connection_id, module_connection);
5074        assert_eq!(abandoned.channel, pending.module_channel);
5075        assert_eq!(abandoned.epoch, pending.module_epoch);
5076        assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
5077        assert!(matches!(
5078            pending.receiver.blocking_recv().unwrap(),
5079            RouteBindRelayOutcome::ModuleGone(_)
5080        ));
5081        // No route was published to a client that is on its way out, and the
5082        // reserved handle pair went back.
5083        assert!(client_rx.try_recv().is_err());
5084        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
5085
5086        // The module endpoint is untouched: still live, and still able to take a
5087        // route from another client.
5088        assert!(forwarding
5089            .has_live_module_connection("closing-client")
5090            .unwrap());
5091        let cotenant = ConnectionId::new(201);
5092        let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
5093        let cotenant_route = begin_test_route(
5094            &forwarding,
5095            cotenant,
5096            FrameSink::new(cotenant_tx),
5097            62,
5098            "closing-client",
5099        );
5100        assert_eq!(cotenant_route.endpoint, endpoint);
5101        forwarding
5102            .complete_pending_relay(
5103                module_connection,
5104                cotenant_route.corr,
5105                RouteBindRelayOutcome::Accepted,
5106            )
5107            .unwrap();
5108        assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
5109        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5110    }
5111
5112    #[test]
5113    fn pending_route_permit_is_released_on_rejection_and_abort() {
5114        let forwarding = ForwardingTable::default();
5115        let module_connection = ConnectionId::new(300);
5116        let client = ConnectionId::new(301);
5117        let (module_tx, _module_rx) = mpsc::channel(1);
5118        let endpoint = forwarding
5119            .register_module_connection(
5120                module_connection,
5121                "permit".into(),
5122                2,
5123                Concurrency::ModuleManaged,
5124                FrameSink::new(module_tx),
5125            )
5126            .unwrap();
5127        let (client_tx, mut client_rx) = mpsc::channel(1);
5128        let sink = FrameSink::new(client_tx);
5129        let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
5130        assert!(sink.try_send(test_ping(999)).is_err());
5131        forwarding
5132            .complete_pending_relay(
5133                module_connection,
5134                rejected.corr,
5135                RouteBindRelayOutcome::Rejected(ErrorBody {
5136                    code: "no".into(),
5137                    message: "rejected".into(),
5138                    detail: None,
5139                }),
5140            )
5141            .unwrap();
5142        sink.try_send(test_ping(1000)).unwrap();
5143        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
5144
5145        let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
5146        assert!(sink.try_send(test_ping(1001)).is_err());
5147        forwarding
5148            .abort_pending_relay(
5149                endpoint,
5150                aborted.corr,
5151                RouteBindRelayOutcome::ModuleGone("abort".into()),
5152            )
5153            .unwrap();
5154        sink.try_send(test_ping(1002)).unwrap();
5155        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
5156
5157        let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
5158        forwarding
5159            .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
5160            .unwrap();
5161        drop(client_rx);
5162        let completion = forwarding
5163            .complete_pending_relay(
5164                module_connection,
5165                receiver_closed.corr,
5166                RouteBindRelayOutcome::Accepted,
5167            )
5168            .unwrap();
5169        assert!(completion.abandoned.is_some());
5170        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5171    }
5172
5173    /// Every connection teardown passes through `cleanup_connection`, and
5174    /// connection ids come from a monotonic counter that never hands an id out
5175    /// twice. If the closing mark survives teardown, the set grows by one entry
5176    /// per connection for the life of the daemon -- the self-watchdog alone
5177    /// reconnects once a minute.
5178    #[test]
5179    fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
5180        let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
5181            route_fixture("closing-set-leak");
5182
5183        const CONNECTIONS: u64 = 32;
5184        for index in 0..CONNECTIONS {
5185            let client = ConnectionId::new(1000 + index);
5186            let (client_tx, _client_rx) = mpsc::channel(8);
5187            let route = begin_test_route(
5188                &forwarding,
5189                client,
5190                FrameSink::new(client_tx),
5191                index + 1,
5192                "closing-set-leak",
5193            );
5194            forwarding
5195                .complete_pending_relay(
5196                    module_connection,
5197                    route.corr,
5198                    RouteBindRelayOutcome::Accepted,
5199                )
5200                .unwrap();
5201            forwarding.cleanup_connection(client).unwrap();
5202        }
5203        forwarding.cleanup_connection(module_connection).unwrap();
5204
5205        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5206    }
5207
5208    /// The closing mark exists to refuse new work for a connection that is on
5209    /// its way out but whose teardown has not run yet: the daemon asks the
5210    /// connection loop to end, and only when the loop reacts does
5211    /// `cleanup_connection` strip the connection's state. Inside that window an
5212    /// operation for the dying connection must still be refused; only after
5213    /// cleanup completes may the mark go.
5214    #[test]
5215    fn closing_connection_is_refused_new_work_until_cleanup_completes() {
5216        let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
5217            route_fixture("closing-gate");
5218
5219        // A published route: escalate_client_delivery_failure only marks a
5220        // connection closing for a route it has already published.
5221        let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
5222        forwarding
5223            .complete_pending_relay(
5224                module_connection,
5225                live.corr,
5226                RouteBindRelayOutcome::Accepted,
5227            )
5228            .unwrap();
5229        client_rx.try_recv().unwrap();
5230
5231        // Mark the connection closing through the production path without
5232        // running teardown, pinning the window open.
5233        assert!(forwarding
5234            .escalate_client_delivery_failure(
5235                client,
5236                live.client_channel,
5237                live.client_epoch,
5238                CloseReason::new(
5239                    "module_to_client_delivery_failed",
5240                    "client egress refused a module frame",
5241                ),
5242                UndeliveredFrame {
5243                    module_id: None,
5244                    sink: &sink,
5245                },
5246            )
5247            .unwrap());
5248        assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
5249
5250        // A late route.open for the closing client is refused ...
5251        assert!(matches!(
5252            forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
5253            Err(ForwardingError::ConnectionClosing { connection_id })
5254                if connection_id == client
5255        ));
5256        // ... and so is a late attempt to register the connection as a module.
5257        let (late_tx, _late_rx) = mpsc::channel(1);
5258        assert!(matches!(
5259            forwarding.register_module_connection(
5260                client,
5261                "late-module".into(),
5262                2,
5263                Concurrency::ModuleManaged,
5264                FrameSink::new(late_tx),
5265            ),
5266            Err(ForwardingError::ConnectionClosing { connection_id })
5267                if connection_id == client
5268        ));
5269
5270        // Teardown is the point that lifts the mark: it has just removed every
5271        // per-connection entry under the same lock, so the gate has nothing
5272        // left to protect for this id.
5273        forwarding.cleanup_connection(client).unwrap();
5274        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5275    }
5276}
5277
5278/// Blue/green swap slots: a candidate registered beside the active endpoint,
5279/// promoted by `cutover_candidate`, with the old incumbent drained by endpoint.
5280#[cfg(test)]
5281mod swap_slot_tests {
5282    use std::time::Duration;
5283
5284    use super::*;
5285    use tokio::sync::mpsc;
5286
5287    const MODULE_ID: &str = "swapped";
5288
5289    struct SwapFixture {
5290        forwarding: ForwardingTable,
5291        incumbent_connection: ConnectionId,
5292        incumbent: ModuleEndpointId,
5293        candidate_connection: ConnectionId,
5294        candidate: ModuleEndpointId,
5295        _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
5296    }
5297
5298    fn swap_fixture() -> SwapFixture {
5299        let forwarding = ForwardingTable::default();
5300        let incumbent_connection = ConnectionId::new(100);
5301        let candidate_connection = ConnectionId::new(110);
5302        let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
5303        let incumbent = forwarding
5304            .register_module_connection(
5305                incumbent_connection,
5306                MODULE_ID.to_string(),
5307                2,
5308                Concurrency::ModuleManaged,
5309                FrameSink::new(incumbent_tx),
5310            )
5311            .unwrap();
5312        let (candidate_tx, candidate_rx) = mpsc::channel(8);
5313        let candidate = forwarding
5314            .register_candidate_module_connection(
5315                candidate_connection,
5316                MODULE_ID.to_string(),
5317                2,
5318                Concurrency::ModuleManaged,
5319                FrameSink::new(candidate_tx),
5320            )
5321            .unwrap();
5322        SwapFixture {
5323            forwarding,
5324            incumbent_connection,
5325            incumbent,
5326            candidate_connection,
5327            candidate,
5328            _module_rxs: vec![incumbent_rx, candidate_rx],
5329        }
5330    }
5331
5332    fn client(
5333        raw: u64,
5334    ) -> (
5335        ConnectionId,
5336        FrameSink,
5337        mpsc::Receiver<crate::router::OutboundFrame>,
5338    ) {
5339        let (tx, rx) = mpsc::channel(8);
5340        (ConnectionId::new(raw), FrameSink::new(tx), rx)
5341    }
5342
5343    fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
5344        forwarding
5345            .read_inner()
5346            .unwrap()
5347            .client_to_module
5348            .values()
5349            .map(|route| route.module_endpoint)
5350            .collect()
5351    }
5352
5353    #[test]
5354    fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
5355        let fixture = swap_fixture();
5356        let forwarding = &fixture.forwarding;
5357        assert_ne!(fixture.incumbent, fixture.candidate);
5358
5359        // Every by-id consumer still resolves the incumbent.
5360        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5361        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5362        let (client_connection, client_sink, _client_rx) = client(200);
5363        let pending = forwarding
5364            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5365            .unwrap();
5366        assert_eq!(pending.endpoint, fixture.incumbent);
5367        let rpc = forwarding
5368            .begin_module_control_rpc_for(
5369                MODULE_ID,
5370                "health.check",
5371                Instant::now() + Duration::from_secs(1),
5372            )
5373            .unwrap();
5374        assert_eq!(rpc.endpoint, fixture.incumbent);
5375        let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
5376        assert_eq!(census.len(), 1, "the census lists one endpoint per id");
5377
5378        // Connection-keyed lookups see the candidate, so its own frames resolve.
5379        assert_eq!(
5380            forwarding
5381                .module_endpoint_for_connection(fixture.candidate_connection)
5382                .unwrap(),
5383            Some(fixture.candidate)
5384        );
5385        assert_eq!(
5386            forwarding
5387                .module_id_for_connection(fixture.candidate_connection)
5388                .unwrap()
5389                .as_deref(),
5390            Some(MODULE_ID)
5391        );
5392
5393        // One candidate per id.
5394        let (other_tx, _other_rx) = mpsc::channel(1);
5395        assert_eq!(
5396            forwarding.register_candidate_module_connection(
5397                ConnectionId::new(120),
5398                MODULE_ID.to_string(),
5399                2,
5400                Concurrency::ModuleManaged,
5401                FrameSink::new(other_tx),
5402            ),
5403            Err(ForwardingError::CandidateSlotOccupied {
5404                module_id: MODULE_ID.to_string()
5405            })
5406        );
5407    }
5408
5409    /// The cutover linearization point. A relay reserved on the incumbent before
5410    /// cutover must never become a route on the incumbent, and every relay
5411    /// reserved after it must land on the promoted candidate.
5412    #[test]
5413    fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
5414        let fixture = swap_fixture();
5415        let forwarding = &fixture.forwarding;
5416        let (early_client, early_sink, _early_rx) = client(200);
5417        let mut early = forwarding
5418            .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
5419            .unwrap();
5420        assert_eq!(early.endpoint, fixture.incumbent);
5421        assert_eq!(
5422            forwarding
5423                .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
5424                .unwrap(),
5425            RouteBindRelayMark::Marked
5426        );
5427
5428        let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5429        assert_eq!(
5430            cutover,
5431            ForwardingCutover {
5432                promoted: fixture.candidate,
5433                incumbent: Some(fixture.incumbent),
5434            }
5435        );
5436
5437        // A route.open reserved after cutover goes to the promoted candidate.
5438        let (late_client, late_sink, _late_rx) = client(201);
5439        let late = forwarding
5440            .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
5441            .unwrap();
5442        assert_eq!(
5443            late.endpoint, fixture.candidate,
5444            "a route.open after cutover was reserved on the incumbent"
5445        );
5446
5447        // The incumbent acks the early relay after cutover.
5448        let completion = forwarding
5449            .complete_pending_relay(
5450                fixture.incumbent_connection,
5451                early.corr,
5452                RouteBindRelayOutcome::Accepted,
5453            )
5454            .expect("a superseded endpoint's ack is not an error on its connection");
5455        assert!(completion.settled);
5456        assert!(
5457            !committed_endpoints(forwarding).contains(&fixture.incumbent),
5458            "a relay reserved before cutover committed a route on the incumbent"
5459        );
5460        let goodbye = completion
5461            .abandoned
5462            .expect("the incumbent is told to drop the binding it just created");
5463        assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
5464        assert_eq!(goodbye.channel, early.module_channel);
5465        assert_eq!(goodbye.epoch, early.module_epoch);
5466        assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
5467        match early.receiver.try_recv() {
5468            Ok(RouteBindRelayOutcome::Rejected(body)) => assert_eq!(body.code, "module_reloading"),
5469            other => panic!("expected a retryable module_reloading answer, got {other:?}"),
5470        }
5471        assert!(matches!(
5472            forwarding
5473                .lookup_data_route(early_client, early.client_channel, early.client_epoch)
5474                .unwrap(),
5475            DataRoute::Client(DataRouteState::Absent)
5476        ));
5477
5478        // The late relay commits on the candidate; only its pair stays reserved
5479        // until then.
5480        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
5481        forwarding
5482            .complete_pending_relay(
5483                fixture.candidate_connection,
5484                late.corr,
5485                RouteBindRelayOutcome::Accepted,
5486            )
5487            .unwrap();
5488        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5489        assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
5490    }
5491
5492    #[test]
5493    fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
5494        let fixture = swap_fixture();
5495        let forwarding = &fixture.forwarding;
5496        // One bound route and one in-flight relay on the incumbent.
5497        let (bound_client, bound_sink, _bound_rx) = client(200);
5498        let bound = forwarding
5499            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5500            .unwrap();
5501        forwarding
5502            .complete_pending_relay(
5503                fixture.incumbent_connection,
5504                bound.corr,
5505                RouteBindRelayOutcome::Accepted,
5506            )
5507            .unwrap();
5508        let (pending_client, pending_sink, _pending_rx) = client(201);
5509        let mut in_flight = forwarding
5510            .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
5511            .unwrap();
5512        forwarding
5513            .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
5514            .unwrap();
5515
5516        let incumbent = forwarding
5517            .cutover_candidate(MODULE_ID)
5518            .unwrap()
5519            .unwrap()
5520            .incumbent
5521            .unwrap();
5522        let target = forwarding
5523            .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
5524            .unwrap()
5525            .expect("the superseded incumbent is still registered");
5526
5527        assert_eq!(target.endpoint, fixture.incumbent);
5528        assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
5529        assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
5530        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5531        assert_eq!(target.abandoned_bindings.len(), 1);
5532        assert_eq!(
5533            target.abandoned_bindings[0].channel,
5534            in_flight.module_channel
5535        );
5536        assert!(matches!(
5537            in_flight.receiver.try_recv(),
5538            Ok(RouteBindRelayOutcome::Rejected(body)) if body.code == "module_reloading"
5539        ));
5540        assert_eq!(
5541            forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
5542            1,
5543            "the incumbent's bound route stays until its drain finishes"
5544        );
5545
5546        let (next_client, next_sink, _next_rx) = client(202);
5547        let next = forwarding
5548            .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
5549            .expect("the promoted candidate keeps accepting routes");
5550        assert_eq!(next.endpoint, fixture.candidate);
5551    }
5552
5553    /// An endpoint replaced WITHOUT a promotion (a successor registered over it
5554    /// as an ordinary active HELLO) is stale, not superseded, and its ack still
5555    /// fails the acking connection exactly as it did before swap slots existed:
5556    /// `StaleModuleEndpoint`, the reservation indexes already stripped, and the
5557    /// waiting client's sender dropped unanswered.
5558    #[test]
5559    fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
5560        let forwarding = ForwardingTable::default();
5561        let first_connection = ConnectionId::new(70);
5562        let (first_tx, _first_rx) = mpsc::channel(8);
5563        forwarding
5564            .register_module_connection(
5565                first_connection,
5566                MODULE_ID.to_string(),
5567                2,
5568                Concurrency::ModuleManaged,
5569                FrameSink::new(first_tx),
5570            )
5571            .unwrap();
5572        let (client_connection, client_sink, _client_rx) = client(200);
5573        let mut pending = forwarding
5574            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5575            .unwrap();
5576        let (second_tx, _second_rx) = mpsc::channel(8);
5577        forwarding
5578            .register_module_connection(
5579                ConnectionId::new(80),
5580                MODULE_ID.to_string(),
5581                2,
5582                Concurrency::ModuleManaged,
5583                FrameSink::new(second_tx),
5584            )
5585            .unwrap();
5586
5587        assert_eq!(
5588            forwarding
5589                .complete_pending_relay(
5590                    first_connection,
5591                    pending.corr,
5592                    RouteBindRelayOutcome::Accepted
5593                )
5594                .unwrap_err(),
5595            ForwardingError::StaleModuleEndpoint
5596        );
5597        assert!(committed_endpoints(&forwarding).is_empty());
5598        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5599        assert!(matches!(
5600            pending.receiver.try_recv(),
5601            Err(oneshot::error::TryRecvError::Closed)
5602        ));
5603    }
5604
5605    #[test]
5606    fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
5607        // A candidate whose connection drops leaves the incumbent routable.
5608        let fixture = swap_fixture();
5609        let forwarding = &fixture.forwarding;
5610        assert!(forwarding
5611            .cleanup_connection(fixture.candidate_connection)
5612            .unwrap()
5613            .is_empty());
5614        assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
5615        let (client_connection, client_sink, _client_rx) = client(200);
5616        assert_eq!(
5617            forwarding
5618                .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5619                .unwrap()
5620                .endpoint,
5621            fixture.incumbent
5622        );
5623
5624        // After a cutover, the incumbent's teardown releases its own routes and
5625        // leaves the promoted candidate in place.
5626        let fixture = swap_fixture();
5627        let forwarding = &fixture.forwarding;
5628        let (bound_client, bound_sink, _bound_rx) = client(200);
5629        let bound = forwarding
5630            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5631            .unwrap();
5632        forwarding
5633            .complete_pending_relay(
5634                fixture.incumbent_connection,
5635                bound.corr,
5636                RouteBindRelayOutcome::Accepted,
5637            )
5638            .unwrap();
5639        forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5640        let released = forwarding
5641            .cleanup_connection(fixture.incumbent_connection)
5642            .unwrap();
5643        assert_eq!(released.len(), 1);
5644        assert_eq!(released[0].connection_id, bound_client);
5645        assert!(forwarding
5646            .read_inner()
5647            .unwrap()
5648            .superseded_endpoints
5649            .is_empty());
5650        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5651        let (next_client, next_sink, _next_rx) = client(201);
5652        assert_eq!(
5653            forwarding
5654                .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
5655                .unwrap()
5656                .endpoint,
5657            fixture.candidate
5658        );
5659    }
5660}