Skip to main content

subc_daemon/
forwarding.rs

1use std::{
2    collections::{BTreeMap, HashMap, HashSet},
3    error::Error,
4    fmt,
5    sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
6    time::Duration,
7};
8
9use subc_control::{ClientControlResponse, RouteCloseReason};
10use subc_protocol::{
11    manifest::Concurrency,
12    session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
13    ErrorBody, Flags, FrameType, Principal, Priority,
14};
15use tokio::sync::{oneshot, Semaphore};
16use tokio::time::Instant;
17use tracing::{debug, info, warn};
18
19use crate::{
20    control::{RouteBindBreakers, RouteBindConcurrency},
21    observability::DaemonCounters,
22    registry::ConnectionId,
23    router::FrameSink,
24    scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
25    Frame, ProjectRootId,
26};
27
28/// Default per-channel request-credit window for modules that schedule internally.
29const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
30
31/// High per-channel cap for stateless modules; this is an OOM guard, not scheduling policy.
32const STATELESS_PARALLEL_WINDOW: usize = 1024;
33
34/// A stopped probe cycle cannot retain its last unanswered correlation forever.
35/// Active endpoints replace the tombstone on their next serial health probe;
36/// this backstop covers endpoints that stop probing altogether.
37const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
38
39/// Module connection identity used in forwarding keys.
40///
41/// The generation is bumped every time a module connection is registered so a future restart cannot
42/// accidentally reuse a stale `(connection_id, route_channel)` binding.
43#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
44pub struct ModuleEndpointId {
45    pub connection_id: ConnectionId,
46    pub generation: u64,
47}
48
49/// Client-local route key. A route channel is unique only within one client connection.
50#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
51pub(crate) struct ClientRouteKey {
52    pub connection_id: ConnectionId,
53    pub channel: u16,
54}
55
56/// Module-local route key. A route channel is unique only within one live module endpoint.
57#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
58pub(crate) struct ModuleRouteKey {
59    pub endpoint: ModuleEndpointId,
60    pub channel: u16,
61}
62
63#[derive(Debug)]
64pub(crate) struct RouteBinding {
65    pub client_connection_id: ConnectionId,
66    pub client_sink: FrameSink,
67    pub client_negotiated_ver: u8,
68    pub client_channel: u16,
69    pub client_epoch: u32,
70    pub module_id: String,
71    pub module_endpoint: ModuleEndpointId,
72    pub module_sink: FrameSink,
73    pub module_negotiated_ver: u8,
74    pub module_channel: u16,
75    pub module_epoch: u32,
76    pub principal: Principal,
77    pub project_root: Option<ProjectRootId>,
78    pub bound_at: Instant,
79    pub flow: Arc<ChannelFlow>,
80    /// The scope the route was admitted under, with the tag its bind was
81    /// stamped at. A scope change finds the routes to close by it.
82    pub scope: Option<BoundScope>,
83}
84
85#[derive(Debug, Clone)]
86pub(crate) enum DataRoute {
87    Client(DataRouteState),
88    Module(DataRouteState),
89}
90
91#[derive(Debug, Clone)]
92pub(crate) enum DataRouteState {
93    Bound(Arc<RouteBinding>),
94    Reserved,
95    EpochMismatch,
96    Absent,
97}
98
99/// Which kind of peer a route GOODBYE is being delivered to. This decides what
100/// happens when the GOODBYE cannot be enqueued (egress full/closed):
101/// - `Client`: escalate to closing that client connection (a socket close is a
102///   stronger teardown signal, and a full client egress means it is the slow
103///   client we would drop anyway).
104/// - `Module`: never close; deliver late instead (see
105///   [`send_module_route_goodbye`]), and drop only if the module still has no
106///   room after [`LATE_MODULE_GOODBYE_DEADLINE`]. A client-disconnect notifies the
107///   SHARED module that one client's route is gone; closing the module on its
108///   egress backpressure would tear down every co-tenant client (the exact
109///   cross-tenant blast radius this never-close rule exists to prevent — observed when a
110///   flooding dead client filled BOTH its own and the module's egress, so its
111///   route-gone GOODBYE to the module failed and closed the shared connection).
112///   subc has already removed the route from its forwarding state and drops
113///   stale module frames for the released channel (see router.rs), so subc's
114///   own routing is correct. The residual: under SUSTAINED module-egress
115///   backpressure a module-targeted route-gone notification can be lost, which a
116///   module using it for client-refcounting (e.g. AFT's session accounting)
117///   would miss. This is INTENTIONALLY ACCEPTED, not a gap. A consuming module
118///   must bound stale bindings with its own idle-activity reaper (last-touched
119///   TTL) independent of route-gone signals — AFT does exactly this, so a lost
120///   GOODBYE degrades to "the
121///   binding stays warm until its idle TTL" (bounded wasted resources), never an
122///   unbounded leak; disk-durable replay is unaffected. A dedicated reliable
123///   module control lane was evaluated and deliberately NOT
124///   built: it would add starvation-avoidance machinery to the thin core for a
125///   bounded warm-resource window that is not a correctness issue. never-close
126///   is the invariant that matters here.
127#[derive(Debug, Clone, Copy, PartialEq, Eq)]
128pub(crate) enum GoodbyeTargetKind {
129    Client,
130    Module,
131}
132
133#[derive(Debug, Clone)]
134pub(crate) struct GoodbyeTarget {
135    pub connection_id: ConnectionId,
136    pub sink: FrameSink,
137    pub negotiated_ver: u8,
138    pub channel: u16,
139    pub epoch: u32,
140    pub kind: GoodbyeTargetKind,
141    /// The module on the other end of the route: for a module-targeted relay
142    /// the receiving module (attributing a dropped relay), for a client target
143    /// the module whose route is going away (naming it when an undeliverable
144    /// relay closes the client).
145    pub module_id: Option<String>,
146}
147
148/// The frame a client connection's egress queue refused, for the diagnosis
149/// logged when that refusal closes the connection.
150#[derive(Debug, Clone, Copy)]
151pub(crate) struct UndeliveredFrame<'a> {
152    /// The module on the other end of the route the frame belonged to, when known.
153    pub module_id: Option<&'a str>,
154    /// The refusing connection's sink, read for what its queue held.
155    pub sink: &'a FrameSink,
156}
157
158/// How a route's principal appears in logs: `direct`, or `reserved:<module>`.
159pub(crate) fn principal_label(principal: &Principal) -> String {
160    match principal {
161        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
162        Principal::Direct => "direct".to_string(),
163        other => format!("{other:?}"),
164    }
165}
166
167/// The distinct principals of the routes bound on one client connection,
168/// sorted and comma-joined, or `none` when it has no bound route.
169fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
170    let labels = inner
171        .client_to_module
172        .iter()
173        .filter(|(key, _)| key.connection_id == connection_id)
174        .map(|(_, route)| principal_label(&route.principal))
175        .collect::<std::collections::BTreeSet<_>>();
176    if labels.is_empty() {
177        "none".to_string()
178    } else {
179        labels.into_iter().collect::<Vec<_>>().join(",")
180    }
181}
182
183impl GoodbyeTarget {
184    /// True only when an undeliverable GOODBYE should escalate to closing the
185    /// target connection. Never escalate for module recipients.
186    pub(crate) fn close_on_delivery_failure(&self) -> bool {
187        matches!(self.kind, GoodbyeTargetKind::Client)
188    }
189}
190
191/// How long a route GOODBYE that a module's egress queue refused keeps waiting
192/// for room before it is given up and counted as dropped.
193///
194/// A module that stops reading for a moment (a GC pause, a long synchronous
195/// handler, a reconnect herd it is still working through) should still learn
196/// that the route is gone, because a module that never hears the GOODBYE keeps
197/// the route and keeps sending on it for as long as its connection lives. The
198/// value is the default module drain timeout: the daemon already treats that as
199/// the longest it is reasonable to wait on a module that is busy but alive, and
200/// a module that cannot free 21 bytes of egress in that time is wedged rather
201/// than slow. Resolving each module's own configured drain timeout here would
202/// need a registry lookup on a path that holds only the module's sink.
203pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
204
205/// Send a route GOODBYE to a module without ever closing the module's shared
206/// connection, which would sever every other route it serves.
207///
208/// The frame is enqueued at once when the module's egress queue has room. When
209/// it does not, a detached task waits for room (bounded by
210/// [`LATE_MODULE_GOODBYE_DEADLINE`]) and sends it then. Arriving late is safe:
211/// the daemon has already released the route, and module SDKs tear a route
212/// down only when both the channel AND the epoch match the route installed on
213/// that channel, so a late GOODBYE for a channel since reused at a newer epoch
214/// is ignored. The GOODBYE is counted in `goodbye_relay_module_dropped` only
215/// when the connection is closed or the deadline passes.
216pub(crate) fn send_module_route_goodbye(
217    counters: &DaemonCounters,
218    sink: &FrameSink,
219    frame: Frame,
220    module_id: Option<&str>,
221    context: &'static str,
222) {
223    let channel = frame.header.channel;
224    let epoch = frame.header.epoch;
225    let Err(err) = sink.try_send(frame.clone()) else {
226        return;
227    };
228    // Waiting is pointless once the module's writer is gone, and impossible
229    // outside a Tokio runtime (cleanup that runs from a destructor at shutdown).
230    let runtime = match tokio::runtime::Handle::try_current() {
231        Ok(runtime) if !sink.is_closed() => runtime,
232        _ => {
233            counters.increment_goodbye_relay_module_dropped(module_id);
234            warn!(
235            module_id = module_id.unwrap_or("unknown"),
236            route_channel = channel,
237            route_epoch = epoch,
238            error = %err,
239            context,
240            "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
241            );
242            return;
243        }
244    };
245    debug!(
246        module_id = module_id.unwrap_or("unknown"),
247        route_channel = channel,
248        route_epoch = epoch,
249        error = %err,
250        context,
251        "module egress queue refused route GOODBYE; delivering it once the module frees room"
252    );
253    let counters = counters.clone();
254    let sink = sink.clone();
255    let module_id = module_id.map(str::to_string);
256    runtime.spawn(async move {
257        let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
258        let why = match outcome {
259            Ok(Ok(())) => {
260                debug!(
261                    module_id = module_id.as_deref().unwrap_or("unknown"),
262                    route_channel = channel,
263                    route_epoch = epoch,
264                    context,
265                    "late route GOODBYE delivered to module"
266                );
267                return;
268            }
269            Ok(Err(err)) => err.to_string(),
270            Err(_) => format!(
271                "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
272            ),
273        };
274        counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
275        warn!(
276            module_id = module_id.as_deref().unwrap_or("unknown"),
277            route_channel = channel,
278            route_epoch = epoch,
279            error = %why,
280            context,
281            "route GOODBYE to module dropped under backpressure; not closing shared module connection"
282        );
283    });
284}
285
286/// One route currently served by a module endpoint.
287///
288/// `goodbye_target` is deliberately retained alongside the census projection so
289/// a draining caller can address exactly the same route set that this read
290/// reports, without a second forwarding-table pass.
291#[derive(Debug, Clone)]
292pub(crate) struct EndpointRoute {
293    pub goodbye_target: GoodbyeTarget,
294    pub principal: Principal,
295    pub bound_at: Instant,
296    pub draining: bool,
297    /// WHY the endpoint is draining, when it is. Carried per-route so the
298    /// census can answer "closing because of what" without a second lookup;
299    /// `None` exactly when `draining` is false (one source: the drain map).
300    pub drain_reason: Option<RouteCloseReason>,
301}
302
303/// One live route a scope change closed: both ends to send GOODBYE to, and the
304/// reason for the client's `route.closed`.
305#[derive(Debug, Clone)]
306pub(crate) struct ScopeDrainedRoute {
307    pub reason: RouteCloseReason,
308    pub scope: BoundScope,
309    pub module_id: String,
310    pub client: GoodbyeTarget,
311    pub module: GoodbyeTarget,
312}
313
314#[derive(Debug)]
315pub(crate) struct PendingRouteBindRelay {
316    pub endpoint: ModuleEndpointId,
317    pub module_sink: FrameSink,
318    pub negotiated_ver: u8,
319    pub client_channel: u16,
320    pub client_epoch: u32,
321    pub module_channel: u16,
322    pub module_epoch: u32,
323    pub corr: u64,
324    pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
325}
326
327#[derive(Debug, Clone)]
328pub(crate) struct ModuleDrainTarget {
329    pub endpoint: ModuleEndpointId,
330    pub sink: FrameSink,
331    pub negotiated_ver: u8,
332    pub abandoned_bindings: Vec<GoodbyeTarget>,
333    pub excluded_subscriptions: u32,
334}
335
336/// One registered module connection, as [`ForwardingTable::module_connections`]
337/// reports it: enough to send it a frame and then close it.
338#[cfg(unix)]
339#[derive(Debug, Clone)]
340pub(crate) struct ModuleConnectionTarget {
341    pub module_id: String,
342    pub endpoint: ModuleEndpointId,
343    pub sink: FrameSink,
344    pub negotiated_ver: u8,
345}
346
347#[derive(Debug, Clone)]
348pub(crate) enum RouteBindRelayOutcome {
349    Accepted,
350    Rejected(ErrorBody),
351    ModuleGone(String),
352}
353
354/// What [`ForwardingTable::cutover_candidate`] did.
355#[derive(Debug, Clone, Copy, PartialEq, Eq)]
356pub(crate) struct ForwardingCutover {
357    /// The endpoint now in the active slot (the former candidate).
358    pub promoted: ModuleEndpointId,
359    /// The endpoint demoted out of the active slot, to be drained by endpoint.
360    /// `None` when the incumbent's connection was already gone.
361    pub incumbent: Option<ModuleEndpointId>,
362}
363
364/// What tearing down one connection released.
365#[derive(Debug)]
366pub(crate) struct ConnectionCleanup {
367    /// Routes whose other end must be sent GOODBYE.
368    pub released: Vec<GoodbyeTarget>,
369    /// Pending route.bind relays to the closed module that were aborted: opens
370    /// that had not been bound yet. Zero when a client connection closed.
371    pub abandoned_relays: u32,
372}
373
374#[derive(Debug, Clone)]
375pub(crate) struct PendingRelayCompletion {
376    pub settled: bool,
377    pub abandoned: Option<GoodbyeTarget>,
378}
379
380#[derive(Debug)]
381pub(crate) struct PendingModuleControlRpc {
382    pub endpoint: ModuleEndpointId,
383    pub module_sink: FrameSink,
384    pub negotiated_ver: u8,
385    pub corr: u64,
386    pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
387}
388
389#[derive(Debug, Clone)]
390pub(crate) enum ModuleControlRpcOutcome {
391    Response(ModuleControlResponse),
392    Rejected(ErrorBody),
393    ModuleGone(String),
394    MalformedResponse(String),
395    UnexpectedOp { expected: String, actual: String },
396    DeadlineElapsed,
397}
398
399#[derive(Debug, Clone, PartialEq, Eq)]
400pub(crate) enum ModuleControlRpcCompletion {
401    Unknown,
402    Settled,
403    LateHealthAnswer {
404        module_id: String,
405        latency: Duration,
406    },
407}
408
409#[derive(Debug)]
410struct PendingModuleControlRpcEntry {
411    expected_op: String,
412    deadline: Instant,
413    health_probe_started_at: Option<Instant>,
414    sender: oneshot::Sender<ModuleControlRpcOutcome>,
415}
416
417#[derive(Debug)]
418struct HealthProbeTombstone {
419    expected_op: String,
420    module_id: String,
421    probe_started_at: Instant,
422    expires_at: Instant,
423}
424
425#[derive(Debug, Clone)]
426struct RouteReservation {
427    client_key: ClientRouteKey,
428    module_key: ModuleRouteKey,
429    client_epoch: u32,
430    module_epoch: u32,
431    project_root: Option<ProjectRootId>,
432}
433
434#[derive(Debug)]
435struct PendingRouteBindRelayEntry {
436    reservation: RouteReservation,
437    client_sink: FrameSink,
438    client_negotiated_ver: u8,
439    client_permit: crate::router::EgressPermit,
440    route_open_frame: Frame,
441    principal: Principal,
442    /// The scope admitted at `route.open`, with the tag captured then. Commit
443    /// compares it with the published tag.
444    scope: Option<BoundScope>,
445    deadline: Instant,
446    /// Set once `route.open` has finished putting the route.bind on the
447    /// module's egress queue. A path that drops this entry WITHOUT a module
448    /// reply (client teardown, module drain, the open's own abort) sends the
449    /// module a GOODBYE for the bind only when this is set, because only then
450    /// is the GOODBYE guaranteed to queue behind the bind. When it is not set,
451    /// the removal is recorded in `relays_released_before_enqueue` instead and
452    /// `route.open` sends that GOODBYE itself once its send has completed.
453    ///
454    /// A module reply does not consult it: an ack proves the module received
455    /// the bind, so the reply path decides about the GOODBYE on its own.
456    relay_enqueued: bool,
457    sender: oneshot::Sender<RouteBindRelayOutcome>,
458}
459
460/// What `mark_route_bind_relay_enqueued` found once `route.open` had put the
461/// route.bind on the module's queue.
462#[derive(Debug, Clone, Copy, PartialEq, Eq)]
463pub(crate) enum RouteBindRelayMark {
464    /// The relay is still pending; later removals handle their own GOODBYE.
465    Marked,
466    /// The module already answered and the reply path settled the relay:
467    /// it committed the route, or it refused it and sent any GOODBYE the
468    /// module needed. `route.open` must send nothing.
469    SettledByReply,
470    /// The relay was dropped without a module reply while the bind was still
471    /// being queued, so nothing has told the module to drop the binding it
472    /// is about to create. `route.open` must send that GOODBYE.
473    ReleasedWithoutReply,
474}
475
476#[derive(Debug, Clone)]
477pub(crate) enum RouteRelease {
478    Removed(GoodbyeTarget),
479    Stale,
480    Absent,
481}
482
483#[derive(Debug, Clone)]
484pub(crate) enum RoutePollSnapshot {
485    Bound {
486        module_id: String,
487        status: Option<String>,
488    },
489    Absent,
490}
491
492#[derive(Debug, Clone)]
493struct ModuleConnection {
494    endpoint: ModuleEndpointId,
495    sink: FrameSink,
496    negotiated_ver: u8,
497    concurrency: Concurrency,
498}
499
500#[derive(Debug, Default)]
501struct ForwardingInner {
502    operator_confirms: Arc<crate::operator_confirm::OperatorConfirms>,
503    daemon_draining: bool,
504    /// Whether forwarding reached quiescence during daemon shutdown, for each
505    /// active endpoint present when shutdown began. A pending wait is false
506    /// because it has not proved quiescence. Endpoint generations prevent a
507    /// result from describing a replacement connection with the same module id.
508    daemon_drain_results: HashMap<ModuleEndpointId, bool>,
509    /// The ACTIVE slot: the one endpoint per module id that routing resolves.
510    /// Every by-id lookup (relay reservation, drain-by-id, liveness, census,
511    /// live roots, module-control RPCs) reads this map and nothing else.
512    modules_by_id: HashMap<String, ModuleConnection>,
513    /// The CANDIDATE slot of a blue/green swap: a second process registered
514    /// under an id that already has an active endpoint. It has a full endpoint
515    /// identity (so its own connection can be looked up and torn down) but no
516    /// by-id lookup sees it, so nothing is routed to it until `cutover_candidate`
517    /// promotes it.
518    candidates_by_id: HashMap<String, ModuleConnection>,
519    /// Former active endpoints demoted by `cutover_candidate`, until their
520    /// connection is removed. Membership is what distinguishes an endpoint that
521    /// was SUPERSEDED by a promotion (still alive, still carrying bound routes,
522    /// about to be drained) from one that is merely STALE (replaced some other
523    /// way, which is treated as a fault on the acking connection).
524    superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
525    endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
526    module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
527    /// Endpoints mid-drain, keyed to the reason the drain was begun with. The
528    /// value serves the census ("draining because restart"); membership alone
529    /// still answers every admission-gate check.
530    draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
531    closing_connections: HashSet<ConnectionId>,
532    next_generation: u64,
533    reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
534    reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
535    next_client_channel: HashMap<ConnectionId, u16>,
536    next_module_channel: HashMap<ModuleEndpointId, u16>,
537    client_slot_epochs: HashMap<ClientRouteKey, u32>,
538    module_slot_epochs: HashMap<ModuleRouteKey, u32>,
539    last_published_epoch: HashMap<ClientRouteKey, u32>,
540    client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
541    module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
542    status: HashMap<(ClientRouteKey, u32), String>,
543    pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
544    /// Pending relays removed without a module reply before `route.open`
545    /// marked them enqueued. `route.open` consumes its key when it marks the
546    /// relay (and sends the GOODBYE the removal could not send safely) or when
547    /// it aborts, so an entry lives only while that open is mid-send.
548    relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
549    /// Each scope's current `(scope_epoch, version)`, keyed `(owner, ref)`,
550    /// published by a scope sync in the same step that changes the record. A
551    /// bind commit reads it here so it never takes the scope table's lock
552    /// while holding this one. Absent means the scope is not live.
553    scope_tags: HashMap<(String, String), ScopeTag>,
554    next_control_corr: HashMap<ModuleEndpointId, u64>,
555    pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
556    health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
557}
558
559#[derive(Debug, Clone)]
560pub(crate) struct CloseReason {
561    code: &'static str,
562    message: String,
563}
564
565impl CloseReason {
566    pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
567        Self {
568            code,
569            message: message.into(),
570        }
571    }
572}
573
574impl fmt::Display for CloseReason {
575    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
576        write!(f, "{}: {}", self.code, self.message)
577    }
578}
579
580pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
581
582/// Dynamic forwarding state shared by the control plane and data-plane router.
583#[derive(Debug, Default)]
584pub struct ForwardingTable {
585    inner: Arc<RwLock<ForwardingInner>>,
586    #[cfg(feature = "test-support")]
587    route_closing_hold: Mutex<Option<oneshot::Receiver<()>>>,
588    close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
589    counters: DaemonCounters,
590    /// Per-target-module bind-relay breaker state. It lives here, beside the
591    /// module connections it describes, because this is where a module
592    /// connection's identity is established and therefore where a stale
593    /// verdict has to be discarded.
594    route_bind_breakers: RouteBindBreakers,
595    /// Current route.bind relays keyed by target module. Admission is shared
596    /// across every client connection that points at the same endpoint.
597    route_bind_concurrency: RouteBindConcurrency,
598    /// Start and end of each module's `route.open` outage. Held here for the
599    /// same reason as the breakers: every handler built over this table must
600    /// see one outage per module, or each would log its own opening line.
601    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
602}
603
604impl ForwardingTable {
605    /// Install scripted operator confirmation for an in-process integration harness,
606    /// without bootstrapping a daemon process. Only available with `test-support`.
607    ///
608    /// The script has one response per line: `approve`, `decline`, `unavailable`,
609    /// `hang` (until withdrawn), or `approve when <path>` (wait for the path to
610    /// exist). Unreadable or exhausted scripts fail closed. If supplied, `events`
611    /// receives the provider's JSON-lines event log. `limits: None` uses
612    /// [`OperatorLimits::default`](crate::operator_confirm::OperatorLimits); use
613    /// `Some(OperatorLimits { queue_wait, prompt_timeout, stuck_grace })` to
614    /// override the queue wait, prompt timeout and stuck-provider grace periods.
615    ///
616    /// The current executable's file name **must start with `ckdev-`**. Otherwise
617    /// this logs a warning and keeps the OS provider, just like bootstrap.
618    /// `cortexkit-test-support`'s `ckdev_binary` copies a harness executable under
619    /// a qualifying name; run that copy to enable scripted confirmation. See
620    /// `crates/subc-client-rs/tests/real_daemon/operator_confirm.rs` for the
621    /// reference end-to-end harness.
622    ///
623    /// Configure the table shared by the harness's `ControlHandler` and `Router`
624    /// before serving any requests. This also configures privileged confirmations
625    /// that use the same table.
626    ///
627    /// # Panics
628    ///
629    /// Panics if operator-confirm processing has already started.
630    ///
631    /// # Example
632    ///
633    /// ```no_run
634    /// use std::{path::Path, sync::Arc};
635    /// use subc_daemon::{ControlHandler, ForwardingTable, Registry};
636    ///
637    /// let registry = Arc::new(Registry::default());
638    /// let forwarding = Arc::new(ForwardingTable::default());
639    /// forwarding.configure_operator_script(
640    ///     Path::new("operator.script"),
641    ///     Some(Path::new("operator.events")),
642    ///     None,
643    /// );
644    /// let handler = ControlHandler::with_forwarding(registry, forwarding);
645    /// ```
646    #[cfg(feature = "test-support")]
647    pub fn configure_operator_script(
648        &self,
649        script: &std::path::Path,
650        events: Option<&std::path::Path>,
651        limits: Option<crate::operator_confirm::OperatorLimits>,
652    ) {
653        self.operator_confirms()
654            .configure_script(script, events, limits);
655    }
656
657    /// Hold the next supervisor drain between enqueueing `module.draining` and
658    /// `route.closing`, so a test can forward a response in that window. Dropping
659    /// the returned sender also releases the hold, including on a test panic.
660    #[cfg(feature = "test-support")]
661    pub fn hold_route_closing_for_test(&self) -> oneshot::Sender<()> {
662        let (release, hold) = oneshot::channel();
663        *self.route_closing_hold.lock().unwrap() = Some(hold);
664        release
665    }
666
667    #[cfg(feature = "test-support")]
668    pub(crate) async fn wait_before_route_closing_for_test(&self) {
669        let hold = self.route_closing_hold.lock().unwrap().take();
670        if let Some(hold) = hold {
671            let _ = hold.await;
672        }
673    }
674
675    pub(crate) fn operator_confirms(&self) -> Arc<crate::operator_confirm::OperatorConfirms> {
676        Arc::clone(
677            &self
678                .inner
679                .read()
680                .unwrap_or_else(|p| p.into_inner())
681                .operator_confirms,
682        )
683    }
684
685    #[cfg(test)]
686    pub(crate) fn inject_operator_principal(&self, key: ModuleRouteKey, principal: Principal) {
687        let mut inner = self.write_inner().unwrap();
688        let old = inner.module_to_client.remove(&key).unwrap();
689        let client = ClientRouteKey {
690            connection_id: old.client_connection_id,
691            channel: old.client_channel,
692        };
693        inner.client_to_module.remove(&client);
694        let mut binding = Arc::try_unwrap(old).expect("test binding has no outstanding readers");
695        binding.principal = principal;
696        let binding = Arc::new(binding);
697        inner.client_to_module.insert(client, Arc::clone(&binding));
698        inner.module_to_client.insert(key, binding);
699    }
700
701    /// Keep the binding protected through confirmation admission. Route release
702    /// takes the write lock and then the same confirm-state lock, so it cannot
703    /// miss an admitted request or leave it waiting on a vanished route.
704    pub(crate) fn with_operator_route<T>(
705        &self,
706        connection_id: ConnectionId,
707        channel: u16,
708        epoch: u32,
709        admit: impl FnOnce(Option<&RouteBinding>) -> T,
710    ) -> Result<T, ForwardingError> {
711        let inner = self.read_inner()?;
712        let binding = inner
713            .endpoint_by_connection
714            .get(&connection_id)
715            .and_then(|endpoint| {
716                inner.module_to_client.get(&ModuleRouteKey {
717                    endpoint: *endpoint,
718                    channel,
719                })
720            })
721            .filter(|binding| binding.module_epoch == epoch);
722        Ok(admit(binding.map(Arc::as_ref)))
723    }
724
725    pub(crate) fn counters(&self) -> DaemonCounters {
726        self.counters.clone()
727    }
728
729    pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
730        self.route_bind_breakers.clone()
731    }
732
733    pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
734        self.route_bind_concurrency.clone()
735    }
736
737    pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
738        Arc::clone(&self.route_outages)
739    }
740
741    pub(crate) fn register_connection_close(
742        &self,
743        connection_id: ConnectionId,
744    ) -> ConnectionCloseReceiver {
745        let (sender, receiver) = oneshot::channel();
746        let replaced = self
747            .lock_close_registry()
748            .insert(connection_id, sender)
749            .is_some();
750        if replaced {
751            warn!(
752                connection_id = connection_id.get(),
753                "replaced existing connection close registration"
754            );
755        }
756        receiver
757    }
758
759    pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
760        self.lock_close_registry().remove(&connection_id);
761    }
762
763    /// Close every established connection, modules and clients alike, as the
764    /// last step of an announced daemon shutdown. By the time this runs each
765    /// registered module has already been sent a module GOODBYE (see
766    /// `Supervisor::end_children_for_daemon_shutdown`), so the EOF that follows
767    /// is a planned stop; EOF with no GOODBYE before it stays reserved for a
768    /// daemon that went away unannounced. Closing while the daemon is still
769    /// alive lets it wait for the modules' own teardowns before it ends
770    /// whatever is left.
771    #[cfg(unix)]
772    pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
773        let senders: Vec<_> = self.lock_close_registry().drain().collect();
774        let count = senders.len();
775        for (_, sender) in senders {
776            let _ = sender.send(reason.clone());
777        }
778        count
779    }
780
781    /// Every registered module connection, whichever slot it occupies (active,
782    /// swap candidate, or superseded incumbent), once each. Daemon shutdown
783    /// uses this to tell each of them it is a planned stop before closing it.
784    #[cfg(unix)]
785    pub(crate) fn module_connections(
786        &self,
787    ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
788        let inner = self.read_inner()?;
789        let mut seen = HashSet::new();
790        Ok(inner
791            .modules_by_id
792            .values()
793            .chain(inner.candidates_by_id.values())
794            .chain(inner.superseded_endpoints.values())
795            .filter(|module| seen.insert(module.endpoint))
796            .map(|module| ModuleConnectionTarget {
797                module_id: inner
798                    .module_id_by_endpoint
799                    .get(&module.endpoint)
800                    .cloned()
801                    .unwrap_or_default(),
802                endpoint: module.endpoint,
803                sink: module.sink.clone(),
804                negotiated_ver: module.negotiated_ver,
805            })
806            .collect())
807    }
808
809    /// Ask a registered connection to close. Returns true only for the request
810    /// that actually reached it; later requests for the same connection, and
811    /// requests for a connection that is not registered, return false.
812    pub(crate) fn request_connection_close(
813        &self,
814        connection_id: ConnectionId,
815        reason: CloseReason,
816    ) -> bool {
817        let sender = self.lock_close_registry().remove(&connection_id);
818        if let Some(sender) = sender {
819            debug!(
820                connection_id = connection_id.get(),
821                close_reason = %reason,
822                "requesting connection close"
823            );
824            let _ = sender.send(reason);
825            true
826        } else {
827            debug!(
828                connection_id = connection_id.get(),
829                close_reason = %reason,
830                "connection close request ignored for inactive connection"
831            );
832            false
833        }
834    }
835
836    pub fn register_module_connection(
837        &self,
838        connection_id: ConnectionId,
839        module_id: String,
840        negotiated_ver: u8,
841        concurrency: Concurrency,
842        sink: FrameSink,
843    ) -> Result<ModuleEndpointId, ForwardingError> {
844        self.register_module_connection_inner(
845            connection_id,
846            module_id,
847            negotiated_ver,
848            concurrency,
849            sink,
850            None,
851        )
852    }
853
854    /// Register a module connection and queue its HELLO_ACK as the first frame
855    /// on its sink, in the same write-lock critical section that makes the
856    /// endpoint visible.
857    ///
858    /// A module reads HELLO_ACK as the first frame after its HELLO and exits on
859    /// anything else. Once the endpoint is in `modules_by_id`, a `route.open`
860    /// on any other connection can queue a `route.bind` request onto this sink,
861    /// so the ack must already be queued by then. Every lookup that can queue a
862    /// frame for the module takes this lock, so nothing can get in ahead of it.
863    ///
864    /// If the ack cannot be queued (the sink is closed or full), the
865    /// registration fails with nothing inserted.
866    pub(crate) fn register_module_connection_acked(
867        &self,
868        connection_id: ConnectionId,
869        module_id: String,
870        negotiated_ver: u8,
871        concurrency: Concurrency,
872        sink: FrameSink,
873        hello_ack: Frame,
874    ) -> Result<ModuleEndpointId, ForwardingError> {
875        self.register_module_connection_inner(
876            connection_id,
877            module_id,
878            negotiated_ver,
879            concurrency,
880            sink,
881            Some(hello_ack),
882        )
883    }
884
885    fn register_module_connection_inner(
886        &self,
887        connection_id: ConnectionId,
888        module_id: String,
889        negotiated_ver: u8,
890        concurrency: Concurrency,
891        sink: FrameSink,
892        hello_ack: Option<Frame>,
893    ) -> Result<ModuleEndpointId, ForwardingError> {
894        let mut inner = self.write_inner()?;
895        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
896            return Err(ForwardingError::ConnectionClosing { connection_id });
897        }
898        check_module_connection_role_locked(&inner, connection_id)?;
899        // Every refusal check has passed and nothing has been mutated yet, so
900        // a failed enqueue leaves the table exactly as it was.
901        enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
902
903        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
904        let endpoint = ModuleEndpointId {
905            connection_id,
906            generation: inner.next_generation,
907        };
908        inner.endpoint_by_connection.insert(connection_id, endpoint);
909        inner
910            .module_id_by_endpoint
911            .insert(endpoint, module_id.clone());
912        inner.next_module_channel.insert(endpoint, 1);
913        inner.next_control_corr.insert(endpoint, 1);
914        inner.modules_by_id.insert(
915            module_id.clone(),
916            ModuleConnection {
917                endpoint,
918                sink,
919                negotiated_ver,
920                concurrency,
921            },
922        );
923        drop(inner);
924
925        // A new module connection has arrived under this id, so anything the
926        // bind-relay breaker learned was learned about a process that is no
927        // longer the one behind this name. See
928        // `RouteBindBreakers::reset_for_new_module_connection`.
929        //
930        // Keyed on ARRIVAL rather than on teardown deliberately: a connection
931        // going away is not evidence about anything, and a module whose
932        // connection drops without coming back should keep its verdict until
933        // something actually registers in its place. This also covers the
934        // unclean replacements -- a module killed mid-bind, or one whose
935        // connection was closing -- because registration is the single path by
936        // which any module connection becomes usable.
937        if let Some(discarded) = self
938            .route_bind_breakers
939            .reset_for_new_module_connection(&module_id)
940        {
941            info!(
942                module_id = %module_id,
943                discarded_consecutive_timeouts = discarded,
944                "route.bind breaker state discarded: a new module connection replaced the process it described"
945            );
946        }
947        Ok(endpoint)
948    }
949
950    /// Register a blue/green swap candidate for `module_id` into the candidate
951    /// slot, alongside whatever endpoint is active for the id.
952    ///
953    /// Unlike [`Self::register_module_connection`], this never touches the
954    /// active slot and never resets the bind-relay breaker: the incumbent is
955    /// still the process serving the id, so what the breaker learned about it
956    /// is still true. The breaker is reset at cutover instead, when the process
957    /// behind the name actually changes. A second candidate for the same id is
958    /// refused.
959    ///
960    /// Production registers candidates through
961    /// [`Self::register_candidate_module_connection_acked`]; this ack-less form
962    /// is for tests that build forwarding state directly.
963    #[cfg(test)]
964    pub(crate) fn register_candidate_module_connection(
965        &self,
966        connection_id: ConnectionId,
967        module_id: String,
968        negotiated_ver: u8,
969        concurrency: Concurrency,
970        sink: FrameSink,
971    ) -> Result<ModuleEndpointId, ForwardingError> {
972        self.register_candidate_module_connection_inner(
973            connection_id,
974            module_id,
975            negotiated_ver,
976            concurrency,
977            sink,
978            None,
979        )
980    }
981
982    /// The swap-candidate counterpart of
983    /// [`Self::register_module_connection_acked`]: the HELLO_ACK is queued on
984    /// the candidate's sink before its endpoint is inserted. No by-id lookup
985    /// sees a candidate, but its own connection's endpoint does become
986    /// resolvable here, and at cutover it becomes routable; the ack has to be
987    /// ahead of anything either can queue.
988    pub(crate) fn register_candidate_module_connection_acked(
989        &self,
990        connection_id: ConnectionId,
991        module_id: String,
992        negotiated_ver: u8,
993        concurrency: Concurrency,
994        sink: FrameSink,
995        hello_ack: Frame,
996    ) -> Result<ModuleEndpointId, ForwardingError> {
997        self.register_candidate_module_connection_inner(
998            connection_id,
999            module_id,
1000            negotiated_ver,
1001            concurrency,
1002            sink,
1003            Some(hello_ack),
1004        )
1005    }
1006
1007    fn register_candidate_module_connection_inner(
1008        &self,
1009        connection_id: ConnectionId,
1010        module_id: String,
1011        negotiated_ver: u8,
1012        concurrency: Concurrency,
1013        sink: FrameSink,
1014        hello_ack: Option<Frame>,
1015    ) -> Result<ModuleEndpointId, ForwardingError> {
1016        let mut inner = self.write_inner()?;
1017        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1018            return Err(ForwardingError::ConnectionClosing { connection_id });
1019        }
1020        check_module_connection_role_locked(&inner, connection_id)?;
1021        if inner.candidates_by_id.contains_key(&module_id) {
1022            return Err(ForwardingError::CandidateSlotOccupied { module_id });
1023        }
1024        // As in `register_module_connection_inner`: after every refusal check,
1025        // before any mutation.
1026        enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1027
1028        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1029        let endpoint = ModuleEndpointId {
1030            connection_id,
1031            generation: inner.next_generation,
1032        };
1033        inner.endpoint_by_connection.insert(connection_id, endpoint);
1034        inner
1035            .module_id_by_endpoint
1036            .insert(endpoint, module_id.clone());
1037        inner.next_module_channel.insert(endpoint, 1);
1038        inner.next_control_corr.insert(endpoint, 1);
1039        inner.candidates_by_id.insert(
1040            module_id,
1041            ModuleConnection {
1042                endpoint,
1043                sink,
1044                negotiated_ver,
1045                concurrency,
1046            },
1047        );
1048        Ok(endpoint)
1049    }
1050
1051    /// Promote `module_id`'s candidate to the active slot, in ONE forwarding
1052    /// write-lock critical section.
1053    ///
1054    /// That single section is the linearization point of a swap. Every relay
1055    /// reservation resolves the active slot under the same lock, so each one
1056    /// lands wholly before cutover (on the incumbent) or wholly after it (on the
1057    /// promoted candidate), never on a mix. A relay reserved on the incumbent
1058    /// before cutover can no longer commit: `commit_route_locked` requires the
1059    /// reservation's endpoint to be the active one, and an ack for it arriving
1060    /// later is answered by the superseded-endpoint arm of
1061    /// `complete_pending_relay` instead of ending the incumbent's connection.
1062    ///
1063    /// Both endpoints keep their identities: nothing is re-keyed, so the
1064    /// incumbent's bound routes and its pending correlation keys stay exactly
1065    /// where they are until it is drained with [`Self::begin_endpoint_drain`],
1066    /// using the incumbent endpoint this returns. Returns `Ok(None)` when there
1067    /// is no candidate for the id.
1068    pub(crate) fn cutover_candidate(
1069        &self,
1070        module_id: &str,
1071    ) -> Result<Option<ForwardingCutover>, ForwardingError> {
1072        let mut inner = self.write_inner()?;
1073        if inner.daemon_draining {
1074            return Err(ForwardingError::ModuleReloading {
1075                module_id: module_id.to_string(),
1076            });
1077        }
1078        let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
1079            return Ok(None);
1080        };
1081        let promoted = candidate.endpoint;
1082        let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
1083        let incumbent = incumbent.map(|incumbent| {
1084            let endpoint = incumbent.endpoint;
1085            inner.superseded_endpoints.insert(endpoint, incumbent);
1086            endpoint
1087        });
1088        drop(inner);
1089
1090        // A different process now answers for this id; see the matching reset
1091        // in `register_module_connection` for why a verdict about the old one
1092        // must not carry over.
1093        if let Some(discarded) = self
1094            .route_bind_breakers
1095            .reset_for_new_module_connection(module_id)
1096        {
1097            info!(
1098                module_id = %module_id,
1099                discarded_consecutive_timeouts = discarded,
1100                "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
1101            );
1102        }
1103        Ok(Some(ForwardingCutover {
1104            promoted,
1105            incumbent,
1106        }))
1107    }
1108
1109    #[allow(clippy::too_many_arguments)]
1110    pub(crate) async fn begin_route_bind_relay_for(
1111        &self,
1112        client_connection_id: ConnectionId,
1113        client_sink: FrameSink,
1114        client_negotiated_ver: u8,
1115        client_corr: u64,
1116        module_id: &str,
1117        principal: Principal,
1118        scope: Option<BoundScope>,
1119        project_root: Option<ProjectRootId>,
1120        deadline: Instant,
1121    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1122        // Reserve egress capacity before taking the forwarding lock. The permit is
1123        // held until the bind reaches one terminal state, so an accepted bind can
1124        // publish its table entry and RouteOpen response in one critical section.
1125        let client_permit =
1126            client_sink
1127                .reserve_owned()
1128                .await
1129                .map_err(|_| ForwardingError::ClientEgressClosed {
1130                    connection_id: client_connection_id,
1131                })?;
1132        self.begin_route_bind_relay_inner(
1133            client_connection_id,
1134            client_sink,
1135            client_negotiated_ver,
1136            client_corr,
1137            module_id,
1138            principal,
1139            scope,
1140            project_root,
1141            deadline,
1142            client_permit,
1143        )
1144    }
1145
1146    #[cfg(test)]
1147    pub(crate) fn begin_route_bind_relay_for_test(
1148        &self,
1149        client_connection_id: ConnectionId,
1150        client_sink: FrameSink,
1151        client_corr: u64,
1152        module_id: &str,
1153    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1154        let permit =
1155            client_sink
1156                .try_reserve_owned()
1157                .map_err(|_| ForwardingError::ClientEgressClosed {
1158                    connection_id: client_connection_id,
1159                })?;
1160        self.begin_route_bind_relay_inner(
1161            client_connection_id,
1162            client_sink,
1163            subc_protocol::PROTOCOL_VERSION,
1164            client_corr,
1165            module_id,
1166            Principal::Direct,
1167            None,
1168            None,
1169            Instant::now() + std::time::Duration::from_secs(60),
1170            permit,
1171        )
1172    }
1173
1174    pub(crate) fn begin_module_control_rpc_for(
1175        &self,
1176        module_id: &str,
1177        expected_op: &str,
1178        deadline: Instant,
1179    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1180        self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1181    }
1182
1183    pub(crate) fn begin_health_probe_rpc_for(
1184        &self,
1185        module_id: &str,
1186        expected_op: &str,
1187        probe_started_at: Instant,
1188        deadline: Instant,
1189    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1190        self.begin_module_control_rpc_inner(
1191            module_id,
1192            expected_op,
1193            deadline,
1194            Some(probe_started_at),
1195            false,
1196        )
1197    }
1198
1199    pub(crate) fn begin_drain_health_probe_rpc_for(
1200        &self,
1201        module_id: &str,
1202        expected_op: &str,
1203        probe_started_at: Instant,
1204        deadline: Instant,
1205    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1206        self.begin_module_control_rpc_inner(
1207            module_id,
1208            expected_op,
1209            deadline,
1210            Some(probe_started_at),
1211            true,
1212        )
1213    }
1214
1215    /// A health probe addressed to one endpoint, in whichever slot it is.
1216    ///
1217    /// Every other control RPC resolves the id's ACTIVE endpoint, which is how
1218    /// a swap candidate (not active until cutover) and a superseded incumbent
1219    /// (not active after it) are unreachable by them. A swap needs to probe
1220    /// exactly those two: the candidate before promoting it, and the incumbent
1221    /// for its busy gauges while it drains.
1222    pub(crate) fn begin_endpoint_health_probe_rpc_for(
1223        &self,
1224        endpoint: ModuleEndpointId,
1225        expected_op: &str,
1226        probe_started_at: Instant,
1227        deadline: Instant,
1228    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1229        let inner = self.write_inner()?;
1230        let module = module_connection_for_endpoint_locked(&inner, endpoint)
1231            .cloned()
1232            .ok_or(ForwardingError::NoModuleConnection)?;
1233        let module_id = inner
1234            .module_id_by_endpoint
1235            .get(&endpoint)
1236            .cloned()
1237            .unwrap_or_default();
1238        // A draining incumbent is exactly what this probes for busy gauges, so
1239        // draining is allowed, as it is for the by-id drain probe.
1240        self.begin_control_rpc_locked(
1241            inner,
1242            &module_id,
1243            module,
1244            expected_op,
1245            deadline,
1246            Some(probe_started_at),
1247            true,
1248        )
1249    }
1250
1251    fn begin_module_control_rpc_inner(
1252        &self,
1253        module_id: &str,
1254        expected_op: &str,
1255        deadline: Instant,
1256        health_probe_started_at: Option<Instant>,
1257        allow_draining: bool,
1258    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1259        let inner = self.write_inner()?;
1260        let module = inner
1261            .modules_by_id
1262            .get(module_id)
1263            .cloned()
1264            .ok_or(ForwardingError::NoModuleConnection)?;
1265        self.begin_control_rpc_locked(
1266            inner,
1267            module_id,
1268            module,
1269            expected_op,
1270            deadline,
1271            health_probe_started_at,
1272            allow_draining,
1273        )
1274    }
1275
1276    #[allow(clippy::too_many_arguments)]
1277    fn begin_control_rpc_locked(
1278        &self,
1279        mut inner: RwLockWriteGuard<'_, ForwardingInner>,
1280        module_id: &str,
1281        module: ModuleConnection,
1282        expected_op: &str,
1283        deadline: Instant,
1284        health_probe_started_at: Option<Instant>,
1285        allow_draining: bool,
1286    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1287        if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1288            return Err(ForwardingError::ModuleReloading {
1289                module_id: module_id.to_string(),
1290            });
1291        }
1292        if inner
1293            .closing_connections
1294            .contains(&module.endpoint.connection_id)
1295        {
1296            return Err(ForwardingError::ConnectionClosing {
1297                connection_id: module.endpoint.connection_id,
1298            });
1299        }
1300        if health_probe_started_at.is_some() {
1301            // Recurring health probes are serial per endpoint. Once the next one
1302            // starts, an older answer can no longer improve the current snapshot,
1303            // so retaining more than the newest unanswered probe has no value.
1304            inner
1305                .health_probe_tombstones
1306                .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1307        }
1308        let corr = match inner.allocate_control_corr(module.endpoint) {
1309            Ok(corr) => corr,
1310            Err(err) => {
1311                drop(inner);
1312                self.request_connection_close(
1313                    module.endpoint.connection_id,
1314                    CloseReason::new(
1315                        "control_correlation_exhausted",
1316                        "daemon-originated channel-0 correlation space exhausted",
1317                    ),
1318                );
1319                return Err(err);
1320            }
1321        };
1322        let (sender, receiver) = oneshot::channel();
1323        inner.pending_control_rpcs.insert(
1324            (module.endpoint, corr),
1325            PendingModuleControlRpcEntry {
1326                expected_op: expected_op.to_string(),
1327                deadline,
1328                health_probe_started_at,
1329                sender,
1330            },
1331        );
1332
1333        Ok(PendingModuleControlRpc {
1334            endpoint: module.endpoint,
1335            module_sink: module.sink,
1336            negotiated_ver: module.negotiated_ver,
1337            corr,
1338            receiver,
1339        })
1340    }
1341
1342    #[allow(clippy::too_many_arguments)]
1343    fn begin_route_bind_relay_inner(
1344        &self,
1345        client_connection_id: ConnectionId,
1346        client_sink: FrameSink,
1347        client_negotiated_ver: u8,
1348        client_corr: u64,
1349        expected_module_id: &str,
1350        principal: Principal,
1351        scope: Option<BoundScope>,
1352        project_root: Option<ProjectRootId>,
1353        deadline: Instant,
1354        client_permit: crate::router::EgressPermit,
1355    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1356        let mut inner = self.write_inner()?;
1357        // HELLO and route.open can run on different tasks. Decide the role under
1358        // the same lock that installs endpoints and reserves client channels so
1359        // neither task can turn one connection into both kinds of owner.
1360        if inner
1361            .endpoint_by_connection
1362            .contains_key(&client_connection_id)
1363        {
1364            return Err(ForwardingError::ConnectionRoleConflict {
1365                connection_id: client_connection_id,
1366            });
1367        }
1368        if inner.closing_connections.contains(&client_connection_id) {
1369            return Err(ForwardingError::ConnectionClosing {
1370                connection_id: client_connection_id,
1371            });
1372        }
1373        let module = inner
1374            .modules_by_id
1375            .get(expected_module_id)
1376            .cloned()
1377            .ok_or(ForwardingError::NoModuleConnection)?;
1378        if inner.draining_endpoints.contains_key(&module.endpoint) {
1379            return Err(ForwardingError::ModuleReloading {
1380                module_id: expected_module_id.to_string(),
1381            });
1382        }
1383        if inner
1384            .closing_connections
1385            .contains(&module.endpoint.connection_id)
1386        {
1387            return Err(ForwardingError::ConnectionClosing {
1388                connection_id: module.endpoint.connection_id,
1389            });
1390        }
1391
1392        let corr = match inner.allocate_control_corr(module.endpoint) {
1393            Ok(corr) => corr,
1394            Err(err) => {
1395                drop(inner);
1396                self.request_connection_close(
1397                    module.endpoint.connection_id,
1398                    CloseReason::new(
1399                        "control_correlation_exhausted",
1400                        "daemon-originated channel-0 correlation space exhausted",
1401                    ),
1402                );
1403                return Err(err);
1404            }
1405        };
1406        let (client_channel, client_epoch, module_channel, module_epoch) =
1407            inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1408        let client_key = ClientRouteKey {
1409            connection_id: client_connection_id,
1410            channel: client_channel,
1411        };
1412        let module_key = ModuleRouteKey {
1413            endpoint: module.endpoint,
1414            channel: module_channel,
1415        };
1416        let reservation = RouteReservation {
1417            client_key,
1418            module_key,
1419            client_epoch,
1420            module_epoch,
1421            project_root,
1422        };
1423        let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
1424            route_channel: client_channel,
1425            route_epoch: client_epoch,
1426        })
1427        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1428        let route_open_frame = Frame::build_with_version(
1429            client_negotiated_ver,
1430            FrameType::Response,
1431            Flags::new(false, Priority::Passive, false),
1432            0,
1433            0,
1434            client_corr,
1435            response_body,
1436        )
1437        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1438        let (sender, receiver) = oneshot::channel();
1439        inner.reserved_client.insert(client_key, module_key);
1440        inner.reserved_module.insert(module_key, client_key);
1441        inner.pending_relays.insert(
1442            (module.endpoint, corr),
1443            PendingRouteBindRelayEntry {
1444                reservation,
1445                client_sink,
1446                client_negotiated_ver,
1447                client_permit,
1448                route_open_frame,
1449                principal,
1450                scope,
1451                deadline,
1452                relay_enqueued: false,
1453                sender,
1454            },
1455        );
1456
1457        Ok(PendingRouteBindRelay {
1458            endpoint: module.endpoint,
1459            module_sink: module.sink,
1460            negotiated_ver: module.negotiated_ver,
1461            client_channel,
1462            client_epoch,
1463            module_channel,
1464            module_epoch,
1465            corr,
1466            receiver,
1467        })
1468    }
1469
1470    /// Called by `route.open` right after the route.bind entered the module's
1471    /// egress queue.
1472    ///
1473    /// The module reader runs concurrently with `route.open`, so a fast module
1474    /// can receive the bind, ack it, and have the reader commit the route
1475    /// before this runs. A missing entry therefore does not mean the bind was
1476    /// abandoned: only a removal recorded in `relays_released_before_enqueue`
1477    /// does. Telling the two apart is what stops `route.open` from sending a
1478    /// GOODBYE for a route that was just committed.
1479    pub(crate) fn mark_route_bind_relay_enqueued(
1480        &self,
1481        endpoint: ModuleEndpointId,
1482        corr: u64,
1483    ) -> Result<RouteBindRelayMark, ForwardingError> {
1484        let mut inner = self.write_inner()?;
1485        if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
1486            pending.relay_enqueued = true;
1487            return Ok(RouteBindRelayMark::Marked);
1488        }
1489        if inner
1490            .relays_released_before_enqueue
1491            .remove(&(endpoint, corr))
1492        {
1493            return Ok(RouteBindRelayMark::ReleasedWithoutReply);
1494        }
1495        Ok(RouteBindRelayMark::SettledByReply)
1496    }
1497
1498    pub(crate) fn release_client_route(
1499        &self,
1500        client_connection_id: ConnectionId,
1501        client_channel: u16,
1502        expected_epoch: u32,
1503    ) -> Result<RouteRelease, ForwardingError> {
1504        let mut inner = self.write_inner()?;
1505        let release = release_client_route_locked(
1506            &mut inner,
1507            ClientRouteKey {
1508                connection_id: client_connection_id,
1509                channel: client_channel,
1510            },
1511            expected_epoch,
1512        );
1513        self.record_route_release(&release);
1514        Ok(release)
1515    }
1516
1517    pub(crate) fn release_module_route(
1518        &self,
1519        module_connection_id: ConnectionId,
1520        module_channel: u16,
1521        expected_epoch: u32,
1522    ) -> Result<RouteRelease, ForwardingError> {
1523        let mut inner = self.write_inner()?;
1524        let Some(endpoint) = inner
1525            .endpoint_by_connection
1526            .get(&module_connection_id)
1527            .copied()
1528        else {
1529            return Ok(RouteRelease::Absent);
1530        };
1531        let release = release_module_route_locked(
1532            &mut inner,
1533            ModuleRouteKey {
1534                endpoint,
1535                channel: module_channel,
1536            },
1537            expected_epoch,
1538        );
1539        self.record_route_release(&release);
1540        Ok(release)
1541    }
1542
1543    pub(crate) fn abort_pending_relay(
1544        &self,
1545        endpoint: ModuleEndpointId,
1546        corr: u64,
1547        outcome: RouteBindRelayOutcome,
1548    ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
1549        let mut inner = self.write_inner()?;
1550        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1551            // If another path dropped the relay while `route.open` was still
1552            // sending the bind, and that open is now giving up without marking
1553            // it (its send failed or it was cancelled), forget the record so it
1554            // cannot outlive the open.
1555            inner
1556                .relays_released_before_enqueue
1557                .remove(&(endpoint, corr));
1558            return Ok(None);
1559        };
1560        release_reserved_route_locked(
1561            &mut inner,
1562            pending.reservation.client_key,
1563            pending.reservation.module_key,
1564        );
1565        let target = pending
1566            .relay_enqueued
1567            .then(|| abandoned_route_target(&inner, &pending.reservation));
1568        let _ = pending.sender.send(outcome);
1569        Ok(target.flatten())
1570    }
1571
1572    pub(crate) fn cancel_module_control_rpc(
1573        &self,
1574        endpoint: ModuleEndpointId,
1575        corr: u64,
1576    ) -> Result<(), ForwardingError> {
1577        self.write_inner()?
1578            .pending_control_rpcs
1579            .remove(&(endpoint, corr));
1580        Ok(())
1581    }
1582
1583    pub(crate) fn tombstone_health_probe_rpc(
1584        &self,
1585        endpoint: ModuleEndpointId,
1586        corr: u64,
1587    ) -> Result<bool, ForwardingError> {
1588        let key = (endpoint, corr);
1589        let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
1590        {
1591            let mut inner = self.write_inner()?;
1592            let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
1593                return Ok(false);
1594            };
1595            let Some(probe_started_at) = pending.health_probe_started_at else {
1596                inner.pending_control_rpcs.insert(key, pending);
1597                return Ok(false);
1598            };
1599            let module_id = inner
1600                .module_id_by_endpoint
1601                .get(&endpoint)
1602                .cloned()
1603                .unwrap_or_else(|| "unknown".to_string());
1604            inner.health_probe_tombstones.insert(
1605                key,
1606                HealthProbeTombstone {
1607                    expected_op: pending.expected_op,
1608                    module_id,
1609                    probe_started_at,
1610                    expires_at,
1611                },
1612            );
1613        }
1614        self.schedule_health_probe_tombstone_expiration(key, expires_at);
1615        Ok(true)
1616    }
1617
1618    fn schedule_health_probe_tombstone_expiration(
1619        &self,
1620        key: (ModuleEndpointId, u64),
1621        expires_at: Instant,
1622    ) {
1623        let inner = Arc::downgrade(&self.inner);
1624        tokio::spawn(async move {
1625            tokio::time::sleep_until(expires_at).await;
1626            let Some(inner) = inner.upgrade() else {
1627                return;
1628            };
1629            let Ok(mut inner) = inner.write() else {
1630                return;
1631            };
1632            let expired = inner
1633                .health_probe_tombstones
1634                .get(&key)
1635                .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
1636            if expired {
1637                inner.health_probe_tombstones.remove(&key);
1638            }
1639        });
1640    }
1641
1642    pub(crate) fn complete_pending_relay(
1643        &self,
1644        connection_id: ConnectionId,
1645        corr: u64,
1646        outcome: RouteBindRelayOutcome,
1647    ) -> Result<PendingRelayCompletion, ForwardingError> {
1648        let mut inner = self.write_inner()?;
1649        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1650            return Ok(PendingRelayCompletion {
1651                settled: false,
1652                abandoned: None,
1653            });
1654        };
1655        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1656            return Ok(PendingRelayCompletion {
1657                settled: false,
1658                abandoned: None,
1659            });
1660        };
1661
1662        if Instant::now() >= pending.deadline {
1663            release_reserved_route_locked(
1664                &mut inner,
1665                pending.reservation.client_key,
1666                pending.reservation.module_key,
1667            );
1668            let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
1669                .then(|| abandoned_route_target(&inner, &pending.reservation))
1670                .flatten();
1671            let _ = pending
1672                .sender
1673                .send(RouteBindRelayOutcome::Rejected(ErrorBody {
1674                    code: "module_timeout".to_string(),
1675                    message: "route.bind response arrived after its daemon deadline".to_string(),
1676                    detail: None,
1677                }));
1678            return Ok(PendingRelayCompletion {
1679                settled: true,
1680                abandoned,
1681            });
1682        }
1683
1684        match outcome {
1685            // Two shapes of "the client is not there to receive this route" that
1686            // must resolve identically: its egress is already closed, or it is
1687            // marked closing (its connection loop has been asked to end, but has
1688            // not drained yet, so the sink is still open).
1689            //
1690            // Only the first used to be caught here. The second fell through to
1691            // `commit_route_locked`, which refuses a closing client with
1692            // `ConnectionClosing` -- and this function is called from the MODULE
1693            // connection's frame handler, so that refusal ended the module's
1694            // connection instead of this one client's route. One dying client's
1695            // route.open then took down a connection carrying every other
1696            // client's routes to that module.
1697            //
1698            // The remedy for both is the same, which is why they share an arm:
1699            // give back the reserved handle pair, tell the waiting route.open the
1700            // route is gone, and report the module-side channel so the caller can
1701            // send a channel-scoped GOODBYE for the binding the module just
1702            // created. Nothing here touches the module connection.
1703            RouteBindRelayOutcome::Accepted
1704                if pending.client_sink.is_closed()
1705                    || inner
1706                        .closing_connections
1707                        .contains(&pending.reservation.client_key.connection_id) =>
1708            {
1709                let reason = if pending.client_sink.is_closed() {
1710                    "client egress closed before route publication"
1711                } else {
1712                    "client connection is closing before route publication"
1713                };
1714                release_reserved_route_locked(
1715                    &mut inner,
1716                    pending.reservation.client_key,
1717                    pending.reservation.module_key,
1718                );
1719                // Not gated on `relay_enqueued`: an ack proves the module
1720                // received the bind, and `route.open` sends nothing for a
1721                // relay a reply already settled, so this is the only GOODBYE.
1722                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1723                let _ = pending
1724                    .sender
1725                    .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
1726                return Ok(PendingRelayCompletion {
1727                    settled: true,
1728                    abandoned,
1729                });
1730            }
1731            // The endpoint that acked this bind is no longer accepting new
1732            // routes, for one of two reasons. Either it was the module's active
1733            // endpoint when this relay was reserved and a blue/green swap has
1734            // since promoted a candidate (the replacement process registered
1735            // beside it) into the active slot, superseding it; or a daemon
1736            // drain has gated it, and the ack arrived before the ordered module
1737            // drain settled pending relays. The module did nothing wrong: it
1738            // bound the route it was asked to bind, and the endpoint still
1739            // carries every other client's routes until it is drained. So the
1740            // ack is settled here, while the pending entry still holds the
1741            // client's sender and the reserved handle pair: release the pair,
1742            // refuse the waiting route.open with `module_reloading`, which
1743            // callers retry (after a swap the retry reserves on the promoted
1744            // endpoint), and hand back the module-side channel so the caller
1745            // sends one channel-scoped GOODBYE for the binding the module just
1746            // created. Returning an error instead would end the module
1747            // connection all those other routes share; nothing here touches it.
1748            //
1749            // An endpoint that stopped being active any other way is stale
1750            // (replaced, but not by a swap and not draining). It still falls
1751            // through to `commit_route_locked`, which refuses it with
1752            // `StaleModuleEndpoint` exactly as before swaps existed.
1753            RouteBindRelayOutcome::Accepted
1754                if inner.superseded_endpoints.contains_key(&endpoint)
1755                    || inner.draining_endpoints.contains_key(&endpoint) =>
1756            {
1757                release_reserved_route_locked(
1758                    &mut inner,
1759                    pending.reservation.client_key,
1760                    pending.reservation.module_key,
1761                );
1762                // Not gated on `relay_enqueued`: an ack proves the module
1763                // received the bind, whether or not the enqueue mark was set.
1764                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1765                let module_id = inner
1766                    .module_id_by_endpoint
1767                    .get(&endpoint)
1768                    .cloned()
1769                    .unwrap_or_else(|| "unknown".to_string());
1770                let _ = pending
1771                    .sender
1772                    .send(RouteBindRelayOutcome::Rejected(ErrorBody::new(
1773                        "module_reloading",
1774                        format!("module_id '{module_id}' is reloading"),
1775                    )));
1776                return Ok(PendingRelayCompletion {
1777                    settled: true,
1778                    abandoned,
1779                });
1780            }
1781            // The scope the open was admitted under ended or changed after
1782            // admission. The module bound a route under a stamp that is no
1783            // longer true, so it must not become routable. Settled here like
1784            // the superseded arm above, never as an error from
1785            // `commit_route_locked`, which would close the module's whole
1786            // connection and every other client's routes to it: release the
1787            // pair, answer the waiting route.open by name, and hand back the
1788            // module-side channel for one channel-scoped GOODBYE.
1789            RouteBindRelayOutcome::Accepted
1790                if pending
1791                    .scope
1792                    .as_ref()
1793                    .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
1794            {
1795                let refusal = pending
1796                    .scope
1797                    .as_ref()
1798                    .and_then(|scope| scope_refusal_locked(&inner, scope))
1799                    .expect("guard matched a refusal under the same lock");
1800                release_reserved_route_locked(
1801                    &mut inner,
1802                    pending.reservation.client_key,
1803                    pending.reservation.module_key,
1804                );
1805                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1806                let _ = pending
1807                    .sender
1808                    .send(RouteBindRelayOutcome::Rejected(refusal));
1809                return Ok(PendingRelayCompletion {
1810                    settled: true,
1811                    abandoned,
1812                });
1813            }
1814            RouteBindRelayOutcome::Accepted => {
1815                let abandoned = commit_route_locked(&mut inner, pending)?;
1816                return Ok(PendingRelayCompletion {
1817                    settled: true,
1818                    abandoned,
1819                });
1820            }
1821            terminal => {
1822                release_reserved_route_locked(
1823                    &mut inner,
1824                    pending.reservation.client_key,
1825                    pending.reservation.module_key,
1826                );
1827                let _ = pending.sender.send(terminal);
1828            }
1829        }
1830        Ok(PendingRelayCompletion {
1831            settled: true,
1832            abandoned: None,
1833        })
1834    }
1835
1836    pub(crate) fn pending_module_control_op(
1837        &self,
1838        connection_id: ConnectionId,
1839        corr: u64,
1840    ) -> Result<Option<String>, ForwardingError> {
1841        let inner = self.read_inner()?;
1842        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1843            return Ok(None);
1844        };
1845        let key = (endpoint, corr);
1846        Ok(inner
1847            .pending_control_rpcs
1848            .get(&key)
1849            .map(|pending| pending.expected_op.clone())
1850            .or_else(|| {
1851                inner
1852                    .health_probe_tombstones
1853                    .get(&key)
1854                    .filter(|tombstone| tombstone.expires_at > Instant::now())
1855                    .map(|tombstone| tombstone.expected_op.clone())
1856            }))
1857    }
1858
1859    pub(crate) fn complete_module_control_rpc(
1860        &self,
1861        connection_id: ConnectionId,
1862        corr: u64,
1863        actual_op: Option<&str>,
1864        outcome: ModuleControlRpcOutcome,
1865    ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
1866        let now = Instant::now();
1867        let mut inner = self.write_inner()?;
1868        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1869            return Ok(ModuleControlRpcCompletion::Unknown);
1870        };
1871        let key = (endpoint, corr);
1872        if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
1873            if now >= pending.deadline {
1874                let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
1875                    ModuleControlRpcCompletion::LateHealthAnswer {
1876                        module_id: inner
1877                            .module_id_by_endpoint
1878                            .get(&endpoint)
1879                            .cloned()
1880                            .unwrap_or_else(|| "unknown".to_string()),
1881                        latency: now.saturating_duration_since(probe_started_at),
1882                    }
1883                });
1884                let _ = pending
1885                    .sender
1886                    .send(ModuleControlRpcOutcome::DeadlineElapsed);
1887                return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
1888            }
1889            let outcome = match actual_op {
1890                Some(actual) if actual != pending.expected_op => {
1891                    ModuleControlRpcOutcome::UnexpectedOp {
1892                        expected: pending.expected_op,
1893                        actual: actual.to_string(),
1894                    }
1895                }
1896                _ => outcome,
1897            };
1898            let _ = pending.sender.send(outcome);
1899            return Ok(ModuleControlRpcCompletion::Settled);
1900        }
1901
1902        let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
1903            return Ok(ModuleControlRpcCompletion::Unknown);
1904        };
1905        if tombstone.expires_at <= now {
1906            return Ok(ModuleControlRpcCompletion::Unknown);
1907        }
1908        Ok(ModuleControlRpcCompletion::LateHealthAnswer {
1909            module_id: tombstone.module_id,
1910            latency: now.saturating_duration_since(tombstone.probe_started_at),
1911        })
1912    }
1913
1914    #[cfg(test)]
1915    pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
1916        Ok(self.read_inner()?.health_probe_tombstones.len())
1917    }
1918
1919    #[cfg(test)]
1920    pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
1921        Ok(self.read_inner()?.closing_connections.len())
1922    }
1923
1924    /// Reserved-but-uncommitted route handles, counted on both index sides, so
1925    /// a test can assert a reservation pair was actually given back.
1926    #[cfg(test)]
1927    pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
1928        let inner = self.read_inner()?;
1929        Ok((inner.reserved_client.len(), inner.reserved_module.len()))
1930    }
1931
1932    pub(crate) fn module_endpoint_for_connection(
1933        &self,
1934        connection_id: ConnectionId,
1935    ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
1936        Ok(self
1937            .read_inner()?
1938            .endpoint_by_connection
1939            .get(&connection_id)
1940            .copied())
1941    }
1942
1943    /// Looks up the module registered on a data-plane connection so route-drop
1944    /// diagnostics name the emitter instead of only reporting a daemon total.
1945    pub(crate) fn module_id_for_connection(
1946        &self,
1947        connection_id: ConnectionId,
1948    ) -> Result<Option<String>, ForwardingError> {
1949        let inner = self.read_inner()?;
1950        Ok(inner
1951            .endpoint_by_connection
1952            .get(&connection_id)
1953            .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
1954            .cloned())
1955    }
1956
1957    /// Whether the daemon ever allocated `(channel, epoch)` on this module
1958    /// connection. Epochs on a module channel are handed out as 1, 2, 3, ...
1959    /// and the last one handed out is remembered until the connection ends, so
1960    /// every epoch from 1 up to that one was a real route at some point. Used
1961    /// only on the drop path, to tell a module still sending on a route the
1962    /// daemon released from one sending on a route that never existed.
1963    pub(crate) fn module_route_epoch_was_allocated(
1964        &self,
1965        connection_id: ConnectionId,
1966        channel: u16,
1967        epoch: u32,
1968    ) -> Result<bool, ForwardingError> {
1969        let inner = self.read_inner()?;
1970        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1971            return Ok(false);
1972        };
1973        Ok(inner
1974            .module_slot_epochs
1975            .get(&ModuleRouteKey { endpoint, channel })
1976            .is_some_and(|last| epoch != 0 && epoch <= *last))
1977    }
1978
1979    pub(crate) fn has_live_module_connection(
1980        &self,
1981        module_id: &str,
1982    ) -> Result<bool, ForwardingError> {
1983        Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
1984    }
1985
1986    pub(crate) fn lookup_data_route(
1987        &self,
1988        connection_id: ConnectionId,
1989        channel: u16,
1990        epoch: u32,
1991    ) -> Result<DataRoute, ForwardingError> {
1992        let inner = self.read_inner()?;
1993        let state = if let Some(endpoint) =
1994            inner.endpoint_by_connection.get(&connection_id).copied()
1995        {
1996            let key = ModuleRouteKey { endpoint, channel };
1997            match inner.module_to_client.get(&key) {
1998                Some(route) if route.module_epoch == epoch => {
1999                    DataRouteState::Bound(Arc::clone(route))
2000                }
2001                Some(_) => DataRouteState::EpochMismatch,
2002                None if inner.reserved_module.contains_key(&key)
2003                    && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
2004                {
2005                    DataRouteState::Reserved
2006                }
2007                None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
2008                None => DataRouteState::Absent,
2009            }
2010        } else {
2011            let key = ClientRouteKey {
2012                connection_id,
2013                channel,
2014            };
2015            match inner.client_to_module.get(&key) {
2016                Some(route) if route.client_epoch == epoch => {
2017                    DataRouteState::Bound(Arc::clone(route))
2018                }
2019                Some(_) => DataRouteState::EpochMismatch,
2020                None if inner.reserved_client.contains_key(&key)
2021                    && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
2022                {
2023                    DataRouteState::Reserved
2024                }
2025                None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
2026                None => DataRouteState::Absent,
2027            }
2028        };
2029        Ok(
2030            if inner.endpoint_by_connection.contains_key(&connection_id) {
2031                DataRoute::Module(state)
2032            } else {
2033                DataRoute::Client(state)
2034            },
2035        )
2036    }
2037
2038    #[cfg(test)]
2039    pub(crate) fn inject_client_slot_epoch(
2040        &self,
2041        connection_id: ConnectionId,
2042        channel: u16,
2043        last_epoch: u32,
2044    ) {
2045        let mut inner = self.write_inner().expect("forwarding lock");
2046        inner.client_slot_epochs.insert(
2047            ClientRouteKey {
2048                connection_id,
2049                channel,
2050            },
2051            last_epoch,
2052        );
2053        inner.next_client_channel.insert(connection_id, channel);
2054    }
2055
2056    #[cfg(test)]
2057    pub(crate) fn inject_module_slot_epoch(
2058        &self,
2059        endpoint: ModuleEndpointId,
2060        channel: u16,
2061        last_epoch: u32,
2062    ) {
2063        let mut inner = self.write_inner().expect("forwarding lock");
2064        inner
2065            .module_slot_epochs
2066            .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
2067        inner.next_module_channel.insert(endpoint, channel);
2068    }
2069
2070    #[cfg(test)]
2071    pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
2072        self.write_inner()
2073            .expect("forwarding lock")
2074            .next_control_corr
2075            .insert(endpoint, next_corr);
2076    }
2077
2078    pub(crate) fn cache_status(
2079        &self,
2080        endpoint: ModuleEndpointId,
2081        module_channel: u16,
2082        module_epoch: u32,
2083        status: String,
2084    ) -> Result<bool, ForwardingError> {
2085        let mut inner = self.write_inner()?;
2086        if !inner.module_id_by_endpoint.contains_key(&endpoint) {
2087            return Err(ForwardingError::StaleModuleEndpoint);
2088        }
2089
2090        let module_key = ModuleRouteKey {
2091            endpoint,
2092            channel: module_channel,
2093        };
2094        let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
2095            (route.module_epoch == module_epoch).then_some((
2096                ClientRouteKey {
2097                    connection_id: route.client_connection_id,
2098                    channel: route.client_channel,
2099                },
2100                route.client_epoch,
2101            ))
2102        } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
2103            (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
2104                client_key,
2105                inner
2106                    .client_slot_epochs
2107                    .get(&client_key)
2108                    .copied()
2109                    .unwrap_or(0),
2110            ))
2111        } else {
2112            None
2113        };
2114
2115        if let Some(handle) = handle {
2116            inner.status.insert(handle, status);
2117            Ok(true)
2118        } else {
2119            debug!(
2120                module_channel,
2121                module_epoch,
2122                generation = endpoint.generation,
2123                connection_id = endpoint.connection_id.get(),
2124                "dropping stale status update for module route handle"
2125            );
2126            Ok(false)
2127        }
2128    }
2129
2130    pub(crate) fn route_poll_snapshot(
2131        &self,
2132        client_connection_id: ConnectionId,
2133        client_channel: u16,
2134        client_epoch: u32,
2135    ) -> Result<RoutePollSnapshot, ForwardingError> {
2136        let inner = self.read_inner()?;
2137        let client_key = ClientRouteKey {
2138            connection_id: client_connection_id,
2139            channel: client_channel,
2140        };
2141        let Some(route) = inner.client_to_module.get(&client_key) else {
2142            return Ok(RoutePollSnapshot::Absent);
2143        };
2144        if route.client_epoch != client_epoch
2145            || !inner
2146                .module_id_by_endpoint
2147                .contains_key(&route.module_endpoint)
2148        {
2149            return Ok(RoutePollSnapshot::Absent);
2150        }
2151        Ok(RoutePollSnapshot::Bound {
2152            module_id: route.module_id.clone(),
2153            status: inner.status.get(&(client_key, client_epoch)).cloned(),
2154        })
2155    }
2156
2157    pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2158        Ok(self.read_inner()?.client_to_module.len())
2159    }
2160
2161    /// How many distinct client connections hold at least one committed route,
2162    /// alongside the largest number of routes any single connection holds.
2163    ///
2164    /// `connected_clients` alone cannot distinguish many clients with a route
2165    /// each from one client accumulating hundreds, and those have opposite
2166    /// causes. Reading it required an out-of-band `lsof` during a live
2167    /// investigation, and the count of connections was mistaken for a count of
2168    /// client processes — which sent two of us after cleanup paths that were
2169    /// working correctly.
2170    pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2171        let inner = self.read_inner()?;
2172        let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2173        for key in inner.client_to_module.keys() {
2174            *per_connection.entry(key.connection_id).or_insert(0) += 1;
2175        }
2176        let max = per_connection.values().copied().max().unwrap_or(0);
2177        Ok((per_connection.len(), max))
2178    }
2179
2180    pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2181        let inner = self.read_inner()?;
2182        Ok(inner
2183            .client_to_module
2184            .keys()
2185            .any(|key| key.channel == route_channel))
2186    }
2187
2188    /// Whether daemon-wide shutdown has begun draining all providers.
2189    /// `None` means normal operation; during daemon shutdown, return the
2190    /// endpoint's measured result, or false if no completed measurement exists.
2191    pub(crate) fn daemon_drain_result_for_connection(
2192        &self,
2193        connection_id: ConnectionId,
2194    ) -> Result<Option<bool>, ForwardingError> {
2195        let inner = self.read_inner()?;
2196        Ok(inner.daemon_draining.then(|| {
2197            inner
2198                .endpoint_by_connection
2199                .get(&connection_id)
2200                .and_then(|endpoint| inner.daemon_drain_results.get(endpoint))
2201                .copied()
2202                .unwrap_or(false)
2203        }))
2204    }
2205
2206    #[cfg(unix)]
2207    pub(crate) fn record_daemon_drain_result(
2208        &self,
2209        endpoint: ModuleEndpointId,
2210        drained: bool,
2211    ) -> Result<(), ForwardingError> {
2212        let mut inner = self.write_inner()?;
2213        // Only update an endpoint captured at the start of daemon shutdown.
2214        // Teardown removes its entry, so a late wait cannot recreate the result
2215        // or attribute it to a replacement connection.
2216        if let Some(result) = inner.daemon_drain_results.get_mut(&endpoint) {
2217            *result = drained;
2218        }
2219        Ok(())
2220    }
2221
2222    /// Gate every provider atomically, including registrations racing shutdown.
2223    /// No supervisor lock is held while taking the forwarding lock.
2224    #[cfg(unix)]
2225    pub(crate) fn begin_daemon_drain(
2226        &self,
2227    ) -> Result<Vec<(String, ModuleEndpointId)>, ForwardingError> {
2228        let mut inner = self.write_inner()?;
2229        if inner.daemon_draining {
2230            return Ok(Vec::new());
2231        }
2232        inner.daemon_draining = true;
2233        let modules = inner
2234            .modules_by_id
2235            .iter()
2236            .map(|(id, module)| (id.clone(), module.endpoint))
2237            .collect::<Vec<_>>();
2238        for (_, endpoint) in &modules {
2239            inner.daemon_drain_results.insert(*endpoint, false);
2240            inner
2241                .draining_endpoints
2242                .insert(*endpoint, RouteCloseReason::Restart);
2243        }
2244        // Swap candidates and superseded incumbents are not routable, but they
2245        // are live endpoints that could still be sent module-control work, so
2246        // the daemon-wide gate covers them too. Only the active endpoints are
2247        // returned to the supervisor for measurement, one per module id.
2248        let off_slot_endpoints = inner
2249            .candidates_by_id
2250            .values()
2251            .map(|module| module.endpoint)
2252            .chain(inner.superseded_endpoints.keys().copied())
2253            .collect::<Vec<_>>();
2254        for endpoint in off_slot_endpoints {
2255            inner
2256                .draining_endpoints
2257                .insert(endpoint, RouteCloseReason::Restart);
2258        }
2259        Ok(modules)
2260    }
2261
2262    /// Begin draining whatever endpoint is ACTIVE for `module_id`.
2263    ///
2264    /// After a swap's cutover the active endpoint is the promoted candidate, so
2265    /// this must not be used to drain the incumbent; use
2266    /// [`Self::begin_endpoint_drain`] with the endpoint `cutover_candidate`
2267    /// returned.
2268    pub(crate) fn begin_module_drain(
2269        &self,
2270        module_id: &str,
2271        reason: RouteCloseReason,
2272    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2273        let mut inner = self.write_inner()?;
2274        let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2275            return Ok(None);
2276        };
2277        Ok(Some(begin_drain_locked(
2278            &mut inner, module_id, module, reason,
2279        )))
2280    }
2281
2282    /// Begin draining one specific endpoint, whichever slot it is in.
2283    ///
2284    /// This is how a swap drains its incumbent after cutover: the incumbent's
2285    /// endpoint is captured by `cutover_candidate`, and resolving it by module id
2286    /// instead would find the promoted candidate and leave neither process
2287    /// routable. Returns `Ok(None)` when the endpoint is no longer registered.
2288    pub(crate) fn begin_endpoint_drain(
2289        &self,
2290        endpoint: ModuleEndpointId,
2291        reason: RouteCloseReason,
2292    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2293        let mut inner = self.write_inner()?;
2294        let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2295            return Ok(None);
2296        };
2297        let module_id = inner
2298            .module_id_by_endpoint
2299            .get(&endpoint)
2300            .cloned()
2301            .expect("an endpoint resolved to a module connection has a module id");
2302        Ok(Some(begin_drain_locked(
2303            &mut inner, &module_id, module, reason,
2304        )))
2305    }
2306}
2307
2308/// Mark `module`'s endpoint draining and settle everything still pending on it.
2309/// Shared by the by-id and by-endpoint drain entry points, which differ only in
2310/// how they find the endpoint.
2311fn begin_drain_locked(
2312    inner: &mut ForwardingInner,
2313    module_id: &str,
2314    module: ModuleConnection,
2315    reason: RouteCloseReason,
2316) -> ModuleDrainTarget {
2317    {
2318        let endpoint = module.endpoint;
2319        inner.draining_endpoints.insert(endpoint, reason);
2320
2321        let flows = inner
2322            .client_to_module
2323            .values()
2324            .filter(|route| route.module_endpoint == endpoint)
2325            .map(|route| Arc::clone(&route.flow))
2326            .collect::<Vec<_>>();
2327        let excluded_subscriptions = flows
2328            .into_iter()
2329            .map(|flow| flow.begin_drain())
2330            .fold(0u32, u32::saturating_add);
2331
2332        let pending_keys = inner
2333            .pending_relays
2334            .keys()
2335            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2336            .copied()
2337            .collect::<Vec<_>>();
2338        let mut abandoned_bindings = Vec::new();
2339        for key in pending_keys {
2340            let Some(pending) = inner.pending_relays.remove(&key) else {
2341                continue;
2342            };
2343            release_reserved_route_locked(
2344                inner,
2345                pending.reservation.client_key,
2346                pending.reservation.module_key,
2347            );
2348            if pending.relay_enqueued {
2349                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2350                    abandoned_bindings.push(target);
2351                }
2352            } else {
2353                // The bind may still be on its way into the module's queue;
2354                // a GOODBYE sent now could overtake it. `route.open` sends
2355                // it once its send completes.
2356                inner.relays_released_before_enqueue.insert(key);
2357            }
2358            let _ = pending
2359                .sender
2360                .send(RouteBindRelayOutcome::Rejected(ErrorBody::new(
2361                    "module_reloading",
2362                    format!("module_id '{module_id}' is reloading"),
2363                )));
2364        }
2365
2366        let pending_control_keys = inner
2367            .pending_control_rpcs
2368            .keys()
2369            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2370            .copied()
2371            .collect::<Vec<_>>();
2372        for key in pending_control_keys {
2373            if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2374                let _ = pending
2375                    .sender
2376                    .send(ModuleControlRpcOutcome::ModuleGone(format!(
2377                        "module '{module_id}' began draining during module-control RPC"
2378                    )));
2379            }
2380        }
2381
2382        ModuleDrainTarget {
2383            endpoint,
2384            sink: module.sink,
2385            negotiated_ver: module.negotiated_ver,
2386            abandoned_bindings,
2387            excluded_subscriptions,
2388        }
2389    }
2390}
2391
2392/// What a drain that timed out was still waiting on, for the log line that
2393/// reports the timeout. Per-request ages are not tracked.
2394#[derive(Debug, Default, PartialEq, Eq)]
2395pub(crate) struct DrainHoldouts {
2396    /// Requests still counted against the drain (subscriptions flagged as
2397    /// such are excluded from the drain and not counted here).
2398    pub(crate) requests: usize,
2399    /// Routes holding at least one of those requests.
2400    pub(crate) routes: usize,
2401    /// Every route on the endpoint, for scale.
2402    pub(crate) total_routes: usize,
2403    /// The client connections holding the most requests, largest first, at
2404    /// most three: enough to name the consumer without listing every route.
2405    pub(crate) top_connections: Vec<(u64, usize)>,
2406    /// The held requests themselves, as the module saw them, so its own log
2407    /// can say what each one was: `(module channel, corr)`, ordered by channel
2408    /// then corr, at most [`DRAIN_HELD_REQUESTS_LISTED`]. The daemon never reads
2409    /// request bodies, so it cannot name a request's method; the module can,
2410    /// from the channel and corr. A request is released only when the module
2411    /// sends a terminal frame (Response, Error or StreamEnd) with its corr on
2412    /// its route, so every pair listed here is one the module ended, if at all,
2413    /// without sending that frame.
2414    pub(crate) held: Vec<(u16, u64)>,
2415}
2416
2417/// How many held requests the drain-timeout line lists by channel and corr.
2418pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
2419
2420impl ForwardingTable {
2421    /// Summarise the requests one endpoint's drain is still waiting on.
2422    pub(crate) fn endpoint_drain_holdouts(
2423        &self,
2424        endpoint: ModuleEndpointId,
2425    ) -> Result<DrainHoldouts, ForwardingError> {
2426        let inner = self.read_inner()?;
2427        let mut holdouts = DrainHoldouts::default();
2428        let mut by_connection: HashMap<u64, usize> = HashMap::new();
2429        for (key, route) in &inner.client_to_module {
2430            if route.module_endpoint != endpoint {
2431                continue;
2432            }
2433            holdouts.total_routes += 1;
2434            let held = route.flow.drain_in_flight();
2435            if held == 0 {
2436                continue;
2437            }
2438            holdouts.requests += held;
2439            holdouts.routes += 1;
2440            *by_connection.entry(key.connection_id.get()).or_default() += held;
2441            holdouts.held.extend(
2442                route
2443                    .flow
2444                    .drain_held_corrs()
2445                    .into_iter()
2446                    .map(|corr| (route.module_channel, corr)),
2447            );
2448        }
2449        holdouts.held.sort_unstable();
2450        holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
2451        let mut connections = by_connection.into_iter().collect::<Vec<_>>();
2452        connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
2453        connections.truncate(3);
2454        holdouts.top_connections = connections;
2455        Ok(holdouts)
2456    }
2457
2458    pub(crate) fn endpoint_in_flight_count(
2459        &self,
2460        endpoint: ModuleEndpointId,
2461    ) -> Result<usize, ForwardingError> {
2462        let inner = self.read_inner()?;
2463        Ok(inner
2464            .client_to_module
2465            .values()
2466            .filter(|route| route.module_endpoint == endpoint)
2467            .map(|route| route.flow.drain_in_flight())
2468            .sum())
2469    }
2470
2471    pub(crate) fn endpoint_is_draining(
2472        &self,
2473        endpoint: ModuleEndpointId,
2474    ) -> Result<bool, ForwardingError> {
2475        Ok(self
2476            .read_inner()?
2477            .draining_endpoints
2478            .contains_key(&endpoint))
2479    }
2480
2481    pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
2482        let inner = self.read_inner()?;
2483        Ok(inner
2484            .modules_by_id
2485            .get(module_id)
2486            .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
2487    }
2488
2489    pub(crate) fn release_module_endpoint_routes(
2490        &self,
2491        endpoint: ModuleEndpointId,
2492    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2493        let mut inner = self.write_inner()?;
2494        let routes = inner
2495            .module_to_client
2496            .iter()
2497            .filter(|(module_key, _)| module_key.endpoint == endpoint)
2498            .map(|(module_key, route)| (*module_key, route.module_epoch))
2499            .collect::<Vec<_>>();
2500        let mut released = Vec::with_capacity(routes.len());
2501        for (module_key, epoch) in routes {
2502            if let RouteRelease::Removed(target) =
2503                release_module_route_locked(&mut inner, module_key, epoch)
2504            {
2505                released.push(target);
2506            }
2507        }
2508        Ok(released)
2509    }
2510
2511    /// Enumerate one endpoint's current routes without contacting the module.
2512    ///
2513    /// The read lock makes this safe while the endpoint drains: the returned
2514    /// `draining` marker describes the same table state that owns the route,
2515    /// rather than inferring liveness from a module that may be stopping.
2516    pub(crate) fn endpoint_routes(
2517        &self,
2518        endpoint: ModuleEndpointId,
2519    ) -> Result<Vec<EndpointRoute>, ForwardingError> {
2520        let inner = self.read_inner()?;
2521        Ok(endpoint_routes_locked(&inner, endpoint))
2522    }
2523
2524    /// Snapshot all live endpoint route sets under one forwarding-table read lock.
2525    pub(crate) fn route_census(
2526        &self,
2527        module_id: Option<&str>,
2528    ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
2529        let inner = self.read_inner()?;
2530        let mut endpoints = inner
2531            .modules_by_id
2532            .iter()
2533            .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
2534            .map(|(id, module)| (id.clone(), module.endpoint))
2535            .collect::<Vec<_>>();
2536        endpoints.sort_by(|left, right| left.0.cmp(&right.0));
2537        Ok(endpoints
2538            .into_iter()
2539            .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
2540            .collect())
2541    }
2542
2543    /// Snapshot only the endpoint currently routable under this module id.
2544    pub(crate) fn live_roots(
2545        &self,
2546        module_id: &str,
2547    ) -> Result<ModuleControlResponseToModule, ForwardingError> {
2548        let inner = self.read_inner()?;
2549        let endpoint = inner
2550            .modules_by_id
2551            .get(module_id)
2552            .map(|module| module.endpoint);
2553        let mut roots = BTreeMap::new();
2554        let mut unknown_root_bindings = 0;
2555        let mut total_bindings = 0;
2556        if let Some(endpoint) = endpoint {
2557            for binding in inner
2558                .module_to_client
2559                .values()
2560                .filter(|binding| binding.module_endpoint == endpoint)
2561            {
2562                total_bindings += 1;
2563                let workdir = binding
2564                    .scope
2565                    .as_ref()
2566                    .and_then(|scope| scope.workdir.as_deref())
2567                    .map(std::path::Path::new);
2568                if let Some(root) = &binding.project_root {
2569                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2570                    entry.0 += 1;
2571                } else if workdir.is_none() {
2572                    unknown_root_bindings += 1;
2573                }
2574                if let Some(workdir) = workdir.filter(|workdir| {
2575                    binding.project_root.as_ref().map(ProjectRootId::as_path) != Some(*workdir)
2576                }) {
2577                    roots.entry(workdir.to_path_buf()).or_insert((0, 0)).0 += 1;
2578                }
2579            }
2580            for pending in inner
2581                .pending_relays
2582                .values()
2583                .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
2584            {
2585                total_bindings += 1;
2586                if let Some(root) = &pending.reservation.project_root {
2587                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2588                    entry.1 += 1;
2589                } else {
2590                    unknown_root_bindings += 1;
2591                }
2592            }
2593        }
2594        Ok(ModuleControlResponseToModule::LiveRoots {
2595            roots: roots
2596                .into_iter()
2597                .map(|(project_root, (bound, pending))| LiveRoot {
2598                    project_root,
2599                    bound,
2600                    pending,
2601                })
2602                .collect(),
2603            unknown_root_bindings,
2604            total_bindings,
2605        })
2606    }
2607
2608    /// True if this connection already owns committed or reserved CLIENT routes.
2609    /// A module registers (HELLO) before serving and never opens client routes, so
2610    /// a connection that has client routes must not also become a module endpoint
2611    /// — otherwise one connection holds both client and module state and cleanup
2612    /// only releases one side.
2613    pub(crate) fn connection_has_client_routes(
2614        &self,
2615        connection_id: ConnectionId,
2616    ) -> Result<bool, ForwardingError> {
2617        let inner = self.read_inner()?;
2618        Ok(connection_has_client_routes_locked(&inner, connection_id))
2619    }
2620
2621    pub(crate) fn cleanup_connection(
2622        &self,
2623        connection_id: ConnectionId,
2624    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2625        self.cleanup_connection_counted(connection_id)
2626            .map(|cleanup| cleanup.released)
2627    }
2628
2629    /// [`Self::cleanup_connection`], also reporting how many pending
2630    /// route.bind relays to the closed module were aborted. That count is the
2631    /// `abandoned` figure of the `route.closed` push sent for a lost module
2632    /// connection; it is always zero for a client connection.
2633    pub(crate) fn cleanup_connection_counted(
2634        &self,
2635        connection_id: ConnectionId,
2636    ) -> Result<ConnectionCleanup, ForwardingError> {
2637        let mut inner = self.write_inner()?;
2638        inner.closing_connections.insert(connection_id);
2639        let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
2640            remove_module_connection_locked(&mut inner, endpoint)
2641        } else {
2642            ConnectionCleanup {
2643                released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
2644                abandoned_relays: 0,
2645            }
2646        };
2647        // The closing mark refuses new work for a connection whose teardown is
2648        // still pending. This is the latest point at which lifting it is safe:
2649        // teardown has just removed every per-connection entry above, under
2650        // this same write lock, so no lookup can still find live state for the
2651        // id; and ids come from a monotonic counter that never reissues one,
2652        // so the id can never name a future connection either. Keeping the
2653        // mark past this point only grew the set by one entry per connection
2654        // for the life of the daemon.
2655        inner.closing_connections.remove(&connection_id);
2656        Ok(cleanup)
2657    }
2658
2659    fn cleanup_client_connection_locked(
2660        inner: &mut ForwardingInner,
2661        connection_id: ConnectionId,
2662    ) -> Vec<GoodbyeTarget> {
2663        let routes = inner
2664            .client_to_module
2665            .iter()
2666            .filter(|(key, _)| key.connection_id == connection_id)
2667            .map(|(key, route)| (*key, route.client_epoch))
2668            .collect::<Vec<_>>();
2669        let mut released = Vec::with_capacity(routes.len());
2670        for (client_key, epoch) in routes {
2671            if let RouteRelease::Removed(target) =
2672                release_client_route_locked(inner, client_key, epoch)
2673            {
2674                released.push(target);
2675            }
2676        }
2677
2678        let pending_keys = inner
2679            .pending_relays
2680            .iter()
2681            .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
2682            .map(|(key, _)| *key)
2683            .collect::<Vec<_>>();
2684        for key in pending_keys {
2685            let Some(pending) = inner.pending_relays.remove(&key) else {
2686                continue;
2687            };
2688            release_reserved_route_locked(
2689                inner,
2690                pending.reservation.client_key,
2691                pending.reservation.module_key,
2692            );
2693            if pending.relay_enqueued {
2694                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2695                    released.push(target);
2696                }
2697            } else {
2698                // The bind may still be on its way into the module's queue;
2699                // a GOODBYE sent now could overtake it. `route.open` sends
2700                // it once its send completes.
2701                inner.relays_released_before_enqueue.insert(key);
2702            }
2703            let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
2704                "client connection closed during route.bind relay".to_string(),
2705            ));
2706        }
2707
2708        let orphaned = inner
2709            .reserved_client
2710            .iter()
2711            .filter(|(key, _)| key.connection_id == connection_id)
2712            .map(|(client, module)| (*client, *module))
2713            .collect::<Vec<_>>();
2714        for (client_key, module_key) in orphaned {
2715            release_reserved_route_locked(inner, client_key, module_key);
2716        }
2717        inner.next_client_channel.remove(&connection_id);
2718        inner
2719            .client_slot_epochs
2720            .retain(|key, _| key.connection_id != connection_id);
2721        inner
2722            .last_published_epoch
2723            .retain(|key, _| key.connection_id != connection_id);
2724        inner
2725            .status
2726            .retain(|(key, _), _| key.connection_id != connection_id);
2727
2728        released
2729    }
2730
2731    /// Close a client connection whose egress queue refused a frame for the
2732    /// route `(connection_id, channel)` at `expected_epoch`. The whole
2733    /// connection closes because its queue is shared by every route on it.
2734    ///
2735    /// The first request that actually closes the connection logs one WARN with
2736    /// the diagnosis: which principals the connection's routes belonged to,
2737    /// which module's frame did not fit and on which client channel, and what
2738    /// the queue held at that moment.
2739    pub(crate) fn escalate_client_delivery_failure(
2740        &self,
2741        connection_id: ConnectionId,
2742        channel: u16,
2743        expected_epoch: u32,
2744        reason: CloseReason,
2745        undelivered: UndeliveredFrame<'_>,
2746    ) -> Result<bool, ForwardingError> {
2747        self.escalate_client_delivery_failure_diagnosed(
2748            connection_id,
2749            channel,
2750            expected_epoch,
2751            reason,
2752            undelivered,
2753            None,
2754        )
2755    }
2756
2757    pub(crate) fn escalate_client_delivery_failure_diagnosed(
2758        &self,
2759        connection_id: ConnectionId,
2760        channel: u16,
2761        expected_epoch: u32,
2762        reason: CloseReason,
2763        undelivered: UndeliveredFrame<'_>,
2764        diagnosis: Option<&crate::router::ClientDeliveryDiagnosis>,
2765    ) -> Result<bool, ForwardingError> {
2766        let principals = {
2767            let mut inner = self.write_inner()?;
2768            let key = ClientRouteKey {
2769                connection_id,
2770                channel,
2771            };
2772            if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
2773                None
2774            } else {
2775                inner.closing_connections.insert(connection_id);
2776                Some(connection_principals_locked(&inner, connection_id))
2777            }
2778        };
2779        let Some(principals) = principals else {
2780            return Ok(false);
2781        };
2782        let backlog = undelivered.sink.backlog();
2783        let close_reason = reason.to_string();
2784        if self.request_connection_close(connection_id, reason) {
2785            warn!(
2786                connection_id = connection_id.get(),
2787                principals = %principals,
2788                module_id = undelivered.module_id.unwrap_or("unknown"),
2789                client_channel = channel,
2790                queued_bytes = backlog.queued_bytes,
2791                queued_frames = backlog.queued_frames,
2792                frame_type = ?diagnosis.map(|failure| failure.frame_type),
2793                frame_flags = ?diagnosis.map(|failure| failure.flags.0),
2794                close_cause = diagnosis.map(|failure| failure.cause).unwrap_or("delivery_failed"),
2795                oldest_queued_ms = backlog
2796                    .oldest_age
2797                    .map(|age| age.as_millis() as u64)
2798                    .unwrap_or(0),
2799                close_reason = %close_reason,
2800                "closing client connection: its egress queue could not take a frame"
2801            );
2802        }
2803        Ok(true)
2804    }
2805
2806    /// Publish the tags a scope sync changed and close the live routes its
2807    /// drain table selects, in one critical section.
2808    ///
2809    /// The caller holds the scope table's write lock (scope table, then this
2810    /// table, always). Publishing and selecting under one lock is what closes
2811    /// the race with a bind commit: a commit before this sees the old tag and
2812    /// its route is selected here like any other; a commit after it sees the
2813    /// new tag and is refused.
2814    ///
2815    /// Every client route index entry is scanned, which includes routes on a
2816    /// swap's superseded endpoint (the old process of a module being replaced
2817    /// blue/green, which keeps its existing routes until they drain): those
2818    /// stay indexed until drained, so ending a scope reaches them too.
2819    pub(crate) fn publish_scope_changes(
2820        &self,
2821        changes: &[ScopeTagChange],
2822    ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
2823        let mut inner = self.write_inner()?;
2824        let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
2825        for change in changes {
2826            let key = (change.owner.clone(), change.scope_ref.clone());
2827            match change.after {
2828                Some(tag) => {
2829                    inner.scope_tags.insert(key, tag);
2830                }
2831                None => {
2832                    inner.scope_tags.remove(&key);
2833                }
2834            }
2835            by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
2836        }
2837        let mut selected = Vec::new();
2838        for (client_key, route) in &inner.client_to_module {
2839            let Some(scope) = &route.scope else {
2840                continue;
2841            };
2842            let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
2843            else {
2844                continue;
2845            };
2846            if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
2847                continue;
2848            }
2849            let reason = match &change.drain {
2850                ScopeDrain::Nothing => continue,
2851                ScopeDrain::All(reason) => *reason,
2852                ScopeDrain::Carriers(narrowed) => {
2853                    let owner = Principal::Reserved {
2854                        module_id: scope.owner.clone(),
2855                    };
2856                    let hit = route.principal != owner
2857                        && narrowed.iter().any(|(principal, allowed)| {
2858                            *principal == route.principal
2859                                && allowed
2860                                    .as_ref()
2861                                    .is_none_or(|targets| !targets.contains(&route.module_id))
2862                        });
2863                    if !hit {
2864                        continue;
2865                    }
2866                    RouteCloseReason::ScopeCarrierRemoved
2867                }
2868            };
2869            selected.push((*client_key, route.client_epoch, reason));
2870        }
2871        let mut drained = Vec::new();
2872        for (client_key, client_epoch, reason) in selected {
2873            let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
2874                continue;
2875            };
2876            let release = release_client_route_locked(&mut inner, client_key, client_epoch);
2877            self.record_route_release(&release);
2878            if let RouteRelease::Removed(module) = release {
2879                drained.push(ScopeDrainedRoute {
2880                    reason,
2881                    scope: route
2882                        .scope
2883                        .clone()
2884                        .expect("selected scope drain has a bound scope"),
2885                    module_id: route.module_id.clone(),
2886                    client: GoodbyeTarget {
2887                        connection_id: route.client_connection_id,
2888                        sink: route.client_sink.clone(),
2889                        negotiated_ver: route.client_negotiated_ver,
2890                        channel: route.client_channel,
2891                        epoch: route.client_epoch,
2892                        kind: GoodbyeTargetKind::Client,
2893                        module_id: Some(route.module_id.clone()),
2894                    },
2895                    module,
2896                });
2897            }
2898        }
2899        Ok(drained)
2900    }
2901
2902    /// The published tag of one scope, for tests of the commit re-check.
2903    #[cfg(test)]
2904    pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
2905        self.read_inner()
2906            .ok()?
2907            .scope_tags
2908            .get(&(owner.to_string(), scope_ref.to_string()))
2909            .copied()
2910    }
2911
2912    fn record_route_release(&self, release: &RouteRelease) {
2913        match release {
2914            RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
2915            RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
2916            RouteRelease::Absent => {}
2917        }
2918    }
2919
2920    fn read_inner(&self) -> Result<RwLockReadGuard<'_, ForwardingInner>, ForwardingError> {
2921        self.inner.read().map_err(|_| ForwardingError::Poisoned)
2922    }
2923
2924    fn write_inner(&self) -> Result<RwLockWriteGuard<'_, ForwardingInner>, ForwardingError> {
2925        self.inner.write().map_err(|_| ForwardingError::Poisoned)
2926    }
2927
2928    fn lock_close_registry(
2929        &self,
2930    ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
2931        self.close_registry
2932            .lock()
2933            .unwrap_or_else(|poisoned| poisoned.into_inner())
2934    }
2935}
2936
2937impl ForwardingInner {
2938    fn allocate_route_slots(
2939        &mut self,
2940        connection_id: ConnectionId,
2941        endpoint: ModuleEndpointId,
2942    ) -> Result<(u16, u32, u16, u32), ForwardingError> {
2943        let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
2944        let mut client_channel = client_start;
2945        let client_channel = loop {
2946            let key = ClientRouteKey {
2947                connection_id,
2948                channel: client_channel,
2949            };
2950            let eligible = !self.client_to_module.contains_key(&key)
2951                && !self.reserved_client.contains_key(&key)
2952                && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2953            if eligible {
2954                break client_channel;
2955            }
2956            client_channel = next_channel(client_channel);
2957            if client_channel == client_start {
2958                return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
2959            }
2960        };
2961
2962        let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
2963        let mut module_channel = module_start;
2964        let module_channel = loop {
2965            let key = ModuleRouteKey {
2966                endpoint,
2967                channel: module_channel,
2968            };
2969            let eligible = !self.module_to_client.contains_key(&key)
2970                && !self.reserved_module.contains_key(&key)
2971                && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2972            if eligible {
2973                break module_channel;
2974            }
2975            module_channel = next_channel(module_channel);
2976            if module_channel == module_start {
2977                return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
2978            }
2979        };
2980
2981        let client_key = ClientRouteKey {
2982            connection_id,
2983            channel: client_channel,
2984        };
2985        let module_key = ModuleRouteKey {
2986            endpoint,
2987            channel: module_channel,
2988        };
2989        let client_epoch = self
2990            .client_slot_epochs
2991            .get(&client_key)
2992            .copied()
2993            .unwrap_or(0)
2994            + 1;
2995        let module_epoch = self
2996            .module_slot_epochs
2997            .get(&module_key)
2998            .copied()
2999            .unwrap_or(0)
3000            + 1;
3001        self.client_slot_epochs.insert(client_key, client_epoch);
3002        self.module_slot_epochs.insert(module_key, module_epoch);
3003        self.next_client_channel
3004            .insert(connection_id, next_channel(client_channel));
3005        self.next_module_channel
3006            .insert(endpoint, next_channel(module_channel));
3007        Ok((client_channel, client_epoch, module_channel, module_epoch))
3008    }
3009
3010    fn allocate_control_corr(
3011        &mut self,
3012        endpoint: ModuleEndpointId,
3013    ) -> Result<u64, ForwardingError> {
3014        let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
3015        if candidate == 0 {
3016            self.closing_connections.insert(endpoint.connection_id);
3017            return Err(ForwardingError::RelayCorrelationExhausted);
3018        }
3019        self.next_control_corr.insert(
3020            endpoint,
3021            if candidate == u64::MAX {
3022                0
3023            } else {
3024                candidate + 1
3025            },
3026        );
3027        Ok(candidate)
3028    }
3029}
3030
3031fn connection_has_client_routes_locked(
3032    inner: &ForwardingInner,
3033    connection_id: ConnectionId,
3034) -> bool {
3035    inner
3036        .client_to_module
3037        .keys()
3038        .any(|key| key.connection_id == connection_id)
3039        || inner
3040            .reserved_client
3041            .keys()
3042            .any(|key| key.connection_id == connection_id)
3043}
3044
3045fn check_module_connection_role_locked(
3046    inner: &ForwardingInner,
3047    connection_id: ConnectionId,
3048) -> Result<(), ForwardingError> {
3049    if inner.endpoint_by_connection.contains_key(&connection_id)
3050        || connection_has_client_routes_locked(inner, connection_id)
3051    {
3052        return Err(ForwardingError::ConnectionRoleConflict { connection_id });
3053    }
3054    Ok(())
3055}
3056
3057fn next_channel(channel: u16) -> u16 {
3058    let next = channel.wrapping_add(1);
3059    if next == 0 {
3060        1
3061    } else {
3062        next
3063    }
3064}
3065
3066fn endpoint_routes_locked(
3067    inner: &ForwardingInner,
3068    endpoint: ModuleEndpointId,
3069) -> Vec<EndpointRoute> {
3070    let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
3071    let draining = drain_reason.is_some();
3072    let mut routes = inner
3073        .module_to_client
3074        .iter()
3075        .filter(|(module_key, _)| module_key.endpoint == endpoint)
3076        .map(|(_, route)| EndpointRoute {
3077            goodbye_target: GoodbyeTarget {
3078                connection_id: route.client_connection_id,
3079                sink: route.client_sink.clone(),
3080                negotiated_ver: route.client_negotiated_ver,
3081                channel: route.client_channel,
3082                epoch: route.client_epoch,
3083                kind: GoodbyeTargetKind::Client,
3084                module_id: Some(route.module_id.clone()),
3085            },
3086            principal: route.principal.clone(),
3087            bound_at: route.bound_at,
3088            draining,
3089            drain_reason,
3090        })
3091        .collect::<Vec<_>>();
3092    routes.sort_by_key(|route| {
3093        (
3094            route.goodbye_target.connection_id.get(),
3095            route.goodbye_target.channel,
3096            route.goodbye_target.epoch,
3097        )
3098    });
3099    routes
3100}
3101
3102/// Why a bind admitted under `scope` may no longer commit, read from the
3103/// published tags: `scope_ended` when the scope is gone or at another epoch,
3104/// `scope_changed` (retryable) when only its content moved.
3105fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
3106    let key = (scope.owner.clone(), scope.scope_ref.clone());
3107    match inner.scope_tags.get(&key) {
3108        Some(current) if *current == scope.tag => None,
3109        Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
3110            subc_protocol::error_codes::SCOPE_CHANGED,
3111            format!(
3112                "scope '{}' of {} changed while the route was being bound; re-open it",
3113                scope.scope_ref, scope.owner
3114            ),
3115        )),
3116        _ => Some(ErrorBody::new(
3117            subc_protocol::error_codes::SCOPE_ENDED,
3118            format!(
3119                "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
3120                scope.scope_ref, scope.owner, scope.tag.scope_epoch
3121            ),
3122        )),
3123    }
3124}
3125
3126fn release_reserved_route_locked(
3127    inner: &mut ForwardingInner,
3128    client_key: ClientRouteKey,
3129    module_key: ModuleRouteKey,
3130) {
3131    if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
3132        inner.reserved_client.remove(&client_key);
3133    }
3134    if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
3135        inner.reserved_module.remove(&module_key);
3136    }
3137    inner.status.retain(|(key, _), _| *key != client_key);
3138}
3139
3140fn release_client_route_locked(
3141    inner: &mut ForwardingInner,
3142    client_key: ClientRouteKey,
3143    expected_epoch: u32,
3144) -> RouteRelease {
3145    let Some(route) = inner.client_to_module.get(&client_key) else {
3146        return RouteRelease::Absent;
3147    };
3148    if route.client_epoch != expected_epoch {
3149        return RouteRelease::Stale;
3150    }
3151    let route = inner
3152        .client_to_module
3153        .remove(&client_key)
3154        .expect("route checked under the same forwarding lock");
3155    route.flow.close();
3156    if let Some(scope) = &route.scope {
3157        scope.release_bind();
3158    }
3159    inner.module_to_client.remove(&ModuleRouteKey {
3160        endpoint: route.module_endpoint,
3161        channel: route.module_channel,
3162    });
3163    inner.operator_confirms.route_closed(ModuleRouteKey {
3164        endpoint: route.module_endpoint,
3165        channel: route.module_channel,
3166    });
3167    inner.status.remove(&(client_key, expected_epoch));
3168    RouteRelease::Removed(GoodbyeTarget {
3169        connection_id: route.module_endpoint.connection_id,
3170        sink: route.module_sink.clone(),
3171        negotiated_ver: route.module_negotiated_ver,
3172        channel: route.module_channel,
3173        epoch: route.module_epoch,
3174        kind: GoodbyeTargetKind::Module,
3175        module_id: Some(route.module_id.clone()),
3176    })
3177}
3178
3179fn release_module_route_locked(
3180    inner: &mut ForwardingInner,
3181    module_key: ModuleRouteKey,
3182    expected_epoch: u32,
3183) -> RouteRelease {
3184    let Some(route) = inner.module_to_client.get(&module_key) else {
3185        return RouteRelease::Absent;
3186    };
3187    if route.module_epoch != expected_epoch {
3188        return RouteRelease::Stale;
3189    }
3190    let route = inner
3191        .module_to_client
3192        .remove(&module_key)
3193        .expect("route checked under the same forwarding lock");
3194    inner.operator_confirms.route_closed(module_key);
3195    route.flow.close();
3196    if let Some(scope) = &route.scope {
3197        scope.release_bind();
3198    }
3199    let client_key = ClientRouteKey {
3200        connection_id: route.client_connection_id,
3201        channel: route.client_channel,
3202    };
3203    inner.client_to_module.remove(&client_key);
3204    inner.status.remove(&(client_key, route.client_epoch));
3205    RouteRelease::Removed(GoodbyeTarget {
3206        connection_id: route.client_connection_id,
3207        sink: route.client_sink.clone(),
3208        negotiated_ver: route.client_negotiated_ver,
3209        channel: route.client_channel,
3210        epoch: route.client_epoch,
3211        kind: GoodbyeTargetKind::Client,
3212        module_id: Some(route.module_id.clone()),
3213    })
3214}
3215
3216fn commit_route_locked(
3217    inner: &mut ForwardingInner,
3218    pending: PendingRouteBindRelayEntry,
3219) -> Result<Option<GoodbyeTarget>, ForwardingError> {
3220    let reservation = pending.reservation;
3221    if inner
3222        .closing_connections
3223        .contains(&reservation.client_key.connection_id)
3224    {
3225        return Err(ForwardingError::ConnectionClosing {
3226            connection_id: reservation.client_key.connection_id,
3227        });
3228    }
3229    let module_id = inner
3230        .module_id_by_endpoint
3231        .get(&reservation.module_key.endpoint)
3232        .cloned()
3233        .ok_or(ForwardingError::StaleModuleEndpoint)?;
3234    if inner
3235        .draining_endpoints
3236        .contains_key(&reservation.module_key.endpoint)
3237    {
3238        return Err(ForwardingError::ModuleReloading { module_id });
3239    }
3240    if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
3241        || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
3242    {
3243        return Err(ForwardingError::UnknownReservation {
3244            client_channel: reservation.client_key.channel,
3245            module_channel: reservation.module_key.channel,
3246        });
3247    }
3248    let module = inner
3249        .modules_by_id
3250        .get(&module_id)
3251        .filter(|module| module.endpoint == reservation.module_key.endpoint)
3252        .cloned()
3253        .ok_or(ForwardingError::StaleModuleEndpoint)?;
3254    let binding = Arc::new(RouteBinding {
3255        client_connection_id: reservation.client_key.connection_id,
3256        client_sink: pending.client_sink,
3257        client_negotiated_ver: pending.client_negotiated_ver,
3258        client_channel: reservation.client_key.channel,
3259        client_epoch: reservation.client_epoch,
3260        module_id,
3261        module_endpoint: reservation.module_key.endpoint,
3262        module_sink: module.sink,
3263        module_negotiated_ver: module.negotiated_ver,
3264        module_channel: reservation.module_key.channel,
3265        module_epoch: reservation.module_epoch,
3266        principal: pending.principal,
3267        project_root: reservation.project_root.clone(),
3268        bound_at: Instant::now(),
3269        flow: Arc::new(ChannelFlow::new(window_for(&module.concurrency))),
3270        scope: pending.scope,
3271    });
3272    inner
3273        .client_to_module
3274        .insert(reservation.client_key, Arc::clone(&binding));
3275    inner
3276        .module_to_client
3277        .insert(reservation.module_key, Arc::clone(&binding));
3278    let previous_published = inner
3279        .last_published_epoch
3280        .insert(reservation.client_key, reservation.client_epoch);
3281
3282    // This point is reachable only through an accepted route.bind acknowledgement.
3283    // Publish history before notifying the opener, so a subsequent
3284    // `scope.served_by` query sees the acknowledged bind.
3285    if let Some(scope) = &binding.scope {
3286        scope.record_served(&binding.module_id);
3287    }
3288
3289    // Sending through the reserved slot cannot fail, but it reports a receiver
3290    // that closed after reservation and before this locked publication point.
3291    // The frame is stamped and charged to the queued-byte count here, when it
3292    // actually enters the queue, not when the slot was reserved.
3293    let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
3294    if client_writer_closed {
3295        // Not gated on `relay_enqueued`: only a module ack reaches this
3296        // commit, which proves the module holds the binding, and `route.open`
3297        // sends nothing for a relay a reply already settled.
3298        let abandoned = abandoned_route_target(inner, &reservation);
3299        if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
3300            route.flow.close();
3301        }
3302        inner.module_to_client.remove(&reservation.module_key);
3303        inner
3304            .status
3305            .remove(&(reservation.client_key, reservation.client_epoch));
3306        match previous_published {
3307            Some(epoch) => {
3308                inner
3309                    .last_published_epoch
3310                    .insert(reservation.client_key, epoch);
3311            }
3312            None => {
3313                inner.last_published_epoch.remove(&reservation.client_key);
3314            }
3315        }
3316        let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3317            "client egress closed during route publication".to_string(),
3318        ));
3319        return Ok(abandoned);
3320    }
3321
3322    if let Some(scope) = &binding.scope {
3323        scope.commit_presence();
3324    }
3325    let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
3326    Ok(None)
3327}
3328
3329/// The live module connection registered under exactly `endpoint`, whichever
3330/// slot holds it: active, swap candidate, or superseded incumbent.
3331///
3332/// Resolving by endpoint rather than by module id is what keeps an incumbent
3333/// addressable after a swap promoted a candidate over its id. An endpoint that
3334/// is in none of the three (a stale one, replaced without a promotion) resolves
3335/// to nothing, as it always has.
3336fn module_connection_for_endpoint_locked(
3337    inner: &ForwardingInner,
3338    endpoint: ModuleEndpointId,
3339) -> Option<&ModuleConnection> {
3340    let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
3341    inner
3342        .modules_by_id
3343        .get(module_id)
3344        .filter(|module| module.endpoint == endpoint)
3345        .or_else(|| {
3346            inner
3347                .candidates_by_id
3348                .get(module_id)
3349                .filter(|module| module.endpoint == endpoint)
3350        })
3351        .or_else(|| inner.superseded_endpoints.get(&endpoint))
3352}
3353
3354fn abandoned_route_target(
3355    inner: &ForwardingInner,
3356    reservation: &RouteReservation,
3357) -> Option<GoodbyeTarget> {
3358    let module_id = inner
3359        .module_id_by_endpoint
3360        .get(&reservation.module_key.endpoint)?;
3361    let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
3362    (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
3363        connection_id: module.endpoint.connection_id,
3364        sink: module.sink.clone(),
3365        negotiated_ver: module.negotiated_ver,
3366        channel: reservation.module_key.channel,
3367        epoch: reservation.module_epoch,
3368        kind: GoodbyeTargetKind::Module,
3369        module_id: Some(module_id.clone()),
3370    })
3371}
3372
3373/// Queue a registering module's HELLO_ACK on its sink. Called with the
3374/// forwarding write lock held, before the endpoint is inserted, which is what
3375/// puts the ack ahead of any `route.bind` or control RPC routed to the module.
3376/// `try_send` never waits, so holding the lock across it is safe.
3377fn enqueue_hello_ack_locked(
3378    sink: &FrameSink,
3379    connection_id: ConnectionId,
3380    hello_ack: Option<Frame>,
3381) -> Result<(), ForwardingError> {
3382    let Some(hello_ack) = hello_ack else {
3383        return Ok(());
3384    };
3385    sink.try_send(hello_ack)
3386        .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
3387}
3388
3389fn remove_module_connection_locked(
3390    inner: &mut ForwardingInner,
3391    endpoint: ModuleEndpointId,
3392) -> ConnectionCleanup {
3393    // Commit the disconnect before its routes are released, so their teardown
3394    // cannot replace module_closed with route_closed.
3395    inner.operator_confirms.module_closed(endpoint);
3396    inner.draining_endpoints.remove(&endpoint);
3397    inner.daemon_drain_results.remove(&endpoint);
3398    let module_id = inner.module_id_by_endpoint.remove(&endpoint);
3399    if let Some(module_id) = module_id.as_ref() {
3400        if inner
3401            .modules_by_id
3402            .get(module_id)
3403            .is_some_and(|module| module.endpoint == endpoint)
3404        {
3405            inner.modules_by_id.remove(module_id);
3406        }
3407        if inner
3408            .candidates_by_id
3409            .get(module_id)
3410            .is_some_and(|module| module.endpoint == endpoint)
3411        {
3412            inner.candidates_by_id.remove(module_id);
3413        }
3414    }
3415    inner.superseded_endpoints.remove(&endpoint);
3416    inner.endpoint_by_connection.remove(&endpoint.connection_id);
3417    inner.next_module_channel.remove(&endpoint);
3418    inner.next_control_corr.remove(&endpoint);
3419    inner
3420        .health_probe_tombstones
3421        .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
3422    inner
3423        .module_slot_epochs
3424        .retain(|key, _| key.endpoint != endpoint);
3425    let reserved_module_keys: Vec<ModuleRouteKey> = inner
3426        .reserved_module
3427        .keys()
3428        .filter(|module_key| module_key.endpoint == endpoint)
3429        .copied()
3430        .collect();
3431    for module_key in reserved_module_keys {
3432        if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
3433            release_reserved_route_locked(inner, client_key, module_key);
3434        }
3435    }
3436
3437    let pending_keys: Vec<_> = inner
3438        .pending_relays
3439        .keys()
3440        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3441        .copied()
3442        .collect();
3443    let pending: Vec<_> = pending_keys
3444        .into_iter()
3445        .filter_map(|key| {
3446            inner
3447                .pending_relays
3448                .remove(&key)
3449                .map(|pending| (key, pending))
3450        })
3451        .collect();
3452    let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
3453    for (key, pending) in pending {
3454        // Recorded like every other removal without a reply, so a `route.open`
3455        // still sending the bind learns the relay was released rather than
3456        // settled by the module.
3457        if !pending.relay_enqueued {
3458            inner.relays_released_before_enqueue.insert(key);
3459        }
3460        let module_label = module_id.as_deref().unwrap_or("unknown");
3461        let _ = pending
3462            .sender
3463            .send(RouteBindRelayOutcome::ModuleGone(format!(
3464                "module '{module_label}' connection closed during route.bind relay"
3465            )));
3466    }
3467
3468    let pending_control_keys: Vec<_> = inner
3469        .pending_control_rpcs
3470        .keys()
3471        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3472        .copied()
3473        .collect();
3474    let pending_control: Vec<_> = pending_control_keys
3475        .into_iter()
3476        .filter_map(|key| inner.pending_control_rpcs.remove(&key))
3477        .collect();
3478    for pending in pending_control {
3479        let module_label = module_id.as_deref().unwrap_or("unknown");
3480        let _ = pending
3481            .sender
3482            .send(ModuleControlRpcOutcome::ModuleGone(format!(
3483                "module '{module_label}' connection closed during module-control RPC"
3484            )));
3485    }
3486
3487    let module_routes = inner
3488        .module_to_client
3489        .iter()
3490        .filter(|(module_key, _)| module_key.endpoint == endpoint)
3491        .map(|(module_key, route)| (*module_key, route.module_epoch))
3492        .collect::<Vec<_>>();
3493    let mut released = Vec::with_capacity(module_routes.len());
3494    for (module_key, epoch) in module_routes {
3495        if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
3496        {
3497            released.push(target);
3498        }
3499    }
3500    ConnectionCleanup {
3501        released,
3502        abandoned_relays,
3503    }
3504}
3505
3506#[derive(Debug, Clone, Copy)]
3507struct RequestCredit {
3508    subscription: bool,
3509    excluded_from_drain: bool,
3510}
3511
3512#[derive(Debug, Default)]
3513struct CreditLedger {
3514    by_corr: HashMap<u64, Vec<RequestCredit>>,
3515}
3516
3517impl CreditLedger {
3518    fn acquire(&mut self, corr: u64, subscription: bool) {
3519        self.by_corr.entry(corr).or_default().push(RequestCredit {
3520            subscription,
3521            excluded_from_drain: false,
3522        });
3523    }
3524
3525    fn release(&mut self, corr: u64) -> bool {
3526        let Some(credits) = self.by_corr.get_mut(&corr) else {
3527            return false;
3528        };
3529        let released = credits.pop().is_some();
3530        if credits.is_empty() {
3531            self.by_corr.remove(&corr);
3532        }
3533        released
3534    }
3535
3536    fn capture_subscription_exclusions(&mut self) -> u32 {
3537        let mut excluded = 0u32;
3538        for credit in self.by_corr.values_mut().flatten() {
3539            if credit.subscription && !credit.excluded_from_drain {
3540                credit.excluded_from_drain = true;
3541                excluded = excluded.saturating_add(1);
3542            }
3543        }
3544        excluded
3545    }
3546
3547    #[cfg(test)]
3548    fn in_flight(&self) -> usize {
3549        self.by_corr.values().map(Vec::len).sum()
3550    }
3551
3552    fn drain_in_flight(&self) -> usize {
3553        self.by_corr
3554            .values()
3555            .flatten()
3556            .filter(|credit| !credit.excluded_from_drain)
3557            .count()
3558    }
3559
3560    /// The correlation ids of the requests a drain still waits on, one entry
3561    /// per held credit, in ascending order.
3562    fn drain_held_corrs(&self) -> Vec<u64> {
3563        let mut corrs = self
3564            .by_corr
3565            .iter()
3566            .flat_map(|(corr, credits)| {
3567                credits
3568                    .iter()
3569                    .filter(|credit| !credit.excluded_from_drain)
3570                    .map(move |_| *corr)
3571            })
3572            .collect::<Vec<_>>();
3573        corrs.sort_unstable();
3574        corrs
3575    }
3576}
3577
3578#[derive(Debug, Default)]
3579struct ChannelFlowState {
3580    closed: bool,
3581    credits: CreditLedger,
3582}
3583
3584/// Per-channel request-credit accounting shared by the client and module route halves.
3585#[derive(Debug)]
3586pub(crate) struct ChannelFlow {
3587    sem: Semaphore,
3588    window: usize,
3589    state: Mutex<ChannelFlowState>,
3590}
3591
3592impl ChannelFlow {
3593    pub(crate) fn new(window: usize) -> Self {
3594        debug_assert!(window > 0, "flow-control window must be non-zero");
3595        Self {
3596            sem: Semaphore::new(window),
3597            window,
3598            state: Mutex::new(ChannelFlowState::default()),
3599        }
3600    }
3601
3602    #[cfg(test)]
3603    pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
3604        self.acquire_tagged(0, false).await
3605    }
3606
3607    pub(crate) async fn acquire_tagged(
3608        &self,
3609        corr: u64,
3610        subscription: bool,
3611    ) -> Result<(), ChannelFlowClosed> {
3612        let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
3613        let mut state = self
3614            .state
3615            .lock()
3616            .unwrap_or_else(|poisoned| poisoned.into_inner());
3617        if state.closed {
3618            return Err(ChannelFlowClosed);
3619        }
3620        state.credits.acquire(corr, subscription);
3621        permit.forget();
3622        Ok(())
3623    }
3624
3625    #[cfg(test)]
3626    pub(crate) fn release(&self) {
3627        self.release_corr(0);
3628    }
3629
3630    pub(crate) fn release_corr(&self, corr: u64) {
3631        let released = self
3632            .state
3633            .lock()
3634            .unwrap_or_else(|poisoned| poisoned.into_inner())
3635            .credits
3636            .release(corr);
3637        if !released {
3638            // Protocol-conforming modules emit exactly one terminal per request.
3639            // This guard is a best-effort safety net against window growth, not a
3640            // security boundary against malicious peers.
3641            warn!(
3642                window = self.window,
3643                available = self.sem.available_permits(),
3644                "flow-control over-release ignored"
3645            );
3646            return;
3647        }
3648        if !self.sem.is_closed() {
3649            self.sem.add_permits(1);
3650        }
3651    }
3652
3653    #[cfg(test)]
3654    pub(crate) fn in_flight(&self) -> usize {
3655        self.state
3656            .lock()
3657            .unwrap_or_else(|poisoned| poisoned.into_inner())
3658            .credits
3659            .in_flight()
3660    }
3661
3662    pub(crate) fn drain_in_flight(&self) -> usize {
3663        self.state
3664            .lock()
3665            .unwrap_or_else(|poisoned| poisoned.into_inner())
3666            .credits
3667            .drain_in_flight()
3668    }
3669
3670    pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
3671        self.state
3672            .lock()
3673            .unwrap_or_else(|poisoned| poisoned.into_inner())
3674            .credits
3675            .drain_held_corrs()
3676    }
3677
3678    #[cfg(test)]
3679    pub(crate) fn available_permits(&self) -> usize {
3680        self.sem.available_permits()
3681    }
3682
3683    pub(crate) fn begin_drain(&self) -> u32 {
3684        let mut state = self
3685            .state
3686            .lock()
3687            .unwrap_or_else(|poisoned| poisoned.into_inner());
3688        state.closed = true;
3689        self.sem.close();
3690        state.credits.capture_subscription_exclusions()
3691    }
3692
3693    pub(crate) fn close(&self) {
3694        self.state
3695            .lock()
3696            .unwrap_or_else(|poisoned| poisoned.into_inner())
3697            .closed = true;
3698        self.sem.close();
3699    }
3700}
3701
3702#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3703pub(crate) struct ChannelFlowClosed;
3704
3705impl fmt::Display for ChannelFlowClosed {
3706    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3707        write!(f, "flow-control window closed")
3708    }
3709}
3710
3711impl Error for ChannelFlowClosed {}
3712
3713fn window_for(concurrency: &Concurrency) -> usize {
3714    match concurrency {
3715        Concurrency::Serial => 1,
3716        Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
3717        Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
3718    }
3719}
3720
3721#[derive(Debug, Clone, PartialEq, Eq)]
3722pub enum ForwardingError {
3723    /// A connection cannot hold both client routes and a module endpoint, or
3724    /// replace a module identity while that endpoint is still registered.
3725    ConnectionRoleConflict {
3726        connection_id: ConnectionId,
3727    },
3728    NoModuleConnection,
3729    ModuleReloading {
3730        module_id: String,
3731    },
3732    StaleModuleEndpoint,
3733    UnknownReservation {
3734        client_channel: u16,
3735        module_channel: u16,
3736    },
3737    ClientRouteChannelExhausted {
3738        connection_id: ConnectionId,
3739    },
3740    ModuleRouteChannelExhausted {
3741        endpoint: ModuleEndpointId,
3742    },
3743    RelayCorrelationExhausted,
3744    ConnectionClosing {
3745        connection_id: ConnectionId,
3746    },
3747    ClientEgressClosed {
3748        connection_id: ConnectionId,
3749    },
3750    RouteOpenBuild(String),
3751    /// A swap candidate is already registered for this module id.
3752    CandidateSlotOccupied {
3753        module_id: String,
3754    },
3755    /// A registering module's HELLO_ACK could not be queued because its
3756    /// outbound queue is closed or full.
3757    ModuleEgressUnavailable {
3758        connection_id: ConnectionId,
3759    },
3760    Poisoned,
3761}
3762
3763impl fmt::Display for ForwardingError {
3764    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3765        match self {
3766            Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
3767            Self::NoModuleConnection => write!(f, "no module connection is registered"),
3768            Self::ModuleReloading { module_id } => {
3769                write!(f, "module_id '{module_id}' is reloading")
3770            }
3771            Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
3772            Self::UnknownReservation {
3773                client_channel,
3774                module_channel,
3775            } => write!(
3776                f,
3777                "route reservation client channel {client_channel} / module channel {module_channel} was not found"
3778            ),
3779            Self::ClientRouteChannelExhausted { connection_id } => write!(
3780                f,
3781                "no client route channels are available for connection {}",
3782                connection_id.get()
3783            ),
3784            Self::ModuleRouteChannelExhausted { endpoint } => write!(
3785                f,
3786                "no module route channels are available for endpoint generation {} on connection {}",
3787                endpoint.generation,
3788                endpoint.connection_id.get()
3789            ),
3790            Self::RelayCorrelationExhausted => {
3791                write!(f, "module control correlation ids are exhausted")
3792            }
3793            Self::ConnectionClosing { connection_id } => write!(
3794                f,
3795                "connection {} is closing and cannot accept route allocation",
3796                connection_id.get()
3797            ),
3798            Self::ClientEgressClosed { connection_id } => write!(
3799                f,
3800                "client connection {} egress is closed",
3801                connection_id.get()
3802            ),
3803            Self::RouteOpenBuild(message) => {
3804                write!(f, "failed to prebuild route.open response: {message}")
3805            }
3806            Self::CandidateSlotOccupied { module_id } => write!(
3807                f,
3808                "module_id '{module_id}' already has a swap candidate registered"
3809            ),
3810            Self::ModuleEgressUnavailable { connection_id } => write!(
3811                f,
3812                "module connection {} egress is unavailable; HELLO_ACK could not be queued",
3813                connection_id.get()
3814            ),
3815            Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
3816        }
3817    }
3818}
3819
3820impl Error for ForwardingError {}
3821
3822#[cfg(test)]
3823mod tests {
3824    use std::time::Duration;
3825
3826    use super::*;
3827    use tokio::sync::mpsc;
3828
3829    #[test]
3830    fn ordinary_long_running_request_is_not_excluded_from_drain() {
3831        let mut ledger = CreditLedger::default();
3832        ledger.acquire(1, false);
3833
3834        assert_eq!(ledger.capture_subscription_exclusions(), 0);
3835        assert_eq!(ledger.drain_in_flight(), 1);
3836    }
3837
3838    #[test]
3839    fn bit_set_subscription_is_excluded_and_counted() {
3840        let mut ledger = CreditLedger::default();
3841        ledger.acquire(1, true);
3842
3843        assert_eq!(ledger.capture_subscription_exclusions(), 1);
3844        assert_eq!(ledger.drain_in_flight(), 0);
3845    }
3846
3847    #[test]
3848    fn subscription_opened_after_drain_snapshot_is_not_excluded() {
3849        let mut ledger = CreditLedger::default();
3850        ledger.acquire(1, true);
3851        assert_eq!(ledger.capture_subscription_exclusions(), 1);
3852
3853        ledger.acquire(2, true);
3854
3855        assert_eq!(ledger.drain_in_flight(), 1);
3856    }
3857
3858    #[test]
3859    fn drain_with_no_subscriptions_reports_zero_excluded() {
3860        let mut ledger = CreditLedger::default();
3861        assert_eq!(ledger.capture_subscription_exclusions(), 0);
3862    }
3863
3864    fn test_hello_ack(corr: u64) -> Frame {
3865        Frame::build(
3866            FrameType::HelloAck,
3867            Flags::new(false, Priority::Passive, false),
3868            0,
3869            0,
3870            corr,
3871            Vec::new(),
3872        )
3873        .unwrap()
3874    }
3875
3876    /// An acked registration whose HELLO_ACK cannot be queued must leave no
3877    /// endpoint behind: a routable module that never got its ack would read a
3878    /// route.bind first and exit. A closed queue and a full one both refuse.
3879    #[test]
3880    fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
3881        let forwarding = ForwardingTable::default();
3882
3883        let (closed_tx, closed_rx) = mpsc::channel(8);
3884        drop(closed_rx);
3885        let closed = ConnectionId::new(1);
3886        assert_eq!(
3887            forwarding.register_module_connection_acked(
3888                closed,
3889                "closed".to_string(),
3890                2,
3891                Concurrency::ModuleManaged,
3892                FrameSink::new(closed_tx),
3893                test_hello_ack(1),
3894            ),
3895            Err(ForwardingError::ModuleEgressUnavailable {
3896                connection_id: closed
3897            })
3898        );
3899
3900        let (full_tx, _full_rx) = mpsc::channel(1);
3901        let full_sink = FrameSink::new(full_tx);
3902        full_sink.try_send(test_hello_ack(99)).unwrap();
3903        let full = ConnectionId::new(2);
3904        assert_eq!(
3905            forwarding.register_module_connection_acked(
3906                full,
3907                "full".to_string(),
3908                2,
3909                Concurrency::ModuleManaged,
3910                full_sink.clone(),
3911                test_hello_ack(2),
3912            ),
3913            Err(ForwardingError::ModuleEgressUnavailable {
3914                connection_id: full
3915            })
3916        );
3917        assert_eq!(
3918            forwarding.register_candidate_module_connection_acked(
3919                full,
3920                "full".to_string(),
3921                2,
3922                Concurrency::ModuleManaged,
3923                full_sink,
3924                test_hello_ack(3),
3925            ),
3926            Err(ForwardingError::ModuleEgressUnavailable {
3927                connection_id: full
3928            })
3929        );
3930
3931        for (connection, module_id) in [(closed, "closed"), (full, "full")] {
3932            assert_eq!(
3933                forwarding
3934                    .module_endpoint_for_connection(connection)
3935                    .unwrap(),
3936                None
3937            );
3938            let (client_tx, _client_rx) = mpsc::channel(8);
3939            assert_eq!(
3940                forwarding
3941                    .begin_route_bind_relay_for_test(
3942                        ConnectionId::new(50),
3943                        FrameSink::new(client_tx),
3944                        1,
3945                        module_id,
3946                    )
3947                    .err(),
3948                Some(ForwardingError::NoModuleConnection)
3949            );
3950        }
3951        assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
3952    }
3953
3954    /// Both acked registration forms put the HELLO_ACK on the module's queue
3955    /// by the time the endpoint can be resolved.
3956    #[test]
3957    fn acked_registration_queues_the_hello_ack_first() {
3958        let forwarding = ForwardingTable::default();
3959        let (active_tx, mut active_rx) = mpsc::channel(8);
3960        forwarding
3961            .register_module_connection_acked(
3962                ConnectionId::new(1),
3963                "acked".to_string(),
3964                2,
3965                Concurrency::ModuleManaged,
3966                FrameSink::new(active_tx),
3967                test_hello_ack(11),
3968            )
3969            .unwrap();
3970        let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
3971        forwarding
3972            .register_candidate_module_connection_acked(
3973                ConnectionId::new(2),
3974                "acked".to_string(),
3975                2,
3976                Concurrency::ModuleManaged,
3977                FrameSink::new(candidate_tx),
3978                test_hello_ack(12),
3979            )
3980            .unwrap();
3981
3982        let active_first = active_rx.try_recv().unwrap().frame;
3983        assert_eq!(active_first.header.ty, FrameType::HelloAck);
3984        assert_eq!(active_first.header.corr, 11);
3985        let candidate_first = candidate_rx.try_recv().unwrap().frame;
3986        assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
3987        assert_eq!(candidate_first.header.corr, 12);
3988    }
3989
3990    #[test]
3991    fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
3992        let forwarding = ForwardingTable::default();
3993        let module_connection = ConnectionId::new(10);
3994        let exhausted_client = ConnectionId::new(20);
3995        let second_client = ConnectionId::new(30);
3996        let (module_tx, _module_rx) = mpsc::channel(1);
3997        let endpoint = forwarding
3998            .register_module_connection(
3999                module_connection,
4000                "route-limit-provider".to_string(),
4001                1,
4002                Concurrency::ModuleManaged,
4003                FrameSink::new(module_tx),
4004            )
4005            .unwrap();
4006
4007        {
4008            let mut inner = forwarding.inner.write().unwrap();
4009            for channel in 1..=u16::MAX {
4010                inner.reserved_client.insert(
4011                    ClientRouteKey {
4012                        connection_id: exhausted_client,
4013                        channel,
4014                    },
4015                    ModuleRouteKey {
4016                        endpoint,
4017                        channel: 1,
4018                    },
4019                );
4020            }
4021        }
4022
4023        let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
4024        let err = forwarding
4025            .begin_route_bind_relay_for_test(
4026                exhausted_client,
4027                FrameSink::new(exhausted_tx),
4028                1,
4029                "route-limit-provider",
4030            )
4031            .unwrap_err();
4032        assert!(matches!(
4033            err,
4034            ForwardingError::ClientRouteChannelExhausted { connection_id }
4035                if connection_id == exhausted_client
4036        ));
4037
4038        let (second_tx, _second_rx) = mpsc::channel(1);
4039        let pending = forwarding
4040            .begin_route_bind_relay_for_test(
4041                second_client,
4042                FrameSink::new(second_tx),
4043                2,
4044                "route-limit-provider",
4045            )
4046            .unwrap();
4047        assert_eq!(pending.client_channel, 1);
4048    }
4049
4050    #[test]
4051    fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
4052        let forwarding = ForwardingTable::default();
4053        let module_connection = ConnectionId::new(40);
4054        let client = ConnectionId::new(50);
4055        let (module_tx, _module_rx) = mpsc::channel(1);
4056        forwarding
4057            .register_module_connection(
4058                module_connection,
4059                "slot-reuse-provider".to_string(),
4060                1,
4061                Concurrency::ModuleManaged,
4062                FrameSink::new(module_tx),
4063            )
4064            .unwrap();
4065
4066        let (client_tx, _client_rx) = mpsc::channel(1);
4067        let client_sink = FrameSink::new(client_tx);
4068        let mut wrapped_channel = None;
4069        for index in 0..=usize::from(u16::MAX) {
4070            let pending = forwarding
4071                .begin_route_bind_relay_for_test(
4072                    client,
4073                    client_sink.clone(),
4074                    index as u64 + 1,
4075                    "slot-reuse-provider",
4076                )
4077                .unwrap();
4078            if index == usize::from(u16::MAX) {
4079                wrapped_channel = Some(pending.module_channel);
4080            }
4081            forwarding
4082                .abort_pending_relay(
4083                    pending.endpoint,
4084                    pending.corr,
4085                    RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
4086                )
4087                .unwrap();
4088        }
4089
4090        assert_eq!(wrapped_channel, Some(1));
4091    }
4092
4093    #[test]
4094    fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
4095        let forwarding = ForwardingTable::default();
4096        let client = ConnectionId::new(60);
4097        forwarding
4098            .inner
4099            .write()
4100            .unwrap()
4101            .next_client_channel
4102            .insert(client, 41);
4103
4104        let released = forwarding.cleanup_connection(client).unwrap();
4105
4106        assert!(released.is_empty());
4107        assert!(!forwarding
4108            .inner
4109            .read()
4110            .unwrap()
4111            .next_client_channel
4112            .contains_key(&client));
4113    }
4114
4115    #[test]
4116    fn stale_module_cleanup_preserves_fast_reconnect_successor() {
4117        let forwarding = ForwardingTable::default();
4118        let module_id = "fast-reconnect-provider";
4119        let first_connection = ConnectionId::new(70);
4120        let second_connection = ConnectionId::new(80);
4121        let (first_tx, _first_rx) = mpsc::channel(1);
4122        let first_endpoint = forwarding
4123            .register_module_connection(
4124                first_connection,
4125                module_id.to_string(),
4126                1,
4127                Concurrency::ModuleManaged,
4128                FrameSink::new(first_tx),
4129            )
4130            .unwrap();
4131        let (second_tx, _second_rx) = mpsc::channel(1);
4132        let second_endpoint = forwarding
4133            .register_module_connection(
4134                second_connection,
4135                module_id.to_string(),
4136                1,
4137                Concurrency::ModuleManaged,
4138                FrameSink::new(second_tx),
4139            )
4140            .unwrap();
4141        assert_ne!(first_endpoint, second_endpoint);
4142
4143        let released = forwarding.cleanup_connection(first_connection).unwrap();
4144
4145        assert!(released.is_empty());
4146        assert_eq!(
4147            forwarding
4148                .inner
4149                .read()
4150                .unwrap()
4151                .modules_by_id
4152                .get(module_id)
4153                .map(|module| module.endpoint),
4154            Some(second_endpoint)
4155        );
4156        assert!(forwarding.has_live_module_connection(module_id).unwrap());
4157        let control_rpc = forwarding
4158            .begin_module_control_rpc_for(
4159                module_id,
4160                "health.check",
4161                Instant::now() + Duration::from_secs(1),
4162            )
4163            .unwrap();
4164        assert_eq!(control_rpc.endpoint, second_endpoint);
4165    }
4166
4167    fn route_fixture(
4168        module_id: &str,
4169    ) -> (
4170        ForwardingTable,
4171        ConnectionId,
4172        ModuleEndpointId,
4173        ConnectionId,
4174        FrameSink,
4175        mpsc::Receiver<crate::router::OutboundFrame>,
4176    ) {
4177        let forwarding = ForwardingTable::default();
4178        let module_connection = ConnectionId::new(100);
4179        let client_connection = ConnectionId::new(200);
4180        let (module_tx, _module_rx) = mpsc::channel(8);
4181        let endpoint = forwarding
4182            .register_module_connection(
4183                module_connection,
4184                module_id.to_string(),
4185                2,
4186                Concurrency::ModuleManaged,
4187                FrameSink::new(module_tx),
4188            )
4189            .unwrap();
4190        let (client_tx, client_rx) = mpsc::channel(8);
4191        (
4192            forwarding,
4193            module_connection,
4194            endpoint,
4195            client_connection,
4196            FrameSink::new(client_tx),
4197            client_rx,
4198        )
4199    }
4200
4201    #[test]
4202    #[cfg(unix)]
4203    fn daemon_drain_gates_current_and_racing_provider_registrations() {
4204        let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
4205        assert_eq!(
4206            forwarding.begin_daemon_drain().unwrap(),
4207            [("provider".to_string(), endpoint)]
4208        );
4209        assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
4210        assert!(matches!(
4211            forwarding.register_module_connection(
4212                ConnectionId::new(300),
4213                "late-provider".into(),
4214                2,
4215                Concurrency::ModuleManaged,
4216                sink,
4217            ),
4218            Err(ForwardingError::ConnectionClosing { .. })
4219        ));
4220    }
4221
4222    #[test]
4223    #[cfg(unix)]
4224    fn daemon_drain_results_are_endpoint_fenced_and_removed_on_cleanup() {
4225        let (forwarding, connection, endpoint, _, _, _) = route_fixture("provider");
4226        assert_eq!(
4227            forwarding
4228                .daemon_drain_result_for_connection(connection)
4229                .unwrap(),
4230            None
4231        );
4232        forwarding
4233            .record_daemon_drain_result(endpoint, true)
4234            .unwrap();
4235        forwarding.begin_daemon_drain().unwrap();
4236        assert_eq!(
4237            forwarding
4238                .daemon_drain_result_for_connection(connection)
4239                .unwrap(),
4240            Some(false)
4241        );
4242        let replacement = ModuleEndpointId {
4243            generation: endpoint.generation + 1,
4244            ..endpoint
4245        };
4246        forwarding
4247            .record_daemon_drain_result(replacement, true)
4248            .unwrap();
4249        assert_eq!(
4250            forwarding
4251                .daemon_drain_result_for_connection(connection)
4252                .unwrap(),
4253            Some(false)
4254        );
4255        forwarding
4256            .record_daemon_drain_result(endpoint, true)
4257            .unwrap();
4258        assert_eq!(
4259            forwarding
4260                .daemon_drain_result_for_connection(connection)
4261                .unwrap(),
4262            Some(true)
4263        );
4264        assert_eq!(
4265            forwarding
4266                .daemon_drain_result_for_connection(ConnectionId::new(300))
4267                .unwrap(),
4268            Some(false)
4269        );
4270        forwarding.cleanup_connection(connection).unwrap();
4271        forwarding
4272            .record_daemon_drain_result(endpoint, true)
4273            .unwrap();
4274        assert_eq!(
4275            forwarding
4276                .daemon_drain_result_for_connection(connection)
4277                .unwrap(),
4278            Some(false)
4279        );
4280        assert!(forwarding
4281            .read_inner()
4282            .unwrap()
4283            .daemon_drain_results
4284            .is_empty());
4285    }
4286
4287    #[test]
4288    #[cfg(unix)]
4289    fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
4290        let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
4291            route_fixture("provider");
4292        let mut pending =
4293            begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
4294        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
4295        forwarding.begin_daemon_drain().unwrap();
4296        let completion = forwarding
4297            .complete_pending_relay(
4298                module_connection,
4299                pending.corr,
4300                RouteBindRelayOutcome::Accepted,
4301            )
4302            .expect("draining admission is not a fatal module error");
4303        assert!(completion.settled);
4304        let abandoned = completion
4305            .abandoned
4306            .expect("late accepted bind needs a route GOODBYE");
4307        assert_eq!(abandoned.channel, pending.module_channel);
4308        assert!(matches!(pending.receiver.try_recv().unwrap(),
4309            RouteBindRelayOutcome::Rejected(error) if error.code == "module_reloading"));
4310        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4311        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4312        assert!(
4313            rx.try_recv().is_err(),
4314            "no route.open may be published during drain"
4315        );
4316        assert_eq!(
4317            forwarding
4318                .module_endpoint_for_connection(module_connection)
4319                .unwrap(),
4320            Some(endpoint)
4321        );
4322        assert!(
4323            !forwarding
4324                .complete_pending_relay(
4325                    module_connection,
4326                    pending.corr,
4327                    RouteBindRelayOutcome::Accepted,
4328                )
4329                .unwrap()
4330                .settled
4331        );
4332    }
4333
4334    fn test_ping(corr: u64) -> Frame {
4335        Frame::build(
4336            FrameType::Ping,
4337            Flags::new(false, Priority::Passive, false),
4338            0,
4339            0,
4340            corr,
4341            Vec::new(),
4342        )
4343        .unwrap()
4344    }
4345
4346    fn begin_test_route(
4347        forwarding: &ForwardingTable,
4348        client_connection: ConnectionId,
4349        client_sink: FrameSink,
4350        corr: u64,
4351        module_id: &str,
4352    ) -> PendingRouteBindRelay {
4353        forwarding
4354            .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
4355            .unwrap()
4356    }
4357
4358    #[test]
4359    fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
4360        for candidate in [false, true] {
4361            for register_first in [false, true] {
4362                let (forwarding, module_connection, _, client_connection, sink, _rx) =
4363                    route_fixture("target");
4364                let register = || {
4365                    if candidate {
4366                        forwarding.register_candidate_module_connection(
4367                            client_connection,
4368                            "source".into(),
4369                            2,
4370                            Concurrency::ModuleManaged,
4371                            sink.clone(),
4372                        )
4373                    } else {
4374                        forwarding.register_module_connection(
4375                            client_connection,
4376                            "source".into(),
4377                            2,
4378                            Concurrency::ModuleManaged,
4379                            sink.clone(),
4380                        )
4381                    }
4382                };
4383                if register_first {
4384                    register().unwrap();
4385                    assert!(
4386                        forwarding
4387                            .begin_route_bind_relay_for_test(
4388                                client_connection,
4389                                sink.clone(),
4390                                1,
4391                                "target",
4392                            )
4393                            .is_err(),
4394                        "a deferred route.open cannot reserve after HELLO"
4395                    );
4396                    assert!(forwarding
4397                        .cleanup_connection(client_connection)
4398                        .unwrap()
4399                        .is_empty());
4400                } else {
4401                    let pending =
4402                        begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
4403                    assert!(
4404                        register().is_err(),
4405                        "HELLO cannot register after a route reservation"
4406                    );
4407                    forwarding
4408                        .complete_pending_relay(
4409                            module_connection,
4410                            pending.corr,
4411                            RouteBindRelayOutcome::Accepted,
4412                        )
4413                        .unwrap();
4414                    assert_eq!(
4415                        forwarding
4416                            .cleanup_connection(client_connection)
4417                            .unwrap()
4418                            .len(),
4419                        1
4420                    );
4421                }
4422                assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4423                assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4424            }
4425        }
4426    }
4427
4428    /// A pending route.open reserves its queue slot with `reserve_owned` and
4429    /// sends its prebuilt response through that slot when the module accepts.
4430    /// A queue already holding many data frames (far more than the old 64-frame
4431    /// queue) must not stop the open from reserving or completing, and the
4432    /// response must be charged to the queue's byte count when it is sent.
4433    #[tokio::test]
4434    async fn pending_route_open_completes_behind_queued_data_frames() {
4435        assert_eq!(
4436            crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
4437            8,
4438            "the per-connection pending route.open limit is its own constant"
4439        );
4440        let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
4441            route_fixture("open-behind-data");
4442        let (sink, mut client_rx) = crate::server::connection_egress();
4443        const DATA_FRAMES: usize = 1_000;
4444        let data = |corr: u64| {
4445            Frame::build(
4446                FrameType::StreamData,
4447                Flags::new(false, Priority::Interactive, false),
4448                9,
4449                1,
4450                corr,
4451                vec![b'x'; 200],
4452            )
4453            .unwrap()
4454        };
4455        for corr in 0..DATA_FRAMES as u64 {
4456            sink.try_send(data(corr)).unwrap();
4457        }
4458        let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
4459        assert_eq!(sink.backlog().queued_bytes, data_bytes);
4460
4461        let pending = tokio::time::timeout(
4462            Duration::from_secs(5),
4463            forwarding.begin_route_bind_relay_for(
4464                client,
4465                sink.clone(),
4466                subc_protocol::PROTOCOL_VERSION,
4467                4_242,
4468                "open-behind-data",
4469                Principal::Direct,
4470                None,
4471                None,
4472                Instant::now() + Duration::from_secs(60),
4473            ),
4474        )
4475        .await
4476        .expect("reserving the route.open slot must not wait behind data frames")
4477        .unwrap();
4478        forwarding
4479            .complete_pending_relay(
4480                module_connection,
4481                pending.corr,
4482                RouteBindRelayOutcome::Accepted,
4483            )
4484            .unwrap();
4485
4486        let backlog = sink.backlog();
4487        assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
4488        assert!(
4489            backlog.queued_bytes > data_bytes,
4490            "the route.open response must be counted in queued bytes: {backlog:?}"
4491        );
4492        for corr in 0..DATA_FRAMES as u64 {
4493            assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
4494        }
4495        let open = client_rx.recv().await.unwrap();
4496        assert_eq!(open.header.corr, 4_242);
4497        assert_eq!(open.header.ty, FrameType::Response);
4498        drop(open);
4499        assert_eq!(sink.backlog().queued_bytes, 0);
4500        assert_eq!(sink.backlog().queued_frames, 0);
4501    }
4502
4503    /// The drain-timeout line reports these numbers, so they must count the
4504    /// requests the drain is waiting on and no others: a route with nothing in
4505    /// flight is not a holdout, and a flagged subscription is excluded from the
4506    /// drain and so from the count.
4507    #[tokio::test]
4508    async fn drain_holdouts_count_held_requests_and_name_the_connection() {
4509        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4510            route_fixture("holdouts");
4511        let mut bound = |corr| {
4512            let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
4513            forwarding
4514                .complete_pending_relay(
4515                    module_connection,
4516                    route.corr,
4517                    RouteBindRelayOutcome::Accepted,
4518                )
4519                .unwrap();
4520            client_rx.try_recv().unwrap();
4521            match forwarding
4522                .lookup_data_route(client, route.client_channel, route.client_epoch)
4523                .unwrap()
4524            {
4525                DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4526                other => panic!("expected live route, got {other:?}"),
4527            }
4528        };
4529        let holding = bound(61);
4530        let _idle = bound(62);
4531        holding.flow.acquire_tagged(7, false).await.unwrap();
4532        holding.flow.acquire_tagged(2, false).await.unwrap();
4533        holding.flow.acquire_tagged(3, true).await.unwrap();
4534        forwarding
4535            .begin_module_drain("holdouts", RouteCloseReason::Restart)
4536            .unwrap();
4537
4538        let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
4539        assert_eq!(
4540            holdouts,
4541            DrainHoldouts {
4542                requests: 2,
4543                routes: 1,
4544                total_routes: 2,
4545                top_connections: vec![(client.get(), 2)],
4546                // The held requests by the module's channel and corr, ascending,
4547                // without the excluded subscription (corr 3).
4548                held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
4549            }
4550        );
4551    }
4552
4553    #[test]
4554    fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
4555        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4556            route_fixture("census");
4557        let pending = begin_test_route(&forwarding, client, sink, 1, "census");
4558        forwarding
4559            .complete_pending_relay(
4560                module_connection,
4561                pending.corr,
4562                RouteBindRelayOutcome::Accepted,
4563            )
4564            .unwrap();
4565
4566        let routes = forwarding.endpoint_routes(endpoint).unwrap();
4567        assert_eq!(routes.len(), 1);
4568        assert!(matches!(routes[0].principal, Principal::Direct));
4569        assert_eq!(routes[0].goodbye_target.connection_id, client);
4570        assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
4571        assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
4572        assert!(!routes[0].draining);
4573
4574        forwarding
4575            .begin_module_drain("census", RouteCloseReason::Restart)
4576            .unwrap();
4577        let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
4578        assert_eq!(draining_routes.len(), 1);
4579        assert!(draining_routes[0].draining);
4580    }
4581
4582    #[test]
4583    fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
4584        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
4585        let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
4586        assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
4587        forwarding
4588            .abort_pending_relay(
4589                first.endpoint,
4590                first.corr,
4591                RouteBindRelayOutcome::ModuleGone("abort".into()),
4592            )
4593            .unwrap();
4594        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4595        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4596
4597        let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
4598        assert_eq!(second.client_channel, first.client_channel);
4599        assert_eq!(second.module_channel, first.module_channel);
4600        assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
4601    }
4602
4603    #[test]
4604    fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
4605        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4606            route_fixture("epoch-release");
4607        let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
4608        forwarding
4609            .complete_pending_relay(
4610                module_connection,
4611                first.corr,
4612                RouteBindRelayOutcome::Accepted,
4613            )
4614            .unwrap();
4615        assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
4616        assert!(matches!(
4617            forwarding
4618                .release_client_route(client, first.client_channel, first.client_epoch)
4619                .unwrap(),
4620            RouteRelease::Removed(_)
4621        ));
4622        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4623        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4624
4625        let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
4626        forwarding
4627            .complete_pending_relay(
4628                module_connection,
4629                second.corr,
4630                RouteBindRelayOutcome::Accepted,
4631            )
4632            .unwrap();
4633        assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
4634        assert!(matches!(
4635            forwarding
4636                .release_client_route(client, second.client_channel, first.client_epoch)
4637                .unwrap(),
4638            RouteRelease::Stale
4639        ));
4640        assert!(!forwarding
4641            .cache_status(
4642                endpoint,
4643                second.module_channel,
4644                first.module_epoch,
4645                "stale".into(),
4646            )
4647            .unwrap());
4648        assert!(forwarding
4649            .cache_status(
4650                endpoint,
4651                second.module_channel,
4652                second.module_epoch,
4653                "current".into(),
4654            )
4655            .unwrap());
4656        match forwarding
4657            .route_poll_snapshot(client, second.client_channel, second.client_epoch)
4658            .unwrap()
4659        {
4660            RoutePollSnapshot::Bound { status, .. } => {
4661                assert_eq!(status.as_deref(), Some("current"));
4662            }
4663            RoutePollSnapshot::Absent => panic!("successor binding was removed"),
4664        }
4665        let counters = forwarding.counters().snapshot();
4666        assert_eq!(counters["route_released_epoch_fenced"], 1);
4667        assert_eq!(counters["route_release_stale_skipped"], 1);
4668    }
4669
4670    #[test]
4671    fn max_epoch_reservation_retires_only_that_slot() {
4672        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
4673        forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
4674        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
4675        let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
4676        assert_eq!(
4677            (final_use.client_channel, final_use.client_epoch),
4678            (7, u32::MAX)
4679        );
4680        assert_eq!(
4681            (final_use.module_channel, final_use.module_epoch),
4682            (9, u32::MAX)
4683        );
4684        forwarding
4685            .abort_pending_relay(
4686                endpoint,
4687                final_use.corr,
4688                RouteBindRelayOutcome::ModuleGone("abort".into()),
4689            )
4690            .unwrap();
4691        forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
4692        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
4693        let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
4694        assert_ne!(next.client_channel, 7);
4695        assert_ne!(next.module_channel, 9);
4696        assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
4697    }
4698
4699    #[test]
4700    fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
4701        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4702            route_fixture("corr-shared");
4703        let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
4704        assert_eq!(bind.corr, 1);
4705        forwarding
4706            .abort_pending_relay(
4707                endpoint,
4708                bind.corr,
4709                RouteBindRelayOutcome::ModuleGone("abort".into()),
4710            )
4711            .unwrap();
4712        let rpc = forwarding
4713            .begin_module_control_rpc_for(
4714                "corr-shared",
4715                "health.check",
4716                Instant::now() - Duration::from_millis(1),
4717            )
4718            .unwrap();
4719        assert_eq!(rpc.corr, 2);
4720        assert_eq!(
4721            forwarding
4722                .complete_module_control_rpc(
4723                    module_connection,
4724                    rpc.corr,
4725                    Some("health.check"),
4726                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
4727                        status: subc_protocol::session::HealthStatus::Ok,
4728                        detail: None,
4729                        metrics: None,
4730                    }),
4731                )
4732                .unwrap(),
4733            ModuleControlRpcCompletion::Settled
4734        );
4735        assert!(matches!(
4736            rpc.receiver.blocking_recv().unwrap(),
4737            ModuleControlRpcOutcome::DeadlineElapsed
4738        ));
4739    }
4740
4741    #[tokio::test(start_paused = true)]
4742    async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
4743        let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
4744        let probe_started_at = Instant::now();
4745        let rpc = forwarding
4746            .begin_health_probe_rpc_for(
4747                "tombstone-ttl",
4748                "health.check",
4749                probe_started_at,
4750                probe_started_at + Duration::from_secs(5),
4751            )
4752            .unwrap();
4753        assert!(forwarding
4754            .tombstone_health_probe_rpc(endpoint, rpc.corr)
4755            .unwrap());
4756        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
4757
4758        tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
4759        tokio::task::yield_now().await;
4760
4761        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
4762    }
4763
4764    #[test]
4765    fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
4766        let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
4767        let mut close = forwarding.register_connection_close(endpoint.connection_id);
4768        forwarding.inject_control_corr(endpoint, u64::MAX);
4769        let final_rpc = forwarding
4770            .begin_module_control_rpc_for(
4771                "corr-max",
4772                "health.check",
4773                Instant::now() + Duration::from_secs(1),
4774            )
4775            .unwrap();
4776        assert_eq!(final_rpc.corr, u64::MAX);
4777        forwarding
4778            .cancel_module_control_rpc(endpoint, final_rpc.corr)
4779            .unwrap();
4780        assert!(matches!(
4781            forwarding.begin_module_control_rpc_for(
4782                "corr-max",
4783                "health.check",
4784                Instant::now() + Duration::from_secs(1),
4785            ),
4786            Err(ForwardingError::RelayCorrelationExhausted)
4787        ));
4788        assert!(close.try_recv().is_ok());
4789    }
4790
4791    #[test]
4792    fn publication_epoch_controls_delivery_failure_escalation() {
4793        fn setup_successor(
4794            commit_successor: Option<bool>,
4795        ) -> (ForwardingTable, ConnectionId, u16, u32) {
4796            let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4797                route_fixture("escalation");
4798            let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
4799            forwarding
4800                .complete_pending_relay(
4801                    module_connection,
4802                    first.corr,
4803                    RouteBindRelayOutcome::Accepted,
4804                )
4805                .unwrap();
4806            client_rx.try_recv().unwrap();
4807            assert!(matches!(
4808                forwarding
4809                    .release_client_route(client, first.client_channel, first.client_epoch)
4810                    .unwrap(),
4811                RouteRelease::Removed(_)
4812            ));
4813            if let Some(commit_successor) = commit_successor {
4814                forwarding.inject_client_slot_epoch(
4815                    client,
4816                    first.client_channel,
4817                    first.client_epoch,
4818                );
4819                forwarding.inject_module_slot_epoch(
4820                    endpoint,
4821                    first.module_channel,
4822                    first.module_epoch,
4823                );
4824                let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
4825                if commit_successor {
4826                    forwarding
4827                        .complete_pending_relay(
4828                            module_connection,
4829                            successor.corr,
4830                            RouteBindRelayOutcome::Accepted,
4831                        )
4832                        .unwrap();
4833                    client_rx.try_recv().unwrap();
4834                } else {
4835                    forwarding
4836                        .abort_pending_relay(
4837                            endpoint,
4838                            successor.corr,
4839                            RouteBindRelayOutcome::ModuleGone("abort".into()),
4840                        )
4841                        .unwrap();
4842                }
4843            }
4844            (forwarding, client, first.client_channel, first.client_epoch)
4845        }
4846
4847        let probe_sink = FrameSink::new(mpsc::channel(1).0);
4848        let (no_successor, client, channel, epoch) = setup_successor(None);
4849        let mut close = no_successor.register_connection_close(client);
4850        assert!(no_successor
4851            .escalate_client_delivery_failure(
4852                client,
4853                channel,
4854                epoch,
4855                CloseReason::new("delivery", "failed"),
4856                UndeliveredFrame {
4857                    module_id: None,
4858                    sink: &probe_sink,
4859                },
4860            )
4861            .unwrap());
4862        assert!(close.try_recv().is_ok());
4863
4864        let (aborted, client, channel, epoch) = setup_successor(Some(false));
4865        let mut close = aborted.register_connection_close(client);
4866        assert!(aborted
4867            .escalate_client_delivery_failure(
4868                client,
4869                channel,
4870                epoch,
4871                CloseReason::new("delivery", "failed"),
4872                UndeliveredFrame {
4873                    module_id: None,
4874                    sink: &probe_sink,
4875                },
4876            )
4877            .unwrap());
4878        assert!(close.try_recv().is_ok());
4879
4880        let (published, client, channel, epoch) = setup_successor(Some(true));
4881        let mut close = published.register_connection_close(client);
4882        assert!(!published
4883            .escalate_client_delivery_failure(
4884                client,
4885                channel,
4886                epoch,
4887                CloseReason::new("delivery", "stale failure"),
4888                UndeliveredFrame {
4889                    module_id: None,
4890                    sink: &probe_sink,
4891                },
4892            )
4893            .unwrap());
4894        assert!(close.try_recv().is_err());
4895    }
4896
4897    #[test]
4898    fn route_concentration_separates_client_count_from_routes_per_client() {
4899        // The distinction this asserts is the one a bare connection count cannot
4900        // make: two connections holding one route each and one connection
4901        // holding two are the same total, and have opposite causes.
4902        let (forwarding, module_connection, _, client, sink, _client_rx) =
4903            route_fixture("concentration");
4904        assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
4905
4906        for corr in [70_u64, 71] {
4907            let pending =
4908                begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
4909            forwarding
4910                .complete_pending_relay(
4911                    module_connection,
4912                    pending.corr,
4913                    RouteBindRelayOutcome::Accepted,
4914                )
4915                .unwrap();
4916        }
4917
4918        // One connection, two routes — not two connections with a route each.
4919        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
4920        assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
4921    }
4922
4923    #[test]
4924    fn cleanup_and_accepted_resolution_have_one_lock_winner() {
4925        let (forwarding, module_connection, _, client, sink, mut client_rx) =
4926            route_fixture("cleanup-race");
4927        let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
4928        forwarding
4929            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4930            .unwrap();
4931        let released = forwarding.cleanup_connection(client).unwrap();
4932        assert_eq!(released.len(), 1);
4933        let completion = forwarding
4934            .complete_pending_relay(
4935                module_connection,
4936                pending.corr,
4937                RouteBindRelayOutcome::Accepted,
4938            )
4939            .unwrap();
4940        assert!(!completion.settled);
4941        assert!(client_rx.try_recv().is_err());
4942        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4943
4944        let (forwarding, module_connection, _, client, sink, mut client_rx) =
4945            route_fixture("accepted-race");
4946        let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
4947        forwarding
4948            .complete_pending_relay(
4949                module_connection,
4950                pending.corr,
4951                RouteBindRelayOutcome::Accepted,
4952            )
4953            .unwrap();
4954        assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
4955        let released = forwarding.cleanup_connection(client).unwrap();
4956        assert_eq!(released.len(), 1);
4957        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4958    }
4959
4960    #[test]
4961    fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
4962        let (forwarding, module_connection, _, client, sink, mut client_rx) =
4963            route_fixture("drain-gap");
4964        let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
4965        forwarding
4966            .complete_pending_relay(
4967                module_connection,
4968                live.corr,
4969                RouteBindRelayOutcome::Accepted,
4970            )
4971            .unwrap();
4972        client_rx.try_recv().unwrap();
4973        let binding = match forwarding
4974            .lookup_data_route(client, live.client_channel, live.client_epoch)
4975            .unwrap()
4976        {
4977            DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4978            other => panic!("expected live route, got {other:?}"),
4979        };
4980
4981        let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
4982        forwarding
4983            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4984            .unwrap();
4985        let control_rpc = forwarding
4986            .begin_module_control_rpc_for(
4987                "drain-gap",
4988                "health.check",
4989                Instant::now() + Duration::from_secs(1),
4990            )
4991            .unwrap();
4992        let target = forwarding
4993            .begin_module_drain("drain-gap", RouteCloseReason::Reload)
4994            .unwrap()
4995            .unwrap();
4996        assert!(matches!(
4997            control_rpc.receiver.blocking_recv().unwrap(),
4998            ModuleControlRpcOutcome::ModuleGone(_)
4999        ));
5000        assert_eq!(target.abandoned_bindings.len(), 1);
5001        assert!(binding.flow.sem.is_closed());
5002        assert!(
5003            !forwarding
5004                .complete_pending_relay(
5005                    module_connection,
5006                    pending.corr,
5007                    RouteBindRelayOutcome::Accepted,
5008                )
5009                .unwrap()
5010                .settled
5011        );
5012        assert!(matches!(
5013            forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
5014            Err(ForwardingError::ModuleReloading { .. })
5015        ));
5016        let released = forwarding
5017            .release_module_endpoint_routes(target.endpoint)
5018            .unwrap();
5019        assert_eq!(released.len(), 1);
5020        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5021    }
5022
5023    /// A client can be marked closing while its egress is still open: the daemon
5024    /// asks a connection to close (here through the production path, a module
5025    /// frame that its egress refused) and the connection loop tears down a moment
5026    /// later. A route.bind ack that lands inside that window is answered on the
5027    /// MODULE connection's frame handler, so resolving it must not produce an
5028    /// error -- an error there ends the module connection, and that connection
5029    /// carries every other client's routes to the module.
5030    #[test]
5031    fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
5032        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5033            route_fixture("closing-client");
5034
5035        // A published route on this client: escalate_client_delivery_failure only
5036        // marks a connection closing for a route it has already published.
5037        let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
5038        forwarding
5039            .complete_pending_relay(
5040                module_connection,
5041                live.corr,
5042                RouteBindRelayOutcome::Accepted,
5043            )
5044            .unwrap();
5045        client_rx.try_recv().unwrap();
5046
5047        // A second route.open from the same client, relayed and awaiting its ack.
5048        let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
5049        forwarding
5050            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5051            .unwrap();
5052
5053        // The window: closing, but the sink is still open.
5054        assert!(forwarding
5055            .escalate_client_delivery_failure(
5056                client,
5057                live.client_channel,
5058                live.client_epoch,
5059                CloseReason::new(
5060                    "module_to_client_delivery_failed",
5061                    "client egress refused a module frame",
5062                ),
5063                UndeliveredFrame {
5064                    module_id: None,
5065                    sink: &sink,
5066                },
5067            )
5068            .unwrap());
5069        assert!(!sink.is_closed());
5070
5071        let completion = forwarding
5072            .complete_pending_relay(
5073                module_connection,
5074                pending.corr,
5075                RouteBindRelayOutcome::Accepted,
5076            )
5077            .expect("a closing client must not turn a module's ack into an error");
5078
5079        assert!(completion.settled);
5080        let abandoned = completion
5081            .abandoned
5082            .expect("the module must be told to drop the binding it just created");
5083        assert_eq!(abandoned.connection_id, module_connection);
5084        assert_eq!(abandoned.channel, pending.module_channel);
5085        assert_eq!(abandoned.epoch, pending.module_epoch);
5086        assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
5087        assert!(matches!(
5088            pending.receiver.blocking_recv().unwrap(),
5089            RouteBindRelayOutcome::ModuleGone(_)
5090        ));
5091        // No route was published to a client that is on its way out, and the
5092        // reserved handle pair went back.
5093        assert!(client_rx.try_recv().is_err());
5094        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
5095
5096        // The module endpoint is untouched: still live, and still able to take a
5097        // route from another client.
5098        assert!(forwarding
5099            .has_live_module_connection("closing-client")
5100            .unwrap());
5101        let cotenant = ConnectionId::new(201);
5102        let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
5103        let cotenant_route = begin_test_route(
5104            &forwarding,
5105            cotenant,
5106            FrameSink::new(cotenant_tx),
5107            62,
5108            "closing-client",
5109        );
5110        assert_eq!(cotenant_route.endpoint, endpoint);
5111        forwarding
5112            .complete_pending_relay(
5113                module_connection,
5114                cotenant_route.corr,
5115                RouteBindRelayOutcome::Accepted,
5116            )
5117            .unwrap();
5118        assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
5119        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5120    }
5121
5122    #[test]
5123    fn pending_route_permit_is_released_on_rejection_and_abort() {
5124        let forwarding = ForwardingTable::default();
5125        let module_connection = ConnectionId::new(300);
5126        let client = ConnectionId::new(301);
5127        let (module_tx, _module_rx) = mpsc::channel(1);
5128        let endpoint = forwarding
5129            .register_module_connection(
5130                module_connection,
5131                "permit".into(),
5132                2,
5133                Concurrency::ModuleManaged,
5134                FrameSink::new(module_tx),
5135            )
5136            .unwrap();
5137        let (client_tx, mut client_rx) = mpsc::channel(1);
5138        let sink = FrameSink::new(client_tx);
5139        let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
5140        assert!(sink.try_send(test_ping(999)).is_err());
5141        forwarding
5142            .complete_pending_relay(
5143                module_connection,
5144                rejected.corr,
5145                RouteBindRelayOutcome::Rejected(ErrorBody {
5146                    code: "no".into(),
5147                    message: "rejected".into(),
5148                    detail: None,
5149                }),
5150            )
5151            .unwrap();
5152        sink.try_send(test_ping(1000)).unwrap();
5153        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
5154
5155        let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
5156        assert!(sink.try_send(test_ping(1001)).is_err());
5157        forwarding
5158            .abort_pending_relay(
5159                endpoint,
5160                aborted.corr,
5161                RouteBindRelayOutcome::ModuleGone("abort".into()),
5162            )
5163            .unwrap();
5164        sink.try_send(test_ping(1002)).unwrap();
5165        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
5166
5167        let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
5168        forwarding
5169            .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
5170            .unwrap();
5171        drop(client_rx);
5172        let completion = forwarding
5173            .complete_pending_relay(
5174                module_connection,
5175                receiver_closed.corr,
5176                RouteBindRelayOutcome::Accepted,
5177            )
5178            .unwrap();
5179        assert!(completion.abandoned.is_some());
5180        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5181    }
5182
5183    /// Every connection teardown passes through `cleanup_connection`, and
5184    /// connection ids come from a monotonic counter that never hands an id out
5185    /// twice. If the closing mark survives teardown, the set grows by one entry
5186    /// per connection for the life of the daemon -- the self-watchdog alone
5187    /// reconnects once a minute.
5188    #[test]
5189    fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
5190        let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
5191            route_fixture("closing-set-leak");
5192
5193        const CONNECTIONS: u64 = 32;
5194        for index in 0..CONNECTIONS {
5195            let client = ConnectionId::new(1000 + index);
5196            let (client_tx, _client_rx) = mpsc::channel(8);
5197            let route = begin_test_route(
5198                &forwarding,
5199                client,
5200                FrameSink::new(client_tx),
5201                index + 1,
5202                "closing-set-leak",
5203            );
5204            forwarding
5205                .complete_pending_relay(
5206                    module_connection,
5207                    route.corr,
5208                    RouteBindRelayOutcome::Accepted,
5209                )
5210                .unwrap();
5211            forwarding.cleanup_connection(client).unwrap();
5212        }
5213        forwarding.cleanup_connection(module_connection).unwrap();
5214
5215        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5216    }
5217
5218    /// The closing mark exists to refuse new work for a connection that is on
5219    /// its way out but whose teardown has not run yet: the daemon asks the
5220    /// connection loop to end, and only when the loop reacts does
5221    /// `cleanup_connection` strip the connection's state. Inside that window an
5222    /// operation for the dying connection must still be refused; only after
5223    /// cleanup completes may the mark go.
5224    #[test]
5225    fn closing_connection_is_refused_new_work_until_cleanup_completes() {
5226        let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
5227            route_fixture("closing-gate");
5228
5229        // A published route: escalate_client_delivery_failure only marks a
5230        // connection closing for a route it has already published.
5231        let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
5232        forwarding
5233            .complete_pending_relay(
5234                module_connection,
5235                live.corr,
5236                RouteBindRelayOutcome::Accepted,
5237            )
5238            .unwrap();
5239        client_rx.try_recv().unwrap();
5240
5241        // Mark the connection closing through the production path without
5242        // running teardown, pinning the window open.
5243        assert!(forwarding
5244            .escalate_client_delivery_failure(
5245                client,
5246                live.client_channel,
5247                live.client_epoch,
5248                CloseReason::new(
5249                    "module_to_client_delivery_failed",
5250                    "client egress refused a module frame",
5251                ),
5252                UndeliveredFrame {
5253                    module_id: None,
5254                    sink: &sink,
5255                },
5256            )
5257            .unwrap());
5258        assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
5259
5260        // A late route.open for the closing client is refused ...
5261        assert!(matches!(
5262            forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
5263            Err(ForwardingError::ConnectionClosing { connection_id })
5264                if connection_id == client
5265        ));
5266        // ... and so is a late attempt to register the connection as a module.
5267        let (late_tx, _late_rx) = mpsc::channel(1);
5268        assert!(matches!(
5269            forwarding.register_module_connection(
5270                client,
5271                "late-module".into(),
5272                2,
5273                Concurrency::ModuleManaged,
5274                FrameSink::new(late_tx),
5275            ),
5276            Err(ForwardingError::ConnectionClosing { connection_id })
5277                if connection_id == client
5278        ));
5279
5280        // Teardown is the point that lifts the mark: it has just removed every
5281        // per-connection entry under the same lock, so the gate has nothing
5282        // left to protect for this id.
5283        forwarding.cleanup_connection(client).unwrap();
5284        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5285    }
5286}
5287
5288/// Blue/green swap slots: a candidate registered beside the active endpoint,
5289/// promoted by `cutover_candidate`, with the old incumbent drained by endpoint.
5290#[cfg(test)]
5291mod swap_slot_tests {
5292    use std::time::Duration;
5293
5294    use super::*;
5295    use tokio::sync::mpsc;
5296
5297    const MODULE_ID: &str = "swapped";
5298
5299    struct SwapFixture {
5300        forwarding: ForwardingTable,
5301        incumbent_connection: ConnectionId,
5302        incumbent: ModuleEndpointId,
5303        candidate_connection: ConnectionId,
5304        candidate: ModuleEndpointId,
5305        _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
5306    }
5307
5308    fn swap_fixture() -> SwapFixture {
5309        let forwarding = ForwardingTable::default();
5310        let incumbent_connection = ConnectionId::new(100);
5311        let candidate_connection = ConnectionId::new(110);
5312        let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
5313        let incumbent = forwarding
5314            .register_module_connection(
5315                incumbent_connection,
5316                MODULE_ID.to_string(),
5317                2,
5318                Concurrency::ModuleManaged,
5319                FrameSink::new(incumbent_tx),
5320            )
5321            .unwrap();
5322        let (candidate_tx, candidate_rx) = mpsc::channel(8);
5323        let candidate = forwarding
5324            .register_candidate_module_connection(
5325                candidate_connection,
5326                MODULE_ID.to_string(),
5327                2,
5328                Concurrency::ModuleManaged,
5329                FrameSink::new(candidate_tx),
5330            )
5331            .unwrap();
5332        SwapFixture {
5333            forwarding,
5334            incumbent_connection,
5335            incumbent,
5336            candidate_connection,
5337            candidate,
5338            _module_rxs: vec![incumbent_rx, candidate_rx],
5339        }
5340    }
5341
5342    fn client(
5343        raw: u64,
5344    ) -> (
5345        ConnectionId,
5346        FrameSink,
5347        mpsc::Receiver<crate::router::OutboundFrame>,
5348    ) {
5349        let (tx, rx) = mpsc::channel(8);
5350        (ConnectionId::new(raw), FrameSink::new(tx), rx)
5351    }
5352
5353    fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
5354        forwarding
5355            .read_inner()
5356            .unwrap()
5357            .client_to_module
5358            .values()
5359            .map(|route| route.module_endpoint)
5360            .collect()
5361    }
5362
5363    #[test]
5364    fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
5365        let fixture = swap_fixture();
5366        let forwarding = &fixture.forwarding;
5367        assert_ne!(fixture.incumbent, fixture.candidate);
5368
5369        // Every by-id consumer still resolves the incumbent.
5370        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5371        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5372        let (client_connection, client_sink, _client_rx) = client(200);
5373        let pending = forwarding
5374            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5375            .unwrap();
5376        assert_eq!(pending.endpoint, fixture.incumbent);
5377        let rpc = forwarding
5378            .begin_module_control_rpc_for(
5379                MODULE_ID,
5380                "health.check",
5381                Instant::now() + Duration::from_secs(1),
5382            )
5383            .unwrap();
5384        assert_eq!(rpc.endpoint, fixture.incumbent);
5385        let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
5386        assert_eq!(census.len(), 1, "the census lists one endpoint per id");
5387
5388        // Connection-keyed lookups see the candidate, so its own frames resolve.
5389        assert_eq!(
5390            forwarding
5391                .module_endpoint_for_connection(fixture.candidate_connection)
5392                .unwrap(),
5393            Some(fixture.candidate)
5394        );
5395        assert_eq!(
5396            forwarding
5397                .module_id_for_connection(fixture.candidate_connection)
5398                .unwrap()
5399                .as_deref(),
5400            Some(MODULE_ID)
5401        );
5402
5403        // One candidate per id.
5404        let (other_tx, _other_rx) = mpsc::channel(1);
5405        assert_eq!(
5406            forwarding.register_candidate_module_connection(
5407                ConnectionId::new(120),
5408                MODULE_ID.to_string(),
5409                2,
5410                Concurrency::ModuleManaged,
5411                FrameSink::new(other_tx),
5412            ),
5413            Err(ForwardingError::CandidateSlotOccupied {
5414                module_id: MODULE_ID.to_string()
5415            })
5416        );
5417    }
5418
5419    /// The cutover linearization point. A relay reserved on the incumbent before
5420    /// cutover must never become a route on the incumbent, and every relay
5421    /// reserved after it must land on the promoted candidate.
5422    #[test]
5423    fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
5424        let fixture = swap_fixture();
5425        let forwarding = &fixture.forwarding;
5426        let (early_client, early_sink, _early_rx) = client(200);
5427        let mut early = forwarding
5428            .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
5429            .unwrap();
5430        assert_eq!(early.endpoint, fixture.incumbent);
5431        assert_eq!(
5432            forwarding
5433                .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
5434                .unwrap(),
5435            RouteBindRelayMark::Marked
5436        );
5437
5438        let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5439        assert_eq!(
5440            cutover,
5441            ForwardingCutover {
5442                promoted: fixture.candidate,
5443                incumbent: Some(fixture.incumbent),
5444            }
5445        );
5446
5447        // A route.open reserved after cutover goes to the promoted candidate.
5448        let (late_client, late_sink, _late_rx) = client(201);
5449        let late = forwarding
5450            .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
5451            .unwrap();
5452        assert_eq!(
5453            late.endpoint, fixture.candidate,
5454            "a route.open after cutover was reserved on the incumbent"
5455        );
5456
5457        // The incumbent acks the early relay after cutover.
5458        let completion = forwarding
5459            .complete_pending_relay(
5460                fixture.incumbent_connection,
5461                early.corr,
5462                RouteBindRelayOutcome::Accepted,
5463            )
5464            .expect("a superseded endpoint's ack is not an error on its connection");
5465        assert!(completion.settled);
5466        assert!(
5467            !committed_endpoints(forwarding).contains(&fixture.incumbent),
5468            "a relay reserved before cutover committed a route on the incumbent"
5469        );
5470        let goodbye = completion
5471            .abandoned
5472            .expect("the incumbent is told to drop the binding it just created");
5473        assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
5474        assert_eq!(goodbye.channel, early.module_channel);
5475        assert_eq!(goodbye.epoch, early.module_epoch);
5476        assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
5477        match early.receiver.try_recv() {
5478            Ok(RouteBindRelayOutcome::Rejected(body)) => assert_eq!(body.code, "module_reloading"),
5479            other => panic!("expected a retryable module_reloading answer, got {other:?}"),
5480        }
5481        assert!(matches!(
5482            forwarding
5483                .lookup_data_route(early_client, early.client_channel, early.client_epoch)
5484                .unwrap(),
5485            DataRoute::Client(DataRouteState::Absent)
5486        ));
5487
5488        // The late relay commits on the candidate; only its pair stays reserved
5489        // until then.
5490        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
5491        forwarding
5492            .complete_pending_relay(
5493                fixture.candidate_connection,
5494                late.corr,
5495                RouteBindRelayOutcome::Accepted,
5496            )
5497            .unwrap();
5498        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5499        assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
5500    }
5501
5502    #[test]
5503    fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
5504        let fixture = swap_fixture();
5505        let forwarding = &fixture.forwarding;
5506        // One bound route and one in-flight relay on the incumbent.
5507        let (bound_client, bound_sink, _bound_rx) = client(200);
5508        let bound = forwarding
5509            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5510            .unwrap();
5511        forwarding
5512            .complete_pending_relay(
5513                fixture.incumbent_connection,
5514                bound.corr,
5515                RouteBindRelayOutcome::Accepted,
5516            )
5517            .unwrap();
5518        let (pending_client, pending_sink, _pending_rx) = client(201);
5519        let mut in_flight = forwarding
5520            .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
5521            .unwrap();
5522        forwarding
5523            .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
5524            .unwrap();
5525
5526        let incumbent = forwarding
5527            .cutover_candidate(MODULE_ID)
5528            .unwrap()
5529            .unwrap()
5530            .incumbent
5531            .unwrap();
5532        let target = forwarding
5533            .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
5534            .unwrap()
5535            .expect("the superseded incumbent is still registered");
5536
5537        assert_eq!(target.endpoint, fixture.incumbent);
5538        assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
5539        assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
5540        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5541        assert_eq!(target.abandoned_bindings.len(), 1);
5542        assert_eq!(
5543            target.abandoned_bindings[0].channel,
5544            in_flight.module_channel
5545        );
5546        assert!(matches!(
5547            in_flight.receiver.try_recv(),
5548            Ok(RouteBindRelayOutcome::Rejected(body)) if body.code == "module_reloading"
5549        ));
5550        assert_eq!(
5551            forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
5552            1,
5553            "the incumbent's bound route stays until its drain finishes"
5554        );
5555
5556        let (next_client, next_sink, _next_rx) = client(202);
5557        let next = forwarding
5558            .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
5559            .expect("the promoted candidate keeps accepting routes");
5560        assert_eq!(next.endpoint, fixture.candidate);
5561    }
5562
5563    /// An endpoint replaced WITHOUT a promotion (a successor registered over it
5564    /// as an ordinary active HELLO) is stale, not superseded, and its ack still
5565    /// fails the acking connection exactly as it did before swap slots existed:
5566    /// `StaleModuleEndpoint`, the reservation indexes already stripped, and the
5567    /// waiting client's sender dropped unanswered.
5568    #[test]
5569    fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
5570        let forwarding = ForwardingTable::default();
5571        let first_connection = ConnectionId::new(70);
5572        let (first_tx, _first_rx) = mpsc::channel(8);
5573        forwarding
5574            .register_module_connection(
5575                first_connection,
5576                MODULE_ID.to_string(),
5577                2,
5578                Concurrency::ModuleManaged,
5579                FrameSink::new(first_tx),
5580            )
5581            .unwrap();
5582        let (client_connection, client_sink, _client_rx) = client(200);
5583        let mut pending = forwarding
5584            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5585            .unwrap();
5586        let (second_tx, _second_rx) = mpsc::channel(8);
5587        forwarding
5588            .register_module_connection(
5589                ConnectionId::new(80),
5590                MODULE_ID.to_string(),
5591                2,
5592                Concurrency::ModuleManaged,
5593                FrameSink::new(second_tx),
5594            )
5595            .unwrap();
5596
5597        assert_eq!(
5598            forwarding
5599                .complete_pending_relay(
5600                    first_connection,
5601                    pending.corr,
5602                    RouteBindRelayOutcome::Accepted
5603                )
5604                .unwrap_err(),
5605            ForwardingError::StaleModuleEndpoint
5606        );
5607        assert!(committed_endpoints(&forwarding).is_empty());
5608        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5609        assert!(matches!(
5610            pending.receiver.try_recv(),
5611            Err(oneshot::error::TryRecvError::Closed)
5612        ));
5613    }
5614
5615    #[test]
5616    fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
5617        // A candidate whose connection drops leaves the incumbent routable.
5618        let fixture = swap_fixture();
5619        let forwarding = &fixture.forwarding;
5620        assert!(forwarding
5621            .cleanup_connection(fixture.candidate_connection)
5622            .unwrap()
5623            .is_empty());
5624        assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
5625        let (client_connection, client_sink, _client_rx) = client(200);
5626        assert_eq!(
5627            forwarding
5628                .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5629                .unwrap()
5630                .endpoint,
5631            fixture.incumbent
5632        );
5633
5634        // After a cutover, the incumbent's teardown releases its own routes and
5635        // leaves the promoted candidate in place.
5636        let fixture = swap_fixture();
5637        let forwarding = &fixture.forwarding;
5638        let (bound_client, bound_sink, _bound_rx) = client(200);
5639        let bound = forwarding
5640            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5641            .unwrap();
5642        forwarding
5643            .complete_pending_relay(
5644                fixture.incumbent_connection,
5645                bound.corr,
5646                RouteBindRelayOutcome::Accepted,
5647            )
5648            .unwrap();
5649        forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5650        let released = forwarding
5651            .cleanup_connection(fixture.incumbent_connection)
5652            .unwrap();
5653        assert_eq!(released.len(), 1);
5654        assert_eq!(released[0].connection_id, bound_client);
5655        assert!(forwarding
5656            .read_inner()
5657            .unwrap()
5658            .superseded_endpoints
5659            .is_empty());
5660        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5661        let (next_client, next_sink, _next_rx) = client(201);
5662        assert_eq!(
5663            forwarding
5664                .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
5665                .unwrap()
5666                .endpoint,
5667            fixture.candidate
5668        );
5669    }
5670}