Skip to main content

subc_daemon/
forwarding.rs

1use std::{
2    collections::{BTreeMap, HashMap, HashSet},
3    error::Error,
4    fmt,
5    sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
6    time::Duration,
7};
8
9use subc_control::{ClientControlResponse, RouteCloseReason};
10use subc_protocol::{
11    manifest::Concurrency,
12    session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
13    ErrorBody, Flags, FrameType, Principal, Priority,
14};
15use tokio::sync::{oneshot, Semaphore};
16use tokio::time::Instant;
17use tracing::{debug, info, warn};
18
19use crate::{
20    control::{RouteBindBreakers, RouteBindConcurrency},
21    observability::DaemonCounters,
22    registry::ConnectionId,
23    router::FrameSink,
24    scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
25    Frame, ProjectRootId,
26};
27
28/// Default per-channel request-credit window for modules that schedule internally.
29const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
30
31/// Watches leave eight request credits available for ordinary calls.
32const DEFAULT_MODULE_MANAGED_WATCH_WINDOW: usize = 24;
33
34/// High per-channel cap for stateless modules; this is an OOM guard, not scheduling policy.
35const STATELESS_PARALLEL_WINDOW: usize = 1024;
36
37/// A stopped probe cycle cannot retain its last unanswered correlation forever.
38/// Active endpoints replace the tombstone on their next serial health probe;
39/// this backstop covers endpoints that stop probing altogether.
40const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
41
42/// Module connection identity used in forwarding keys.
43///
44/// The generation is bumped every time a module connection is registered so a future restart cannot
45/// accidentally reuse a stale `(connection_id, route_channel)` binding.
46#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
47pub struct ModuleEndpointId {
48    pub connection_id: ConnectionId,
49    pub generation: u64,
50}
51
52/// Client-local route key. A route channel is unique only within one client connection.
53#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
54pub(crate) struct ClientRouteKey {
55    pub connection_id: ConnectionId,
56    pub channel: u16,
57}
58
59/// Module-local route key. A route channel is unique only within one live module endpoint.
60#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
61pub(crate) struct ModuleRouteKey {
62    pub endpoint: ModuleEndpointId,
63    pub channel: u16,
64}
65
66#[derive(Debug)]
67pub(crate) struct RouteBinding {
68    pub client_connection_id: ConnectionId,
69    pub client_sink: FrameSink,
70    pub client_negotiated_ver: u8,
71    pub client_channel: u16,
72    pub client_epoch: u32,
73    pub module_id: String,
74    pub module_endpoint: ModuleEndpointId,
75    pub module_sink: FrameSink,
76    pub module_negotiated_ver: u8,
77    pub module_channel: u16,
78    pub module_epoch: u32,
79    pub principal: Principal,
80    /// Opener attributes captured at admission for daemon-side prompt policy.
81    #[allow(dead_code)]
82    pub harness: String,
83    /// Only Callosum's exact production claim is retained; never relayed.
84    #[allow(dead_code)]
85    pub build_class: Option<String>,
86    /// Captured at connect, carried through admission; not standing authority.
87    #[allow(dead_code)]
88    pub operator_app: Option<crate::operator_apps::OperatorAppStamp>,
89    pub project_root: Option<ProjectRootId>,
90    pub bound_at: Instant,
91    pub flow: Arc<ChannelFlow>,
92    /// The scope the route was admitted under, with the tag its bind was
93    /// stamped at. A scope change finds the routes to close by it.
94    pub scope: Option<BoundScope>,
95}
96
97#[derive(Debug, Clone)]
98pub(crate) enum DataRoute {
99    Client(DataRouteState),
100    Module(DataRouteState),
101}
102
103#[derive(Debug, Clone)]
104pub(crate) enum DataRouteState {
105    Bound(Arc<RouteBinding>),
106    Reserved,
107    EpochMismatch,
108    Absent,
109}
110
111/// Which kind of peer a route GOODBYE is being delivered to. This decides what
112/// happens when the GOODBYE cannot be enqueued (egress full/closed):
113/// - `Client`: escalate to closing that client connection (a socket close is a
114///   stronger teardown signal, and a full client egress means it is the slow
115///   client we would drop anyway).
116/// - `Module`: never close; deliver late instead (see
117///   [`send_module_route_goodbye`]), and drop only if the module still has no
118///   room after [`LATE_MODULE_GOODBYE_DEADLINE`]. A client-disconnect notifies the
119///   SHARED module that one client's route is gone; closing the module on its
120///   egress backpressure would tear down every co-tenant client (the exact
121///   cross-tenant blast radius this never-close rule exists to prevent — observed when a
122///   flooding dead client filled BOTH its own and the module's egress, so its
123///   route-gone GOODBYE to the module failed and closed the shared connection).
124///   subc has already removed the route from its forwarding state and drops
125///   stale module frames for the released channel (see router.rs), so subc's
126///   own routing is correct. The residual: under SUSTAINED module-egress
127///   backpressure a module-targeted route-gone notification can be lost, which a
128///   module using it for client-refcounting (e.g. AFT's session accounting)
129///   would miss. This is INTENTIONALLY ACCEPTED, not a gap. A consuming module
130///   must bound stale bindings with its own idle-activity reaper (last-touched
131///   TTL) independent of route-gone signals — AFT does exactly this, so a lost
132///   GOODBYE degrades to "the
133///   binding stays warm until its idle TTL" (bounded wasted resources), never an
134///   unbounded leak; disk-durable replay is unaffected. A dedicated reliable
135///   module control lane was evaluated and deliberately NOT
136///   built: it would add starvation-avoidance machinery to the thin core for a
137///   bounded warm-resource window that is not a correctness issue. never-close
138///   is the invariant that matters here.
139#[derive(Debug, Clone, Copy, PartialEq, Eq)]
140pub(crate) enum GoodbyeTargetKind {
141    Client,
142    Module,
143}
144
145#[derive(Debug, Clone)]
146pub(crate) struct GoodbyeTarget {
147    pub connection_id: ConnectionId,
148    pub sink: FrameSink,
149    pub negotiated_ver: u8,
150    pub channel: u16,
151    pub epoch: u32,
152    pub kind: GoodbyeTargetKind,
153    /// The module on the other end of the route: for a module-targeted relay
154    /// the receiving module (attributing a dropped relay), for a client target
155    /// the module whose route is going away (naming it when an undeliverable
156    /// relay closes the client).
157    pub module_id: Option<String>,
158}
159
160/// The frame a client connection's egress queue refused, for the diagnosis
161/// logged when that refusal closes the connection.
162#[derive(Debug, Clone, Copy)]
163pub(crate) struct UndeliveredFrame<'a> {
164    /// The module on the other end of the route the frame belonged to, when known.
165    pub module_id: Option<&'a str>,
166    /// The refusing connection's sink, read for what its queue held.
167    pub sink: &'a FrameSink,
168}
169
170/// How a route's principal appears in logs: `direct`, or `reserved:<module>`.
171pub(crate) fn principal_label(principal: &Principal) -> String {
172    match principal {
173        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
174        Principal::Direct => "direct".to_string(),
175        other => format!("{other:?}"),
176    }
177}
178
179/// The distinct principals of the routes bound on one client connection,
180/// sorted and comma-joined, or `none` when it has no bound route.
181fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
182    let labels = inner
183        .client_to_module
184        .iter()
185        .filter(|(key, _)| key.connection_id == connection_id)
186        .map(|(_, route)| principal_label(&route.principal))
187        .collect::<std::collections::BTreeSet<_>>();
188    if labels.is_empty() {
189        "none".to_string()
190    } else {
191        labels.into_iter().collect::<Vec<_>>().join(",")
192    }
193}
194
195impl GoodbyeTarget {
196    /// True only when an undeliverable GOODBYE should escalate to closing the
197    /// target connection. Never escalate for module recipients.
198    pub(crate) fn close_on_delivery_failure(&self) -> bool {
199        matches!(self.kind, GoodbyeTargetKind::Client)
200    }
201}
202
203/// How long a route GOODBYE that a module's egress queue refused keeps waiting
204/// for room before it is given up and counted as dropped.
205///
206/// A module that stops reading for a moment (a GC pause, a long synchronous
207/// handler, a reconnect herd it is still working through) should still learn
208/// that the route is gone, because a module that never hears the GOODBYE keeps
209/// the route and keeps sending on it for as long as its connection lives. The
210/// value is the default module drain timeout: the daemon already treats that as
211/// the longest it is reasonable to wait on a module that is busy but alive, and
212/// a module that cannot free 21 bytes of egress in that time is wedged rather
213/// than slow. Resolving each module's own configured drain timeout here would
214/// need a registry lookup on a path that holds only the module's sink.
215pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
216
217/// Send a route GOODBYE to a module without ever closing the module's shared
218/// connection, which would sever every other route it serves.
219///
220/// The frame is enqueued at once when the module's egress queue has room. When
221/// it does not, a detached task waits for room (bounded by
222/// [`LATE_MODULE_GOODBYE_DEADLINE`]) and sends it then. Arriving late is safe:
223/// the daemon has already released the route, and module SDKs tear a route
224/// down only when both the channel AND the epoch match the route installed on
225/// that channel, so a late GOODBYE for a channel since reused at a newer epoch
226/// is ignored. The GOODBYE is counted in `goodbye_relay_module_dropped` only
227/// when the connection is closed or the deadline passes.
228pub(crate) fn send_module_route_goodbye(
229    counters: &DaemonCounters,
230    sink: &FrameSink,
231    frame: Frame,
232    module_id: Option<&str>,
233    context: &'static str,
234) {
235    let channel = frame.header.channel;
236    let epoch = frame.header.epoch;
237    let Err(err) = sink.try_send(frame.clone()) else {
238        return;
239    };
240    // Waiting is pointless once the module's writer is gone, and impossible
241    // outside a Tokio runtime (cleanup that runs from a destructor at shutdown).
242    let runtime = match tokio::runtime::Handle::try_current() {
243        Ok(runtime) if !sink.is_closed() => runtime,
244        _ => {
245            counters.increment_goodbye_relay_module_dropped(module_id);
246            warn!(
247            module_id = module_id.unwrap_or("unknown"),
248            route_channel = channel,
249            route_epoch = epoch,
250            error = %err,
251            context,
252            "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
253            );
254            return;
255        }
256    };
257    debug!(
258        module_id = module_id.unwrap_or("unknown"),
259        route_channel = channel,
260        route_epoch = epoch,
261        error = %err,
262        context,
263        "module egress queue refused route GOODBYE; delivering it once the module frees room"
264    );
265    let counters = counters.clone();
266    let sink = sink.clone();
267    let module_id = module_id.map(str::to_string);
268    runtime.spawn(async move {
269        let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
270        let why = match outcome {
271            Ok(Ok(())) => {
272                debug!(
273                    module_id = module_id.as_deref().unwrap_or("unknown"),
274                    route_channel = channel,
275                    route_epoch = epoch,
276                    context,
277                    "late route GOODBYE delivered to module"
278                );
279                return;
280            }
281            Ok(Err(err)) => err.to_string(),
282            Err(_) => format!(
283                "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
284            ),
285        };
286        counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
287        warn!(
288            module_id = module_id.as_deref().unwrap_or("unknown"),
289            route_channel = channel,
290            route_epoch = epoch,
291            error = %why,
292            context,
293            "route GOODBYE to module dropped under backpressure; not closing shared module connection"
294        );
295    });
296}
297
298/// One route currently served by a module endpoint.
299///
300/// `goodbye_target` is deliberately retained alongside the census projection so
301/// a draining caller can address exactly the same route set that this read
302/// reports, without a second forwarding-table pass.
303#[derive(Debug, Clone)]
304pub(crate) struct EndpointRoute {
305    pub goodbye_target: GoodbyeTarget,
306    pub principal: Principal,
307    pub bound_at: Instant,
308    pub draining: bool,
309    /// WHY the endpoint is draining, when it is. Carried per-route so the
310    /// census can answer "closing because of what" without a second lookup;
311    /// `None` exactly when `draining` is false (one source: the drain map).
312    pub drain_reason: Option<RouteCloseReason>,
313}
314
315/// One live route a scope change closed: both ends to send GOODBYE to, and the
316/// reason for the client's `route.closed`.
317#[derive(Debug, Clone)]
318pub(crate) struct ScopeDrainedRoute {
319    pub reason: RouteCloseReason,
320    pub scope: BoundScope,
321    pub module_id: String,
322    pub client: GoodbyeTarget,
323    pub module: GoodbyeTarget,
324}
325
326#[derive(Debug)]
327pub(crate) struct PendingRouteBindRelay {
328    pub endpoint: ModuleEndpointId,
329    pub module_sink: FrameSink,
330    pub negotiated_ver: u8,
331    pub client_channel: u16,
332    pub client_epoch: u32,
333    pub module_channel: u16,
334    pub module_epoch: u32,
335    pub corr: u64,
336    pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
337}
338
339#[derive(Debug, Clone)]
340pub(crate) struct ModuleDrainTarget {
341    pub endpoint: ModuleEndpointId,
342    pub sink: FrameSink,
343    pub negotiated_ver: u8,
344    pub abandoned_bindings: Vec<GoodbyeTarget>,
345    pub excluded_subscriptions: u32,
346}
347
348/// One registered module connection, as [`ForwardingTable::module_connections`]
349/// reports it: enough to send it a frame and then close it.
350#[cfg(unix)]
351#[derive(Debug, Clone)]
352pub(crate) struct ModuleConnectionTarget {
353    pub module_id: String,
354    pub endpoint: ModuleEndpointId,
355    pub sink: FrameSink,
356    pub negotiated_ver: u8,
357}
358
359#[derive(Debug, Clone)]
360pub(crate) enum RouteBindRelayOutcome {
361    Accepted,
362    /// The module refused the bind with its own body. Relayed to the client
363    /// without the daemon-origin bit: the code and message are module-chosen,
364    /// so the frame must not claim daemon authorship, even when the module
365    /// picks a daemon code like `module_reloading`.
366    Rejected(ErrorBody),
367    /// The daemon refused the bind itself — a drain or superseded endpoint
368    /// (`module_reloading`), a reply that arrived after the deadline
369    /// (`module_timeout`), or a scope that ended mid-bind. Carries the
370    /// daemon-origin bit because the daemon wrote it. The module may still
371    /// have received the bind (a late reply, or a drain that began after it
372    /// accepted); the daemon then ends that binding with a GOODBYE, so no
373    /// call runs on it.
374    DaemonRejected(ErrorBody),
375    ModuleGone(String),
376}
377
378/// What [`ForwardingTable::cutover_candidate`] did.
379#[derive(Debug, Clone, Copy, PartialEq, Eq)]
380pub(crate) struct ForwardingCutover {
381    /// The endpoint now in the active slot (the former candidate).
382    pub promoted: ModuleEndpointId,
383    /// The endpoint demoted out of the active slot, to be drained by endpoint.
384    /// `None` when the incumbent's connection was already gone.
385    pub incumbent: Option<ModuleEndpointId>,
386}
387
388/// What tearing down one connection released.
389#[derive(Debug)]
390pub(crate) struct ConnectionCleanup {
391    /// Routes whose other end must be sent GOODBYE.
392    pub released: Vec<GoodbyeTarget>,
393    /// Pending route.bind relays to the closed module that were aborted: opens
394    /// that had not been bound yet. Zero when a client connection closed.
395    pub abandoned_relays: u32,
396}
397
398#[derive(Debug, Clone)]
399pub(crate) struct PendingRelayCompletion {
400    pub settled: bool,
401    pub abandoned: Option<GoodbyeTarget>,
402}
403
404#[derive(Debug)]
405pub(crate) struct PendingModuleControlRpc {
406    pub endpoint: ModuleEndpointId,
407    pub module_sink: FrameSink,
408    pub negotiated_ver: u8,
409    pub corr: u64,
410    pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
411}
412
413#[derive(Debug, Clone)]
414pub(crate) enum ModuleControlRpcOutcome {
415    Response(ModuleControlResponse),
416    Rejected(ErrorBody),
417    ModuleGone(String),
418    MalformedResponse(String),
419    UnexpectedOp { expected: String, actual: String },
420    DeadlineElapsed,
421}
422
423#[derive(Debug, Clone, PartialEq, Eq)]
424pub(crate) enum ModuleControlRpcCompletion {
425    Unknown,
426    Settled,
427    LateHealthAnswer {
428        module_id: String,
429        latency: Duration,
430    },
431}
432
433#[derive(Debug)]
434struct PendingModuleControlRpcEntry {
435    expected_op: String,
436    deadline: Instant,
437    health_probe_started_at: Option<Instant>,
438    sender: oneshot::Sender<ModuleControlRpcOutcome>,
439}
440
441#[derive(Debug)]
442struct HealthProbeTombstone {
443    expected_op: String,
444    module_id: String,
445    probe_started_at: Instant,
446    expires_at: Instant,
447}
448
449#[derive(Debug, Clone)]
450struct RouteReservation {
451    client_key: ClientRouteKey,
452    module_key: ModuleRouteKey,
453    client_epoch: u32,
454    module_epoch: u32,
455    project_root: Option<ProjectRootId>,
456}
457
458#[derive(Debug)]
459struct PendingRouteBindRelayEntry {
460    reservation: RouteReservation,
461    client_sink: FrameSink,
462    client_negotiated_ver: u8,
463    client_permit: crate::router::EgressPermit,
464    route_open_frame: Frame,
465    busy_refusals: bool,
466    request_window: usize,
467    watch_window: Option<usize>,
468    principal: Principal,
469    /// The scope admitted at `route.open`, with the tag captured then. Commit
470    /// compares it with the published tag.
471    scope: Option<BoundScope>,
472    harness: String,
473    build_class: Option<String>,
474    operator_app: Option<crate::operator_apps::OperatorAppStamp>,
475    deadline: Instant,
476    /// Set once `route.open` has finished putting the route.bind on the
477    /// module's egress queue. A path that drops this entry WITHOUT a module
478    /// reply (client teardown, module drain, the open's own abort) sends the
479    /// module a GOODBYE for the bind only when this is set, because only then
480    /// is the GOODBYE guaranteed to queue behind the bind. When it is not set,
481    /// the removal is recorded in `relays_released_before_enqueue` instead and
482    /// `route.open` sends that GOODBYE itself once its send has completed.
483    ///
484    /// A module reply does not consult it: an ack proves the module received
485    /// the bind, so the reply path decides about the GOODBYE on its own.
486    relay_enqueued: bool,
487    sender: oneshot::Sender<RouteBindRelayOutcome>,
488}
489
490/// What `mark_route_bind_relay_enqueued` found once `route.open` had put the
491/// route.bind on the module's queue.
492#[derive(Debug, Clone, Copy, PartialEq, Eq)]
493pub(crate) enum RouteBindRelayMark {
494    /// The relay is still pending; later removals handle their own GOODBYE.
495    Marked,
496    /// The module already answered and the reply path settled the relay:
497    /// it committed the route, or it refused it and sent any GOODBYE the
498    /// module needed. `route.open` must send nothing.
499    SettledByReply,
500    /// The relay was dropped without a module reply while the bind was still
501    /// being queued, so nothing has told the module to drop the binding it
502    /// is about to create. `route.open` must send that GOODBYE.
503    ReleasedWithoutReply,
504}
505
506#[derive(Debug, Clone)]
507pub(crate) enum RouteRelease {
508    Removed(GoodbyeTarget),
509    Stale,
510    Absent,
511}
512
513#[derive(Debug, Clone)]
514pub(crate) enum RoutePollSnapshot {
515    Bound {
516        module_id: String,
517        status: Option<String>,
518    },
519    Absent,
520}
521
522#[derive(Debug, Clone)]
523struct ModuleConnection {
524    endpoint: ModuleEndpointId,
525    sink: FrameSink,
526    negotiated_ver: u8,
527    concurrency: Concurrency,
528}
529
530#[derive(Debug, Default)]
531struct ForwardingInner {
532    operator_confirms: Arc<crate::operator_confirm::OperatorConfirms>,
533    daemon_draining: bool,
534    /// Whether forwarding reached quiescence during daemon shutdown, for each
535    /// active endpoint present when shutdown began. A pending wait is false
536    /// because it has not proved quiescence. Endpoint generations prevent a
537    /// result from describing a replacement connection with the same module id.
538    daemon_drain_results: HashMap<ModuleEndpointId, bool>,
539    /// The ACTIVE slot: the one endpoint per module id that routing resolves.
540    /// Every by-id lookup (relay reservation, drain-by-id, liveness, census,
541    /// live roots, module-control RPCs) reads this map and nothing else.
542    modules_by_id: HashMap<String, ModuleConnection>,
543    /// The CANDIDATE slot of a blue/green swap: a second process registered
544    /// under an id that already has an active endpoint. It has a full endpoint
545    /// identity (so its own connection can be looked up and torn down) but no
546    /// by-id lookup sees it, so nothing is routed to it until `cutover_candidate`
547    /// promotes it.
548    candidates_by_id: HashMap<String, ModuleConnection>,
549    /// Former active endpoints demoted by `cutover_candidate`, until their
550    /// connection is removed. Membership is what distinguishes an endpoint that
551    /// was SUPERSEDED by a promotion (still alive, still carrying bound routes,
552    /// about to be drained) from one that is merely STALE (replaced some other
553    /// way, which is treated as a fault on the acking connection).
554    superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
555    endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
556    module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
557    /// Endpoints mid-drain, keyed to the reason the drain was begun with. The
558    /// value serves the census ("draining because restart"); membership alone
559    /// still answers every admission-gate check.
560    draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
561    closing_connections: HashSet<ConnectionId>,
562    next_generation: u64,
563    reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
564    reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
565    next_client_channel: HashMap<ConnectionId, u16>,
566    next_module_channel: HashMap<ModuleEndpointId, u16>,
567    client_slot_epochs: HashMap<ClientRouteKey, u32>,
568    module_slot_epochs: HashMap<ModuleRouteKey, u32>,
569    last_published_epoch: HashMap<ClientRouteKey, u32>,
570    client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
571    module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
572    status: HashMap<(ClientRouteKey, u32), String>,
573    pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
574    /// Pending relays removed without a module reply before `route.open`
575    /// marked them enqueued. `route.open` consumes its key when it marks the
576    /// relay (and sends the GOODBYE the removal could not send safely) or when
577    /// it aborts, so an entry lives only while that open is mid-send.
578    relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
579    /// Each scope's current `(scope_epoch, version)`, keyed `(owner, ref)`,
580    /// published by a scope sync in the same step that changes the record. A
581    /// bind commit reads it here so it never takes the scope table's lock
582    /// while holding this one. Absent means the scope is not live.
583    scope_tags: HashMap<(String, String), ScopeTag>,
584    next_control_corr: HashMap<ModuleEndpointId, u64>,
585    pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
586    health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
587}
588
589#[derive(Debug, Clone)]
590pub(crate) struct CloseReason {
591    code: &'static str,
592    message: String,
593}
594
595impl CloseReason {
596    pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
597        Self {
598            code,
599            message: message.into(),
600        }
601    }
602}
603
604impl fmt::Display for CloseReason {
605    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
606        write!(f, "{}: {}", self.code, self.message)
607    }
608}
609
610pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
611
612/// Dynamic forwarding state shared by the control plane and data-plane router.
613#[derive(Debug, Default)]
614pub struct ForwardingTable {
615    inner: Arc<RwLock<ForwardingInner>>,
616    frame_drop_warnings: Mutex<HashMap<ConnectionId, ChannelDropWarnings>>,
617    #[cfg(feature = "test-support")]
618    route_closing_hold: Mutex<Option<oneshot::Receiver<()>>>,
619    close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
620    counters: DaemonCounters,
621    /// Per-target-module bind-relay breaker state. It lives here, beside the
622    /// module connections it describes, because this is where a module
623    /// connection's identity is established and therefore where a stale
624    /// verdict has to be discarded.
625    route_bind_breakers: RouteBindBreakers,
626    /// Current route.bind relays keyed by target module. Admission is shared
627    /// across every client connection that points at the same endpoint.
628    route_bind_concurrency: RouteBindConcurrency,
629    /// Start and end of each module's `route.open` outage. Held here for the
630    /// same reason as the breakers: every handler built over this table must
631    /// see one outage per module, or each would log its own opening line.
632    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
633}
634
635#[derive(Debug)]
636struct FrameDropWarning {
637    last_warned: tokio::time::Instant,
638    suppressed: u64,
639}
640
641type ChannelDropWarnings = HashMap<(u16, &'static str), FrameDropWarning>;
642
643impl ForwardingTable {
644    /// Keep a silent route diagnosable without letting a broken peer flood logs.
645    /// Epoch and direction describe the dropped frame, but do not split its limit.
646    pub(crate) fn warn_frame_drop(
647        &self,
648        connection_id: ConnectionId,
649        channel: u16,
650        epoch: u32,
651        direction: &'static str,
652        reason: &'static str,
653    ) {
654        let now = tokio::time::Instant::now();
655        let mut warnings = self
656            .frame_drop_warnings
657            .lock()
658            .unwrap_or_else(std::sync::PoisonError::into_inner);
659        let entry = warnings
660            .entry(connection_id)
661            .or_default()
662            .entry((channel, reason));
663        let suppressed = match entry {
664            std::collections::hash_map::Entry::Occupied(mut entry) => {
665                let state = entry.get_mut();
666                if now.saturating_duration_since(state.last_warned) < Duration::from_secs(60) {
667                    state.suppressed = state.suppressed.saturating_add(1);
668                    return;
669                }
670                state.last_warned = now;
671                std::mem::take(&mut state.suppressed)
672            }
673            std::collections::hash_map::Entry::Vacant(entry) => {
674                entry.insert(FrameDropWarning {
675                    last_warned: now,
676                    suppressed: 0,
677                });
678                0
679            }
680        };
681        warn!(
682            connection_id = connection_id.get(),
683            channel, epoch, direction, reason, suppressed, "dropping data-plane frame"
684        );
685    }
686
687    /// Install scripted operator confirmation for an in-process integration harness,
688    /// without bootstrapping a daemon process. Only available with `test-support`.
689    ///
690    /// The script has one response per line: `approve`, `decline`, `unavailable`,
691    /// `hang` (until withdrawn), or `approve when <path>` (wait for the path to
692    /// exist). Unreadable or exhausted scripts fail closed. If supplied, `events`
693    /// receives the provider's JSON-lines event log. `limits: None` uses
694    /// [`OperatorLimits::default`](crate::operator_confirm::OperatorLimits); use
695    /// `Some(OperatorLimits { queue_wait, prompt_timeout, stuck_grace })` to
696    /// override the queue wait, prompt timeout and stuck-provider grace periods.
697    ///
698    /// The current executable's file name **must start with `ckdev-`**. Otherwise
699    /// this logs a warning and keeps the OS provider, just like bootstrap.
700    /// `cortexkit-test-support`'s `ckdev_binary` copies a harness executable under
701    /// a qualifying name; run that copy to enable scripted confirmation. See
702    /// `crates/subc-client-rs/tests/real_daemon/operator_confirm.rs` for the
703    /// reference end-to-end harness.
704    ///
705    /// Configure the table shared by the harness's `ControlHandler` and `Router`
706    /// before serving any requests. This also configures privileged confirmations
707    /// that use the same table.
708    ///
709    /// # Panics
710    ///
711    /// Panics if operator-confirm processing has already started.
712    ///
713    /// # Example
714    ///
715    /// ```no_run
716    /// use std::{path::Path, sync::Arc};
717    /// use subc_daemon::{ControlHandler, ForwardingTable, Registry};
718    ///
719    /// let registry = Arc::new(Registry::default());
720    /// let forwarding = Arc::new(ForwardingTable::default());
721    /// forwarding.configure_operator_script(
722    ///     Path::new("operator.script"),
723    ///     Some(Path::new("operator.events")),
724    ///     None,
725    /// );
726    /// let handler = ControlHandler::with_forwarding(registry, forwarding);
727    /// ```
728    #[cfg(feature = "test-support")]
729    pub fn configure_operator_script(
730        &self,
731        script: &std::path::Path,
732        events: Option<&std::path::Path>,
733        limits: Option<crate::operator_confirm::OperatorLimits>,
734    ) {
735        self.operator_confirms()
736            .configure_script(script, events, limits);
737    }
738
739    /// Hold the next supervisor drain between enqueueing `module.draining` and
740    /// `route.closing`, so a test can forward a response in that window. Dropping
741    /// the returned sender also releases the hold, including on a test panic.
742    #[cfg(feature = "test-support")]
743    pub fn hold_route_closing_for_test(&self) -> oneshot::Sender<()> {
744        let (release, hold) = oneshot::channel();
745        *self.route_closing_hold.lock().unwrap() = Some(hold);
746        release
747    }
748
749    #[cfg(feature = "test-support")]
750    pub(crate) async fn wait_before_route_closing_for_test(&self) {
751        let hold = self.route_closing_hold.lock().unwrap().take();
752        if let Some(hold) = hold {
753            let _ = hold.await;
754        }
755    }
756
757    pub(crate) fn operator_confirms(&self) -> Arc<crate::operator_confirm::OperatorConfirms> {
758        Arc::clone(
759            &self
760                .inner
761                .read()
762                .unwrap_or_else(|p| p.into_inner())
763                .operator_confirms,
764        )
765    }
766
767    #[cfg(test)]
768    pub(crate) fn inject_operator_principal(&self, key: ModuleRouteKey, principal: Principal) {
769        self.alter_operator_binding_for_test(key, |binding| binding.principal = principal);
770    }
771
772    #[cfg(test)]
773    pub(crate) fn alter_operator_binding_for_test(
774        &self,
775        key: ModuleRouteKey,
776        alter: impl FnOnce(&mut RouteBinding),
777    ) {
778        let mut inner = self.write_inner().unwrap();
779        let old = inner.module_to_client.remove(&key).unwrap();
780        let client = ClientRouteKey {
781            connection_id: old.client_connection_id,
782            channel: old.client_channel,
783        };
784        inner.client_to_module.remove(&client);
785        let mut binding = Arc::try_unwrap(old).expect("test binding has no outstanding readers");
786        alter(&mut binding);
787        let binding = Arc::new(binding);
788        inner.client_to_module.insert(client, Arc::clone(&binding));
789        inner.module_to_client.insert(key, binding);
790    }
791
792    /// Mutate the route without delivering its withdrawal notice, so tests can
793    /// exercise the commit-point route check rather than the withdrawal path.
794    #[cfg(test)]
795    pub(crate) fn invalidate_operator_route_for_test(&self, key: ModuleRouteKey, stale: bool) {
796        let mut inner = self.write_inner().unwrap();
797        let old = inner.module_to_client.remove(&key).unwrap();
798        let client = ClientRouteKey {
799            connection_id: old.client_connection_id,
800            channel: old.client_channel,
801        };
802        inner.client_to_module.remove(&client);
803        if stale {
804            let mut binding = Arc::try_unwrap(old).expect("no binding readers at the test pause");
805            binding.module_epoch += 1;
806            let binding = Arc::new(binding);
807            inner.client_to_module.insert(client, binding.clone());
808            inner.module_to_client.insert(key, binding);
809        }
810    }
811
812    /// Keep the binding protected through confirmation admission. Route release
813    /// takes the write lock and then the same confirm-state lock, so it cannot
814    /// miss an admitted request or leave it waiting on a vanished route.
815    pub(crate) fn with_operator_route<T>(
816        &self,
817        connection_id: ConnectionId,
818        channel: u16,
819        epoch: u32,
820        admit: impl FnOnce(Option<&RouteBinding>) -> T,
821    ) -> Result<T, ForwardingError> {
822        self.with_operator_route_epoch(connection_id, channel, |binding| {
823            admit(binding.filter(|binding| binding.module_epoch == epoch))
824        })
825    }
826
827    /// Like admission, but retains a mismatched epoch so a shortcut can audit
828    /// a stale channel separately from a closed one, under the same read lock.
829    pub(crate) fn with_operator_route_epoch<T>(
830        &self,
831        connection_id: ConnectionId,
832        channel: u16,
833        decide: impl FnOnce(Option<&RouteBinding>) -> T,
834    ) -> Result<T, ForwardingError> {
835        let inner = self.read_inner()?;
836        let binding = inner
837            .endpoint_by_connection
838            .get(&connection_id)
839            .and_then(|endpoint| {
840                inner.module_to_client.get(&ModuleRouteKey {
841                    endpoint: *endpoint,
842                    channel,
843                })
844            });
845        Ok(decide(binding.map(Arc::as_ref)))
846    }
847
848    pub(crate) fn counters(&self) -> DaemonCounters {
849        self.counters.clone()
850    }
851
852    pub(crate) fn counters_ref(&self) -> &DaemonCounters {
853        &self.counters
854    }
855
856    pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
857        self.route_bind_breakers.clone()
858    }
859
860    pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
861        self.route_bind_concurrency.clone()
862    }
863
864    pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
865        Arc::clone(&self.route_outages)
866    }
867
868    pub(crate) fn register_connection_close(
869        &self,
870        connection_id: ConnectionId,
871    ) -> ConnectionCloseReceiver {
872        let (sender, receiver) = oneshot::channel();
873        let replaced = self
874            .lock_close_registry()
875            .insert(connection_id, sender)
876            .is_some();
877        if replaced {
878            warn!(
879                connection_id = connection_id.get(),
880                "replaced existing connection close registration"
881            );
882        }
883        receiver
884    }
885
886    pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
887        self.lock_close_registry().remove(&connection_id);
888    }
889
890    /// Close every established connection, modules and clients alike, as the
891    /// last step of an announced daemon shutdown. By the time this runs each
892    /// registered module has already been sent a module GOODBYE (see
893    /// `Supervisor::end_children_for_daemon_shutdown`), so the EOF that follows
894    /// is a planned stop; EOF with no GOODBYE before it stays reserved for a
895    /// daemon that went away unannounced. Closing while the daemon is still
896    /// alive lets it wait for the modules' own teardowns before it ends
897    /// whatever is left.
898    #[cfg(unix)]
899    pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
900        let senders: Vec<_> = self.lock_close_registry().drain().collect();
901        let count = senders.len();
902        for (_, sender) in senders {
903            let _ = sender.send(reason.clone());
904        }
905        count
906    }
907
908    /// Every registered module connection, whichever slot it occupies (active,
909    /// swap candidate, or superseded incumbent), once each. Daemon shutdown
910    /// uses this to tell each of them it is a planned stop before closing it.
911    #[cfg(unix)]
912    pub(crate) fn module_connections(
913        &self,
914    ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
915        let inner = self.read_inner()?;
916        let mut seen = HashSet::new();
917        Ok(inner
918            .modules_by_id
919            .values()
920            .chain(inner.candidates_by_id.values())
921            .chain(inner.superseded_endpoints.values())
922            .filter(|module| seen.insert(module.endpoint))
923            .map(|module| ModuleConnectionTarget {
924                module_id: inner
925                    .module_id_by_endpoint
926                    .get(&module.endpoint)
927                    .cloned()
928                    .unwrap_or_default(),
929                endpoint: module.endpoint,
930                sink: module.sink.clone(),
931                negotiated_ver: module.negotiated_ver,
932            })
933            .collect())
934    }
935
936    /// Ask a registered connection to close. Returns true only for the request
937    /// that actually reached it; later requests for the same connection, and
938    /// requests for a connection that is not registered, return false.
939    pub(crate) fn request_connection_close(
940        &self,
941        connection_id: ConnectionId,
942        reason: CloseReason,
943    ) -> bool {
944        let sender = self.lock_close_registry().remove(&connection_id);
945        if let Some(sender) = sender {
946            debug!(
947                connection_id = connection_id.get(),
948                close_reason = %reason,
949                "requesting connection close"
950            );
951            let _ = sender.send(reason);
952            true
953        } else {
954            debug!(
955                connection_id = connection_id.get(),
956                close_reason = %reason,
957                "connection close request ignored for inactive connection"
958            );
959            false
960        }
961    }
962
963    pub fn register_module_connection(
964        &self,
965        connection_id: ConnectionId,
966        module_id: String,
967        negotiated_ver: u8,
968        concurrency: Concurrency,
969        sink: FrameSink,
970    ) -> Result<ModuleEndpointId, ForwardingError> {
971        self.register_module_connection_inner(
972            connection_id,
973            module_id,
974            negotiated_ver,
975            concurrency,
976            sink,
977            None,
978        )
979    }
980
981    /// Register a module connection and queue its HELLO_ACK as the first frame
982    /// on its sink, in the same write-lock critical section that makes the
983    /// endpoint visible.
984    ///
985    /// A module reads HELLO_ACK as the first frame after its HELLO and exits on
986    /// anything else. Once the endpoint is in `modules_by_id`, a `route.open`
987    /// on any other connection can queue a `route.bind` request onto this sink,
988    /// so the ack must already be queued by then. Every lookup that can queue a
989    /// frame for the module takes this lock, so nothing can get in ahead of it.
990    ///
991    /// If the ack cannot be queued (the sink is closed or full), the
992    /// registration fails with nothing inserted.
993    pub(crate) fn register_module_connection_acked(
994        &self,
995        connection_id: ConnectionId,
996        module_id: String,
997        negotiated_ver: u8,
998        concurrency: Concurrency,
999        sink: FrameSink,
1000        hello_ack: Frame,
1001    ) -> Result<ModuleEndpointId, ForwardingError> {
1002        self.register_module_connection_inner(
1003            connection_id,
1004            module_id,
1005            negotiated_ver,
1006            concurrency,
1007            sink,
1008            Some(hello_ack),
1009        )
1010    }
1011
1012    fn register_module_connection_inner(
1013        &self,
1014        connection_id: ConnectionId,
1015        module_id: String,
1016        negotiated_ver: u8,
1017        concurrency: Concurrency,
1018        sink: FrameSink,
1019        hello_ack: Option<Frame>,
1020    ) -> Result<ModuleEndpointId, ForwardingError> {
1021        let mut inner = self.write_inner()?;
1022        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1023            return Err(ForwardingError::ConnectionClosing { connection_id });
1024        }
1025        check_module_connection_role_locked(&inner, connection_id)?;
1026        // Every refusal check has passed and nothing has been mutated yet, so
1027        // a failed enqueue leaves the table exactly as it was.
1028        enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1029
1030        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1031        let endpoint = ModuleEndpointId {
1032            connection_id,
1033            generation: inner.next_generation,
1034        };
1035        inner.endpoint_by_connection.insert(connection_id, endpoint);
1036        inner
1037            .module_id_by_endpoint
1038            .insert(endpoint, module_id.clone());
1039        inner.next_module_channel.insert(endpoint, 1);
1040        inner.next_control_corr.insert(endpoint, 1);
1041        inner.modules_by_id.insert(
1042            module_id.clone(),
1043            ModuleConnection {
1044                endpoint,
1045                sink,
1046                negotiated_ver,
1047                concurrency,
1048            },
1049        );
1050        drop(inner);
1051
1052        // A new module connection has arrived under this id, so anything the
1053        // bind-relay breaker learned was learned about a process that is no
1054        // longer the one behind this name. See
1055        // `RouteBindBreakers::reset_for_new_module_connection`.
1056        //
1057        // Keyed on ARRIVAL rather than on teardown deliberately: a connection
1058        // going away is not evidence about anything, and a module whose
1059        // connection drops without coming back should keep its verdict until
1060        // something actually registers in its place. This also covers the
1061        // unclean replacements -- a module killed mid-bind, or one whose
1062        // connection was closing -- because registration is the single path by
1063        // which any module connection becomes usable.
1064        if let Some(discarded) = self
1065            .route_bind_breakers
1066            .reset_for_new_module_connection(&module_id)
1067        {
1068            info!(
1069                module_id = %module_id,
1070                discarded_consecutive_timeouts = discarded,
1071                "route.bind breaker state discarded: a new module connection replaced the process it described"
1072            );
1073        }
1074        Ok(endpoint)
1075    }
1076
1077    /// Register a blue/green swap candidate for `module_id` into the candidate
1078    /// slot, alongside whatever endpoint is active for the id.
1079    ///
1080    /// Unlike [`Self::register_module_connection`], this never touches the
1081    /// active slot and never resets the bind-relay breaker: the incumbent is
1082    /// still the process serving the id, so what the breaker learned about it
1083    /// is still true. The breaker is reset at cutover instead, when the process
1084    /// behind the name actually changes. A second candidate for the same id is
1085    /// refused.
1086    ///
1087    /// Production registers candidates through
1088    /// [`Self::register_candidate_module_connection_acked`]; this ack-less form
1089    /// is for tests that build forwarding state directly.
1090    #[cfg(test)]
1091    pub(crate) fn register_candidate_module_connection(
1092        &self,
1093        connection_id: ConnectionId,
1094        module_id: String,
1095        negotiated_ver: u8,
1096        concurrency: Concurrency,
1097        sink: FrameSink,
1098    ) -> Result<ModuleEndpointId, ForwardingError> {
1099        self.register_candidate_module_connection_inner(
1100            connection_id,
1101            module_id,
1102            negotiated_ver,
1103            concurrency,
1104            sink,
1105            None,
1106        )
1107    }
1108
1109    /// The swap-candidate counterpart of
1110    /// [`Self::register_module_connection_acked`]: the HELLO_ACK is queued on
1111    /// the candidate's sink before its endpoint is inserted. No by-id lookup
1112    /// sees a candidate, but its own connection's endpoint does become
1113    /// resolvable here, and at cutover it becomes routable; the ack has to be
1114    /// ahead of anything either can queue.
1115    pub(crate) fn register_candidate_module_connection_acked(
1116        &self,
1117        connection_id: ConnectionId,
1118        module_id: String,
1119        negotiated_ver: u8,
1120        concurrency: Concurrency,
1121        sink: FrameSink,
1122        hello_ack: Frame,
1123    ) -> Result<ModuleEndpointId, ForwardingError> {
1124        self.register_candidate_module_connection_inner(
1125            connection_id,
1126            module_id,
1127            negotiated_ver,
1128            concurrency,
1129            sink,
1130            Some(hello_ack),
1131        )
1132    }
1133
1134    fn register_candidate_module_connection_inner(
1135        &self,
1136        connection_id: ConnectionId,
1137        module_id: String,
1138        negotiated_ver: u8,
1139        concurrency: Concurrency,
1140        sink: FrameSink,
1141        hello_ack: Option<Frame>,
1142    ) -> Result<ModuleEndpointId, ForwardingError> {
1143        let mut inner = self.write_inner()?;
1144        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1145            return Err(ForwardingError::ConnectionClosing { connection_id });
1146        }
1147        check_module_connection_role_locked(&inner, connection_id)?;
1148        if inner.candidates_by_id.contains_key(&module_id) {
1149            return Err(ForwardingError::CandidateSlotOccupied { module_id });
1150        }
1151        // As in `register_module_connection_inner`: after every refusal check,
1152        // before any mutation.
1153        enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1154
1155        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1156        let endpoint = ModuleEndpointId {
1157            connection_id,
1158            generation: inner.next_generation,
1159        };
1160        inner.endpoint_by_connection.insert(connection_id, endpoint);
1161        inner
1162            .module_id_by_endpoint
1163            .insert(endpoint, module_id.clone());
1164        inner.next_module_channel.insert(endpoint, 1);
1165        inner.next_control_corr.insert(endpoint, 1);
1166        inner.candidates_by_id.insert(
1167            module_id,
1168            ModuleConnection {
1169                endpoint,
1170                sink,
1171                negotiated_ver,
1172                concurrency,
1173            },
1174        );
1175        Ok(endpoint)
1176    }
1177
1178    /// Promote `module_id`'s candidate to the active slot, in ONE forwarding
1179    /// write-lock critical section.
1180    ///
1181    /// That single section is the linearization point of a swap. Every relay
1182    /// reservation resolves the active slot under the same lock, so each one
1183    /// lands wholly before cutover (on the incumbent) or wholly after it (on the
1184    /// promoted candidate), never on a mix. A relay reserved on the incumbent
1185    /// before cutover can no longer commit: `commit_route_locked` requires the
1186    /// reservation's endpoint to be the active one, and an ack for it arriving
1187    /// later is answered by the superseded-endpoint arm of
1188    /// `complete_pending_relay` instead of ending the incumbent's connection.
1189    ///
1190    /// Both endpoints keep their identities: nothing is re-keyed, so the
1191    /// incumbent's bound routes and its pending correlation keys stay exactly
1192    /// where they are until it is drained with [`Self::begin_endpoint_drain`],
1193    /// using the incumbent endpoint this returns. Returns `Ok(None)` when there
1194    /// is no candidate for the id.
1195    pub(crate) fn cutover_candidate(
1196        &self,
1197        module_id: &str,
1198    ) -> Result<Option<ForwardingCutover>, ForwardingError> {
1199        let mut inner = self.write_inner()?;
1200        if inner.daemon_draining {
1201            return Err(ForwardingError::ModuleReloading {
1202                module_id: module_id.to_string(),
1203            });
1204        }
1205        let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
1206            return Ok(None);
1207        };
1208        let promoted = candidate.endpoint;
1209        let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
1210        let incumbent = incumbent.map(|incumbent| {
1211            let endpoint = incumbent.endpoint;
1212            inner.superseded_endpoints.insert(endpoint, incumbent);
1213            endpoint
1214        });
1215        drop(inner);
1216
1217        // A different process now answers for this id; see the matching reset
1218        // in `register_module_connection` for why a verdict about the old one
1219        // must not carry over.
1220        if let Some(discarded) = self
1221            .route_bind_breakers
1222            .reset_for_new_module_connection(module_id)
1223        {
1224            info!(
1225                module_id = %module_id,
1226                discarded_consecutive_timeouts = discarded,
1227                "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
1228            );
1229        }
1230        Ok(Some(ForwardingCutover {
1231            promoted,
1232            incumbent,
1233        }))
1234    }
1235
1236    #[allow(clippy::too_many_arguments)]
1237    pub(crate) async fn begin_route_bind_relay_for(
1238        &self,
1239        client_connection_id: ConnectionId,
1240        client_sink: FrameSink,
1241        client_negotiated_ver: u8,
1242        client_corr: u64,
1243        module_id: &str,
1244        principal: Principal,
1245        scope: Option<BoundScope>,
1246        project_root: Option<ProjectRootId>,
1247        harness: String,
1248        build_class: Option<String>,
1249        operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1250        deadline: Instant,
1251        busy_refusals: bool,
1252    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1253        // Reserve egress capacity before taking the forwarding lock. The permit is
1254        // held until the bind reaches one terminal state, so an accepted bind can
1255        // publish its table entry and RouteOpen response in one critical section.
1256        let client_permit =
1257            client_sink
1258                .reserve_owned()
1259                .await
1260                .map_err(|_| ForwardingError::ClientEgressClosed {
1261                    connection_id: client_connection_id,
1262                })?;
1263        self.begin_route_bind_relay_inner(
1264            client_connection_id,
1265            client_sink,
1266            client_negotiated_ver,
1267            client_corr,
1268            module_id,
1269            principal,
1270            scope,
1271            project_root,
1272            harness,
1273            build_class,
1274            operator_app,
1275            deadline,
1276            client_permit,
1277            busy_refusals,
1278        )
1279    }
1280
1281    #[cfg(test)]
1282    pub(crate) fn begin_route_bind_relay_for_test(
1283        &self,
1284        client_connection_id: ConnectionId,
1285        client_sink: FrameSink,
1286        client_corr: u64,
1287        module_id: &str,
1288    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1289        self.begin_route_bind_relay_with_busy_for_test(
1290            client_connection_id,
1291            client_sink,
1292            client_corr,
1293            module_id,
1294            false,
1295        )
1296    }
1297
1298    #[cfg(test)]
1299    pub(crate) fn begin_route_bind_relay_with_busy_for_test(
1300        &self,
1301        client_connection_id: ConnectionId,
1302        client_sink: FrameSink,
1303        client_corr: u64,
1304        module_id: &str,
1305        busy_refusals: bool,
1306    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1307        let permit =
1308            client_sink
1309                .try_reserve_owned()
1310                .map_err(|_| ForwardingError::ClientEgressClosed {
1311                    connection_id: client_connection_id,
1312                })?;
1313        self.begin_route_bind_relay_inner(
1314            client_connection_id,
1315            client_sink,
1316            subc_protocol::PROTOCOL_VERSION,
1317            client_corr,
1318            module_id,
1319            Principal::Direct,
1320            None,
1321            None,
1322            String::new(),
1323            None,
1324            None,
1325            Instant::now() + std::time::Duration::from_secs(60),
1326            permit,
1327            busy_refusals,
1328        )
1329    }
1330
1331    pub(crate) fn begin_module_control_rpc_for(
1332        &self,
1333        module_id: &str,
1334        expected_op: &str,
1335        deadline: Instant,
1336    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1337        self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1338    }
1339
1340    pub(crate) fn begin_health_probe_rpc_for(
1341        &self,
1342        module_id: &str,
1343        expected_op: &str,
1344        probe_started_at: Instant,
1345        deadline: Instant,
1346    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1347        self.begin_module_control_rpc_inner(
1348            module_id,
1349            expected_op,
1350            deadline,
1351            Some(probe_started_at),
1352            false,
1353        )
1354    }
1355
1356    pub(crate) fn begin_drain_health_probe_rpc_for(
1357        &self,
1358        module_id: &str,
1359        expected_op: &str,
1360        probe_started_at: Instant,
1361        deadline: Instant,
1362    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1363        self.begin_module_control_rpc_inner(
1364            module_id,
1365            expected_op,
1366            deadline,
1367            Some(probe_started_at),
1368            true,
1369        )
1370    }
1371
1372    /// A health probe addressed to one endpoint, in whichever slot it is.
1373    ///
1374    /// Every other control RPC resolves the id's ACTIVE endpoint, which is how
1375    /// a swap candidate (not active until cutover) and a superseded incumbent
1376    /// (not active after it) are unreachable by them. A swap needs to probe
1377    /// exactly those two: the candidate before promoting it, and the incumbent
1378    /// for its busy gauges while it drains.
1379    pub(crate) fn begin_endpoint_health_probe_rpc_for(
1380        &self,
1381        endpoint: ModuleEndpointId,
1382        expected_op: &str,
1383        probe_started_at: Instant,
1384        deadline: Instant,
1385    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1386        let inner = self.write_inner()?;
1387        let module = module_connection_for_endpoint_locked(&inner, endpoint)
1388            .cloned()
1389            .ok_or(ForwardingError::NoModuleConnection)?;
1390        let module_id = inner
1391            .module_id_by_endpoint
1392            .get(&endpoint)
1393            .cloned()
1394            .unwrap_or_default();
1395        // A draining incumbent is exactly what this probes for busy gauges, so
1396        // draining is allowed, as it is for the by-id drain probe.
1397        self.begin_control_rpc_locked(
1398            inner,
1399            &module_id,
1400            module,
1401            expected_op,
1402            deadline,
1403            Some(probe_started_at),
1404            true,
1405        )
1406    }
1407
1408    fn begin_module_control_rpc_inner(
1409        &self,
1410        module_id: &str,
1411        expected_op: &str,
1412        deadline: Instant,
1413        health_probe_started_at: Option<Instant>,
1414        allow_draining: bool,
1415    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1416        let inner = self.write_inner()?;
1417        let module = inner
1418            .modules_by_id
1419            .get(module_id)
1420            .cloned()
1421            .ok_or(ForwardingError::NoModuleConnection)?;
1422        self.begin_control_rpc_locked(
1423            inner,
1424            module_id,
1425            module,
1426            expected_op,
1427            deadline,
1428            health_probe_started_at,
1429            allow_draining,
1430        )
1431    }
1432
1433    #[allow(clippy::too_many_arguments)]
1434    fn begin_control_rpc_locked(
1435        &self,
1436        mut inner: RwLockWriteGuard<'_, ForwardingInner>,
1437        module_id: &str,
1438        module: ModuleConnection,
1439        expected_op: &str,
1440        deadline: Instant,
1441        health_probe_started_at: Option<Instant>,
1442        allow_draining: bool,
1443    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1444        if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1445            return Err(ForwardingError::ModuleReloading {
1446                module_id: module_id.to_string(),
1447            });
1448        }
1449        if inner
1450            .closing_connections
1451            .contains(&module.endpoint.connection_id)
1452        {
1453            return Err(ForwardingError::ConnectionClosing {
1454                connection_id: module.endpoint.connection_id,
1455            });
1456        }
1457        if health_probe_started_at.is_some() {
1458            // Recurring health probes are serial per endpoint. Once the next one
1459            // starts, an older answer can no longer improve the current snapshot,
1460            // so retaining more than the newest unanswered probe has no value.
1461            inner
1462                .health_probe_tombstones
1463                .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1464        }
1465        let corr = match inner.allocate_control_corr(module.endpoint) {
1466            Ok(corr) => corr,
1467            Err(err) => {
1468                drop(inner);
1469                self.request_connection_close(
1470                    module.endpoint.connection_id,
1471                    CloseReason::new(
1472                        "control_correlation_exhausted",
1473                        "daemon-originated channel-0 correlation space exhausted",
1474                    ),
1475                );
1476                return Err(err);
1477            }
1478        };
1479        let (sender, receiver) = oneshot::channel();
1480        inner.pending_control_rpcs.insert(
1481            (module.endpoint, corr),
1482            PendingModuleControlRpcEntry {
1483                expected_op: expected_op.to_string(),
1484                deadline,
1485                health_probe_started_at,
1486                sender,
1487            },
1488        );
1489
1490        Ok(PendingModuleControlRpc {
1491            endpoint: module.endpoint,
1492            module_sink: module.sink,
1493            negotiated_ver: module.negotiated_ver,
1494            corr,
1495            receiver,
1496        })
1497    }
1498
1499    #[allow(clippy::too_many_arguments)]
1500    fn begin_route_bind_relay_inner(
1501        &self,
1502        client_connection_id: ConnectionId,
1503        client_sink: FrameSink,
1504        client_negotiated_ver: u8,
1505        client_corr: u64,
1506        expected_module_id: &str,
1507        principal: Principal,
1508        scope: Option<BoundScope>,
1509        project_root: Option<ProjectRootId>,
1510        harness: String,
1511        build_class: Option<String>,
1512        operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1513        deadline: Instant,
1514        client_permit: crate::router::EgressPermit,
1515        busy_refusals: bool,
1516    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1517        let mut inner = self.write_inner()?;
1518        // HELLO and route.open can run on different tasks. Decide the role under
1519        // the same lock that installs endpoints and reserves client channels so
1520        // neither task can turn one connection into both kinds of owner.
1521        if inner
1522            .endpoint_by_connection
1523            .contains_key(&client_connection_id)
1524        {
1525            return Err(ForwardingError::ConnectionRoleConflict {
1526                connection_id: client_connection_id,
1527            });
1528        }
1529        if inner.closing_connections.contains(&client_connection_id) {
1530            return Err(ForwardingError::ConnectionClosing {
1531                connection_id: client_connection_id,
1532            });
1533        }
1534        let module = inner
1535            .modules_by_id
1536            .get(expected_module_id)
1537            .cloned()
1538            .ok_or(ForwardingError::NoModuleConnection)?;
1539        if inner.draining_endpoints.contains_key(&module.endpoint) {
1540            return Err(ForwardingError::ModuleReloading {
1541                module_id: expected_module_id.to_string(),
1542            });
1543        }
1544        if inner
1545            .closing_connections
1546            .contains(&module.endpoint.connection_id)
1547        {
1548            return Err(ForwardingError::ConnectionClosing {
1549                connection_id: module.endpoint.connection_id,
1550            });
1551        }
1552
1553        let corr = match inner.allocate_control_corr(module.endpoint) {
1554            Ok(corr) => corr,
1555            Err(err) => {
1556                drop(inner);
1557                self.request_connection_close(
1558                    module.endpoint.connection_id,
1559                    CloseReason::new(
1560                        "control_correlation_exhausted",
1561                        "daemon-originated channel-0 correlation space exhausted",
1562                    ),
1563                );
1564                return Err(err);
1565            }
1566        };
1567        let (client_channel, client_epoch, module_channel, module_epoch) =
1568            inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1569        let client_key = ClientRouteKey {
1570            connection_id: client_connection_id,
1571            channel: client_channel,
1572        };
1573        let module_key = ModuleRouteKey {
1574            endpoint: module.endpoint,
1575            channel: module_channel,
1576        };
1577        let reservation = RouteReservation {
1578            client_key,
1579            module_key,
1580            client_epoch,
1581            module_epoch,
1582            project_root,
1583        };
1584        let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
1585            request_window: None,
1586            watch_window: None,
1587            route_channel: client_channel,
1588            route_epoch: client_epoch,
1589        })
1590        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1591        let route_open_frame = Frame::build_with_version(
1592            client_negotiated_ver,
1593            FrameType::Response,
1594            Flags::new(false, Priority::Passive, false),
1595            0,
1596            0,
1597            client_corr,
1598            response_body,
1599        )
1600        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1601        let (sender, receiver) = oneshot::channel();
1602        inner.reserved_client.insert(client_key, module_key);
1603        inner.reserved_module.insert(module_key, client_key);
1604        inner.pending_relays.insert(
1605            (module.endpoint, corr),
1606            PendingRouteBindRelayEntry {
1607                reservation,
1608                client_sink,
1609                client_negotiated_ver,
1610                client_permit,
1611                route_open_frame,
1612                busy_refusals,
1613                request_window: window_for(&module.concurrency),
1614                watch_window: (busy_refusals && module.concurrency == Concurrency::ModuleManaged)
1615                    .then_some(DEFAULT_MODULE_MANAGED_WATCH_WINDOW),
1616                principal,
1617                scope,
1618                harness,
1619                build_class,
1620                operator_app,
1621                deadline,
1622                relay_enqueued: false,
1623                sender,
1624            },
1625        );
1626
1627        Ok(PendingRouteBindRelay {
1628            endpoint: module.endpoint,
1629            module_sink: module.sink,
1630            negotiated_ver: module.negotiated_ver,
1631            client_channel,
1632            client_epoch,
1633            module_channel,
1634            module_epoch,
1635            corr,
1636            receiver,
1637        })
1638    }
1639
1640    /// Called by `route.open` right after the route.bind entered the module's
1641    /// egress queue.
1642    ///
1643    /// The module reader runs concurrently with `route.open`, so a fast module
1644    /// can receive the bind, ack it, and have the reader commit the route
1645    /// before this runs. A missing entry therefore does not mean the bind was
1646    /// abandoned: only a removal recorded in `relays_released_before_enqueue`
1647    /// does. Telling the two apart is what stops `route.open` from sending a
1648    /// GOODBYE for a route that was just committed.
1649    pub(crate) fn mark_route_bind_relay_enqueued(
1650        &self,
1651        endpoint: ModuleEndpointId,
1652        corr: u64,
1653    ) -> Result<RouteBindRelayMark, ForwardingError> {
1654        let mut inner = self.write_inner()?;
1655        if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
1656            pending.relay_enqueued = true;
1657            return Ok(RouteBindRelayMark::Marked);
1658        }
1659        if inner
1660            .relays_released_before_enqueue
1661            .remove(&(endpoint, corr))
1662        {
1663            return Ok(RouteBindRelayMark::ReleasedWithoutReply);
1664        }
1665        Ok(RouteBindRelayMark::SettledByReply)
1666    }
1667
1668    pub(crate) fn release_client_route(
1669        &self,
1670        client_connection_id: ConnectionId,
1671        client_channel: u16,
1672        expected_epoch: u32,
1673    ) -> Result<RouteRelease, ForwardingError> {
1674        let mut inner = self.write_inner()?;
1675        let release = release_client_route_locked(
1676            &mut inner,
1677            ClientRouteKey {
1678                connection_id: client_connection_id,
1679                channel: client_channel,
1680            },
1681            expected_epoch,
1682        );
1683        self.record_route_release(&release);
1684        Ok(release)
1685    }
1686
1687    pub(crate) fn release_module_route(
1688        &self,
1689        module_connection_id: ConnectionId,
1690        module_channel: u16,
1691        expected_epoch: u32,
1692    ) -> Result<RouteRelease, ForwardingError> {
1693        let mut inner = self.write_inner()?;
1694        let Some(endpoint) = inner
1695            .endpoint_by_connection
1696            .get(&module_connection_id)
1697            .copied()
1698        else {
1699            return Ok(RouteRelease::Absent);
1700        };
1701        let release = release_module_route_locked(
1702            &mut inner,
1703            ModuleRouteKey {
1704                endpoint,
1705                channel: module_channel,
1706            },
1707            expected_epoch,
1708        );
1709        self.record_route_release(&release);
1710        Ok(release)
1711    }
1712
1713    pub(crate) fn abort_pending_relay(
1714        &self,
1715        endpoint: ModuleEndpointId,
1716        corr: u64,
1717        outcome: RouteBindRelayOutcome,
1718    ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
1719        let mut inner = self.write_inner()?;
1720        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1721            // If another path dropped the relay while `route.open` was still
1722            // sending the bind, and that open is now giving up without marking
1723            // it (its send failed or it was cancelled), forget the record so it
1724            // cannot outlive the open.
1725            inner
1726                .relays_released_before_enqueue
1727                .remove(&(endpoint, corr));
1728            return Ok(None);
1729        };
1730        release_reserved_route_locked(
1731            &mut inner,
1732            pending.reservation.client_key,
1733            pending.reservation.module_key,
1734        );
1735        let target = pending
1736            .relay_enqueued
1737            .then(|| abandoned_route_target(&inner, &pending.reservation));
1738        let _ = pending.sender.send(outcome);
1739        Ok(target.flatten())
1740    }
1741
1742    pub(crate) fn cancel_module_control_rpc(
1743        &self,
1744        endpoint: ModuleEndpointId,
1745        corr: u64,
1746    ) -> Result<(), ForwardingError> {
1747        self.write_inner()?
1748            .pending_control_rpcs
1749            .remove(&(endpoint, corr));
1750        Ok(())
1751    }
1752
1753    pub(crate) fn tombstone_health_probe_rpc(
1754        &self,
1755        endpoint: ModuleEndpointId,
1756        corr: u64,
1757    ) -> Result<bool, ForwardingError> {
1758        let key = (endpoint, corr);
1759        let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
1760        {
1761            let mut inner = self.write_inner()?;
1762            let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
1763                return Ok(false);
1764            };
1765            let Some(probe_started_at) = pending.health_probe_started_at else {
1766                inner.pending_control_rpcs.insert(key, pending);
1767                return Ok(false);
1768            };
1769            let module_id = inner
1770                .module_id_by_endpoint
1771                .get(&endpoint)
1772                .cloned()
1773                .unwrap_or_else(|| "unknown".to_string());
1774            inner.health_probe_tombstones.insert(
1775                key,
1776                HealthProbeTombstone {
1777                    expected_op: pending.expected_op,
1778                    module_id,
1779                    probe_started_at,
1780                    expires_at,
1781                },
1782            );
1783        }
1784        self.schedule_health_probe_tombstone_expiration(key, expires_at);
1785        Ok(true)
1786    }
1787
1788    fn schedule_health_probe_tombstone_expiration(
1789        &self,
1790        key: (ModuleEndpointId, u64),
1791        expires_at: Instant,
1792    ) {
1793        let inner = Arc::downgrade(&self.inner);
1794        tokio::spawn(async move {
1795            tokio::time::sleep_until(expires_at).await;
1796            let Some(inner) = inner.upgrade() else {
1797                return;
1798            };
1799            let Ok(mut inner) = inner.write() else {
1800                return;
1801            };
1802            let expired = inner
1803                .health_probe_tombstones
1804                .get(&key)
1805                .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
1806            if expired {
1807                inner.health_probe_tombstones.remove(&key);
1808            }
1809        });
1810    }
1811
1812    pub(crate) fn complete_pending_relay(
1813        &self,
1814        connection_id: ConnectionId,
1815        corr: u64,
1816        outcome: RouteBindRelayOutcome,
1817    ) -> Result<PendingRelayCompletion, ForwardingError> {
1818        let mut inner = self.write_inner()?;
1819        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1820            return Ok(PendingRelayCompletion {
1821                settled: false,
1822                abandoned: None,
1823            });
1824        };
1825        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1826            return Ok(PendingRelayCompletion {
1827                settled: false,
1828                abandoned: None,
1829            });
1830        };
1831
1832        if Instant::now() >= pending.deadline {
1833            release_reserved_route_locked(
1834                &mut inner,
1835                pending.reservation.client_key,
1836                pending.reservation.module_key,
1837            );
1838            let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
1839                .then(|| abandoned_route_target(&inner, &pending.reservation))
1840                .flatten();
1841            let _ = pending
1842                .sender
1843                .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody {
1844                    code: "module_timeout".to_string(),
1845                    message: "route.bind response arrived after its daemon deadline".to_string(),
1846                    detail: None,
1847                }));
1848            return Ok(PendingRelayCompletion {
1849                settled: true,
1850                abandoned,
1851            });
1852        }
1853
1854        match outcome {
1855            // Two shapes of "the client is not there to receive this route" that
1856            // must resolve identically: its egress is already closed, or it is
1857            // marked closing (its connection loop has been asked to end, but has
1858            // not drained yet, so the sink is still open).
1859            //
1860            // Only the first used to be caught here. The second fell through to
1861            // `commit_route_locked`, which refuses a closing client with
1862            // `ConnectionClosing` -- and this function is called from the MODULE
1863            // connection's frame handler, so that refusal ended the module's
1864            // connection instead of this one client's route. One dying client's
1865            // route.open then took down a connection carrying every other
1866            // client's routes to that module.
1867            //
1868            // The remedy for both is the same, which is why they share an arm:
1869            // give back the reserved handle pair, tell the waiting route.open the
1870            // route is gone, and report the module-side channel so the caller can
1871            // send a channel-scoped GOODBYE for the binding the module just
1872            // created. Nothing here touches the module connection.
1873            RouteBindRelayOutcome::Accepted
1874                if pending.client_sink.is_closed()
1875                    || inner
1876                        .closing_connections
1877                        .contains(&pending.reservation.client_key.connection_id) =>
1878            {
1879                let reason = if pending.client_sink.is_closed() {
1880                    "client egress closed before route publication"
1881                } else {
1882                    "client connection is closing before route publication"
1883                };
1884                release_reserved_route_locked(
1885                    &mut inner,
1886                    pending.reservation.client_key,
1887                    pending.reservation.module_key,
1888                );
1889                // Not gated on `relay_enqueued`: an ack proves the module
1890                // received the bind, and `route.open` sends nothing for a
1891                // relay a reply already settled, so this is the only GOODBYE.
1892                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1893                let _ = pending
1894                    .sender
1895                    .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
1896                return Ok(PendingRelayCompletion {
1897                    settled: true,
1898                    abandoned,
1899                });
1900            }
1901            // The endpoint that acked this bind is no longer accepting new
1902            // routes, for one of two reasons. Either it was the module's active
1903            // endpoint when this relay was reserved and a blue/green swap has
1904            // since promoted a candidate (the replacement process registered
1905            // beside it) into the active slot, superseding it; or a daemon
1906            // drain has gated it, and the ack arrived before the ordered module
1907            // drain settled pending relays. The module did nothing wrong: it
1908            // bound the route it was asked to bind, and the endpoint still
1909            // carries every other client's routes until it is drained. So the
1910            // ack is settled here, while the pending entry still holds the
1911            // client's sender and the reserved handle pair: release the pair,
1912            // refuse the waiting route.open with `module_reloading`, which
1913            // callers retry (after a swap the retry reserves on the promoted
1914            // endpoint), and hand back the module-side channel so the caller
1915            // sends one channel-scoped GOODBYE for the binding the module just
1916            // created. Returning an error instead would end the module
1917            // connection all those other routes share; nothing here touches it.
1918            //
1919            // An endpoint that stopped being active any other way is stale
1920            // (replaced, but not by a swap and not draining). It still falls
1921            // through to `commit_route_locked`, which refuses it with
1922            // `StaleModuleEndpoint` exactly as before swaps existed.
1923            RouteBindRelayOutcome::Accepted
1924                if inner.superseded_endpoints.contains_key(&endpoint)
1925                    || inner.draining_endpoints.contains_key(&endpoint) =>
1926            {
1927                release_reserved_route_locked(
1928                    &mut inner,
1929                    pending.reservation.client_key,
1930                    pending.reservation.module_key,
1931                );
1932                // Not gated on `relay_enqueued`: an ack proves the module
1933                // received the bind, whether or not the enqueue mark was set.
1934                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1935                let module_id = inner
1936                    .module_id_by_endpoint
1937                    .get(&endpoint)
1938                    .cloned()
1939                    .unwrap_or_else(|| "unknown".to_string());
1940                let _ = pending
1941                    .sender
1942                    .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
1943                        "module_reloading",
1944                        format!("module_id '{module_id}' is reloading"),
1945                    )));
1946                return Ok(PendingRelayCompletion {
1947                    settled: true,
1948                    abandoned,
1949                });
1950            }
1951            // The scope the open was admitted under ended or changed after
1952            // admission. The module bound a route under a stamp that is no
1953            // longer true, so it must not become routable. Settled here like
1954            // the superseded arm above, never as an error from
1955            // `commit_route_locked`, which would close the module's whole
1956            // connection and every other client's routes to it: release the
1957            // pair, answer the waiting route.open by name, and hand back the
1958            // module-side channel for one channel-scoped GOODBYE.
1959            RouteBindRelayOutcome::Accepted
1960                if pending
1961                    .scope
1962                    .as_ref()
1963                    .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
1964            {
1965                let refusal = pending
1966                    .scope
1967                    .as_ref()
1968                    .and_then(|scope| scope_refusal_locked(&inner, scope))
1969                    .expect("guard matched a refusal under the same lock");
1970                release_reserved_route_locked(
1971                    &mut inner,
1972                    pending.reservation.client_key,
1973                    pending.reservation.module_key,
1974                );
1975                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1976                let _ = pending
1977                    .sender
1978                    .send(RouteBindRelayOutcome::DaemonRejected(refusal));
1979                return Ok(PendingRelayCompletion {
1980                    settled: true,
1981                    abandoned,
1982                });
1983            }
1984            RouteBindRelayOutcome::Accepted => {
1985                let abandoned = commit_route_locked(&mut inner, pending)?;
1986                return Ok(PendingRelayCompletion {
1987                    settled: true,
1988                    abandoned,
1989                });
1990            }
1991            terminal => {
1992                release_reserved_route_locked(
1993                    &mut inner,
1994                    pending.reservation.client_key,
1995                    pending.reservation.module_key,
1996                );
1997                let _ = pending.sender.send(terminal);
1998            }
1999        }
2000        Ok(PendingRelayCompletion {
2001            settled: true,
2002            abandoned: None,
2003        })
2004    }
2005
2006    pub(crate) fn pending_module_control_op(
2007        &self,
2008        connection_id: ConnectionId,
2009        corr: u64,
2010    ) -> Result<Option<String>, ForwardingError> {
2011        let inner = self.read_inner()?;
2012        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2013            return Ok(None);
2014        };
2015        let key = (endpoint, corr);
2016        Ok(inner
2017            .pending_control_rpcs
2018            .get(&key)
2019            .map(|pending| pending.expected_op.clone())
2020            .or_else(|| {
2021                inner
2022                    .health_probe_tombstones
2023                    .get(&key)
2024                    .filter(|tombstone| tombstone.expires_at > Instant::now())
2025                    .map(|tombstone| tombstone.expected_op.clone())
2026            }))
2027    }
2028
2029    pub(crate) fn complete_module_control_rpc(
2030        &self,
2031        connection_id: ConnectionId,
2032        corr: u64,
2033        actual_op: Option<&str>,
2034        outcome: ModuleControlRpcOutcome,
2035    ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
2036        let now = Instant::now();
2037        let mut inner = self.write_inner()?;
2038        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2039            return Ok(ModuleControlRpcCompletion::Unknown);
2040        };
2041        let key = (endpoint, corr);
2042        if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2043            if now >= pending.deadline {
2044                let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
2045                    ModuleControlRpcCompletion::LateHealthAnswer {
2046                        module_id: inner
2047                            .module_id_by_endpoint
2048                            .get(&endpoint)
2049                            .cloned()
2050                            .unwrap_or_else(|| "unknown".to_string()),
2051                        latency: now.saturating_duration_since(probe_started_at),
2052                    }
2053                });
2054                let _ = pending
2055                    .sender
2056                    .send(ModuleControlRpcOutcome::DeadlineElapsed);
2057                return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
2058            }
2059            let outcome = match actual_op {
2060                Some(actual) if actual != pending.expected_op => {
2061                    ModuleControlRpcOutcome::UnexpectedOp {
2062                        expected: pending.expected_op,
2063                        actual: actual.to_string(),
2064                    }
2065                }
2066                _ => outcome,
2067            };
2068            let _ = pending.sender.send(outcome);
2069            return Ok(ModuleControlRpcCompletion::Settled);
2070        }
2071
2072        let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
2073            return Ok(ModuleControlRpcCompletion::Unknown);
2074        };
2075        if tombstone.expires_at <= now {
2076            return Ok(ModuleControlRpcCompletion::Unknown);
2077        }
2078        Ok(ModuleControlRpcCompletion::LateHealthAnswer {
2079            module_id: tombstone.module_id,
2080            latency: now.saturating_duration_since(tombstone.probe_started_at),
2081        })
2082    }
2083
2084    #[cfg(test)]
2085    pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
2086        Ok(self.read_inner()?.health_probe_tombstones.len())
2087    }
2088
2089    #[cfg(test)]
2090    pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
2091        Ok(self.read_inner()?.closing_connections.len())
2092    }
2093
2094    /// Reserved-but-uncommitted route handles, counted on both index sides, so
2095    /// a test can assert a reservation pair was actually given back.
2096    #[cfg(test)]
2097    pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
2098        let inner = self.read_inner()?;
2099        Ok((inner.reserved_client.len(), inner.reserved_module.len()))
2100    }
2101
2102    pub(crate) fn module_endpoint_for_connection(
2103        &self,
2104        connection_id: ConnectionId,
2105    ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
2106        Ok(self
2107            .read_inner()?
2108            .endpoint_by_connection
2109            .get(&connection_id)
2110            .copied())
2111    }
2112
2113    /// Looks up the module registered on a data-plane connection so route-drop
2114    /// diagnostics name the emitter instead of only reporting a daemon total.
2115    pub(crate) fn module_id_for_connection(
2116        &self,
2117        connection_id: ConnectionId,
2118    ) -> Result<Option<String>, ForwardingError> {
2119        let inner = self.read_inner()?;
2120        Ok(inner
2121            .endpoint_by_connection
2122            .get(&connection_id)
2123            .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
2124            .cloned())
2125    }
2126
2127    /// Whether the daemon ever allocated `(channel, epoch)` on this module
2128    /// connection. Epochs on a module channel are handed out as 1, 2, 3, ...
2129    /// and the last one handed out is remembered until the connection ends, so
2130    /// every epoch from 1 up to that one was a real route at some point. Used
2131    /// only on the drop path, to tell a module still sending on a route the
2132    /// daemon released from one sending on a route that never existed.
2133    pub(crate) fn module_route_epoch_was_allocated(
2134        &self,
2135        connection_id: ConnectionId,
2136        channel: u16,
2137        epoch: u32,
2138    ) -> Result<bool, ForwardingError> {
2139        let inner = self.read_inner()?;
2140        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2141            return Ok(false);
2142        };
2143        Ok(inner
2144            .module_slot_epochs
2145            .get(&ModuleRouteKey { endpoint, channel })
2146            .is_some_and(|last| epoch != 0 && epoch <= *last))
2147    }
2148
2149    pub(crate) fn has_live_module_connection(
2150        &self,
2151        module_id: &str,
2152    ) -> Result<bool, ForwardingError> {
2153        Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
2154    }
2155
2156    /// Admission and route teardown share the forwarding-then-flow lock order.
2157    /// No asynchronous work may split validation from recording the credits.
2158    pub(crate) fn try_admit_request(
2159        &self,
2160        route: &Arc<RouteBinding>,
2161        epoch: u32,
2162        corr: u64,
2163        subscription: bool,
2164    ) -> Result<RequestAdmission, ForwardingError> {
2165        let inner = self.read_inner()?;
2166        let key = ClientRouteKey {
2167            connection_id: route.client_connection_id,
2168            channel: route.client_channel,
2169        };
2170        if epoch != route.client_epoch
2171            || !inner
2172                .client_to_module
2173                .get(&key)
2174                .is_some_and(|current| Arc::ptr_eq(current, route))
2175        {
2176            return Ok(RequestAdmission::Stale);
2177        }
2178        let admission = route.flow.try_acquire_tagged(corr, subscription);
2179        Ok(match admission {
2180            RequestAdmission::Stale
2181                if inner
2182                    .draining_endpoints
2183                    .contains_key(&route.module_endpoint) =>
2184            {
2185                RequestAdmission::ModuleReloading
2186            }
2187            admission => admission,
2188        })
2189    }
2190
2191    pub(crate) fn lookup_data_route(
2192        &self,
2193        connection_id: ConnectionId,
2194        channel: u16,
2195        epoch: u32,
2196    ) -> Result<DataRoute, ForwardingError> {
2197        let inner = self.read_inner()?;
2198        let state = if let Some(endpoint) =
2199            inner.endpoint_by_connection.get(&connection_id).copied()
2200        {
2201            let key = ModuleRouteKey { endpoint, channel };
2202            match inner.module_to_client.get(&key) {
2203                Some(route) if route.module_epoch == epoch => {
2204                    DataRouteState::Bound(Arc::clone(route))
2205                }
2206                Some(_) => DataRouteState::EpochMismatch,
2207                None if inner.reserved_module.contains_key(&key)
2208                    && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
2209                {
2210                    DataRouteState::Reserved
2211                }
2212                None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
2213                None => DataRouteState::Absent,
2214            }
2215        } else {
2216            let key = ClientRouteKey {
2217                connection_id,
2218                channel,
2219            };
2220            match inner.client_to_module.get(&key) {
2221                Some(route) if route.client_epoch == epoch => {
2222                    DataRouteState::Bound(Arc::clone(route))
2223                }
2224                Some(_) => DataRouteState::EpochMismatch,
2225                None if inner.reserved_client.contains_key(&key)
2226                    && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
2227                {
2228                    DataRouteState::Reserved
2229                }
2230                None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
2231                None => DataRouteState::Absent,
2232            }
2233        };
2234        Ok(
2235            if inner.endpoint_by_connection.contains_key(&connection_id) {
2236                DataRoute::Module(state)
2237            } else {
2238                DataRoute::Client(state)
2239            },
2240        )
2241    }
2242
2243    #[cfg(test)]
2244    pub(crate) fn inject_client_slot_epoch(
2245        &self,
2246        connection_id: ConnectionId,
2247        channel: u16,
2248        last_epoch: u32,
2249    ) {
2250        let mut inner = self.write_inner().expect("forwarding lock");
2251        inner.client_slot_epochs.insert(
2252            ClientRouteKey {
2253                connection_id,
2254                channel,
2255            },
2256            last_epoch,
2257        );
2258        inner.next_client_channel.insert(connection_id, channel);
2259    }
2260
2261    #[cfg(test)]
2262    pub(crate) fn inject_module_slot_epoch(
2263        &self,
2264        endpoint: ModuleEndpointId,
2265        channel: u16,
2266        last_epoch: u32,
2267    ) {
2268        let mut inner = self.write_inner().expect("forwarding lock");
2269        inner
2270            .module_slot_epochs
2271            .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
2272        inner.next_module_channel.insert(endpoint, channel);
2273    }
2274
2275    #[cfg(test)]
2276    pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
2277        self.write_inner()
2278            .expect("forwarding lock")
2279            .next_control_corr
2280            .insert(endpoint, next_corr);
2281    }
2282
2283    pub(crate) fn cache_status(
2284        &self,
2285        endpoint: ModuleEndpointId,
2286        module_channel: u16,
2287        module_epoch: u32,
2288        status: String,
2289    ) -> Result<bool, ForwardingError> {
2290        let mut inner = self.write_inner()?;
2291        if !inner.module_id_by_endpoint.contains_key(&endpoint) {
2292            return Err(ForwardingError::StaleModuleEndpoint);
2293        }
2294
2295        let module_key = ModuleRouteKey {
2296            endpoint,
2297            channel: module_channel,
2298        };
2299        let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
2300            (route.module_epoch == module_epoch).then_some((
2301                ClientRouteKey {
2302                    connection_id: route.client_connection_id,
2303                    channel: route.client_channel,
2304                },
2305                route.client_epoch,
2306            ))
2307        } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
2308            (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
2309                client_key,
2310                inner
2311                    .client_slot_epochs
2312                    .get(&client_key)
2313                    .copied()
2314                    .unwrap_or(0),
2315            ))
2316        } else {
2317            None
2318        };
2319
2320        if let Some(handle) = handle {
2321            inner.status.insert(handle, status);
2322            Ok(true)
2323        } else {
2324            debug!(
2325                module_channel,
2326                module_epoch,
2327                generation = endpoint.generation,
2328                connection_id = endpoint.connection_id.get(),
2329                "dropping stale status update for module route handle"
2330            );
2331            Ok(false)
2332        }
2333    }
2334
2335    pub(crate) fn route_poll_snapshot(
2336        &self,
2337        client_connection_id: ConnectionId,
2338        client_channel: u16,
2339        client_epoch: u32,
2340    ) -> Result<RoutePollSnapshot, ForwardingError> {
2341        let inner = self.read_inner()?;
2342        let client_key = ClientRouteKey {
2343            connection_id: client_connection_id,
2344            channel: client_channel,
2345        };
2346        let Some(route) = inner.client_to_module.get(&client_key) else {
2347            return Ok(RoutePollSnapshot::Absent);
2348        };
2349        if route.client_epoch != client_epoch
2350            || !inner
2351                .module_id_by_endpoint
2352                .contains_key(&route.module_endpoint)
2353        {
2354            return Ok(RoutePollSnapshot::Absent);
2355        }
2356        Ok(RoutePollSnapshot::Bound {
2357            module_id: route.module_id.clone(),
2358            status: inner.status.get(&(client_key, client_epoch)).cloned(),
2359        })
2360    }
2361
2362    pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2363        Ok(self.read_inner()?.client_to_module.len())
2364    }
2365
2366    /// How many distinct client connections hold at least one committed route,
2367    /// alongside the largest number of routes any single connection holds.
2368    ///
2369    /// `connected_clients` alone cannot distinguish many clients with a route
2370    /// each from one client accumulating hundreds, and those have opposite
2371    /// causes. Reading it required an out-of-band `lsof` during a live
2372    /// investigation, and the count of connections was mistaken for a count of
2373    /// client processes — which sent two of us after cleanup paths that were
2374    /// working correctly.
2375    pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2376        let inner = self.read_inner()?;
2377        let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2378        for key in inner.client_to_module.keys() {
2379            *per_connection.entry(key.connection_id).or_insert(0) += 1;
2380        }
2381        let max = per_connection.values().copied().max().unwrap_or(0);
2382        Ok((per_connection.len(), max))
2383    }
2384
2385    pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2386        let inner = self.read_inner()?;
2387        Ok(inner
2388            .client_to_module
2389            .keys()
2390            .any(|key| key.channel == route_channel))
2391    }
2392
2393    /// Whether daemon-wide shutdown has begun draining all providers.
2394    /// `None` means normal operation; during daemon shutdown, return the
2395    /// endpoint's measured result, or false if no completed measurement exists.
2396    pub(crate) fn daemon_drain_result_for_connection(
2397        &self,
2398        connection_id: ConnectionId,
2399    ) -> Result<Option<bool>, ForwardingError> {
2400        let inner = self.read_inner()?;
2401        Ok(inner.daemon_draining.then(|| {
2402            inner
2403                .endpoint_by_connection
2404                .get(&connection_id)
2405                .and_then(|endpoint| inner.daemon_drain_results.get(endpoint))
2406                .copied()
2407                .unwrap_or(false)
2408        }))
2409    }
2410
2411    #[cfg(unix)]
2412    pub(crate) fn record_daemon_drain_result(
2413        &self,
2414        endpoint: ModuleEndpointId,
2415        drained: bool,
2416    ) -> Result<(), ForwardingError> {
2417        let mut inner = self.write_inner()?;
2418        // Only update an endpoint captured at the start of daemon shutdown.
2419        // Teardown removes its entry, so a late wait cannot recreate the result
2420        // or attribute it to a replacement connection.
2421        if let Some(result) = inner.daemon_drain_results.get_mut(&endpoint) {
2422            *result = drained;
2423        }
2424        Ok(())
2425    }
2426
2427    /// Gate every provider atomically, including registrations racing shutdown.
2428    /// No supervisor lock is held while taking the forwarding lock.
2429    #[cfg(unix)]
2430    pub(crate) fn begin_daemon_drain(
2431        &self,
2432    ) -> Result<Vec<(String, ModuleEndpointId)>, ForwardingError> {
2433        let mut inner = self.write_inner()?;
2434        if inner.daemon_draining {
2435            return Ok(Vec::new());
2436        }
2437        inner.daemon_draining = true;
2438        let modules = inner
2439            .modules_by_id
2440            .iter()
2441            .map(|(id, module)| (id.clone(), module.endpoint))
2442            .collect::<Vec<_>>();
2443        for (_, endpoint) in &modules {
2444            inner.daemon_drain_results.insert(*endpoint, false);
2445            inner
2446                .draining_endpoints
2447                .insert(*endpoint, RouteCloseReason::Restart);
2448        }
2449        // Swap candidates and superseded incumbents are not routable, but they
2450        // are live endpoints that could still be sent module-control work, so
2451        // the daemon-wide gate covers them too. Only the active endpoints are
2452        // returned to the supervisor for measurement, one per module id.
2453        let off_slot_endpoints = inner
2454            .candidates_by_id
2455            .values()
2456            .map(|module| module.endpoint)
2457            .chain(inner.superseded_endpoints.keys().copied())
2458            .collect::<Vec<_>>();
2459        for endpoint in off_slot_endpoints {
2460            inner
2461                .draining_endpoints
2462                .insert(endpoint, RouteCloseReason::Restart);
2463        }
2464        Ok(modules)
2465    }
2466
2467    /// Begin draining whatever endpoint is ACTIVE for `module_id`.
2468    ///
2469    /// After a swap's cutover the active endpoint is the promoted candidate, so
2470    /// this must not be used to drain the incumbent; use
2471    /// [`Self::begin_endpoint_drain`] with the endpoint `cutover_candidate`
2472    /// returned.
2473    pub(crate) fn begin_module_drain(
2474        &self,
2475        module_id: &str,
2476        reason: RouteCloseReason,
2477    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2478        let mut inner = self.write_inner()?;
2479        let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2480            return Ok(None);
2481        };
2482        Ok(Some(begin_drain_locked(
2483            &mut inner, module_id, module, reason,
2484        )))
2485    }
2486
2487    /// Begin draining one specific endpoint, whichever slot it is in.
2488    ///
2489    /// This is how a swap drains its incumbent after cutover: the incumbent's
2490    /// endpoint is captured by `cutover_candidate`, and resolving it by module id
2491    /// instead would find the promoted candidate and leave neither process
2492    /// routable. Returns `Ok(None)` when the endpoint is no longer registered.
2493    pub(crate) fn begin_endpoint_drain(
2494        &self,
2495        endpoint: ModuleEndpointId,
2496        reason: RouteCloseReason,
2497    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2498        let mut inner = self.write_inner()?;
2499        let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2500            return Ok(None);
2501        };
2502        let module_id = inner
2503            .module_id_by_endpoint
2504            .get(&endpoint)
2505            .cloned()
2506            .expect("an endpoint resolved to a module connection has a module id");
2507        Ok(Some(begin_drain_locked(
2508            &mut inner, &module_id, module, reason,
2509        )))
2510    }
2511}
2512
2513/// Mark `module`'s endpoint draining and settle everything still pending on it.
2514/// Shared by the by-id and by-endpoint drain entry points, which differ only in
2515/// how they find the endpoint.
2516fn begin_drain_locked(
2517    inner: &mut ForwardingInner,
2518    module_id: &str,
2519    module: ModuleConnection,
2520    reason: RouteCloseReason,
2521) -> ModuleDrainTarget {
2522    {
2523        let endpoint = module.endpoint;
2524        inner.draining_endpoints.insert(endpoint, reason);
2525
2526        let flows = inner
2527            .client_to_module
2528            .values()
2529            .filter(|route| route.module_endpoint == endpoint)
2530            .map(|route| Arc::clone(&route.flow))
2531            .collect::<Vec<_>>();
2532        let excluded_subscriptions = flows
2533            .into_iter()
2534            .map(|flow| flow.begin_drain())
2535            .fold(0u32, u32::saturating_add);
2536
2537        let pending_keys = inner
2538            .pending_relays
2539            .keys()
2540            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2541            .copied()
2542            .collect::<Vec<_>>();
2543        let mut abandoned_bindings = Vec::new();
2544        for key in pending_keys {
2545            let Some(pending) = inner.pending_relays.remove(&key) else {
2546                continue;
2547            };
2548            release_reserved_route_locked(
2549                inner,
2550                pending.reservation.client_key,
2551                pending.reservation.module_key,
2552            );
2553            if pending.relay_enqueued {
2554                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2555                    abandoned_bindings.push(target);
2556                }
2557            } else {
2558                // The bind may still be on its way into the module's queue;
2559                // a GOODBYE sent now could overtake it. `route.open` sends
2560                // it once its send completes.
2561                inner.relays_released_before_enqueue.insert(key);
2562            }
2563            let _ = pending
2564                .sender
2565                .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
2566                    "module_reloading",
2567                    format!("module_id '{module_id}' is reloading"),
2568                )));
2569        }
2570
2571        let pending_control_keys = inner
2572            .pending_control_rpcs
2573            .keys()
2574            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2575            .copied()
2576            .collect::<Vec<_>>();
2577        for key in pending_control_keys {
2578            if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2579                let _ = pending
2580                    .sender
2581                    .send(ModuleControlRpcOutcome::ModuleGone(format!(
2582                        "module '{module_id}' began draining during module-control RPC"
2583                    )));
2584            }
2585        }
2586
2587        ModuleDrainTarget {
2588            endpoint,
2589            sink: module.sink,
2590            negotiated_ver: module.negotiated_ver,
2591            abandoned_bindings,
2592            excluded_subscriptions,
2593        }
2594    }
2595}
2596
2597/// What a drain that timed out was still waiting on, for the log line that
2598/// reports the timeout. Per-request ages are not tracked.
2599#[derive(Debug, Default, PartialEq, Eq)]
2600pub(crate) struct DrainHoldouts {
2601    /// Requests still counted against the drain (subscriptions flagged as
2602    /// such are excluded from the drain and not counted here).
2603    pub(crate) requests: usize,
2604    /// Routes holding at least one of those requests.
2605    pub(crate) routes: usize,
2606    /// Every route on the endpoint, for scale.
2607    pub(crate) total_routes: usize,
2608    /// The client connections holding the most requests, largest first, at
2609    /// most three: enough to name the consumer without listing every route.
2610    pub(crate) top_connections: Vec<(u64, usize)>,
2611    /// The held requests themselves, as the module saw them, so its own log
2612    /// can say what each one was: `(module channel, corr)`, ordered by channel
2613    /// then corr, at most [`DRAIN_HELD_REQUESTS_LISTED`]. The daemon never reads
2614    /// request bodies, so it cannot name a request's method; the module can,
2615    /// from the channel and corr. A request is released only when the module
2616    /// sends a terminal frame (Response, Error or StreamEnd) with its corr on
2617    /// its route, so every pair listed here is one the module ended, if at all,
2618    /// without sending that frame.
2619    pub(crate) held: Vec<(u16, u64)>,
2620}
2621
2622/// How many held requests the drain-timeout line lists by channel and corr.
2623pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
2624
2625impl ForwardingTable {
2626    /// Summarise the requests one endpoint's drain is still waiting on.
2627    pub(crate) fn endpoint_drain_holdouts(
2628        &self,
2629        endpoint: ModuleEndpointId,
2630    ) -> Result<DrainHoldouts, ForwardingError> {
2631        let inner = self.read_inner()?;
2632        let mut holdouts = DrainHoldouts::default();
2633        let mut by_connection: HashMap<u64, usize> = HashMap::new();
2634        for (key, route) in &inner.client_to_module {
2635            if route.module_endpoint != endpoint {
2636                continue;
2637            }
2638            holdouts.total_routes += 1;
2639            let held = route.flow.drain_in_flight();
2640            if held == 0 {
2641                continue;
2642            }
2643            holdouts.requests += held;
2644            holdouts.routes += 1;
2645            *by_connection.entry(key.connection_id.get()).or_default() += held;
2646            holdouts.held.extend(
2647                route
2648                    .flow
2649                    .drain_held_corrs()
2650                    .into_iter()
2651                    .map(|corr| (route.module_channel, corr)),
2652            );
2653        }
2654        holdouts.held.sort_unstable();
2655        holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
2656        let mut connections = by_connection.into_iter().collect::<Vec<_>>();
2657        connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
2658        connections.truncate(3);
2659        holdouts.top_connections = connections;
2660        Ok(holdouts)
2661    }
2662
2663    pub(crate) fn endpoint_in_flight_count(
2664        &self,
2665        endpoint: ModuleEndpointId,
2666    ) -> Result<usize, ForwardingError> {
2667        let inner = self.read_inner()?;
2668        Ok(inner
2669            .client_to_module
2670            .values()
2671            .filter(|route| route.module_endpoint == endpoint)
2672            .map(|route| route.flow.drain_in_flight())
2673            .sum())
2674    }
2675
2676    pub(crate) fn endpoint_is_draining(
2677        &self,
2678        endpoint: ModuleEndpointId,
2679    ) -> Result<bool, ForwardingError> {
2680        Ok(self
2681            .read_inner()?
2682            .draining_endpoints
2683            .contains_key(&endpoint))
2684    }
2685
2686    pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
2687        let inner = self.read_inner()?;
2688        Ok(inner
2689            .modules_by_id
2690            .get(module_id)
2691            .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
2692    }
2693
2694    pub(crate) fn release_module_endpoint_routes(
2695        &self,
2696        endpoint: ModuleEndpointId,
2697    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2698        let mut inner = self.write_inner()?;
2699        let routes = inner
2700            .module_to_client
2701            .iter()
2702            .filter(|(module_key, _)| module_key.endpoint == endpoint)
2703            .map(|(module_key, route)| (*module_key, route.module_epoch))
2704            .collect::<Vec<_>>();
2705        let mut released = Vec::with_capacity(routes.len());
2706        for (module_key, epoch) in routes {
2707            if let RouteRelease::Removed(target) =
2708                release_module_route_locked(&mut inner, module_key, epoch)
2709            {
2710                released.push(target);
2711            }
2712        }
2713        Ok(released)
2714    }
2715
2716    /// Enumerate one endpoint's current routes without contacting the module.
2717    ///
2718    /// The read lock makes this safe while the endpoint drains: the returned
2719    /// `draining` marker describes the same table state that owns the route,
2720    /// rather than inferring liveness from a module that may be stopping.
2721    pub(crate) fn endpoint_routes(
2722        &self,
2723        endpoint: ModuleEndpointId,
2724    ) -> Result<Vec<EndpointRoute>, ForwardingError> {
2725        let inner = self.read_inner()?;
2726        Ok(endpoint_routes_locked(&inner, endpoint))
2727    }
2728
2729    /// Snapshot all live endpoint route sets under one forwarding-table read lock.
2730    pub(crate) fn route_census(
2731        &self,
2732        module_id: Option<&str>,
2733    ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
2734        let inner = self.read_inner()?;
2735        let mut endpoints = inner
2736            .modules_by_id
2737            .iter()
2738            .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
2739            .map(|(id, module)| (id.clone(), module.endpoint))
2740            .collect::<Vec<_>>();
2741        endpoints.sort_by(|left, right| left.0.cmp(&right.0));
2742        Ok(endpoints
2743            .into_iter()
2744            .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
2745            .collect())
2746    }
2747
2748    /// Snapshot only the endpoint currently routable under this module id.
2749    pub(crate) fn live_roots(
2750        &self,
2751        module_id: &str,
2752    ) -> Result<ModuleControlResponseToModule, ForwardingError> {
2753        let inner = self.read_inner()?;
2754        let endpoint = inner
2755            .modules_by_id
2756            .get(module_id)
2757            .map(|module| module.endpoint);
2758        let mut roots = BTreeMap::new();
2759        let mut unknown_root_bindings = 0;
2760        let mut total_bindings = 0;
2761        if let Some(endpoint) = endpoint {
2762            for binding in inner
2763                .module_to_client
2764                .values()
2765                .filter(|binding| binding.module_endpoint == endpoint)
2766            {
2767                total_bindings += 1;
2768                let workdir = binding
2769                    .scope
2770                    .as_ref()
2771                    .and_then(|scope| scope.workdir.as_deref())
2772                    .map(std::path::Path::new);
2773                if let Some(root) = &binding.project_root {
2774                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2775                    entry.0 += 1;
2776                } else if workdir.is_none() {
2777                    unknown_root_bindings += 1;
2778                }
2779                if let Some(workdir) = workdir.filter(|workdir| {
2780                    binding.project_root.as_ref().map(ProjectRootId::as_path) != Some(*workdir)
2781                }) {
2782                    roots.entry(workdir.to_path_buf()).or_insert((0, 0)).0 += 1;
2783                }
2784            }
2785            for pending in inner
2786                .pending_relays
2787                .values()
2788                .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
2789            {
2790                total_bindings += 1;
2791                if let Some(root) = &pending.reservation.project_root {
2792                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2793                    entry.1 += 1;
2794                } else {
2795                    unknown_root_bindings += 1;
2796                }
2797            }
2798        }
2799        Ok(ModuleControlResponseToModule::LiveRoots {
2800            roots: roots
2801                .into_iter()
2802                .map(|(project_root, (bound, pending))| LiveRoot {
2803                    project_root,
2804                    bound,
2805                    pending,
2806                })
2807                .collect(),
2808            unknown_root_bindings,
2809            total_bindings,
2810        })
2811    }
2812
2813    /// True if this connection already owns committed or reserved CLIENT routes.
2814    /// A module registers (HELLO) before serving and never opens client routes, so
2815    /// a connection that has client routes must not also become a module endpoint
2816    /// — otherwise one connection holds both client and module state and cleanup
2817    /// only releases one side.
2818    pub(crate) fn connection_has_client_routes(
2819        &self,
2820        connection_id: ConnectionId,
2821    ) -> Result<bool, ForwardingError> {
2822        let inner = self.read_inner()?;
2823        Ok(connection_has_client_routes_locked(&inner, connection_id))
2824    }
2825
2826    pub(crate) fn cleanup_connection(
2827        &self,
2828        connection_id: ConnectionId,
2829    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2830        self.cleanup_connection_counted(connection_id)
2831            .map(|cleanup| cleanup.released)
2832    }
2833
2834    /// [`Self::cleanup_connection`], also reporting how many pending
2835    /// route.bind relays to the closed module were aborted. That count is the
2836    /// `abandoned` figure of the `route.closed` push sent for a lost module
2837    /// connection; it is always zero for a client connection.
2838    pub(crate) fn cleanup_connection_counted(
2839        &self,
2840        connection_id: ConnectionId,
2841    ) -> Result<ConnectionCleanup, ForwardingError> {
2842        self.frame_drop_warnings
2843            .lock()
2844            .unwrap_or_else(std::sync::PoisonError::into_inner)
2845            .remove(&connection_id);
2846        let mut inner = self.write_inner()?;
2847        inner.closing_connections.insert(connection_id);
2848        let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
2849            remove_module_connection_locked(&mut inner, endpoint)
2850        } else {
2851            ConnectionCleanup {
2852                released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
2853                abandoned_relays: 0,
2854            }
2855        };
2856        // The closing mark refuses new work for a connection whose teardown is
2857        // still pending. This is the latest point at which lifting it is safe:
2858        // teardown has just removed every per-connection entry above, under
2859        // this same write lock, so no lookup can still find live state for the
2860        // id; and ids come from a monotonic counter that never reissues one,
2861        // so the id can never name a future connection either. Keeping the
2862        // mark past this point only grew the set by one entry per connection
2863        // for the life of the daemon.
2864        inner.closing_connections.remove(&connection_id);
2865        Ok(cleanup)
2866    }
2867
2868    fn cleanup_client_connection_locked(
2869        inner: &mut ForwardingInner,
2870        connection_id: ConnectionId,
2871    ) -> Vec<GoodbyeTarget> {
2872        let routes = inner
2873            .client_to_module
2874            .iter()
2875            .filter(|(key, _)| key.connection_id == connection_id)
2876            .map(|(key, route)| (*key, route.client_epoch))
2877            .collect::<Vec<_>>();
2878        let mut released = Vec::with_capacity(routes.len());
2879        for (client_key, epoch) in routes {
2880            if let RouteRelease::Removed(target) =
2881                release_client_route_locked(inner, client_key, epoch)
2882            {
2883                released.push(target);
2884            }
2885        }
2886
2887        let pending_keys = inner
2888            .pending_relays
2889            .iter()
2890            .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
2891            .map(|(key, _)| *key)
2892            .collect::<Vec<_>>();
2893        for key in pending_keys {
2894            let Some(pending) = inner.pending_relays.remove(&key) else {
2895                continue;
2896            };
2897            release_reserved_route_locked(
2898                inner,
2899                pending.reservation.client_key,
2900                pending.reservation.module_key,
2901            );
2902            if pending.relay_enqueued {
2903                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2904                    released.push(target);
2905                }
2906            } else {
2907                // The bind may still be on its way into the module's queue;
2908                // a GOODBYE sent now could overtake it. `route.open` sends
2909                // it once its send completes.
2910                inner.relays_released_before_enqueue.insert(key);
2911            }
2912            let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
2913                "client connection closed during route.bind relay".to_string(),
2914            ));
2915        }
2916
2917        let orphaned = inner
2918            .reserved_client
2919            .iter()
2920            .filter(|(key, _)| key.connection_id == connection_id)
2921            .map(|(client, module)| (*client, *module))
2922            .collect::<Vec<_>>();
2923        for (client_key, module_key) in orphaned {
2924            release_reserved_route_locked(inner, client_key, module_key);
2925        }
2926        inner.next_client_channel.remove(&connection_id);
2927        inner
2928            .client_slot_epochs
2929            .retain(|key, _| key.connection_id != connection_id);
2930        inner
2931            .last_published_epoch
2932            .retain(|key, _| key.connection_id != connection_id);
2933        inner
2934            .status
2935            .retain(|(key, _), _| key.connection_id != connection_id);
2936
2937        released
2938    }
2939
2940    /// Close a client connection whose egress queue refused a frame for the
2941    /// route `(connection_id, channel)` at `expected_epoch`. The whole
2942    /// connection closes because its queue is shared by every route on it.
2943    ///
2944    /// The first request that actually closes the connection logs one WARN with
2945    /// the diagnosis: which principals the connection's routes belonged to,
2946    /// which module's frame did not fit and on which client channel, and what
2947    /// the queue held at that moment.
2948    pub(crate) fn escalate_client_delivery_failure(
2949        &self,
2950        connection_id: ConnectionId,
2951        channel: u16,
2952        expected_epoch: u32,
2953        reason: CloseReason,
2954        undelivered: UndeliveredFrame<'_>,
2955    ) -> Result<bool, ForwardingError> {
2956        self.escalate_client_delivery_failure_diagnosed(
2957            connection_id,
2958            channel,
2959            expected_epoch,
2960            reason,
2961            undelivered,
2962            None,
2963        )
2964    }
2965
2966    pub(crate) fn escalate_client_delivery_failure_diagnosed(
2967        &self,
2968        connection_id: ConnectionId,
2969        channel: u16,
2970        expected_epoch: u32,
2971        reason: CloseReason,
2972        undelivered: UndeliveredFrame<'_>,
2973        diagnosis: Option<&crate::router::ClientDeliveryDiagnosis>,
2974    ) -> Result<bool, ForwardingError> {
2975        let principals = {
2976            let mut inner = self.write_inner()?;
2977            let key = ClientRouteKey {
2978                connection_id,
2979                channel,
2980            };
2981            if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
2982                None
2983            } else {
2984                inner.closing_connections.insert(connection_id);
2985                Some(connection_principals_locked(&inner, connection_id))
2986            }
2987        };
2988        let Some(principals) = principals else {
2989            // A concurrent release can remove the epoch before escalation.
2990            // The frame was still lost, even though this route can no longer
2991            // authorize closing the shared client connection.
2992            self.warn_frame_drop(
2993                connection_id,
2994                channel,
2995                expected_epoch,
2996                "module_to_client",
2997                diagnosis
2998                    .map(|failure| failure.cause)
2999                    .unwrap_or("delivery_failed"),
3000            );
3001            return Ok(false);
3002        };
3003        let backlog = undelivered.sink.backlog();
3004        let close_reason = reason.to_string();
3005        if self.request_connection_close(connection_id, reason) {
3006            warn!(
3007                connection_id = connection_id.get(),
3008                principals = %principals,
3009                module_id = undelivered.module_id.unwrap_or("unknown"),
3010                client_channel = channel,
3011                queued_bytes = backlog.queued_bytes,
3012                queued_frames = backlog.queued_frames,
3013                frame_type = ?diagnosis.map(|failure| failure.frame_type),
3014                frame_flags = ?diagnosis.map(|failure| failure.flags.0),
3015                close_cause = diagnosis.map(|failure| failure.cause).unwrap_or("delivery_failed"),
3016                oldest_queued_ms = backlog
3017                    .oldest_age
3018                    .map(|age| age.as_millis() as u64)
3019                    .unwrap_or(0),
3020                close_reason = %close_reason,
3021                "closing client connection: its egress queue could not take a frame"
3022            );
3023        }
3024        Ok(true)
3025    }
3026
3027    /// Publish the tags a scope sync changed and close the live routes its
3028    /// drain table selects, in one critical section.
3029    ///
3030    /// The caller holds the scope table's write lock (scope table, then this
3031    /// table, always). Publishing and selecting under one lock is what closes
3032    /// the race with a bind commit: a commit before this sees the old tag and
3033    /// its route is selected here like any other; a commit after it sees the
3034    /// new tag and is refused.
3035    ///
3036    /// Every client route index entry is scanned, which includes routes on a
3037    /// swap's superseded endpoint (the old process of a module being replaced
3038    /// blue/green, which keeps its existing routes until they drain): those
3039    /// stay indexed until drained, so ending a scope reaches them too.
3040    pub(crate) fn publish_scope_changes(
3041        &self,
3042        changes: &[ScopeTagChange],
3043    ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
3044        let mut inner = self.write_inner()?;
3045        let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
3046        for change in changes {
3047            let key = (change.owner.clone(), change.scope_ref.clone());
3048            match change.after {
3049                Some(tag) => {
3050                    inner.scope_tags.insert(key, tag);
3051                }
3052                None => {
3053                    inner.scope_tags.remove(&key);
3054                }
3055            }
3056            by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
3057        }
3058        let mut selected = Vec::new();
3059        for (client_key, route) in &inner.client_to_module {
3060            let Some(scope) = &route.scope else {
3061                continue;
3062            };
3063            let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
3064            else {
3065                continue;
3066            };
3067            if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
3068                continue;
3069            }
3070            let reason = match &change.drain {
3071                ScopeDrain::Nothing => continue,
3072                ScopeDrain::All(reason) => *reason,
3073                ScopeDrain::Carriers(narrowed) => {
3074                    let owner = Principal::Reserved {
3075                        module_id: scope.owner.clone(),
3076                    };
3077                    let hit = route.principal != owner
3078                        && narrowed.iter().any(|(principal, allowed)| {
3079                            *principal == route.principal
3080                                && allowed
3081                                    .as_ref()
3082                                    .is_none_or(|targets| !targets.contains(&route.module_id))
3083                        });
3084                    if !hit {
3085                        continue;
3086                    }
3087                    RouteCloseReason::ScopeCarrierRemoved
3088                }
3089            };
3090            selected.push((*client_key, route.client_epoch, reason));
3091        }
3092        let mut drained = Vec::new();
3093        for (client_key, client_epoch, reason) in selected {
3094            let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
3095                continue;
3096            };
3097            let release = release_client_route_locked(&mut inner, client_key, client_epoch);
3098            self.record_route_release(&release);
3099            if let RouteRelease::Removed(module) = release {
3100                drained.push(ScopeDrainedRoute {
3101                    reason,
3102                    scope: route
3103                        .scope
3104                        .clone()
3105                        .expect("selected scope drain has a bound scope"),
3106                    module_id: route.module_id.clone(),
3107                    client: GoodbyeTarget {
3108                        connection_id: route.client_connection_id,
3109                        sink: route.client_sink.clone(),
3110                        negotiated_ver: route.client_negotiated_ver,
3111                        channel: route.client_channel,
3112                        epoch: route.client_epoch,
3113                        kind: GoodbyeTargetKind::Client,
3114                        module_id: Some(route.module_id.clone()),
3115                    },
3116                    module,
3117                });
3118            }
3119        }
3120        Ok(drained)
3121    }
3122
3123    /// The published tag of one scope, for tests of the commit re-check.
3124    #[cfg(test)]
3125    pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
3126        self.read_inner()
3127            .ok()?
3128            .scope_tags
3129            .get(&(owner.to_string(), scope_ref.to_string()))
3130            .copied()
3131    }
3132
3133    fn record_route_release(&self, release: &RouteRelease) {
3134        match release {
3135            RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
3136            RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
3137            RouteRelease::Absent => {}
3138        }
3139    }
3140
3141    fn read_inner(&self) -> Result<RwLockReadGuard<'_, ForwardingInner>, ForwardingError> {
3142        self.inner.read().map_err(|_| ForwardingError::Poisoned)
3143    }
3144
3145    fn write_inner(&self) -> Result<RwLockWriteGuard<'_, ForwardingInner>, ForwardingError> {
3146        self.inner.write().map_err(|_| ForwardingError::Poisoned)
3147    }
3148
3149    fn lock_close_registry(
3150        &self,
3151    ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
3152        self.close_registry
3153            .lock()
3154            .unwrap_or_else(|poisoned| poisoned.into_inner())
3155    }
3156}
3157
3158impl ForwardingInner {
3159    fn allocate_route_slots(
3160        &mut self,
3161        connection_id: ConnectionId,
3162        endpoint: ModuleEndpointId,
3163    ) -> Result<(u16, u32, u16, u32), ForwardingError> {
3164        let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
3165        let mut client_channel = client_start;
3166        let client_channel = loop {
3167            let key = ClientRouteKey {
3168                connection_id,
3169                channel: client_channel,
3170            };
3171            let eligible = !self.client_to_module.contains_key(&key)
3172                && !self.reserved_client.contains_key(&key)
3173                && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3174            if eligible {
3175                break client_channel;
3176            }
3177            client_channel = next_channel(client_channel);
3178            if client_channel == client_start {
3179                return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
3180            }
3181        };
3182
3183        let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
3184        let mut module_channel = module_start;
3185        let module_channel = loop {
3186            let key = ModuleRouteKey {
3187                endpoint,
3188                channel: module_channel,
3189            };
3190            let eligible = !self.module_to_client.contains_key(&key)
3191                && !self.reserved_module.contains_key(&key)
3192                && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3193            if eligible {
3194                break module_channel;
3195            }
3196            module_channel = next_channel(module_channel);
3197            if module_channel == module_start {
3198                return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
3199            }
3200        };
3201
3202        let client_key = ClientRouteKey {
3203            connection_id,
3204            channel: client_channel,
3205        };
3206        let module_key = ModuleRouteKey {
3207            endpoint,
3208            channel: module_channel,
3209        };
3210        let client_epoch = self
3211            .client_slot_epochs
3212            .get(&client_key)
3213            .copied()
3214            .unwrap_or(0)
3215            + 1;
3216        let module_epoch = self
3217            .module_slot_epochs
3218            .get(&module_key)
3219            .copied()
3220            .unwrap_or(0)
3221            + 1;
3222        self.client_slot_epochs.insert(client_key, client_epoch);
3223        self.module_slot_epochs.insert(module_key, module_epoch);
3224        self.next_client_channel
3225            .insert(connection_id, next_channel(client_channel));
3226        self.next_module_channel
3227            .insert(endpoint, next_channel(module_channel));
3228        Ok((client_channel, client_epoch, module_channel, module_epoch))
3229    }
3230
3231    fn allocate_control_corr(
3232        &mut self,
3233        endpoint: ModuleEndpointId,
3234    ) -> Result<u64, ForwardingError> {
3235        let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
3236        if candidate == 0 {
3237            self.closing_connections.insert(endpoint.connection_id);
3238            return Err(ForwardingError::RelayCorrelationExhausted);
3239        }
3240        self.next_control_corr.insert(
3241            endpoint,
3242            if candidate == u64::MAX {
3243                0
3244            } else {
3245                candidate + 1
3246            },
3247        );
3248        Ok(candidate)
3249    }
3250}
3251
3252fn connection_has_client_routes_locked(
3253    inner: &ForwardingInner,
3254    connection_id: ConnectionId,
3255) -> bool {
3256    inner
3257        .client_to_module
3258        .keys()
3259        .any(|key| key.connection_id == connection_id)
3260        || inner
3261            .reserved_client
3262            .keys()
3263            .any(|key| key.connection_id == connection_id)
3264}
3265
3266fn check_module_connection_role_locked(
3267    inner: &ForwardingInner,
3268    connection_id: ConnectionId,
3269) -> Result<(), ForwardingError> {
3270    if inner.endpoint_by_connection.contains_key(&connection_id)
3271        || connection_has_client_routes_locked(inner, connection_id)
3272    {
3273        return Err(ForwardingError::ConnectionRoleConflict { connection_id });
3274    }
3275    Ok(())
3276}
3277
3278fn next_channel(channel: u16) -> u16 {
3279    let next = channel.wrapping_add(1);
3280    if next == 0 {
3281        1
3282    } else {
3283        next
3284    }
3285}
3286
3287fn endpoint_routes_locked(
3288    inner: &ForwardingInner,
3289    endpoint: ModuleEndpointId,
3290) -> Vec<EndpointRoute> {
3291    let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
3292    let draining = drain_reason.is_some();
3293    let mut routes = inner
3294        .module_to_client
3295        .iter()
3296        .filter(|(module_key, _)| module_key.endpoint == endpoint)
3297        .map(|(_, route)| EndpointRoute {
3298            goodbye_target: GoodbyeTarget {
3299                connection_id: route.client_connection_id,
3300                sink: route.client_sink.clone(),
3301                negotiated_ver: route.client_negotiated_ver,
3302                channel: route.client_channel,
3303                epoch: route.client_epoch,
3304                kind: GoodbyeTargetKind::Client,
3305                module_id: Some(route.module_id.clone()),
3306            },
3307            principal: route.principal.clone(),
3308            bound_at: route.bound_at,
3309            draining,
3310            drain_reason,
3311        })
3312        .collect::<Vec<_>>();
3313    routes.sort_by_key(|route| {
3314        (
3315            route.goodbye_target.connection_id.get(),
3316            route.goodbye_target.channel,
3317            route.goodbye_target.epoch,
3318        )
3319    });
3320    routes
3321}
3322
3323/// Why a bind admitted under `scope` may no longer commit, read from the
3324/// published tags: `scope_ended` when the scope is gone or at another epoch,
3325/// `scope_changed` (retryable) when only its content moved.
3326fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
3327    let key = (scope.owner.clone(), scope.scope_ref.clone());
3328    match inner.scope_tags.get(&key) {
3329        Some(current) if *current == scope.tag => None,
3330        Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
3331            subc_protocol::error_codes::SCOPE_CHANGED,
3332            format!(
3333                "scope '{}' of {} changed while the route was being bound; re-open it",
3334                scope.scope_ref, scope.owner
3335            ),
3336        )),
3337        _ => Some(ErrorBody::new(
3338            subc_protocol::error_codes::SCOPE_ENDED,
3339            format!(
3340                "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
3341                scope.scope_ref, scope.owner, scope.tag.scope_epoch
3342            ),
3343        )),
3344    }
3345}
3346
3347fn release_reserved_route_locked(
3348    inner: &mut ForwardingInner,
3349    client_key: ClientRouteKey,
3350    module_key: ModuleRouteKey,
3351) {
3352    if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
3353        inner.reserved_client.remove(&client_key);
3354    }
3355    if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
3356        inner.reserved_module.remove(&module_key);
3357    }
3358    inner.status.retain(|(key, _), _| *key != client_key);
3359}
3360
3361fn release_client_route_locked(
3362    inner: &mut ForwardingInner,
3363    client_key: ClientRouteKey,
3364    expected_epoch: u32,
3365) -> RouteRelease {
3366    let Some(route) = inner.client_to_module.get(&client_key) else {
3367        return RouteRelease::Absent;
3368    };
3369    if route.client_epoch != expected_epoch {
3370        return RouteRelease::Stale;
3371    }
3372    let route = inner
3373        .client_to_module
3374        .remove(&client_key)
3375        .expect("route checked under the same forwarding lock");
3376    route.flow.close();
3377    if let Some(scope) = &route.scope {
3378        scope.release_bind();
3379    }
3380    inner.module_to_client.remove(&ModuleRouteKey {
3381        endpoint: route.module_endpoint,
3382        channel: route.module_channel,
3383    });
3384    inner.operator_confirms.route_closed(ModuleRouteKey {
3385        endpoint: route.module_endpoint,
3386        channel: route.module_channel,
3387    });
3388    inner.status.remove(&(client_key, expected_epoch));
3389    RouteRelease::Removed(GoodbyeTarget {
3390        connection_id: route.module_endpoint.connection_id,
3391        sink: route.module_sink.clone(),
3392        negotiated_ver: route.module_negotiated_ver,
3393        channel: route.module_channel,
3394        epoch: route.module_epoch,
3395        kind: GoodbyeTargetKind::Module,
3396        module_id: Some(route.module_id.clone()),
3397    })
3398}
3399
3400fn release_module_route_locked(
3401    inner: &mut ForwardingInner,
3402    module_key: ModuleRouteKey,
3403    expected_epoch: u32,
3404) -> RouteRelease {
3405    let Some(route) = inner.module_to_client.get(&module_key) else {
3406        return RouteRelease::Absent;
3407    };
3408    if route.module_epoch != expected_epoch {
3409        return RouteRelease::Stale;
3410    }
3411    let route = inner
3412        .module_to_client
3413        .remove(&module_key)
3414        .expect("route checked under the same forwarding lock");
3415    inner.operator_confirms.route_closed(module_key);
3416    route.flow.close();
3417    if let Some(scope) = &route.scope {
3418        scope.release_bind();
3419    }
3420    let client_key = ClientRouteKey {
3421        connection_id: route.client_connection_id,
3422        channel: route.client_channel,
3423    };
3424    inner.client_to_module.remove(&client_key);
3425    inner.status.remove(&(client_key, route.client_epoch));
3426    RouteRelease::Removed(GoodbyeTarget {
3427        connection_id: route.client_connection_id,
3428        sink: route.client_sink.clone(),
3429        negotiated_ver: route.client_negotiated_ver,
3430        channel: route.client_channel,
3431        epoch: route.client_epoch,
3432        kind: GoodbyeTargetKind::Client,
3433        module_id: Some(route.module_id.clone()),
3434    })
3435}
3436
3437fn commit_route_locked(
3438    inner: &mut ForwardingInner,
3439    mut pending: PendingRouteBindRelayEntry,
3440) -> Result<Option<GoodbyeTarget>, ForwardingError> {
3441    let reservation = pending.reservation;
3442    if inner
3443        .closing_connections
3444        .contains(&reservation.client_key.connection_id)
3445    {
3446        return Err(ForwardingError::ConnectionClosing {
3447            connection_id: reservation.client_key.connection_id,
3448        });
3449    }
3450    let module_id = inner
3451        .module_id_by_endpoint
3452        .get(&reservation.module_key.endpoint)
3453        .cloned()
3454        .ok_or(ForwardingError::StaleModuleEndpoint)?;
3455    if inner
3456        .draining_endpoints
3457        .contains_key(&reservation.module_key.endpoint)
3458    {
3459        return Err(ForwardingError::ModuleReloading { module_id });
3460    }
3461    if pending.busy_refusals {
3462        let body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
3463            route_channel: reservation.client_key.channel,
3464            route_epoch: reservation.client_epoch,
3465            request_window: Some(pending.request_window),
3466            watch_window: pending.watch_window,
3467        })
3468        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
3469        pending.route_open_frame = Frame::build_with_version(
3470            pending.client_negotiated_ver,
3471            FrameType::Response,
3472            pending.route_open_frame.header.flags,
3473            0,
3474            0,
3475            pending.route_open_frame.header.corr,
3476            body,
3477        )
3478        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
3479    }
3480    if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
3481        || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
3482    {
3483        return Err(ForwardingError::UnknownReservation {
3484            client_channel: reservation.client_key.channel,
3485            module_channel: reservation.module_key.channel,
3486        });
3487    }
3488    let module = inner
3489        .modules_by_id
3490        .get(&module_id)
3491        .filter(|module| module.endpoint == reservation.module_key.endpoint)
3492        .cloned()
3493        .ok_or(ForwardingError::StaleModuleEndpoint)?;
3494    let binding = Arc::new(RouteBinding {
3495        client_connection_id: reservation.client_key.connection_id,
3496        client_sink: pending.client_sink,
3497        client_negotiated_ver: pending.client_negotiated_ver,
3498        client_channel: reservation.client_key.channel,
3499        client_epoch: reservation.client_epoch,
3500        module_id,
3501        module_endpoint: reservation.module_key.endpoint,
3502        module_sink: module.sink,
3503        module_negotiated_ver: module.negotiated_ver,
3504        module_channel: reservation.module_key.channel,
3505        module_epoch: reservation.module_epoch,
3506        principal: pending.principal,
3507        harness: pending.harness,
3508        build_class: pending.build_class,
3509        operator_app: pending.operator_app,
3510        project_root: reservation.project_root.clone(),
3511        bound_at: Instant::now(),
3512        flow: Arc::new(if pending.busy_refusals {
3513            ChannelFlow::with_admission(pending.request_window, true, pending.watch_window)
3514        } else {
3515            ChannelFlow::new(pending.request_window)
3516        }),
3517        scope: pending.scope,
3518    });
3519    inner
3520        .client_to_module
3521        .insert(reservation.client_key, Arc::clone(&binding));
3522    inner
3523        .module_to_client
3524        .insert(reservation.module_key, Arc::clone(&binding));
3525    let previous_published = inner
3526        .last_published_epoch
3527        .insert(reservation.client_key, reservation.client_epoch);
3528
3529    // This point is reachable only through an accepted route.bind acknowledgement.
3530    // Publish history before notifying the opener, so a subsequent
3531    // `scope.served_by` query sees the acknowledged bind.
3532    if let Some(scope) = &binding.scope {
3533        scope.record_served(&binding.module_id);
3534    }
3535
3536    // Sending through the reserved slot cannot fail, but it reports a receiver
3537    // that closed after reservation and before this locked publication point.
3538    // The frame is stamped and charged to the queued-byte count here, when it
3539    // actually enters the queue, not when the slot was reserved.
3540    let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
3541    if client_writer_closed {
3542        // Not gated on `relay_enqueued`: only a module ack reaches this
3543        // commit, which proves the module holds the binding, and `route.open`
3544        // sends nothing for a relay a reply already settled.
3545        let abandoned = abandoned_route_target(inner, &reservation);
3546        if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
3547            route.flow.close();
3548        }
3549        inner.module_to_client.remove(&reservation.module_key);
3550        inner
3551            .status
3552            .remove(&(reservation.client_key, reservation.client_epoch));
3553        match previous_published {
3554            Some(epoch) => {
3555                inner
3556                    .last_published_epoch
3557                    .insert(reservation.client_key, epoch);
3558            }
3559            None => {
3560                inner.last_published_epoch.remove(&reservation.client_key);
3561            }
3562        }
3563        let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3564            "client egress closed during route publication".to_string(),
3565        ));
3566        return Ok(abandoned);
3567    }
3568
3569    if let Some(scope) = &binding.scope {
3570        scope.commit_presence();
3571    }
3572    let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
3573    Ok(None)
3574}
3575
3576/// The live module connection registered under exactly `endpoint`, whichever
3577/// slot holds it: active, swap candidate, or superseded incumbent.
3578///
3579/// Resolving by endpoint rather than by module id is what keeps an incumbent
3580/// addressable after a swap promoted a candidate over its id. An endpoint that
3581/// is in none of the three (a stale one, replaced without a promotion) resolves
3582/// to nothing, as it always has.
3583fn module_connection_for_endpoint_locked(
3584    inner: &ForwardingInner,
3585    endpoint: ModuleEndpointId,
3586) -> Option<&ModuleConnection> {
3587    let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
3588    inner
3589        .modules_by_id
3590        .get(module_id)
3591        .filter(|module| module.endpoint == endpoint)
3592        .or_else(|| {
3593            inner
3594                .candidates_by_id
3595                .get(module_id)
3596                .filter(|module| module.endpoint == endpoint)
3597        })
3598        .or_else(|| inner.superseded_endpoints.get(&endpoint))
3599}
3600
3601fn abandoned_route_target(
3602    inner: &ForwardingInner,
3603    reservation: &RouteReservation,
3604) -> Option<GoodbyeTarget> {
3605    let module_id = inner
3606        .module_id_by_endpoint
3607        .get(&reservation.module_key.endpoint)?;
3608    let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
3609    (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
3610        connection_id: module.endpoint.connection_id,
3611        sink: module.sink.clone(),
3612        negotiated_ver: module.negotiated_ver,
3613        channel: reservation.module_key.channel,
3614        epoch: reservation.module_epoch,
3615        kind: GoodbyeTargetKind::Module,
3616        module_id: Some(module_id.clone()),
3617    })
3618}
3619
3620/// Queue a registering module's HELLO_ACK on its sink. Called with the
3621/// forwarding write lock held, before the endpoint is inserted, which is what
3622/// puts the ack ahead of any `route.bind` or control RPC routed to the module.
3623/// `try_send` never waits, so holding the lock across it is safe.
3624fn enqueue_hello_ack_locked(
3625    sink: &FrameSink,
3626    connection_id: ConnectionId,
3627    hello_ack: Option<Frame>,
3628) -> Result<(), ForwardingError> {
3629    let Some(hello_ack) = hello_ack else {
3630        return Ok(());
3631    };
3632    sink.try_send(hello_ack)
3633        .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
3634}
3635
3636fn remove_module_connection_locked(
3637    inner: &mut ForwardingInner,
3638    endpoint: ModuleEndpointId,
3639) -> ConnectionCleanup {
3640    // Commit the disconnect before its routes are released, so their teardown
3641    // cannot replace module_closed with route_closed.
3642    inner.operator_confirms.module_closed(endpoint);
3643    inner.draining_endpoints.remove(&endpoint);
3644    inner.daemon_drain_results.remove(&endpoint);
3645    let module_id = inner.module_id_by_endpoint.remove(&endpoint);
3646    if let Some(module_id) = module_id.as_ref() {
3647        if inner
3648            .modules_by_id
3649            .get(module_id)
3650            .is_some_and(|module| module.endpoint == endpoint)
3651        {
3652            inner.modules_by_id.remove(module_id);
3653        }
3654        if inner
3655            .candidates_by_id
3656            .get(module_id)
3657            .is_some_and(|module| module.endpoint == endpoint)
3658        {
3659            inner.candidates_by_id.remove(module_id);
3660        }
3661    }
3662    inner.superseded_endpoints.remove(&endpoint);
3663    inner.endpoint_by_connection.remove(&endpoint.connection_id);
3664    inner.next_module_channel.remove(&endpoint);
3665    inner.next_control_corr.remove(&endpoint);
3666    inner
3667        .health_probe_tombstones
3668        .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
3669    inner
3670        .module_slot_epochs
3671        .retain(|key, _| key.endpoint != endpoint);
3672    let reserved_module_keys: Vec<ModuleRouteKey> = inner
3673        .reserved_module
3674        .keys()
3675        .filter(|module_key| module_key.endpoint == endpoint)
3676        .copied()
3677        .collect();
3678    for module_key in reserved_module_keys {
3679        if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
3680            release_reserved_route_locked(inner, client_key, module_key);
3681        }
3682    }
3683
3684    let pending_keys: Vec<_> = inner
3685        .pending_relays
3686        .keys()
3687        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3688        .copied()
3689        .collect();
3690    let pending: Vec<_> = pending_keys
3691        .into_iter()
3692        .filter_map(|key| {
3693            inner
3694                .pending_relays
3695                .remove(&key)
3696                .map(|pending| (key, pending))
3697        })
3698        .collect();
3699    let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
3700    for (key, pending) in pending {
3701        // Recorded like every other removal without a reply, so a `route.open`
3702        // still sending the bind learns the relay was released rather than
3703        // settled by the module.
3704        if !pending.relay_enqueued {
3705            inner.relays_released_before_enqueue.insert(key);
3706        }
3707        let module_label = module_id.as_deref().unwrap_or("unknown");
3708        let _ = pending
3709            .sender
3710            .send(RouteBindRelayOutcome::ModuleGone(format!(
3711                "module '{module_label}' connection closed during route.bind relay"
3712            )));
3713    }
3714
3715    let pending_control_keys: Vec<_> = inner
3716        .pending_control_rpcs
3717        .keys()
3718        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3719        .copied()
3720        .collect();
3721    let pending_control: Vec<_> = pending_control_keys
3722        .into_iter()
3723        .filter_map(|key| inner.pending_control_rpcs.remove(&key))
3724        .collect();
3725    for pending in pending_control {
3726        let module_label = module_id.as_deref().unwrap_or("unknown");
3727        let _ = pending
3728            .sender
3729            .send(ModuleControlRpcOutcome::ModuleGone(format!(
3730                "module '{module_label}' connection closed during module-control RPC"
3731            )));
3732    }
3733
3734    let module_routes = inner
3735        .module_to_client
3736        .iter()
3737        .filter(|(module_key, _)| module_key.endpoint == endpoint)
3738        .map(|(module_key, route)| (*module_key, route.module_epoch))
3739        .collect::<Vec<_>>();
3740    let mut released = Vec::with_capacity(module_routes.len());
3741    for (module_key, epoch) in module_routes {
3742        if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
3743        {
3744            released.push(target);
3745        }
3746    }
3747    ConnectionCleanup {
3748        released,
3749        abandoned_relays,
3750    }
3751}
3752
3753#[derive(Debug, Clone, Copy)]
3754struct RequestCredit {
3755    subscription: bool,
3756    watch_credit: bool,
3757    excluded_from_drain: bool,
3758}
3759
3760#[derive(Debug, Default)]
3761struct CreditLedger {
3762    by_corr: HashMap<u64, Vec<RequestCredit>>,
3763}
3764
3765impl CreditLedger {
3766    fn acquire(&mut self, corr: u64, subscription: bool, watch_credit: bool) {
3767        self.by_corr.entry(corr).or_default().push(RequestCredit {
3768            subscription,
3769            watch_credit,
3770            excluded_from_drain: false,
3771        });
3772    }
3773
3774    fn release(&mut self, corr: u64) -> Option<RequestCredit> {
3775        let credits = self.by_corr.get_mut(&corr)?;
3776        let released = credits.pop();
3777        if credits.is_empty() {
3778            self.by_corr.remove(&corr);
3779        }
3780        released
3781    }
3782
3783    fn capture_subscription_exclusions(&mut self) -> u32 {
3784        let mut excluded = 0u32;
3785        for credit in self.by_corr.values_mut().flatten() {
3786            if credit.subscription && !credit.excluded_from_drain {
3787                credit.excluded_from_drain = true;
3788                excluded = excluded.saturating_add(1);
3789            }
3790        }
3791        excluded
3792    }
3793
3794    fn in_flight(&self) -> usize {
3795        self.by_corr.values().map(Vec::len).sum()
3796    }
3797
3798    fn drain_in_flight(&self) -> usize {
3799        self.by_corr
3800            .values()
3801            .flatten()
3802            .filter(|credit| !credit.excluded_from_drain)
3803            .count()
3804    }
3805
3806    /// The correlation ids of the requests a drain still waits on, one entry
3807    /// per held credit, in ascending order.
3808    fn drain_held_corrs(&self) -> Vec<u64> {
3809        let mut corrs = self
3810            .by_corr
3811            .iter()
3812            .flat_map(|(corr, credits)| {
3813                credits
3814                    .iter()
3815                    .filter(|credit| !credit.excluded_from_drain)
3816                    .map(move |_| *corr)
3817            })
3818            .collect::<Vec<_>>();
3819        corrs.sort_unstable();
3820        corrs
3821    }
3822}
3823
3824#[derive(Debug, Default)]
3825struct ChannelFlowState {
3826    closed: bool,
3827    credits: CreditLedger,
3828    last_busy_warn: Option<tokio::time::Instant>,
3829}
3830
3831/// Per-channel request-credit accounting shared by the client and module route halves.
3832#[derive(Debug)]
3833pub(crate) struct ChannelFlow {
3834    sem: Semaphore,
3835    window: usize,
3836    busy_refusals: bool,
3837    watch_sem: Option<Semaphore>,
3838    watch_window: Option<usize>,
3839    state: Mutex<ChannelFlowState>,
3840}
3841
3842impl ChannelFlow {
3843    pub(crate) fn new(window: usize) -> Self {
3844        Self::with_admission(window, false, None)
3845    }
3846
3847    fn with_admission(window: usize, busy_refusals: bool, watch_window: Option<usize>) -> Self {
3848        debug_assert!(window > 0, "flow-control window must be non-zero");
3849        Self {
3850            sem: Semaphore::new(window),
3851            window,
3852            busy_refusals,
3853            watch_sem: watch_window.map(Semaphore::new),
3854            watch_window,
3855            state: Mutex::new(ChannelFlowState::default()),
3856        }
3857    }
3858
3859    pub(crate) fn busy_refusals(&self) -> bool {
3860        self.busy_refusals
3861    }
3862
3863    fn try_acquire_tagged(&self, corr: u64, subscription: bool) -> RequestAdmission {
3864        let mut state = self
3865            .state
3866            .lock()
3867            .unwrap_or_else(|poisoned| poisoned.into_inner());
3868        if state.closed {
3869            return RequestAdmission::Stale;
3870        }
3871        let busy = |state: &ChannelFlowState| {
3872            RequestAdmission::Busy(subc_protocol::RouteBusyDetail {
3873                reason: "no_credit".into(),
3874                request_in_flight: state.credits.in_flight(),
3875                watch_in_flight: state
3876                    .credits
3877                    .by_corr
3878                    .values()
3879                    .flatten()
3880                    .filter(|credit| credit.subscription)
3881                    .count(),
3882                request_window: self.window,
3883                watch_window: self.watch_window,
3884            })
3885        };
3886        let Ok(total) = self.sem.try_acquire() else {
3887            return busy(&state);
3888        };
3889        // A watch takes the total credit first. Dropping it on watch-lane
3890        // exhaustion rolls back before any ledger entry becomes visible.
3891        let watch = if subscription {
3892            if let Some(sem) = &self.watch_sem {
3893                match sem.try_acquire() {
3894                    Ok(permit) => Some(permit),
3895                    Err(_) => return busy(&state),
3896                }
3897            } else {
3898                None
3899            }
3900        } else {
3901            None
3902        };
3903        state.credits.acquire(corr, subscription, watch.is_some());
3904        total.forget();
3905        if let Some(watch) = watch {
3906            watch.forget();
3907        }
3908        RequestAdmission::Admitted
3909    }
3910
3911    pub(crate) fn should_warn_busy(&self) -> bool {
3912        let mut state = self
3913            .state
3914            .lock()
3915            .unwrap_or_else(|poisoned| poisoned.into_inner());
3916        let now = tokio::time::Instant::now();
3917        if state
3918            .last_busy_warn
3919            .is_some_and(|last| now.duration_since(last) < std::time::Duration::from_secs(60))
3920        {
3921            return false;
3922        }
3923        state.last_busy_warn = Some(now);
3924        true
3925    }
3926
3927    #[cfg(test)]
3928    pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
3929        self.acquire_tagged(0, false).await
3930    }
3931
3932    pub(crate) async fn acquire_tagged(
3933        &self,
3934        corr: u64,
3935        subscription: bool,
3936    ) -> Result<(), ChannelFlowClosed> {
3937        let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
3938        let mut state = self
3939            .state
3940            .lock()
3941            .unwrap_or_else(|poisoned| poisoned.into_inner());
3942        if state.closed {
3943            return Err(ChannelFlowClosed);
3944        }
3945        state.credits.acquire(corr, subscription, false);
3946        permit.forget();
3947        Ok(())
3948    }
3949
3950    #[cfg(test)]
3951    pub(crate) fn release(&self) {
3952        self.release_corr(0);
3953    }
3954
3955    pub(crate) fn release_corr(&self, corr: u64) {
3956        let mut state = self
3957            .state
3958            .lock()
3959            .unwrap_or_else(|poisoned| poisoned.into_inner());
3960        let Some(released) = state.credits.release(corr) else {
3961            // Protocol-conforming modules emit exactly one terminal per request.
3962            // This guard is a best-effort safety net against window growth, not a
3963            // security boundary against malicious peers.
3964            warn!(
3965                window = self.window,
3966                available = self.sem.available_permits(),
3967                "flow-control over-release ignored"
3968            );
3969            return;
3970        };
3971        if !self.sem.is_closed() {
3972            self.sem.add_permits(1);
3973        }
3974        if released.watch_credit {
3975            if let Some(watch_sem) = self.watch_sem.as_ref().filter(|sem| !sem.is_closed()) {
3976                watch_sem.add_permits(1);
3977            }
3978        }
3979    }
3980
3981    #[cfg(test)]
3982    pub(crate) fn in_flight(&self) -> usize {
3983        self.state
3984            .lock()
3985            .unwrap_or_else(|poisoned| poisoned.into_inner())
3986            .credits
3987            .in_flight()
3988    }
3989
3990    pub(crate) fn drain_in_flight(&self) -> usize {
3991        self.state
3992            .lock()
3993            .unwrap_or_else(|poisoned| poisoned.into_inner())
3994            .credits
3995            .drain_in_flight()
3996    }
3997
3998    pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
3999        self.state
4000            .lock()
4001            .unwrap_or_else(|poisoned| poisoned.into_inner())
4002            .credits
4003            .drain_held_corrs()
4004    }
4005
4006    #[cfg(test)]
4007    pub(crate) fn available_permits(&self) -> usize {
4008        self.sem.available_permits()
4009    }
4010
4011    pub(crate) fn begin_drain(&self) -> u32 {
4012        let mut state = self
4013            .state
4014            .lock()
4015            .unwrap_or_else(|poisoned| poisoned.into_inner());
4016        state.closed = true;
4017        self.sem.close();
4018        if let Some(sem) = &self.watch_sem {
4019            sem.close();
4020        }
4021        state.credits.capture_subscription_exclusions()
4022    }
4023
4024    pub(crate) fn close(&self) {
4025        let mut state = self
4026            .state
4027            .lock()
4028            .unwrap_or_else(|poisoned| poisoned.into_inner());
4029        state.closed = true;
4030        self.sem.close();
4031        if let Some(sem) = &self.watch_sem {
4032            sem.close();
4033        }
4034    }
4035}
4036
4037#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4038pub(crate) struct ChannelFlowClosed;
4039
4040#[derive(Debug, Clone, PartialEq, Eq)]
4041pub(crate) enum RequestAdmission {
4042    Admitted,
4043    Busy(subc_protocol::RouteBusyDetail),
4044    ModuleReloading,
4045    Stale,
4046}
4047
4048impl fmt::Display for ChannelFlowClosed {
4049    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4050        write!(f, "flow-control window closed")
4051    }
4052}
4053
4054impl Error for ChannelFlowClosed {}
4055
4056#[cfg(test)]
4057#[path = "route_busy_flow_tests.rs"]
4058mod route_busy_flow_tests;
4059
4060fn window_for(concurrency: &Concurrency) -> usize {
4061    match concurrency {
4062        Concurrency::Serial => 1,
4063        Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
4064        Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
4065    }
4066}
4067
4068#[derive(Debug, Clone, PartialEq, Eq)]
4069pub enum ForwardingError {
4070    /// A connection cannot hold both client routes and a module endpoint, or
4071    /// replace a module identity while that endpoint is still registered.
4072    ConnectionRoleConflict {
4073        connection_id: ConnectionId,
4074    },
4075    NoModuleConnection,
4076    ModuleReloading {
4077        module_id: String,
4078    },
4079    StaleModuleEndpoint,
4080    UnknownReservation {
4081        client_channel: u16,
4082        module_channel: u16,
4083    },
4084    ClientRouteChannelExhausted {
4085        connection_id: ConnectionId,
4086    },
4087    ModuleRouteChannelExhausted {
4088        endpoint: ModuleEndpointId,
4089    },
4090    RelayCorrelationExhausted,
4091    ConnectionClosing {
4092        connection_id: ConnectionId,
4093    },
4094    ClientEgressClosed {
4095        connection_id: ConnectionId,
4096    },
4097    RouteOpenBuild(String),
4098    /// A swap candidate is already registered for this module id.
4099    CandidateSlotOccupied {
4100        module_id: String,
4101    },
4102    /// A registering module's HELLO_ACK could not be queued because its
4103    /// outbound queue is closed or full.
4104    ModuleEgressUnavailable {
4105        connection_id: ConnectionId,
4106    },
4107    Poisoned,
4108}
4109
4110impl fmt::Display for ForwardingError {
4111    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4112        match self {
4113            Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
4114            Self::NoModuleConnection => write!(f, "no module connection is registered"),
4115            Self::ModuleReloading { module_id } => {
4116                write!(f, "module_id '{module_id}' is reloading")
4117            }
4118            Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
4119            Self::UnknownReservation {
4120                client_channel,
4121                module_channel,
4122            } => write!(
4123                f,
4124                "route reservation client channel {client_channel} / module channel {module_channel} was not found"
4125            ),
4126            Self::ClientRouteChannelExhausted { connection_id } => write!(
4127                f,
4128                "no client route channels are available for connection {}",
4129                connection_id.get()
4130            ),
4131            Self::ModuleRouteChannelExhausted { endpoint } => write!(
4132                f,
4133                "no module route channels are available for endpoint generation {} on connection {}",
4134                endpoint.generation,
4135                endpoint.connection_id.get()
4136            ),
4137            Self::RelayCorrelationExhausted => {
4138                write!(f, "module control correlation ids are exhausted")
4139            }
4140            Self::ConnectionClosing { connection_id } => write!(
4141                f,
4142                "connection {} is closing and cannot accept route allocation",
4143                connection_id.get()
4144            ),
4145            Self::ClientEgressClosed { connection_id } => write!(
4146                f,
4147                "client connection {} egress is closed",
4148                connection_id.get()
4149            ),
4150            Self::RouteOpenBuild(message) => {
4151                write!(f, "failed to prebuild route.open response: {message}")
4152            }
4153            Self::CandidateSlotOccupied { module_id } => write!(
4154                f,
4155                "module_id '{module_id}' already has a swap candidate registered"
4156            ),
4157            Self::ModuleEgressUnavailable { connection_id } => write!(
4158                f,
4159                "module connection {} egress is unavailable; HELLO_ACK could not be queued",
4160                connection_id.get()
4161            ),
4162            Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
4163        }
4164    }
4165}
4166
4167impl Error for ForwardingError {}
4168
4169#[cfg(test)]
4170mod tests {
4171    use std::time::Duration;
4172
4173    use super::*;
4174    use tokio::sync::mpsc;
4175
4176    #[test]
4177    fn ordinary_long_running_request_is_not_excluded_from_drain() {
4178        let mut ledger = CreditLedger::default();
4179        ledger.acquire(1, false, false);
4180
4181        assert_eq!(ledger.capture_subscription_exclusions(), 0);
4182        assert_eq!(ledger.drain_in_flight(), 1);
4183    }
4184
4185    #[test]
4186    fn bit_set_subscription_is_excluded_and_counted() {
4187        let mut ledger = CreditLedger::default();
4188        ledger.acquire(1, true, false);
4189
4190        assert_eq!(ledger.capture_subscription_exclusions(), 1);
4191        assert_eq!(ledger.drain_in_flight(), 0);
4192    }
4193
4194    #[test]
4195    fn subscription_opened_after_drain_snapshot_is_not_excluded() {
4196        let mut ledger = CreditLedger::default();
4197        ledger.acquire(1, true, false);
4198        assert_eq!(ledger.capture_subscription_exclusions(), 1);
4199
4200        ledger.acquire(2, true, false);
4201
4202        assert_eq!(ledger.drain_in_flight(), 1);
4203    }
4204
4205    #[test]
4206    fn drain_with_no_subscriptions_reports_zero_excluded() {
4207        let mut ledger = CreditLedger::default();
4208        assert_eq!(ledger.capture_subscription_exclusions(), 0);
4209    }
4210
4211    fn test_hello_ack(corr: u64) -> Frame {
4212        Frame::build(
4213            FrameType::HelloAck,
4214            Flags::new(false, Priority::Passive, false),
4215            0,
4216            0,
4217            corr,
4218            Vec::new(),
4219        )
4220        .unwrap()
4221    }
4222
4223    /// An acked registration whose HELLO_ACK cannot be queued must leave no
4224    /// endpoint behind: a routable module that never got its ack would read a
4225    /// route.bind first and exit. A closed queue and a full one both refuse.
4226    #[test]
4227    fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
4228        let forwarding = ForwardingTable::default();
4229
4230        let (closed_tx, closed_rx) = mpsc::channel(8);
4231        drop(closed_rx);
4232        let closed = ConnectionId::new(1);
4233        assert_eq!(
4234            forwarding.register_module_connection_acked(
4235                closed,
4236                "closed".to_string(),
4237                2,
4238                Concurrency::ModuleManaged,
4239                FrameSink::new(closed_tx),
4240                test_hello_ack(1),
4241            ),
4242            Err(ForwardingError::ModuleEgressUnavailable {
4243                connection_id: closed
4244            })
4245        );
4246
4247        let (full_tx, _full_rx) = mpsc::channel(1);
4248        let full_sink = FrameSink::new(full_tx);
4249        full_sink.try_send(test_hello_ack(99)).unwrap();
4250        let full = ConnectionId::new(2);
4251        assert_eq!(
4252            forwarding.register_module_connection_acked(
4253                full,
4254                "full".to_string(),
4255                2,
4256                Concurrency::ModuleManaged,
4257                full_sink.clone(),
4258                test_hello_ack(2),
4259            ),
4260            Err(ForwardingError::ModuleEgressUnavailable {
4261                connection_id: full
4262            })
4263        );
4264        assert_eq!(
4265            forwarding.register_candidate_module_connection_acked(
4266                full,
4267                "full".to_string(),
4268                2,
4269                Concurrency::ModuleManaged,
4270                full_sink,
4271                test_hello_ack(3),
4272            ),
4273            Err(ForwardingError::ModuleEgressUnavailable {
4274                connection_id: full
4275            })
4276        );
4277
4278        for (connection, module_id) in [(closed, "closed"), (full, "full")] {
4279            assert_eq!(
4280                forwarding
4281                    .module_endpoint_for_connection(connection)
4282                    .unwrap(),
4283                None
4284            );
4285            let (client_tx, _client_rx) = mpsc::channel(8);
4286            assert_eq!(
4287                forwarding
4288                    .begin_route_bind_relay_for_test(
4289                        ConnectionId::new(50),
4290                        FrameSink::new(client_tx),
4291                        1,
4292                        module_id,
4293                    )
4294                    .err(),
4295                Some(ForwardingError::NoModuleConnection)
4296            );
4297        }
4298        assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
4299    }
4300
4301    /// Both acked registration forms put the HELLO_ACK on the module's queue
4302    /// by the time the endpoint can be resolved.
4303    #[test]
4304    fn acked_registration_queues_the_hello_ack_first() {
4305        let forwarding = ForwardingTable::default();
4306        let (active_tx, mut active_rx) = mpsc::channel(8);
4307        forwarding
4308            .register_module_connection_acked(
4309                ConnectionId::new(1),
4310                "acked".to_string(),
4311                2,
4312                Concurrency::ModuleManaged,
4313                FrameSink::new(active_tx),
4314                test_hello_ack(11),
4315            )
4316            .unwrap();
4317        let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
4318        forwarding
4319            .register_candidate_module_connection_acked(
4320                ConnectionId::new(2),
4321                "acked".to_string(),
4322                2,
4323                Concurrency::ModuleManaged,
4324                FrameSink::new(candidate_tx),
4325                test_hello_ack(12),
4326            )
4327            .unwrap();
4328
4329        let active_first = active_rx.try_recv().unwrap().frame;
4330        assert_eq!(active_first.header.ty, FrameType::HelloAck);
4331        assert_eq!(active_first.header.corr, 11);
4332        let candidate_first = candidate_rx.try_recv().unwrap().frame;
4333        assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
4334        assert_eq!(candidate_first.header.corr, 12);
4335    }
4336
4337    #[test]
4338    fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
4339        let forwarding = ForwardingTable::default();
4340        let module_connection = ConnectionId::new(10);
4341        let exhausted_client = ConnectionId::new(20);
4342        let second_client = ConnectionId::new(30);
4343        let (module_tx, _module_rx) = mpsc::channel(1);
4344        let endpoint = forwarding
4345            .register_module_connection(
4346                module_connection,
4347                "route-limit-provider".to_string(),
4348                1,
4349                Concurrency::ModuleManaged,
4350                FrameSink::new(module_tx),
4351            )
4352            .unwrap();
4353
4354        {
4355            let mut inner = forwarding.inner.write().unwrap();
4356            for channel in 1..=u16::MAX {
4357                inner.reserved_client.insert(
4358                    ClientRouteKey {
4359                        connection_id: exhausted_client,
4360                        channel,
4361                    },
4362                    ModuleRouteKey {
4363                        endpoint,
4364                        channel: 1,
4365                    },
4366                );
4367            }
4368        }
4369
4370        let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
4371        let err = forwarding
4372            .begin_route_bind_relay_for_test(
4373                exhausted_client,
4374                FrameSink::new(exhausted_tx),
4375                1,
4376                "route-limit-provider",
4377            )
4378            .unwrap_err();
4379        assert!(matches!(
4380            err,
4381            ForwardingError::ClientRouteChannelExhausted { connection_id }
4382                if connection_id == exhausted_client
4383        ));
4384
4385        let (second_tx, _second_rx) = mpsc::channel(1);
4386        let pending = forwarding
4387            .begin_route_bind_relay_for_test(
4388                second_client,
4389                FrameSink::new(second_tx),
4390                2,
4391                "route-limit-provider",
4392            )
4393            .unwrap();
4394        assert_eq!(pending.client_channel, 1);
4395    }
4396
4397    #[test]
4398    fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
4399        let forwarding = ForwardingTable::default();
4400        let module_connection = ConnectionId::new(40);
4401        let client = ConnectionId::new(50);
4402        let (module_tx, _module_rx) = mpsc::channel(1);
4403        forwarding
4404            .register_module_connection(
4405                module_connection,
4406                "slot-reuse-provider".to_string(),
4407                1,
4408                Concurrency::ModuleManaged,
4409                FrameSink::new(module_tx),
4410            )
4411            .unwrap();
4412
4413        let (client_tx, _client_rx) = mpsc::channel(1);
4414        let client_sink = FrameSink::new(client_tx);
4415        let mut wrapped_channel = None;
4416        for index in 0..=usize::from(u16::MAX) {
4417            let pending = forwarding
4418                .begin_route_bind_relay_for_test(
4419                    client,
4420                    client_sink.clone(),
4421                    index as u64 + 1,
4422                    "slot-reuse-provider",
4423                )
4424                .unwrap();
4425            if index == usize::from(u16::MAX) {
4426                wrapped_channel = Some(pending.module_channel);
4427            }
4428            forwarding
4429                .abort_pending_relay(
4430                    pending.endpoint,
4431                    pending.corr,
4432                    RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
4433                )
4434                .unwrap();
4435        }
4436
4437        assert_eq!(wrapped_channel, Some(1));
4438    }
4439
4440    #[test]
4441    fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
4442        let forwarding = ForwardingTable::default();
4443        let client = ConnectionId::new(60);
4444        forwarding
4445            .inner
4446            .write()
4447            .unwrap()
4448            .next_client_channel
4449            .insert(client, 41);
4450
4451        let released = forwarding.cleanup_connection(client).unwrap();
4452
4453        assert!(released.is_empty());
4454        assert!(!forwarding
4455            .inner
4456            .read()
4457            .unwrap()
4458            .next_client_channel
4459            .contains_key(&client));
4460    }
4461
4462    #[test]
4463    fn stale_module_cleanup_preserves_fast_reconnect_successor() {
4464        let forwarding = ForwardingTable::default();
4465        let module_id = "fast-reconnect-provider";
4466        let first_connection = ConnectionId::new(70);
4467        let second_connection = ConnectionId::new(80);
4468        let (first_tx, _first_rx) = mpsc::channel(1);
4469        let first_endpoint = forwarding
4470            .register_module_connection(
4471                first_connection,
4472                module_id.to_string(),
4473                1,
4474                Concurrency::ModuleManaged,
4475                FrameSink::new(first_tx),
4476            )
4477            .unwrap();
4478        let (second_tx, _second_rx) = mpsc::channel(1);
4479        let second_endpoint = forwarding
4480            .register_module_connection(
4481                second_connection,
4482                module_id.to_string(),
4483                1,
4484                Concurrency::ModuleManaged,
4485                FrameSink::new(second_tx),
4486            )
4487            .unwrap();
4488        assert_ne!(first_endpoint, second_endpoint);
4489
4490        let released = forwarding.cleanup_connection(first_connection).unwrap();
4491
4492        assert!(released.is_empty());
4493        assert_eq!(
4494            forwarding
4495                .inner
4496                .read()
4497                .unwrap()
4498                .modules_by_id
4499                .get(module_id)
4500                .map(|module| module.endpoint),
4501            Some(second_endpoint)
4502        );
4503        assert!(forwarding.has_live_module_connection(module_id).unwrap());
4504        let control_rpc = forwarding
4505            .begin_module_control_rpc_for(
4506                module_id,
4507                "health.check",
4508                Instant::now() + Duration::from_secs(1),
4509            )
4510            .unwrap();
4511        assert_eq!(control_rpc.endpoint, second_endpoint);
4512    }
4513
4514    fn route_fixture(
4515        module_id: &str,
4516    ) -> (
4517        ForwardingTable,
4518        ConnectionId,
4519        ModuleEndpointId,
4520        ConnectionId,
4521        FrameSink,
4522        mpsc::Receiver<crate::router::OutboundFrame>,
4523    ) {
4524        let forwarding = ForwardingTable::default();
4525        let module_connection = ConnectionId::new(100);
4526        let client_connection = ConnectionId::new(200);
4527        let (module_tx, _module_rx) = mpsc::channel(8);
4528        let endpoint = forwarding
4529            .register_module_connection(
4530                module_connection,
4531                module_id.to_string(),
4532                2,
4533                Concurrency::ModuleManaged,
4534                FrameSink::new(module_tx),
4535            )
4536            .unwrap();
4537        let (client_tx, client_rx) = mpsc::channel(8);
4538        (
4539            forwarding,
4540            module_connection,
4541            endpoint,
4542            client_connection,
4543            FrameSink::new(client_tx),
4544            client_rx,
4545        )
4546    }
4547
4548    #[test]
4549    #[cfg(unix)]
4550    fn daemon_drain_gates_current_and_racing_provider_registrations() {
4551        let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
4552        assert_eq!(
4553            forwarding.begin_daemon_drain().unwrap(),
4554            [("provider".to_string(), endpoint)]
4555        );
4556        assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
4557        assert!(matches!(
4558            forwarding.register_module_connection(
4559                ConnectionId::new(300),
4560                "late-provider".into(),
4561                2,
4562                Concurrency::ModuleManaged,
4563                sink,
4564            ),
4565            Err(ForwardingError::ConnectionClosing { .. })
4566        ));
4567    }
4568
4569    #[test]
4570    #[cfg(unix)]
4571    fn daemon_drain_results_are_endpoint_fenced_and_removed_on_cleanup() {
4572        let (forwarding, connection, endpoint, _, _, _) = route_fixture("provider");
4573        assert_eq!(
4574            forwarding
4575                .daemon_drain_result_for_connection(connection)
4576                .unwrap(),
4577            None
4578        );
4579        forwarding
4580            .record_daemon_drain_result(endpoint, true)
4581            .unwrap();
4582        forwarding.begin_daemon_drain().unwrap();
4583        assert_eq!(
4584            forwarding
4585                .daemon_drain_result_for_connection(connection)
4586                .unwrap(),
4587            Some(false)
4588        );
4589        let replacement = ModuleEndpointId {
4590            generation: endpoint.generation + 1,
4591            ..endpoint
4592        };
4593        forwarding
4594            .record_daemon_drain_result(replacement, true)
4595            .unwrap();
4596        assert_eq!(
4597            forwarding
4598                .daemon_drain_result_for_connection(connection)
4599                .unwrap(),
4600            Some(false)
4601        );
4602        forwarding
4603            .record_daemon_drain_result(endpoint, true)
4604            .unwrap();
4605        assert_eq!(
4606            forwarding
4607                .daemon_drain_result_for_connection(connection)
4608                .unwrap(),
4609            Some(true)
4610        );
4611        assert_eq!(
4612            forwarding
4613                .daemon_drain_result_for_connection(ConnectionId::new(300))
4614                .unwrap(),
4615            Some(false)
4616        );
4617        forwarding.cleanup_connection(connection).unwrap();
4618        forwarding
4619            .record_daemon_drain_result(endpoint, true)
4620            .unwrap();
4621        assert_eq!(
4622            forwarding
4623                .daemon_drain_result_for_connection(connection)
4624                .unwrap(),
4625            Some(false)
4626        );
4627        assert!(forwarding
4628            .read_inner()
4629            .unwrap()
4630            .daemon_drain_results
4631            .is_empty());
4632    }
4633
4634    #[test]
4635    #[cfg(unix)]
4636    fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
4637        let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
4638            route_fixture("provider");
4639        let mut pending =
4640            begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
4641        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
4642        forwarding.begin_daemon_drain().unwrap();
4643        let completion = forwarding
4644            .complete_pending_relay(
4645                module_connection,
4646                pending.corr,
4647                RouteBindRelayOutcome::Accepted,
4648            )
4649            .expect("draining admission is not a fatal module error");
4650        assert!(completion.settled);
4651        let abandoned = completion
4652            .abandoned
4653            .expect("late accepted bind needs a route GOODBYE");
4654        assert_eq!(abandoned.channel, pending.module_channel);
4655        assert!(matches!(pending.receiver.try_recv().unwrap(),
4656            RouteBindRelayOutcome::DaemonRejected(error) if error.code == "module_reloading"));
4657        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4658        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4659        assert!(
4660            rx.try_recv().is_err(),
4661            "no route.open may be published during drain"
4662        );
4663        assert_eq!(
4664            forwarding
4665                .module_endpoint_for_connection(module_connection)
4666                .unwrap(),
4667            Some(endpoint)
4668        );
4669        assert!(
4670            !forwarding
4671                .complete_pending_relay(
4672                    module_connection,
4673                    pending.corr,
4674                    RouteBindRelayOutcome::Accepted,
4675                )
4676                .unwrap()
4677                .settled
4678        );
4679    }
4680
4681    fn test_ping(corr: u64) -> Frame {
4682        Frame::build(
4683            FrameType::Ping,
4684            Flags::new(false, Priority::Passive, false),
4685            0,
4686            0,
4687            corr,
4688            Vec::new(),
4689        )
4690        .unwrap()
4691    }
4692
4693    fn begin_test_route(
4694        forwarding: &ForwardingTable,
4695        client_connection: ConnectionId,
4696        client_sink: FrameSink,
4697        corr: u64,
4698        module_id: &str,
4699    ) -> PendingRouteBindRelay {
4700        forwarding
4701            .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
4702            .unwrap()
4703    }
4704
4705    #[test]
4706    fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
4707        for candidate in [false, true] {
4708            for register_first in [false, true] {
4709                let (forwarding, module_connection, _, client_connection, sink, _rx) =
4710                    route_fixture("target");
4711                let register = || {
4712                    if candidate {
4713                        forwarding.register_candidate_module_connection(
4714                            client_connection,
4715                            "source".into(),
4716                            2,
4717                            Concurrency::ModuleManaged,
4718                            sink.clone(),
4719                        )
4720                    } else {
4721                        forwarding.register_module_connection(
4722                            client_connection,
4723                            "source".into(),
4724                            2,
4725                            Concurrency::ModuleManaged,
4726                            sink.clone(),
4727                        )
4728                    }
4729                };
4730                if register_first {
4731                    register().unwrap();
4732                    assert!(
4733                        forwarding
4734                            .begin_route_bind_relay_for_test(
4735                                client_connection,
4736                                sink.clone(),
4737                                1,
4738                                "target",
4739                            )
4740                            .is_err(),
4741                        "a deferred route.open cannot reserve after HELLO"
4742                    );
4743                    assert!(forwarding
4744                        .cleanup_connection(client_connection)
4745                        .unwrap()
4746                        .is_empty());
4747                } else {
4748                    let pending =
4749                        begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
4750                    assert!(
4751                        register().is_err(),
4752                        "HELLO cannot register after a route reservation"
4753                    );
4754                    forwarding
4755                        .complete_pending_relay(
4756                            module_connection,
4757                            pending.corr,
4758                            RouteBindRelayOutcome::Accepted,
4759                        )
4760                        .unwrap();
4761                    assert_eq!(
4762                        forwarding
4763                            .cleanup_connection(client_connection)
4764                            .unwrap()
4765                            .len(),
4766                        1
4767                    );
4768                }
4769                assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4770                assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4771            }
4772        }
4773    }
4774
4775    /// A pending route.open reserves its queue slot with `reserve_owned` and
4776    /// sends its prebuilt response through that slot when the module accepts.
4777    /// A queue already holding many data frames (far more than the old 64-frame
4778    /// queue) must not stop the open from reserving or completing, and the
4779    /// response must be charged to the queue's byte count when it is sent.
4780    #[tokio::test]
4781    async fn pending_route_open_completes_behind_queued_data_frames() {
4782        assert_eq!(
4783            crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
4784            8,
4785            "the per-connection pending route.open limit is its own constant"
4786        );
4787        let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
4788            route_fixture("open-behind-data");
4789        let (sink, mut client_rx) = crate::server::connection_egress();
4790        const DATA_FRAMES: usize = 1_000;
4791        let data = |corr: u64| {
4792            Frame::build(
4793                FrameType::StreamData,
4794                Flags::new(false, Priority::Interactive, false),
4795                9,
4796                1,
4797                corr,
4798                vec![b'x'; 200],
4799            )
4800            .unwrap()
4801        };
4802        for corr in 0..DATA_FRAMES as u64 {
4803            sink.try_send(data(corr)).unwrap();
4804        }
4805        let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
4806        assert_eq!(sink.backlog().queued_bytes, data_bytes);
4807
4808        let pending = tokio::time::timeout(
4809            Duration::from_secs(5),
4810            forwarding.begin_route_bind_relay_for(
4811                client,
4812                sink.clone(),
4813                subc_protocol::PROTOCOL_VERSION,
4814                4_242,
4815                "open-behind-data",
4816                Principal::Direct,
4817                None,
4818                None,
4819                "fixture".into(),
4820                None,
4821                None,
4822                Instant::now() + Duration::from_secs(60),
4823                false,
4824            ),
4825        )
4826        .await
4827        .expect("reserving the route.open slot must not wait behind data frames")
4828        .unwrap();
4829        forwarding
4830            .complete_pending_relay(
4831                module_connection,
4832                pending.corr,
4833                RouteBindRelayOutcome::Accepted,
4834            )
4835            .unwrap();
4836
4837        let backlog = sink.backlog();
4838        assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
4839        assert!(
4840            backlog.queued_bytes > data_bytes,
4841            "the route.open response must be counted in queued bytes: {backlog:?}"
4842        );
4843        for corr in 0..DATA_FRAMES as u64 {
4844            assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
4845        }
4846        let open = client_rx.recv().await.unwrap();
4847        assert_eq!(open.header.corr, 4_242);
4848        assert_eq!(open.header.ty, FrameType::Response);
4849        drop(open);
4850        assert_eq!(sink.backlog().queued_bytes, 0);
4851        assert_eq!(sink.backlog().queued_frames, 0);
4852    }
4853
4854    /// The drain-timeout line reports these numbers, so they must count the
4855    /// requests the drain is waiting on and no others: a route with nothing in
4856    /// flight is not a holdout, and a flagged subscription is excluded from the
4857    /// drain and so from the count.
4858    #[tokio::test]
4859    async fn drain_holdouts_count_held_requests_and_name_the_connection() {
4860        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4861            route_fixture("holdouts");
4862        let mut bound = |corr| {
4863            let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
4864            forwarding
4865                .complete_pending_relay(
4866                    module_connection,
4867                    route.corr,
4868                    RouteBindRelayOutcome::Accepted,
4869                )
4870                .unwrap();
4871            client_rx.try_recv().unwrap();
4872            match forwarding
4873                .lookup_data_route(client, route.client_channel, route.client_epoch)
4874                .unwrap()
4875            {
4876                DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4877                other => panic!("expected live route, got {other:?}"),
4878            }
4879        };
4880        let holding = bound(61);
4881        let _idle = bound(62);
4882        holding.flow.acquire_tagged(7, false).await.unwrap();
4883        holding.flow.acquire_tagged(2, false).await.unwrap();
4884        holding.flow.acquire_tagged(3, true).await.unwrap();
4885        forwarding
4886            .begin_module_drain("holdouts", RouteCloseReason::Restart)
4887            .unwrap();
4888
4889        let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
4890        assert_eq!(
4891            holdouts,
4892            DrainHoldouts {
4893                requests: 2,
4894                routes: 1,
4895                total_routes: 2,
4896                top_connections: vec![(client.get(), 2)],
4897                // The held requests by the module's channel and corr, ascending,
4898                // without the excluded subscription (corr 3).
4899                held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
4900            }
4901        );
4902    }
4903
4904    #[test]
4905    fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
4906        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4907            route_fixture("census");
4908        let pending = begin_test_route(&forwarding, client, sink, 1, "census");
4909        forwarding
4910            .complete_pending_relay(
4911                module_connection,
4912                pending.corr,
4913                RouteBindRelayOutcome::Accepted,
4914            )
4915            .unwrap();
4916
4917        let routes = forwarding.endpoint_routes(endpoint).unwrap();
4918        assert_eq!(routes.len(), 1);
4919        assert!(matches!(routes[0].principal, Principal::Direct));
4920        assert_eq!(routes[0].goodbye_target.connection_id, client);
4921        assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
4922        assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
4923        assert!(!routes[0].draining);
4924
4925        forwarding
4926            .begin_module_drain("census", RouteCloseReason::Restart)
4927            .unwrap();
4928        let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
4929        assert_eq!(draining_routes.len(), 1);
4930        assert!(draining_routes[0].draining);
4931    }
4932
4933    #[test]
4934    fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
4935        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
4936        let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
4937        assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
4938        forwarding
4939            .abort_pending_relay(
4940                first.endpoint,
4941                first.corr,
4942                RouteBindRelayOutcome::ModuleGone("abort".into()),
4943            )
4944            .unwrap();
4945        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4946        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4947
4948        let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
4949        assert_eq!(second.client_channel, first.client_channel);
4950        assert_eq!(second.module_channel, first.module_channel);
4951        assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
4952    }
4953
4954    #[test]
4955    fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
4956        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4957            route_fixture("epoch-release");
4958        let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
4959        forwarding
4960            .complete_pending_relay(
4961                module_connection,
4962                first.corr,
4963                RouteBindRelayOutcome::Accepted,
4964            )
4965            .unwrap();
4966        assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
4967        assert!(matches!(
4968            forwarding
4969                .release_client_route(client, first.client_channel, first.client_epoch)
4970                .unwrap(),
4971            RouteRelease::Removed(_)
4972        ));
4973        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4974        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4975
4976        let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
4977        forwarding
4978            .complete_pending_relay(
4979                module_connection,
4980                second.corr,
4981                RouteBindRelayOutcome::Accepted,
4982            )
4983            .unwrap();
4984        assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
4985        assert!(matches!(
4986            forwarding
4987                .release_client_route(client, second.client_channel, first.client_epoch)
4988                .unwrap(),
4989            RouteRelease::Stale
4990        ));
4991        assert!(!forwarding
4992            .cache_status(
4993                endpoint,
4994                second.module_channel,
4995                first.module_epoch,
4996                "stale".into(),
4997            )
4998            .unwrap());
4999        assert!(forwarding
5000            .cache_status(
5001                endpoint,
5002                second.module_channel,
5003                second.module_epoch,
5004                "current".into(),
5005            )
5006            .unwrap());
5007        match forwarding
5008            .route_poll_snapshot(client, second.client_channel, second.client_epoch)
5009            .unwrap()
5010        {
5011            RoutePollSnapshot::Bound { status, .. } => {
5012                assert_eq!(status.as_deref(), Some("current"));
5013            }
5014            RoutePollSnapshot::Absent => panic!("successor binding was removed"),
5015        }
5016        let counters = forwarding.counters().snapshot();
5017        assert_eq!(counters["route_released_epoch_fenced"], 1);
5018        assert_eq!(counters["route_release_stale_skipped"], 1);
5019    }
5020
5021    #[test]
5022    fn max_epoch_reservation_retires_only_that_slot() {
5023        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
5024        forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
5025        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
5026        let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
5027        assert_eq!(
5028            (final_use.client_channel, final_use.client_epoch),
5029            (7, u32::MAX)
5030        );
5031        assert_eq!(
5032            (final_use.module_channel, final_use.module_epoch),
5033            (9, u32::MAX)
5034        );
5035        forwarding
5036            .abort_pending_relay(
5037                endpoint,
5038                final_use.corr,
5039                RouteBindRelayOutcome::ModuleGone("abort".into()),
5040            )
5041            .unwrap();
5042        forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
5043        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
5044        let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
5045        assert_ne!(next.client_channel, 7);
5046        assert_ne!(next.module_channel, 9);
5047        assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
5048    }
5049
5050    #[test]
5051    fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
5052        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
5053            route_fixture("corr-shared");
5054        let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
5055        assert_eq!(bind.corr, 1);
5056        forwarding
5057            .abort_pending_relay(
5058                endpoint,
5059                bind.corr,
5060                RouteBindRelayOutcome::ModuleGone("abort".into()),
5061            )
5062            .unwrap();
5063        let rpc = forwarding
5064            .begin_module_control_rpc_for(
5065                "corr-shared",
5066                "health.check",
5067                Instant::now() - Duration::from_millis(1),
5068            )
5069            .unwrap();
5070        assert_eq!(rpc.corr, 2);
5071        assert_eq!(
5072            forwarding
5073                .complete_module_control_rpc(
5074                    module_connection,
5075                    rpc.corr,
5076                    Some("health.check"),
5077                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
5078                        status: subc_protocol::session::HealthStatus::Ok,
5079                        detail: None,
5080                        metrics: None,
5081                    }),
5082                )
5083                .unwrap(),
5084            ModuleControlRpcCompletion::Settled
5085        );
5086        assert!(matches!(
5087            rpc.receiver.blocking_recv().unwrap(),
5088            ModuleControlRpcOutcome::DeadlineElapsed
5089        ));
5090    }
5091
5092    #[tokio::test(start_paused = true)]
5093    async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
5094        let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
5095        let probe_started_at = Instant::now();
5096        let rpc = forwarding
5097            .begin_health_probe_rpc_for(
5098                "tombstone-ttl",
5099                "health.check",
5100                probe_started_at,
5101                probe_started_at + Duration::from_secs(5),
5102            )
5103            .unwrap();
5104        assert!(forwarding
5105            .tombstone_health_probe_rpc(endpoint, rpc.corr)
5106            .unwrap());
5107        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
5108
5109        tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
5110        tokio::task::yield_now().await;
5111
5112        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
5113    }
5114
5115    #[test]
5116    fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
5117        let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
5118        let mut close = forwarding.register_connection_close(endpoint.connection_id);
5119        forwarding.inject_control_corr(endpoint, u64::MAX);
5120        let final_rpc = forwarding
5121            .begin_module_control_rpc_for(
5122                "corr-max",
5123                "health.check",
5124                Instant::now() + Duration::from_secs(1),
5125            )
5126            .unwrap();
5127        assert_eq!(final_rpc.corr, u64::MAX);
5128        forwarding
5129            .cancel_module_control_rpc(endpoint, final_rpc.corr)
5130            .unwrap();
5131        assert!(matches!(
5132            forwarding.begin_module_control_rpc_for(
5133                "corr-max",
5134                "health.check",
5135                Instant::now() + Duration::from_secs(1),
5136            ),
5137            Err(ForwardingError::RelayCorrelationExhausted)
5138        ));
5139        assert!(close.try_recv().is_ok());
5140    }
5141
5142    #[test]
5143    fn publication_epoch_controls_delivery_failure_escalation() {
5144        fn setup_successor(
5145            commit_successor: Option<bool>,
5146        ) -> (ForwardingTable, ConnectionId, u16, u32) {
5147            let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5148                route_fixture("escalation");
5149            let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
5150            forwarding
5151                .complete_pending_relay(
5152                    module_connection,
5153                    first.corr,
5154                    RouteBindRelayOutcome::Accepted,
5155                )
5156                .unwrap();
5157            client_rx.try_recv().unwrap();
5158            assert!(matches!(
5159                forwarding
5160                    .release_client_route(client, first.client_channel, first.client_epoch)
5161                    .unwrap(),
5162                RouteRelease::Removed(_)
5163            ));
5164            if let Some(commit_successor) = commit_successor {
5165                forwarding.inject_client_slot_epoch(
5166                    client,
5167                    first.client_channel,
5168                    first.client_epoch,
5169                );
5170                forwarding.inject_module_slot_epoch(
5171                    endpoint,
5172                    first.module_channel,
5173                    first.module_epoch,
5174                );
5175                let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
5176                if commit_successor {
5177                    forwarding
5178                        .complete_pending_relay(
5179                            module_connection,
5180                            successor.corr,
5181                            RouteBindRelayOutcome::Accepted,
5182                        )
5183                        .unwrap();
5184                    client_rx.try_recv().unwrap();
5185                } else {
5186                    forwarding
5187                        .abort_pending_relay(
5188                            endpoint,
5189                            successor.corr,
5190                            RouteBindRelayOutcome::ModuleGone("abort".into()),
5191                        )
5192                        .unwrap();
5193                }
5194            }
5195            (forwarding, client, first.client_channel, first.client_epoch)
5196        }
5197
5198        let probe_sink = FrameSink::new(mpsc::channel(1).0);
5199        let (no_successor, client, channel, epoch) = setup_successor(None);
5200        let mut close = no_successor.register_connection_close(client);
5201        assert!(no_successor
5202            .escalate_client_delivery_failure(
5203                client,
5204                channel,
5205                epoch,
5206                CloseReason::new("delivery", "failed"),
5207                UndeliveredFrame {
5208                    module_id: None,
5209                    sink: &probe_sink,
5210                },
5211            )
5212            .unwrap());
5213        assert!(close.try_recv().is_ok());
5214
5215        let (aborted, client, channel, epoch) = setup_successor(Some(false));
5216        let mut close = aborted.register_connection_close(client);
5217        assert!(aborted
5218            .escalate_client_delivery_failure(
5219                client,
5220                channel,
5221                epoch,
5222                CloseReason::new("delivery", "failed"),
5223                UndeliveredFrame {
5224                    module_id: None,
5225                    sink: &probe_sink,
5226                },
5227            )
5228            .unwrap());
5229        assert!(close.try_recv().is_ok());
5230
5231        let (published, client, channel, epoch) = setup_successor(Some(true));
5232        let mut close = published.register_connection_close(client);
5233        assert!(!published
5234            .escalate_client_delivery_failure(
5235                client,
5236                channel,
5237                epoch,
5238                CloseReason::new("delivery", "stale failure"),
5239                UndeliveredFrame {
5240                    module_id: None,
5241                    sink: &probe_sink,
5242                },
5243            )
5244            .unwrap());
5245        assert!(close.try_recv().is_err());
5246    }
5247
5248    #[test]
5249    fn route_concentration_separates_client_count_from_routes_per_client() {
5250        // The distinction this asserts is the one a bare connection count cannot
5251        // make: two connections holding one route each and one connection
5252        // holding two are the same total, and have opposite causes.
5253        let (forwarding, module_connection, _, client, sink, _client_rx) =
5254            route_fixture("concentration");
5255        assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
5256
5257        for corr in [70_u64, 71] {
5258            let pending =
5259                begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
5260            forwarding
5261                .complete_pending_relay(
5262                    module_connection,
5263                    pending.corr,
5264                    RouteBindRelayOutcome::Accepted,
5265                )
5266                .unwrap();
5267        }
5268
5269        // One connection, two routes — not two connections with a route each.
5270        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5271        assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
5272    }
5273
5274    #[test]
5275    fn cleanup_and_accepted_resolution_have_one_lock_winner() {
5276        let (forwarding, module_connection, _, client, sink, mut client_rx) =
5277            route_fixture("cleanup-race");
5278        let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
5279        forwarding
5280            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5281            .unwrap();
5282        let released = forwarding.cleanup_connection(client).unwrap();
5283        assert_eq!(released.len(), 1);
5284        let completion = forwarding
5285            .complete_pending_relay(
5286                module_connection,
5287                pending.corr,
5288                RouteBindRelayOutcome::Accepted,
5289            )
5290            .unwrap();
5291        assert!(!completion.settled);
5292        assert!(client_rx.try_recv().is_err());
5293        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5294
5295        let (forwarding, module_connection, _, client, sink, mut client_rx) =
5296            route_fixture("accepted-race");
5297        let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
5298        forwarding
5299            .complete_pending_relay(
5300                module_connection,
5301                pending.corr,
5302                RouteBindRelayOutcome::Accepted,
5303            )
5304            .unwrap();
5305        assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
5306        let released = forwarding.cleanup_connection(client).unwrap();
5307        assert_eq!(released.len(), 1);
5308        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5309    }
5310
5311    #[test]
5312    fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
5313        let (forwarding, module_connection, _, client, sink, mut client_rx) =
5314            route_fixture("drain-gap");
5315        let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
5316        forwarding
5317            .complete_pending_relay(
5318                module_connection,
5319                live.corr,
5320                RouteBindRelayOutcome::Accepted,
5321            )
5322            .unwrap();
5323        client_rx.try_recv().unwrap();
5324        let binding = match forwarding
5325            .lookup_data_route(client, live.client_channel, live.client_epoch)
5326            .unwrap()
5327        {
5328            DataRoute::Client(DataRouteState::Bound(binding)) => binding,
5329            other => panic!("expected live route, got {other:?}"),
5330        };
5331
5332        let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
5333        forwarding
5334            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5335            .unwrap();
5336        let control_rpc = forwarding
5337            .begin_module_control_rpc_for(
5338                "drain-gap",
5339                "health.check",
5340                Instant::now() + Duration::from_secs(1),
5341            )
5342            .unwrap();
5343        let target = forwarding
5344            .begin_module_drain("drain-gap", RouteCloseReason::Reload)
5345            .unwrap()
5346            .unwrap();
5347        assert!(matches!(
5348            control_rpc.receiver.blocking_recv().unwrap(),
5349            ModuleControlRpcOutcome::ModuleGone(_)
5350        ));
5351        assert_eq!(target.abandoned_bindings.len(), 1);
5352        assert!(binding.flow.sem.is_closed());
5353        assert!(
5354            !forwarding
5355                .complete_pending_relay(
5356                    module_connection,
5357                    pending.corr,
5358                    RouteBindRelayOutcome::Accepted,
5359                )
5360                .unwrap()
5361                .settled
5362        );
5363        assert!(matches!(
5364            forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
5365            Err(ForwardingError::ModuleReloading { .. })
5366        ));
5367        let released = forwarding
5368            .release_module_endpoint_routes(target.endpoint)
5369            .unwrap();
5370        assert_eq!(released.len(), 1);
5371        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5372    }
5373
5374    /// A client can be marked closing while its egress is still open: the daemon
5375    /// asks a connection to close (here through the production path, a module
5376    /// frame that its egress refused) and the connection loop tears down a moment
5377    /// later. A route.bind ack that lands inside that window is answered on the
5378    /// MODULE connection's frame handler, so resolving it must not produce an
5379    /// error -- an error there ends the module connection, and that connection
5380    /// carries every other client's routes to the module.
5381    #[test]
5382    fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
5383        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5384            route_fixture("closing-client");
5385
5386        // A published route on this client: escalate_client_delivery_failure only
5387        // marks a connection closing for a route it has already published.
5388        let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
5389        forwarding
5390            .complete_pending_relay(
5391                module_connection,
5392                live.corr,
5393                RouteBindRelayOutcome::Accepted,
5394            )
5395            .unwrap();
5396        client_rx.try_recv().unwrap();
5397
5398        // A second route.open from the same client, relayed and awaiting its ack.
5399        let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
5400        forwarding
5401            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5402            .unwrap();
5403
5404        // The window: closing, but the sink is still open.
5405        assert!(forwarding
5406            .escalate_client_delivery_failure(
5407                client,
5408                live.client_channel,
5409                live.client_epoch,
5410                CloseReason::new(
5411                    "module_to_client_delivery_failed",
5412                    "client egress refused a module frame",
5413                ),
5414                UndeliveredFrame {
5415                    module_id: None,
5416                    sink: &sink,
5417                },
5418            )
5419            .unwrap());
5420        assert!(!sink.is_closed());
5421
5422        let completion = forwarding
5423            .complete_pending_relay(
5424                module_connection,
5425                pending.corr,
5426                RouteBindRelayOutcome::Accepted,
5427            )
5428            .expect("a closing client must not turn a module's ack into an error");
5429
5430        assert!(completion.settled);
5431        let abandoned = completion
5432            .abandoned
5433            .expect("the module must be told to drop the binding it just created");
5434        assert_eq!(abandoned.connection_id, module_connection);
5435        assert_eq!(abandoned.channel, pending.module_channel);
5436        assert_eq!(abandoned.epoch, pending.module_epoch);
5437        assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
5438        assert!(matches!(
5439            pending.receiver.blocking_recv().unwrap(),
5440            RouteBindRelayOutcome::ModuleGone(_)
5441        ));
5442        // No route was published to a client that is on its way out, and the
5443        // reserved handle pair went back.
5444        assert!(client_rx.try_recv().is_err());
5445        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
5446
5447        // The module endpoint is untouched: still live, and still able to take a
5448        // route from another client.
5449        assert!(forwarding
5450            .has_live_module_connection("closing-client")
5451            .unwrap());
5452        let cotenant = ConnectionId::new(201);
5453        let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
5454        let cotenant_route = begin_test_route(
5455            &forwarding,
5456            cotenant,
5457            FrameSink::new(cotenant_tx),
5458            62,
5459            "closing-client",
5460        );
5461        assert_eq!(cotenant_route.endpoint, endpoint);
5462        forwarding
5463            .complete_pending_relay(
5464                module_connection,
5465                cotenant_route.corr,
5466                RouteBindRelayOutcome::Accepted,
5467            )
5468            .unwrap();
5469        assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
5470        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5471    }
5472
5473    #[test]
5474    fn pending_route_permit_is_released_on_rejection_and_abort() {
5475        let forwarding = ForwardingTable::default();
5476        let module_connection = ConnectionId::new(300);
5477        let client = ConnectionId::new(301);
5478        let (module_tx, _module_rx) = mpsc::channel(1);
5479        let endpoint = forwarding
5480            .register_module_connection(
5481                module_connection,
5482                "permit".into(),
5483                2,
5484                Concurrency::ModuleManaged,
5485                FrameSink::new(module_tx),
5486            )
5487            .unwrap();
5488        let (client_tx, mut client_rx) = mpsc::channel(1);
5489        let sink = FrameSink::new(client_tx);
5490        let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
5491        assert!(sink.try_send(test_ping(999)).is_err());
5492        forwarding
5493            .complete_pending_relay(
5494                module_connection,
5495                rejected.corr,
5496                RouteBindRelayOutcome::Rejected(ErrorBody {
5497                    code: "no".into(),
5498                    message: "rejected".into(),
5499                    detail: None,
5500                }),
5501            )
5502            .unwrap();
5503        sink.try_send(test_ping(1000)).unwrap();
5504        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
5505
5506        let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
5507        assert!(sink.try_send(test_ping(1001)).is_err());
5508        forwarding
5509            .abort_pending_relay(
5510                endpoint,
5511                aborted.corr,
5512                RouteBindRelayOutcome::ModuleGone("abort".into()),
5513            )
5514            .unwrap();
5515        sink.try_send(test_ping(1002)).unwrap();
5516        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
5517
5518        let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
5519        forwarding
5520            .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
5521            .unwrap();
5522        drop(client_rx);
5523        let completion = forwarding
5524            .complete_pending_relay(
5525                module_connection,
5526                receiver_closed.corr,
5527                RouteBindRelayOutcome::Accepted,
5528            )
5529            .unwrap();
5530        assert!(completion.abandoned.is_some());
5531        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5532    }
5533
5534    /// Every connection teardown passes through `cleanup_connection`, and
5535    /// connection ids come from a monotonic counter that never hands an id out
5536    /// twice. If the closing mark survives teardown, the set grows by one entry
5537    /// per connection for the life of the daemon -- the self-watchdog alone
5538    /// reconnects once a minute.
5539    #[test]
5540    fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
5541        let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
5542            route_fixture("closing-set-leak");
5543
5544        const CONNECTIONS: u64 = 32;
5545        for index in 0..CONNECTIONS {
5546            let client = ConnectionId::new(1000 + index);
5547            let (client_tx, _client_rx) = mpsc::channel(8);
5548            let route = begin_test_route(
5549                &forwarding,
5550                client,
5551                FrameSink::new(client_tx),
5552                index + 1,
5553                "closing-set-leak",
5554            );
5555            forwarding
5556                .complete_pending_relay(
5557                    module_connection,
5558                    route.corr,
5559                    RouteBindRelayOutcome::Accepted,
5560                )
5561                .unwrap();
5562            forwarding.cleanup_connection(client).unwrap();
5563        }
5564        forwarding.cleanup_connection(module_connection).unwrap();
5565
5566        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5567    }
5568
5569    /// The closing mark exists to refuse new work for a connection that is on
5570    /// its way out but whose teardown has not run yet: the daemon asks the
5571    /// connection loop to end, and only when the loop reacts does
5572    /// `cleanup_connection` strip the connection's state. Inside that window an
5573    /// operation for the dying connection must still be refused; only after
5574    /// cleanup completes may the mark go.
5575    #[test]
5576    fn closing_connection_is_refused_new_work_until_cleanup_completes() {
5577        let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
5578            route_fixture("closing-gate");
5579
5580        // A published route: escalate_client_delivery_failure only marks a
5581        // connection closing for a route it has already published.
5582        let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
5583        forwarding
5584            .complete_pending_relay(
5585                module_connection,
5586                live.corr,
5587                RouteBindRelayOutcome::Accepted,
5588            )
5589            .unwrap();
5590        client_rx.try_recv().unwrap();
5591
5592        // Mark the connection closing through the production path without
5593        // running teardown, pinning the window open.
5594        assert!(forwarding
5595            .escalate_client_delivery_failure(
5596                client,
5597                live.client_channel,
5598                live.client_epoch,
5599                CloseReason::new(
5600                    "module_to_client_delivery_failed",
5601                    "client egress refused a module frame",
5602                ),
5603                UndeliveredFrame {
5604                    module_id: None,
5605                    sink: &sink,
5606                },
5607            )
5608            .unwrap());
5609        assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
5610
5611        // A late route.open for the closing client is refused ...
5612        assert!(matches!(
5613            forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
5614            Err(ForwardingError::ConnectionClosing { connection_id })
5615                if connection_id == client
5616        ));
5617        // ... and so is a late attempt to register the connection as a module.
5618        let (late_tx, _late_rx) = mpsc::channel(1);
5619        assert!(matches!(
5620            forwarding.register_module_connection(
5621                client,
5622                "late-module".into(),
5623                2,
5624                Concurrency::ModuleManaged,
5625                FrameSink::new(late_tx),
5626            ),
5627            Err(ForwardingError::ConnectionClosing { connection_id })
5628                if connection_id == client
5629        ));
5630
5631        // Teardown is the point that lifts the mark: it has just removed every
5632        // per-connection entry under the same lock, so the gate has nothing
5633        // left to protect for this id.
5634        forwarding.cleanup_connection(client).unwrap();
5635        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5636    }
5637}
5638
5639/// Blue/green swap slots: a candidate registered beside the active endpoint,
5640/// promoted by `cutover_candidate`, with the old incumbent drained by endpoint.
5641#[cfg(test)]
5642mod swap_slot_tests {
5643    use std::time::Duration;
5644
5645    use super::*;
5646    use tokio::sync::mpsc;
5647
5648    const MODULE_ID: &str = "swapped";
5649
5650    struct SwapFixture {
5651        forwarding: ForwardingTable,
5652        incumbent_connection: ConnectionId,
5653        incumbent: ModuleEndpointId,
5654        candidate_connection: ConnectionId,
5655        candidate: ModuleEndpointId,
5656        _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
5657    }
5658
5659    fn swap_fixture() -> SwapFixture {
5660        let forwarding = ForwardingTable::default();
5661        let incumbent_connection = ConnectionId::new(100);
5662        let candidate_connection = ConnectionId::new(110);
5663        let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
5664        let incumbent = forwarding
5665            .register_module_connection(
5666                incumbent_connection,
5667                MODULE_ID.to_string(),
5668                2,
5669                Concurrency::ModuleManaged,
5670                FrameSink::new(incumbent_tx),
5671            )
5672            .unwrap();
5673        let (candidate_tx, candidate_rx) = mpsc::channel(8);
5674        let candidate = forwarding
5675            .register_candidate_module_connection(
5676                candidate_connection,
5677                MODULE_ID.to_string(),
5678                2,
5679                Concurrency::ModuleManaged,
5680                FrameSink::new(candidate_tx),
5681            )
5682            .unwrap();
5683        SwapFixture {
5684            forwarding,
5685            incumbent_connection,
5686            incumbent,
5687            candidate_connection,
5688            candidate,
5689            _module_rxs: vec![incumbent_rx, candidate_rx],
5690        }
5691    }
5692
5693    fn client(
5694        raw: u64,
5695    ) -> (
5696        ConnectionId,
5697        FrameSink,
5698        mpsc::Receiver<crate::router::OutboundFrame>,
5699    ) {
5700        let (tx, rx) = mpsc::channel(8);
5701        (ConnectionId::new(raw), FrameSink::new(tx), rx)
5702    }
5703
5704    fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
5705        forwarding
5706            .read_inner()
5707            .unwrap()
5708            .client_to_module
5709            .values()
5710            .map(|route| route.module_endpoint)
5711            .collect()
5712    }
5713
5714    #[test]
5715    fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
5716        let fixture = swap_fixture();
5717        let forwarding = &fixture.forwarding;
5718        assert_ne!(fixture.incumbent, fixture.candidate);
5719
5720        // Every by-id consumer still resolves the incumbent.
5721        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5722        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5723        let (client_connection, client_sink, _client_rx) = client(200);
5724        let pending = forwarding
5725            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5726            .unwrap();
5727        assert_eq!(pending.endpoint, fixture.incumbent);
5728        let rpc = forwarding
5729            .begin_module_control_rpc_for(
5730                MODULE_ID,
5731                "health.check",
5732                Instant::now() + Duration::from_secs(1),
5733            )
5734            .unwrap();
5735        assert_eq!(rpc.endpoint, fixture.incumbent);
5736        let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
5737        assert_eq!(census.len(), 1, "the census lists one endpoint per id");
5738
5739        // Connection-keyed lookups see the candidate, so its own frames resolve.
5740        assert_eq!(
5741            forwarding
5742                .module_endpoint_for_connection(fixture.candidate_connection)
5743                .unwrap(),
5744            Some(fixture.candidate)
5745        );
5746        assert_eq!(
5747            forwarding
5748                .module_id_for_connection(fixture.candidate_connection)
5749                .unwrap()
5750                .as_deref(),
5751            Some(MODULE_ID)
5752        );
5753
5754        // One candidate per id.
5755        let (other_tx, _other_rx) = mpsc::channel(1);
5756        assert_eq!(
5757            forwarding.register_candidate_module_connection(
5758                ConnectionId::new(120),
5759                MODULE_ID.to_string(),
5760                2,
5761                Concurrency::ModuleManaged,
5762                FrameSink::new(other_tx),
5763            ),
5764            Err(ForwardingError::CandidateSlotOccupied {
5765                module_id: MODULE_ID.to_string()
5766            })
5767        );
5768    }
5769
5770    /// The cutover linearization point. A relay reserved on the incumbent before
5771    /// cutover must never become a route on the incumbent, and every relay
5772    /// reserved after it must land on the promoted candidate.
5773    #[test]
5774    fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
5775        let fixture = swap_fixture();
5776        let forwarding = &fixture.forwarding;
5777        let (early_client, early_sink, _early_rx) = client(200);
5778        let mut early = forwarding
5779            .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
5780            .unwrap();
5781        assert_eq!(early.endpoint, fixture.incumbent);
5782        assert_eq!(
5783            forwarding
5784                .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
5785                .unwrap(),
5786            RouteBindRelayMark::Marked
5787        );
5788
5789        let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5790        assert_eq!(
5791            cutover,
5792            ForwardingCutover {
5793                promoted: fixture.candidate,
5794                incumbent: Some(fixture.incumbent),
5795            }
5796        );
5797
5798        // A route.open reserved after cutover goes to the promoted candidate.
5799        let (late_client, late_sink, _late_rx) = client(201);
5800        let late = forwarding
5801            .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
5802            .unwrap();
5803        assert_eq!(
5804            late.endpoint, fixture.candidate,
5805            "a route.open after cutover was reserved on the incumbent"
5806        );
5807
5808        // The incumbent acks the early relay after cutover.
5809        let completion = forwarding
5810            .complete_pending_relay(
5811                fixture.incumbent_connection,
5812                early.corr,
5813                RouteBindRelayOutcome::Accepted,
5814            )
5815            .expect("a superseded endpoint's ack is not an error on its connection");
5816        assert!(completion.settled);
5817        assert!(
5818            !committed_endpoints(forwarding).contains(&fixture.incumbent),
5819            "a relay reserved before cutover committed a route on the incumbent"
5820        );
5821        let goodbye = completion
5822            .abandoned
5823            .expect("the incumbent is told to drop the binding it just created");
5824        assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
5825        assert_eq!(goodbye.channel, early.module_channel);
5826        assert_eq!(goodbye.epoch, early.module_epoch);
5827        assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
5828        match early.receiver.try_recv() {
5829            Ok(RouteBindRelayOutcome::DaemonRejected(body)) => {
5830                assert_eq!(body.code, "module_reloading")
5831            }
5832            other => panic!("expected a retryable module_reloading answer, got {other:?}"),
5833        }
5834        assert!(matches!(
5835            forwarding
5836                .lookup_data_route(early_client, early.client_channel, early.client_epoch)
5837                .unwrap(),
5838            DataRoute::Client(DataRouteState::Absent)
5839        ));
5840
5841        // The late relay commits on the candidate; only its pair stays reserved
5842        // until then.
5843        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
5844        forwarding
5845            .complete_pending_relay(
5846                fixture.candidate_connection,
5847                late.corr,
5848                RouteBindRelayOutcome::Accepted,
5849            )
5850            .unwrap();
5851        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5852        assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
5853    }
5854
5855    #[test]
5856    fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
5857        let fixture = swap_fixture();
5858        let forwarding = &fixture.forwarding;
5859        // One bound route and one in-flight relay on the incumbent.
5860        let (bound_client, bound_sink, _bound_rx) = client(200);
5861        let bound = forwarding
5862            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5863            .unwrap();
5864        forwarding
5865            .complete_pending_relay(
5866                fixture.incumbent_connection,
5867                bound.corr,
5868                RouteBindRelayOutcome::Accepted,
5869            )
5870            .unwrap();
5871        let (pending_client, pending_sink, _pending_rx) = client(201);
5872        let mut in_flight = forwarding
5873            .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
5874            .unwrap();
5875        forwarding
5876            .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
5877            .unwrap();
5878
5879        let incumbent = forwarding
5880            .cutover_candidate(MODULE_ID)
5881            .unwrap()
5882            .unwrap()
5883            .incumbent
5884            .unwrap();
5885        let target = forwarding
5886            .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
5887            .unwrap()
5888            .expect("the superseded incumbent is still registered");
5889
5890        assert_eq!(target.endpoint, fixture.incumbent);
5891        assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
5892        assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
5893        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5894        assert_eq!(target.abandoned_bindings.len(), 1);
5895        assert_eq!(
5896            target.abandoned_bindings[0].channel,
5897            in_flight.module_channel
5898        );
5899        assert!(matches!(
5900            in_flight.receiver.try_recv(),
5901            Ok(RouteBindRelayOutcome::DaemonRejected(body)) if body.code == "module_reloading"
5902        ));
5903        assert_eq!(
5904            forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
5905            1,
5906            "the incumbent's bound route stays until its drain finishes"
5907        );
5908
5909        let (next_client, next_sink, _next_rx) = client(202);
5910        let next = forwarding
5911            .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
5912            .expect("the promoted candidate keeps accepting routes");
5913        assert_eq!(next.endpoint, fixture.candidate);
5914    }
5915
5916    /// An endpoint replaced WITHOUT a promotion (a successor registered over it
5917    /// as an ordinary active HELLO) is stale, not superseded, and its ack still
5918    /// fails the acking connection exactly as it did before swap slots existed:
5919    /// `StaleModuleEndpoint`, the reservation indexes already stripped, and the
5920    /// waiting client's sender dropped unanswered.
5921    #[test]
5922    fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
5923        let forwarding = ForwardingTable::default();
5924        let first_connection = ConnectionId::new(70);
5925        let (first_tx, _first_rx) = mpsc::channel(8);
5926        forwarding
5927            .register_module_connection(
5928                first_connection,
5929                MODULE_ID.to_string(),
5930                2,
5931                Concurrency::ModuleManaged,
5932                FrameSink::new(first_tx),
5933            )
5934            .unwrap();
5935        let (client_connection, client_sink, _client_rx) = client(200);
5936        let mut pending = forwarding
5937            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5938            .unwrap();
5939        let (second_tx, _second_rx) = mpsc::channel(8);
5940        forwarding
5941            .register_module_connection(
5942                ConnectionId::new(80),
5943                MODULE_ID.to_string(),
5944                2,
5945                Concurrency::ModuleManaged,
5946                FrameSink::new(second_tx),
5947            )
5948            .unwrap();
5949
5950        assert_eq!(
5951            forwarding
5952                .complete_pending_relay(
5953                    first_connection,
5954                    pending.corr,
5955                    RouteBindRelayOutcome::Accepted
5956                )
5957                .unwrap_err(),
5958            ForwardingError::StaleModuleEndpoint
5959        );
5960        assert!(committed_endpoints(&forwarding).is_empty());
5961        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5962        assert!(matches!(
5963            pending.receiver.try_recv(),
5964            Err(oneshot::error::TryRecvError::Closed)
5965        ));
5966    }
5967
5968    #[test]
5969    fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
5970        // A candidate whose connection drops leaves the incumbent routable.
5971        let fixture = swap_fixture();
5972        let forwarding = &fixture.forwarding;
5973        assert!(forwarding
5974            .cleanup_connection(fixture.candidate_connection)
5975            .unwrap()
5976            .is_empty());
5977        assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
5978        let (client_connection, client_sink, _client_rx) = client(200);
5979        assert_eq!(
5980            forwarding
5981                .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5982                .unwrap()
5983                .endpoint,
5984            fixture.incumbent
5985        );
5986
5987        // After a cutover, the incumbent's teardown releases its own routes and
5988        // leaves the promoted candidate in place.
5989        let fixture = swap_fixture();
5990        let forwarding = &fixture.forwarding;
5991        let (bound_client, bound_sink, _bound_rx) = client(200);
5992        let bound = forwarding
5993            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5994            .unwrap();
5995        forwarding
5996            .complete_pending_relay(
5997                fixture.incumbent_connection,
5998                bound.corr,
5999                RouteBindRelayOutcome::Accepted,
6000            )
6001            .unwrap();
6002        forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
6003        let released = forwarding
6004            .cleanup_connection(fixture.incumbent_connection)
6005            .unwrap();
6006        assert_eq!(released.len(), 1);
6007        assert_eq!(released[0].connection_id, bound_client);
6008        assert!(forwarding
6009            .read_inner()
6010            .unwrap()
6011            .superseded_endpoints
6012            .is_empty());
6013        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
6014        let (next_client, next_sink, _next_rx) = client(201);
6015        assert_eq!(
6016            forwarding
6017                .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
6018                .unwrap()
6019                .endpoint,
6020            fixture.candidate
6021        );
6022    }
6023}