Skip to main content

subc_daemon/
forwarding.rs

1use std::{
2    collections::{BTreeMap, HashMap, HashSet},
3    error::Error,
4    fmt,
5    sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
6    time::Duration,
7};
8
9use subc_control::{ClientControlResponse, RouteCloseReason};
10use subc_protocol::{
11    manifest::Concurrency,
12    session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
13    ErrorBody, Flags, FrameType, Principal, Priority,
14};
15use tokio::sync::{oneshot, Semaphore};
16use tokio::time::Instant;
17use tracing::{debug, info, warn};
18
19use crate::{
20    control::{RouteBindBreakers, RouteBindConcurrency},
21    observability::DaemonCounters,
22    registry::ConnectionId,
23    router::FrameSink,
24    scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
25    Frame, ProjectRootId,
26};
27
28/// Default per-channel request-credit window for modules that schedule internally.
29const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
30
31/// High per-channel cap for stateless modules; this is an OOM guard, not scheduling policy.
32const STATELESS_PARALLEL_WINDOW: usize = 1024;
33
34/// A stopped probe cycle cannot retain its last unanswered correlation forever.
35/// Active endpoints replace the tombstone on their next serial health probe;
36/// this backstop covers endpoints that stop probing altogether.
37const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
38
39/// Module connection identity used in forwarding keys.
40///
41/// The generation is bumped every time a module connection is registered so a future restart cannot
42/// accidentally reuse a stale `(connection_id, route_channel)` binding.
43#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
44pub struct ModuleEndpointId {
45    pub connection_id: ConnectionId,
46    pub generation: u64,
47}
48
49/// Client-local route key. A route channel is unique only within one client connection.
50#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
51pub(crate) struct ClientRouteKey {
52    pub connection_id: ConnectionId,
53    pub channel: u16,
54}
55
56/// Module-local route key. A route channel is unique only within one live module endpoint.
57#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
58pub(crate) struct ModuleRouteKey {
59    pub endpoint: ModuleEndpointId,
60    pub channel: u16,
61}
62
63#[derive(Debug)]
64pub(crate) struct RouteBinding {
65    pub client_connection_id: ConnectionId,
66    pub client_sink: FrameSink,
67    pub client_negotiated_ver: u8,
68    pub client_channel: u16,
69    pub client_epoch: u32,
70    pub module_id: String,
71    pub module_endpoint: ModuleEndpointId,
72    pub module_sink: FrameSink,
73    pub module_negotiated_ver: u8,
74    pub module_channel: u16,
75    pub module_epoch: u32,
76    pub principal: Principal,
77    /// Opener attributes captured at admission for daemon-side prompt policy.
78    #[allow(dead_code)]
79    pub harness: String,
80    /// Only Callosum's exact production claim is retained; never relayed.
81    #[allow(dead_code)]
82    pub build_class: Option<String>,
83    /// Captured at connect, carried through admission; not standing authority.
84    #[allow(dead_code)]
85    pub operator_app: Option<crate::operator_apps::OperatorAppStamp>,
86    pub project_root: Option<ProjectRootId>,
87    pub bound_at: Instant,
88    pub flow: Arc<ChannelFlow>,
89    /// The scope the route was admitted under, with the tag its bind was
90    /// stamped at. A scope change finds the routes to close by it.
91    pub scope: Option<BoundScope>,
92}
93
94#[derive(Debug, Clone)]
95pub(crate) enum DataRoute {
96    Client(DataRouteState),
97    Module(DataRouteState),
98}
99
100#[derive(Debug, Clone)]
101pub(crate) enum DataRouteState {
102    Bound(Arc<RouteBinding>),
103    Reserved,
104    EpochMismatch,
105    Absent,
106}
107
108/// Which kind of peer a route GOODBYE is being delivered to. This decides what
109/// happens when the GOODBYE cannot be enqueued (egress full/closed):
110/// - `Client`: escalate to closing that client connection (a socket close is a
111///   stronger teardown signal, and a full client egress means it is the slow
112///   client we would drop anyway).
113/// - `Module`: never close; deliver late instead (see
114///   [`send_module_route_goodbye`]), and drop only if the module still has no
115///   room after [`LATE_MODULE_GOODBYE_DEADLINE`]. A client-disconnect notifies the
116///   SHARED module that one client's route is gone; closing the module on its
117///   egress backpressure would tear down every co-tenant client (the exact
118///   cross-tenant blast radius this never-close rule exists to prevent — observed when a
119///   flooding dead client filled BOTH its own and the module's egress, so its
120///   route-gone GOODBYE to the module failed and closed the shared connection).
121///   subc has already removed the route from its forwarding state and drops
122///   stale module frames for the released channel (see router.rs), so subc's
123///   own routing is correct. The residual: under SUSTAINED module-egress
124///   backpressure a module-targeted route-gone notification can be lost, which a
125///   module using it for client-refcounting (e.g. AFT's session accounting)
126///   would miss. This is INTENTIONALLY ACCEPTED, not a gap. A consuming module
127///   must bound stale bindings with its own idle-activity reaper (last-touched
128///   TTL) independent of route-gone signals — AFT does exactly this, so a lost
129///   GOODBYE degrades to "the
130///   binding stays warm until its idle TTL" (bounded wasted resources), never an
131///   unbounded leak; disk-durable replay is unaffected. A dedicated reliable
132///   module control lane was evaluated and deliberately NOT
133///   built: it would add starvation-avoidance machinery to the thin core for a
134///   bounded warm-resource window that is not a correctness issue. never-close
135///   is the invariant that matters here.
136#[derive(Debug, Clone, Copy, PartialEq, Eq)]
137pub(crate) enum GoodbyeTargetKind {
138    Client,
139    Module,
140}
141
142#[derive(Debug, Clone)]
143pub(crate) struct GoodbyeTarget {
144    pub connection_id: ConnectionId,
145    pub sink: FrameSink,
146    pub negotiated_ver: u8,
147    pub channel: u16,
148    pub epoch: u32,
149    pub kind: GoodbyeTargetKind,
150    /// The module on the other end of the route: for a module-targeted relay
151    /// the receiving module (attributing a dropped relay), for a client target
152    /// the module whose route is going away (naming it when an undeliverable
153    /// relay closes the client).
154    pub module_id: Option<String>,
155}
156
157/// The frame a client connection's egress queue refused, for the diagnosis
158/// logged when that refusal closes the connection.
159#[derive(Debug, Clone, Copy)]
160pub(crate) struct UndeliveredFrame<'a> {
161    /// The module on the other end of the route the frame belonged to, when known.
162    pub module_id: Option<&'a str>,
163    /// The refusing connection's sink, read for what its queue held.
164    pub sink: &'a FrameSink,
165}
166
167/// How a route's principal appears in logs: `direct`, or `reserved:<module>`.
168pub(crate) fn principal_label(principal: &Principal) -> String {
169    match principal {
170        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
171        Principal::Direct => "direct".to_string(),
172        other => format!("{other:?}"),
173    }
174}
175
176/// The distinct principals of the routes bound on one client connection,
177/// sorted and comma-joined, or `none` when it has no bound route.
178fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
179    let labels = inner
180        .client_to_module
181        .iter()
182        .filter(|(key, _)| key.connection_id == connection_id)
183        .map(|(_, route)| principal_label(&route.principal))
184        .collect::<std::collections::BTreeSet<_>>();
185    if labels.is_empty() {
186        "none".to_string()
187    } else {
188        labels.into_iter().collect::<Vec<_>>().join(",")
189    }
190}
191
192impl GoodbyeTarget {
193    /// True only when an undeliverable GOODBYE should escalate to closing the
194    /// target connection. Never escalate for module recipients.
195    pub(crate) fn close_on_delivery_failure(&self) -> bool {
196        matches!(self.kind, GoodbyeTargetKind::Client)
197    }
198}
199
200/// How long a route GOODBYE that a module's egress queue refused keeps waiting
201/// for room before it is given up and counted as dropped.
202///
203/// A module that stops reading for a moment (a GC pause, a long synchronous
204/// handler, a reconnect herd it is still working through) should still learn
205/// that the route is gone, because a module that never hears the GOODBYE keeps
206/// the route and keeps sending on it for as long as its connection lives. The
207/// value is the default module drain timeout: the daemon already treats that as
208/// the longest it is reasonable to wait on a module that is busy but alive, and
209/// a module that cannot free 21 bytes of egress in that time is wedged rather
210/// than slow. Resolving each module's own configured drain timeout here would
211/// need a registry lookup on a path that holds only the module's sink.
212pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
213
214/// Send a route GOODBYE to a module without ever closing the module's shared
215/// connection, which would sever every other route it serves.
216///
217/// The frame is enqueued at once when the module's egress queue has room. When
218/// it does not, a detached task waits for room (bounded by
219/// [`LATE_MODULE_GOODBYE_DEADLINE`]) and sends it then. Arriving late is safe:
220/// the daemon has already released the route, and module SDKs tear a route
221/// down only when both the channel AND the epoch match the route installed on
222/// that channel, so a late GOODBYE for a channel since reused at a newer epoch
223/// is ignored. The GOODBYE is counted in `goodbye_relay_module_dropped` only
224/// when the connection is closed or the deadline passes.
225pub(crate) fn send_module_route_goodbye(
226    counters: &DaemonCounters,
227    sink: &FrameSink,
228    frame: Frame,
229    module_id: Option<&str>,
230    context: &'static str,
231) {
232    let channel = frame.header.channel;
233    let epoch = frame.header.epoch;
234    let Err(err) = sink.try_send(frame.clone()) else {
235        return;
236    };
237    // Waiting is pointless once the module's writer is gone, and impossible
238    // outside a Tokio runtime (cleanup that runs from a destructor at shutdown).
239    let runtime = match tokio::runtime::Handle::try_current() {
240        Ok(runtime) if !sink.is_closed() => runtime,
241        _ => {
242            counters.increment_goodbye_relay_module_dropped(module_id);
243            warn!(
244            module_id = module_id.unwrap_or("unknown"),
245            route_channel = channel,
246            route_epoch = epoch,
247            error = %err,
248            context,
249            "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
250            );
251            return;
252        }
253    };
254    debug!(
255        module_id = module_id.unwrap_or("unknown"),
256        route_channel = channel,
257        route_epoch = epoch,
258        error = %err,
259        context,
260        "module egress queue refused route GOODBYE; delivering it once the module frees room"
261    );
262    let counters = counters.clone();
263    let sink = sink.clone();
264    let module_id = module_id.map(str::to_string);
265    runtime.spawn(async move {
266        let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
267        let why = match outcome {
268            Ok(Ok(())) => {
269                debug!(
270                    module_id = module_id.as_deref().unwrap_or("unknown"),
271                    route_channel = channel,
272                    route_epoch = epoch,
273                    context,
274                    "late route GOODBYE delivered to module"
275                );
276                return;
277            }
278            Ok(Err(err)) => err.to_string(),
279            Err(_) => format!(
280                "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
281            ),
282        };
283        counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
284        warn!(
285            module_id = module_id.as_deref().unwrap_or("unknown"),
286            route_channel = channel,
287            route_epoch = epoch,
288            error = %why,
289            context,
290            "route GOODBYE to module dropped under backpressure; not closing shared module connection"
291        );
292    });
293}
294
295/// One route currently served by a module endpoint.
296///
297/// `goodbye_target` is deliberately retained alongside the census projection so
298/// a draining caller can address exactly the same route set that this read
299/// reports, without a second forwarding-table pass.
300#[derive(Debug, Clone)]
301pub(crate) struct EndpointRoute {
302    pub goodbye_target: GoodbyeTarget,
303    pub principal: Principal,
304    pub bound_at: Instant,
305    pub draining: bool,
306    /// WHY the endpoint is draining, when it is. Carried per-route so the
307    /// census can answer "closing because of what" without a second lookup;
308    /// `None` exactly when `draining` is false (one source: the drain map).
309    pub drain_reason: Option<RouteCloseReason>,
310}
311
312/// One live route a scope change closed: both ends to send GOODBYE to, and the
313/// reason for the client's `route.closed`.
314#[derive(Debug, Clone)]
315pub(crate) struct ScopeDrainedRoute {
316    pub reason: RouteCloseReason,
317    pub scope: BoundScope,
318    pub module_id: String,
319    pub client: GoodbyeTarget,
320    pub module: GoodbyeTarget,
321}
322
323#[derive(Debug)]
324pub(crate) struct PendingRouteBindRelay {
325    pub endpoint: ModuleEndpointId,
326    pub module_sink: FrameSink,
327    pub negotiated_ver: u8,
328    pub client_channel: u16,
329    pub client_epoch: u32,
330    pub module_channel: u16,
331    pub module_epoch: u32,
332    pub corr: u64,
333    pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
334}
335
336#[derive(Debug, Clone)]
337pub(crate) struct ModuleDrainTarget {
338    pub endpoint: ModuleEndpointId,
339    pub sink: FrameSink,
340    pub negotiated_ver: u8,
341    pub abandoned_bindings: Vec<GoodbyeTarget>,
342    pub excluded_subscriptions: u32,
343}
344
345/// One registered module connection, as [`ForwardingTable::module_connections`]
346/// reports it: enough to send it a frame and then close it.
347#[cfg(unix)]
348#[derive(Debug, Clone)]
349pub(crate) struct ModuleConnectionTarget {
350    pub module_id: String,
351    pub endpoint: ModuleEndpointId,
352    pub sink: FrameSink,
353    pub negotiated_ver: u8,
354}
355
356#[derive(Debug, Clone)]
357pub(crate) enum RouteBindRelayOutcome {
358    Accepted,
359    /// The module refused the bind with its own body. Relayed to the client
360    /// without the daemon-origin bit: the code and message are module-chosen,
361    /// so the frame must not claim daemon authorship, even when the module
362    /// picks a daemon code like `module_reloading`.
363    Rejected(ErrorBody),
364    /// The daemon refused the bind itself — a drain or superseded endpoint
365    /// (`module_reloading`), a reply that arrived after the deadline
366    /// (`module_timeout`), or a scope that ended mid-bind. Carries the
367    /// daemon-origin bit because the daemon wrote it. The module may still
368    /// have received the bind (a late reply, or a drain that began after it
369    /// accepted); the daemon then ends that binding with a GOODBYE, so no
370    /// call runs on it.
371    DaemonRejected(ErrorBody),
372    ModuleGone(String),
373}
374
375/// What [`ForwardingTable::cutover_candidate`] did.
376#[derive(Debug, Clone, Copy, PartialEq, Eq)]
377pub(crate) struct ForwardingCutover {
378    /// The endpoint now in the active slot (the former candidate).
379    pub promoted: ModuleEndpointId,
380    /// The endpoint demoted out of the active slot, to be drained by endpoint.
381    /// `None` when the incumbent's connection was already gone.
382    pub incumbent: Option<ModuleEndpointId>,
383}
384
385/// What tearing down one connection released.
386#[derive(Debug)]
387pub(crate) struct ConnectionCleanup {
388    /// Routes whose other end must be sent GOODBYE.
389    pub released: Vec<GoodbyeTarget>,
390    /// Pending route.bind relays to the closed module that were aborted: opens
391    /// that had not been bound yet. Zero when a client connection closed.
392    pub abandoned_relays: u32,
393}
394
395#[derive(Debug, Clone)]
396pub(crate) struct PendingRelayCompletion {
397    pub settled: bool,
398    pub abandoned: Option<GoodbyeTarget>,
399}
400
401#[derive(Debug)]
402pub(crate) struct PendingModuleControlRpc {
403    pub endpoint: ModuleEndpointId,
404    pub module_sink: FrameSink,
405    pub negotiated_ver: u8,
406    pub corr: u64,
407    pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
408}
409
410#[derive(Debug, Clone)]
411pub(crate) enum ModuleControlRpcOutcome {
412    Response(ModuleControlResponse),
413    Rejected(ErrorBody),
414    ModuleGone(String),
415    MalformedResponse(String),
416    UnexpectedOp { expected: String, actual: String },
417    DeadlineElapsed,
418}
419
420#[derive(Debug, Clone, PartialEq, Eq)]
421pub(crate) enum ModuleControlRpcCompletion {
422    Unknown,
423    Settled,
424    LateHealthAnswer {
425        module_id: String,
426        latency: Duration,
427    },
428}
429
430#[derive(Debug)]
431struct PendingModuleControlRpcEntry {
432    expected_op: String,
433    deadline: Instant,
434    health_probe_started_at: Option<Instant>,
435    sender: oneshot::Sender<ModuleControlRpcOutcome>,
436}
437
438#[derive(Debug)]
439struct HealthProbeTombstone {
440    expected_op: String,
441    module_id: String,
442    probe_started_at: Instant,
443    expires_at: Instant,
444}
445
446#[derive(Debug, Clone)]
447struct RouteReservation {
448    client_key: ClientRouteKey,
449    module_key: ModuleRouteKey,
450    client_epoch: u32,
451    module_epoch: u32,
452    project_root: Option<ProjectRootId>,
453}
454
455#[derive(Debug)]
456struct PendingRouteBindRelayEntry {
457    reservation: RouteReservation,
458    client_sink: FrameSink,
459    client_negotiated_ver: u8,
460    client_permit: crate::router::EgressPermit,
461    route_open_frame: Frame,
462    principal: Principal,
463    /// The scope admitted at `route.open`, with the tag captured then. Commit
464    /// compares it with the published tag.
465    scope: Option<BoundScope>,
466    harness: String,
467    build_class: Option<String>,
468    operator_app: Option<crate::operator_apps::OperatorAppStamp>,
469    deadline: Instant,
470    /// Set once `route.open` has finished putting the route.bind on the
471    /// module's egress queue. A path that drops this entry WITHOUT a module
472    /// reply (client teardown, module drain, the open's own abort) sends the
473    /// module a GOODBYE for the bind only when this is set, because only then
474    /// is the GOODBYE guaranteed to queue behind the bind. When it is not set,
475    /// the removal is recorded in `relays_released_before_enqueue` instead and
476    /// `route.open` sends that GOODBYE itself once its send has completed.
477    ///
478    /// A module reply does not consult it: an ack proves the module received
479    /// the bind, so the reply path decides about the GOODBYE on its own.
480    relay_enqueued: bool,
481    sender: oneshot::Sender<RouteBindRelayOutcome>,
482}
483
484/// What `mark_route_bind_relay_enqueued` found once `route.open` had put the
485/// route.bind on the module's queue.
486#[derive(Debug, Clone, Copy, PartialEq, Eq)]
487pub(crate) enum RouteBindRelayMark {
488    /// The relay is still pending; later removals handle their own GOODBYE.
489    Marked,
490    /// The module already answered and the reply path settled the relay:
491    /// it committed the route, or it refused it and sent any GOODBYE the
492    /// module needed. `route.open` must send nothing.
493    SettledByReply,
494    /// The relay was dropped without a module reply while the bind was still
495    /// being queued, so nothing has told the module to drop the binding it
496    /// is about to create. `route.open` must send that GOODBYE.
497    ReleasedWithoutReply,
498}
499
500#[derive(Debug, Clone)]
501pub(crate) enum RouteRelease {
502    Removed(GoodbyeTarget),
503    Stale,
504    Absent,
505}
506
507#[derive(Debug, Clone)]
508pub(crate) enum RoutePollSnapshot {
509    Bound {
510        module_id: String,
511        status: Option<String>,
512    },
513    Absent,
514}
515
516#[derive(Debug, Clone)]
517struct ModuleConnection {
518    endpoint: ModuleEndpointId,
519    sink: FrameSink,
520    negotiated_ver: u8,
521    concurrency: Concurrency,
522}
523
524#[derive(Debug, Default)]
525struct ForwardingInner {
526    operator_confirms: Arc<crate::operator_confirm::OperatorConfirms>,
527    daemon_draining: bool,
528    /// Whether forwarding reached quiescence during daemon shutdown, for each
529    /// active endpoint present when shutdown began. A pending wait is false
530    /// because it has not proved quiescence. Endpoint generations prevent a
531    /// result from describing a replacement connection with the same module id.
532    daemon_drain_results: HashMap<ModuleEndpointId, bool>,
533    /// The ACTIVE slot: the one endpoint per module id that routing resolves.
534    /// Every by-id lookup (relay reservation, drain-by-id, liveness, census,
535    /// live roots, module-control RPCs) reads this map and nothing else.
536    modules_by_id: HashMap<String, ModuleConnection>,
537    /// The CANDIDATE slot of a blue/green swap: a second process registered
538    /// under an id that already has an active endpoint. It has a full endpoint
539    /// identity (so its own connection can be looked up and torn down) but no
540    /// by-id lookup sees it, so nothing is routed to it until `cutover_candidate`
541    /// promotes it.
542    candidates_by_id: HashMap<String, ModuleConnection>,
543    /// Former active endpoints demoted by `cutover_candidate`, until their
544    /// connection is removed. Membership is what distinguishes an endpoint that
545    /// was SUPERSEDED by a promotion (still alive, still carrying bound routes,
546    /// about to be drained) from one that is merely STALE (replaced some other
547    /// way, which is treated as a fault on the acking connection).
548    superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
549    endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
550    module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
551    /// Endpoints mid-drain, keyed to the reason the drain was begun with. The
552    /// value serves the census ("draining because restart"); membership alone
553    /// still answers every admission-gate check.
554    draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
555    closing_connections: HashSet<ConnectionId>,
556    next_generation: u64,
557    reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
558    reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
559    next_client_channel: HashMap<ConnectionId, u16>,
560    next_module_channel: HashMap<ModuleEndpointId, u16>,
561    client_slot_epochs: HashMap<ClientRouteKey, u32>,
562    module_slot_epochs: HashMap<ModuleRouteKey, u32>,
563    last_published_epoch: HashMap<ClientRouteKey, u32>,
564    client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
565    module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
566    status: HashMap<(ClientRouteKey, u32), String>,
567    pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
568    /// Pending relays removed without a module reply before `route.open`
569    /// marked them enqueued. `route.open` consumes its key when it marks the
570    /// relay (and sends the GOODBYE the removal could not send safely) or when
571    /// it aborts, so an entry lives only while that open is mid-send.
572    relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
573    /// Each scope's current `(scope_epoch, version)`, keyed `(owner, ref)`,
574    /// published by a scope sync in the same step that changes the record. A
575    /// bind commit reads it here so it never takes the scope table's lock
576    /// while holding this one. Absent means the scope is not live.
577    scope_tags: HashMap<(String, String), ScopeTag>,
578    next_control_corr: HashMap<ModuleEndpointId, u64>,
579    pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
580    health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
581}
582
583#[derive(Debug, Clone)]
584pub(crate) struct CloseReason {
585    code: &'static str,
586    message: String,
587}
588
589impl CloseReason {
590    pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
591        Self {
592            code,
593            message: message.into(),
594        }
595    }
596}
597
598impl fmt::Display for CloseReason {
599    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
600        write!(f, "{}: {}", self.code, self.message)
601    }
602}
603
604pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
605
606/// Dynamic forwarding state shared by the control plane and data-plane router.
607#[derive(Debug, Default)]
608pub struct ForwardingTable {
609    inner: Arc<RwLock<ForwardingInner>>,
610    frame_drop_warnings: Mutex<HashMap<ConnectionId, ChannelDropWarnings>>,
611    #[cfg(feature = "test-support")]
612    route_closing_hold: Mutex<Option<oneshot::Receiver<()>>>,
613    close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
614    counters: DaemonCounters,
615    /// Per-target-module bind-relay breaker state. It lives here, beside the
616    /// module connections it describes, because this is where a module
617    /// connection's identity is established and therefore where a stale
618    /// verdict has to be discarded.
619    route_bind_breakers: RouteBindBreakers,
620    /// Current route.bind relays keyed by target module. Admission is shared
621    /// across every client connection that points at the same endpoint.
622    route_bind_concurrency: RouteBindConcurrency,
623    /// Start and end of each module's `route.open` outage. Held here for the
624    /// same reason as the breakers: every handler built over this table must
625    /// see one outage per module, or each would log its own opening line.
626    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
627}
628
629#[derive(Debug)]
630struct FrameDropWarning {
631    last_warned: tokio::time::Instant,
632    suppressed: u64,
633}
634
635type ChannelDropWarnings = HashMap<(u16, &'static str), FrameDropWarning>;
636
637impl ForwardingTable {
638    /// Keep a silent route diagnosable without letting a broken peer flood logs.
639    /// Epoch and direction describe the dropped frame, but do not split its limit.
640    pub(crate) fn warn_frame_drop(
641        &self,
642        connection_id: ConnectionId,
643        channel: u16,
644        epoch: u32,
645        direction: &'static str,
646        reason: &'static str,
647    ) {
648        let now = tokio::time::Instant::now();
649        let mut warnings = self
650            .frame_drop_warnings
651            .lock()
652            .unwrap_or_else(std::sync::PoisonError::into_inner);
653        let entry = warnings
654            .entry(connection_id)
655            .or_default()
656            .entry((channel, reason));
657        let suppressed = match entry {
658            std::collections::hash_map::Entry::Occupied(mut entry) => {
659                let state = entry.get_mut();
660                if now.saturating_duration_since(state.last_warned) < Duration::from_secs(60) {
661                    state.suppressed = state.suppressed.saturating_add(1);
662                    return;
663                }
664                state.last_warned = now;
665                std::mem::take(&mut state.suppressed)
666            }
667            std::collections::hash_map::Entry::Vacant(entry) => {
668                entry.insert(FrameDropWarning {
669                    last_warned: now,
670                    suppressed: 0,
671                });
672                0
673            }
674        };
675        warn!(
676            connection_id = connection_id.get(),
677            channel, epoch, direction, reason, suppressed, "dropping data-plane frame"
678        );
679    }
680
681    /// Install scripted operator confirmation for an in-process integration harness,
682    /// without bootstrapping a daemon process. Only available with `test-support`.
683    ///
684    /// The script has one response per line: `approve`, `decline`, `unavailable`,
685    /// `hang` (until withdrawn), or `approve when <path>` (wait for the path to
686    /// exist). Unreadable or exhausted scripts fail closed. If supplied, `events`
687    /// receives the provider's JSON-lines event log. `limits: None` uses
688    /// [`OperatorLimits::default`](crate::operator_confirm::OperatorLimits); use
689    /// `Some(OperatorLimits { queue_wait, prompt_timeout, stuck_grace })` to
690    /// override the queue wait, prompt timeout and stuck-provider grace periods.
691    ///
692    /// The current executable's file name **must start with `ckdev-`**. Otherwise
693    /// this logs a warning and keeps the OS provider, just like bootstrap.
694    /// `cortexkit-test-support`'s `ckdev_binary` copies a harness executable under
695    /// a qualifying name; run that copy to enable scripted confirmation. See
696    /// `crates/subc-client-rs/tests/real_daemon/operator_confirm.rs` for the
697    /// reference end-to-end harness.
698    ///
699    /// Configure the table shared by the harness's `ControlHandler` and `Router`
700    /// before serving any requests. This also configures privileged confirmations
701    /// that use the same table.
702    ///
703    /// # Panics
704    ///
705    /// Panics if operator-confirm processing has already started.
706    ///
707    /// # Example
708    ///
709    /// ```no_run
710    /// use std::{path::Path, sync::Arc};
711    /// use subc_daemon::{ControlHandler, ForwardingTable, Registry};
712    ///
713    /// let registry = Arc::new(Registry::default());
714    /// let forwarding = Arc::new(ForwardingTable::default());
715    /// forwarding.configure_operator_script(
716    ///     Path::new("operator.script"),
717    ///     Some(Path::new("operator.events")),
718    ///     None,
719    /// );
720    /// let handler = ControlHandler::with_forwarding(registry, forwarding);
721    /// ```
722    #[cfg(feature = "test-support")]
723    pub fn configure_operator_script(
724        &self,
725        script: &std::path::Path,
726        events: Option<&std::path::Path>,
727        limits: Option<crate::operator_confirm::OperatorLimits>,
728    ) {
729        self.operator_confirms()
730            .configure_script(script, events, limits);
731    }
732
733    /// Hold the next supervisor drain between enqueueing `module.draining` and
734    /// `route.closing`, so a test can forward a response in that window. Dropping
735    /// the returned sender also releases the hold, including on a test panic.
736    #[cfg(feature = "test-support")]
737    pub fn hold_route_closing_for_test(&self) -> oneshot::Sender<()> {
738        let (release, hold) = oneshot::channel();
739        *self.route_closing_hold.lock().unwrap() = Some(hold);
740        release
741    }
742
743    #[cfg(feature = "test-support")]
744    pub(crate) async fn wait_before_route_closing_for_test(&self) {
745        let hold = self.route_closing_hold.lock().unwrap().take();
746        if let Some(hold) = hold {
747            let _ = hold.await;
748        }
749    }
750
751    pub(crate) fn operator_confirms(&self) -> Arc<crate::operator_confirm::OperatorConfirms> {
752        Arc::clone(
753            &self
754                .inner
755                .read()
756                .unwrap_or_else(|p| p.into_inner())
757                .operator_confirms,
758        )
759    }
760
761    #[cfg(test)]
762    pub(crate) fn inject_operator_principal(&self, key: ModuleRouteKey, principal: Principal) {
763        self.alter_operator_binding_for_test(key, |binding| binding.principal = principal);
764    }
765
766    #[cfg(test)]
767    pub(crate) fn alter_operator_binding_for_test(
768        &self,
769        key: ModuleRouteKey,
770        alter: impl FnOnce(&mut RouteBinding),
771    ) {
772        let mut inner = self.write_inner().unwrap();
773        let old = inner.module_to_client.remove(&key).unwrap();
774        let client = ClientRouteKey {
775            connection_id: old.client_connection_id,
776            channel: old.client_channel,
777        };
778        inner.client_to_module.remove(&client);
779        let mut binding = Arc::try_unwrap(old).expect("test binding has no outstanding readers");
780        alter(&mut binding);
781        let binding = Arc::new(binding);
782        inner.client_to_module.insert(client, Arc::clone(&binding));
783        inner.module_to_client.insert(key, binding);
784    }
785
786    /// Mutate the route without delivering its withdrawal notice, so tests can
787    /// exercise the commit-point route check rather than the withdrawal path.
788    #[cfg(test)]
789    pub(crate) fn invalidate_operator_route_for_test(&self, key: ModuleRouteKey, stale: bool) {
790        let mut inner = self.write_inner().unwrap();
791        let old = inner.module_to_client.remove(&key).unwrap();
792        let client = ClientRouteKey {
793            connection_id: old.client_connection_id,
794            channel: old.client_channel,
795        };
796        inner.client_to_module.remove(&client);
797        if stale {
798            let mut binding = Arc::try_unwrap(old).expect("no binding readers at the test pause");
799            binding.module_epoch += 1;
800            let binding = Arc::new(binding);
801            inner.client_to_module.insert(client, binding.clone());
802            inner.module_to_client.insert(key, binding);
803        }
804    }
805
806    /// Keep the binding protected through confirmation admission. Route release
807    /// takes the write lock and then the same confirm-state lock, so it cannot
808    /// miss an admitted request or leave it waiting on a vanished route.
809    pub(crate) fn with_operator_route<T>(
810        &self,
811        connection_id: ConnectionId,
812        channel: u16,
813        epoch: u32,
814        admit: impl FnOnce(Option<&RouteBinding>) -> T,
815    ) -> Result<T, ForwardingError> {
816        self.with_operator_route_epoch(connection_id, channel, |binding| {
817            admit(binding.filter(|binding| binding.module_epoch == epoch))
818        })
819    }
820
821    /// Like admission, but retains a mismatched epoch so a shortcut can audit
822    /// a stale channel separately from a closed one, under the same read lock.
823    pub(crate) fn with_operator_route_epoch<T>(
824        &self,
825        connection_id: ConnectionId,
826        channel: u16,
827        decide: impl FnOnce(Option<&RouteBinding>) -> T,
828    ) -> Result<T, ForwardingError> {
829        let inner = self.read_inner()?;
830        let binding = inner
831            .endpoint_by_connection
832            .get(&connection_id)
833            .and_then(|endpoint| {
834                inner.module_to_client.get(&ModuleRouteKey {
835                    endpoint: *endpoint,
836                    channel,
837                })
838            });
839        Ok(decide(binding.map(Arc::as_ref)))
840    }
841
842    pub(crate) fn counters(&self) -> DaemonCounters {
843        self.counters.clone()
844    }
845
846    pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
847        self.route_bind_breakers.clone()
848    }
849
850    pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
851        self.route_bind_concurrency.clone()
852    }
853
854    pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
855        Arc::clone(&self.route_outages)
856    }
857
858    pub(crate) fn register_connection_close(
859        &self,
860        connection_id: ConnectionId,
861    ) -> ConnectionCloseReceiver {
862        let (sender, receiver) = oneshot::channel();
863        let replaced = self
864            .lock_close_registry()
865            .insert(connection_id, sender)
866            .is_some();
867        if replaced {
868            warn!(
869                connection_id = connection_id.get(),
870                "replaced existing connection close registration"
871            );
872        }
873        receiver
874    }
875
876    pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
877        self.lock_close_registry().remove(&connection_id);
878    }
879
880    /// Close every established connection, modules and clients alike, as the
881    /// last step of an announced daemon shutdown. By the time this runs each
882    /// registered module has already been sent a module GOODBYE (see
883    /// `Supervisor::end_children_for_daemon_shutdown`), so the EOF that follows
884    /// is a planned stop; EOF with no GOODBYE before it stays reserved for a
885    /// daemon that went away unannounced. Closing while the daemon is still
886    /// alive lets it wait for the modules' own teardowns before it ends
887    /// whatever is left.
888    #[cfg(unix)]
889    pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
890        let senders: Vec<_> = self.lock_close_registry().drain().collect();
891        let count = senders.len();
892        for (_, sender) in senders {
893            let _ = sender.send(reason.clone());
894        }
895        count
896    }
897
898    /// Every registered module connection, whichever slot it occupies (active,
899    /// swap candidate, or superseded incumbent), once each. Daemon shutdown
900    /// uses this to tell each of them it is a planned stop before closing it.
901    #[cfg(unix)]
902    pub(crate) fn module_connections(
903        &self,
904    ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
905        let inner = self.read_inner()?;
906        let mut seen = HashSet::new();
907        Ok(inner
908            .modules_by_id
909            .values()
910            .chain(inner.candidates_by_id.values())
911            .chain(inner.superseded_endpoints.values())
912            .filter(|module| seen.insert(module.endpoint))
913            .map(|module| ModuleConnectionTarget {
914                module_id: inner
915                    .module_id_by_endpoint
916                    .get(&module.endpoint)
917                    .cloned()
918                    .unwrap_or_default(),
919                endpoint: module.endpoint,
920                sink: module.sink.clone(),
921                negotiated_ver: module.negotiated_ver,
922            })
923            .collect())
924    }
925
926    /// Ask a registered connection to close. Returns true only for the request
927    /// that actually reached it; later requests for the same connection, and
928    /// requests for a connection that is not registered, return false.
929    pub(crate) fn request_connection_close(
930        &self,
931        connection_id: ConnectionId,
932        reason: CloseReason,
933    ) -> bool {
934        let sender = self.lock_close_registry().remove(&connection_id);
935        if let Some(sender) = sender {
936            debug!(
937                connection_id = connection_id.get(),
938                close_reason = %reason,
939                "requesting connection close"
940            );
941            let _ = sender.send(reason);
942            true
943        } else {
944            debug!(
945                connection_id = connection_id.get(),
946                close_reason = %reason,
947                "connection close request ignored for inactive connection"
948            );
949            false
950        }
951    }
952
953    pub fn register_module_connection(
954        &self,
955        connection_id: ConnectionId,
956        module_id: String,
957        negotiated_ver: u8,
958        concurrency: Concurrency,
959        sink: FrameSink,
960    ) -> Result<ModuleEndpointId, ForwardingError> {
961        self.register_module_connection_inner(
962            connection_id,
963            module_id,
964            negotiated_ver,
965            concurrency,
966            sink,
967            None,
968        )
969    }
970
971    /// Register a module connection and queue its HELLO_ACK as the first frame
972    /// on its sink, in the same write-lock critical section that makes the
973    /// endpoint visible.
974    ///
975    /// A module reads HELLO_ACK as the first frame after its HELLO and exits on
976    /// anything else. Once the endpoint is in `modules_by_id`, a `route.open`
977    /// on any other connection can queue a `route.bind` request onto this sink,
978    /// so the ack must already be queued by then. Every lookup that can queue a
979    /// frame for the module takes this lock, so nothing can get in ahead of it.
980    ///
981    /// If the ack cannot be queued (the sink is closed or full), the
982    /// registration fails with nothing inserted.
983    pub(crate) fn register_module_connection_acked(
984        &self,
985        connection_id: ConnectionId,
986        module_id: String,
987        negotiated_ver: u8,
988        concurrency: Concurrency,
989        sink: FrameSink,
990        hello_ack: Frame,
991    ) -> Result<ModuleEndpointId, ForwardingError> {
992        self.register_module_connection_inner(
993            connection_id,
994            module_id,
995            negotiated_ver,
996            concurrency,
997            sink,
998            Some(hello_ack),
999        )
1000    }
1001
1002    fn register_module_connection_inner(
1003        &self,
1004        connection_id: ConnectionId,
1005        module_id: String,
1006        negotiated_ver: u8,
1007        concurrency: Concurrency,
1008        sink: FrameSink,
1009        hello_ack: Option<Frame>,
1010    ) -> Result<ModuleEndpointId, ForwardingError> {
1011        let mut inner = self.write_inner()?;
1012        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1013            return Err(ForwardingError::ConnectionClosing { connection_id });
1014        }
1015        check_module_connection_role_locked(&inner, connection_id)?;
1016        // Every refusal check has passed and nothing has been mutated yet, so
1017        // a failed enqueue leaves the table exactly as it was.
1018        enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1019
1020        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1021        let endpoint = ModuleEndpointId {
1022            connection_id,
1023            generation: inner.next_generation,
1024        };
1025        inner.endpoint_by_connection.insert(connection_id, endpoint);
1026        inner
1027            .module_id_by_endpoint
1028            .insert(endpoint, module_id.clone());
1029        inner.next_module_channel.insert(endpoint, 1);
1030        inner.next_control_corr.insert(endpoint, 1);
1031        inner.modules_by_id.insert(
1032            module_id.clone(),
1033            ModuleConnection {
1034                endpoint,
1035                sink,
1036                negotiated_ver,
1037                concurrency,
1038            },
1039        );
1040        drop(inner);
1041
1042        // A new module connection has arrived under this id, so anything the
1043        // bind-relay breaker learned was learned about a process that is no
1044        // longer the one behind this name. See
1045        // `RouteBindBreakers::reset_for_new_module_connection`.
1046        //
1047        // Keyed on ARRIVAL rather than on teardown deliberately: a connection
1048        // going away is not evidence about anything, and a module whose
1049        // connection drops without coming back should keep its verdict until
1050        // something actually registers in its place. This also covers the
1051        // unclean replacements -- a module killed mid-bind, or one whose
1052        // connection was closing -- because registration is the single path by
1053        // which any module connection becomes usable.
1054        if let Some(discarded) = self
1055            .route_bind_breakers
1056            .reset_for_new_module_connection(&module_id)
1057        {
1058            info!(
1059                module_id = %module_id,
1060                discarded_consecutive_timeouts = discarded,
1061                "route.bind breaker state discarded: a new module connection replaced the process it described"
1062            );
1063        }
1064        Ok(endpoint)
1065    }
1066
1067    /// Register a blue/green swap candidate for `module_id` into the candidate
1068    /// slot, alongside whatever endpoint is active for the id.
1069    ///
1070    /// Unlike [`Self::register_module_connection`], this never touches the
1071    /// active slot and never resets the bind-relay breaker: the incumbent is
1072    /// still the process serving the id, so what the breaker learned about it
1073    /// is still true. The breaker is reset at cutover instead, when the process
1074    /// behind the name actually changes. A second candidate for the same id is
1075    /// refused.
1076    ///
1077    /// Production registers candidates through
1078    /// [`Self::register_candidate_module_connection_acked`]; this ack-less form
1079    /// is for tests that build forwarding state directly.
1080    #[cfg(test)]
1081    pub(crate) fn register_candidate_module_connection(
1082        &self,
1083        connection_id: ConnectionId,
1084        module_id: String,
1085        negotiated_ver: u8,
1086        concurrency: Concurrency,
1087        sink: FrameSink,
1088    ) -> Result<ModuleEndpointId, ForwardingError> {
1089        self.register_candidate_module_connection_inner(
1090            connection_id,
1091            module_id,
1092            negotiated_ver,
1093            concurrency,
1094            sink,
1095            None,
1096        )
1097    }
1098
1099    /// The swap-candidate counterpart of
1100    /// [`Self::register_module_connection_acked`]: the HELLO_ACK is queued on
1101    /// the candidate's sink before its endpoint is inserted. No by-id lookup
1102    /// sees a candidate, but its own connection's endpoint does become
1103    /// resolvable here, and at cutover it becomes routable; the ack has to be
1104    /// ahead of anything either can queue.
1105    pub(crate) fn register_candidate_module_connection_acked(
1106        &self,
1107        connection_id: ConnectionId,
1108        module_id: String,
1109        negotiated_ver: u8,
1110        concurrency: Concurrency,
1111        sink: FrameSink,
1112        hello_ack: Frame,
1113    ) -> Result<ModuleEndpointId, ForwardingError> {
1114        self.register_candidate_module_connection_inner(
1115            connection_id,
1116            module_id,
1117            negotiated_ver,
1118            concurrency,
1119            sink,
1120            Some(hello_ack),
1121        )
1122    }
1123
1124    fn register_candidate_module_connection_inner(
1125        &self,
1126        connection_id: ConnectionId,
1127        module_id: String,
1128        negotiated_ver: u8,
1129        concurrency: Concurrency,
1130        sink: FrameSink,
1131        hello_ack: Option<Frame>,
1132    ) -> Result<ModuleEndpointId, ForwardingError> {
1133        let mut inner = self.write_inner()?;
1134        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1135            return Err(ForwardingError::ConnectionClosing { connection_id });
1136        }
1137        check_module_connection_role_locked(&inner, connection_id)?;
1138        if inner.candidates_by_id.contains_key(&module_id) {
1139            return Err(ForwardingError::CandidateSlotOccupied { module_id });
1140        }
1141        // As in `register_module_connection_inner`: after every refusal check,
1142        // before any mutation.
1143        enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1144
1145        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1146        let endpoint = ModuleEndpointId {
1147            connection_id,
1148            generation: inner.next_generation,
1149        };
1150        inner.endpoint_by_connection.insert(connection_id, endpoint);
1151        inner
1152            .module_id_by_endpoint
1153            .insert(endpoint, module_id.clone());
1154        inner.next_module_channel.insert(endpoint, 1);
1155        inner.next_control_corr.insert(endpoint, 1);
1156        inner.candidates_by_id.insert(
1157            module_id,
1158            ModuleConnection {
1159                endpoint,
1160                sink,
1161                negotiated_ver,
1162                concurrency,
1163            },
1164        );
1165        Ok(endpoint)
1166    }
1167
1168    /// Promote `module_id`'s candidate to the active slot, in ONE forwarding
1169    /// write-lock critical section.
1170    ///
1171    /// That single section is the linearization point of a swap. Every relay
1172    /// reservation resolves the active slot under the same lock, so each one
1173    /// lands wholly before cutover (on the incumbent) or wholly after it (on the
1174    /// promoted candidate), never on a mix. A relay reserved on the incumbent
1175    /// before cutover can no longer commit: `commit_route_locked` requires the
1176    /// reservation's endpoint to be the active one, and an ack for it arriving
1177    /// later is answered by the superseded-endpoint arm of
1178    /// `complete_pending_relay` instead of ending the incumbent's connection.
1179    ///
1180    /// Both endpoints keep their identities: nothing is re-keyed, so the
1181    /// incumbent's bound routes and its pending correlation keys stay exactly
1182    /// where they are until it is drained with [`Self::begin_endpoint_drain`],
1183    /// using the incumbent endpoint this returns. Returns `Ok(None)` when there
1184    /// is no candidate for the id.
1185    pub(crate) fn cutover_candidate(
1186        &self,
1187        module_id: &str,
1188    ) -> Result<Option<ForwardingCutover>, ForwardingError> {
1189        let mut inner = self.write_inner()?;
1190        if inner.daemon_draining {
1191            return Err(ForwardingError::ModuleReloading {
1192                module_id: module_id.to_string(),
1193            });
1194        }
1195        let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
1196            return Ok(None);
1197        };
1198        let promoted = candidate.endpoint;
1199        let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
1200        let incumbent = incumbent.map(|incumbent| {
1201            let endpoint = incumbent.endpoint;
1202            inner.superseded_endpoints.insert(endpoint, incumbent);
1203            endpoint
1204        });
1205        drop(inner);
1206
1207        // A different process now answers for this id; see the matching reset
1208        // in `register_module_connection` for why a verdict about the old one
1209        // must not carry over.
1210        if let Some(discarded) = self
1211            .route_bind_breakers
1212            .reset_for_new_module_connection(module_id)
1213        {
1214            info!(
1215                module_id = %module_id,
1216                discarded_consecutive_timeouts = discarded,
1217                "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
1218            );
1219        }
1220        Ok(Some(ForwardingCutover {
1221            promoted,
1222            incumbent,
1223        }))
1224    }
1225
1226    #[allow(clippy::too_many_arguments)]
1227    pub(crate) async fn begin_route_bind_relay_for(
1228        &self,
1229        client_connection_id: ConnectionId,
1230        client_sink: FrameSink,
1231        client_negotiated_ver: u8,
1232        client_corr: u64,
1233        module_id: &str,
1234        principal: Principal,
1235        scope: Option<BoundScope>,
1236        project_root: Option<ProjectRootId>,
1237        harness: String,
1238        build_class: Option<String>,
1239        operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1240        deadline: Instant,
1241    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1242        // Reserve egress capacity before taking the forwarding lock. The permit is
1243        // held until the bind reaches one terminal state, so an accepted bind can
1244        // publish its table entry and RouteOpen response in one critical section.
1245        let client_permit =
1246            client_sink
1247                .reserve_owned()
1248                .await
1249                .map_err(|_| ForwardingError::ClientEgressClosed {
1250                    connection_id: client_connection_id,
1251                })?;
1252        self.begin_route_bind_relay_inner(
1253            client_connection_id,
1254            client_sink,
1255            client_negotiated_ver,
1256            client_corr,
1257            module_id,
1258            principal,
1259            scope,
1260            project_root,
1261            harness,
1262            build_class,
1263            operator_app,
1264            deadline,
1265            client_permit,
1266        )
1267    }
1268
1269    #[cfg(test)]
1270    pub(crate) fn begin_route_bind_relay_for_test(
1271        &self,
1272        client_connection_id: ConnectionId,
1273        client_sink: FrameSink,
1274        client_corr: u64,
1275        module_id: &str,
1276    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1277        let permit =
1278            client_sink
1279                .try_reserve_owned()
1280                .map_err(|_| ForwardingError::ClientEgressClosed {
1281                    connection_id: client_connection_id,
1282                })?;
1283        self.begin_route_bind_relay_inner(
1284            client_connection_id,
1285            client_sink,
1286            subc_protocol::PROTOCOL_VERSION,
1287            client_corr,
1288            module_id,
1289            Principal::Direct,
1290            None,
1291            None,
1292            String::new(),
1293            None,
1294            None,
1295            Instant::now() + std::time::Duration::from_secs(60),
1296            permit,
1297        )
1298    }
1299
1300    pub(crate) fn begin_module_control_rpc_for(
1301        &self,
1302        module_id: &str,
1303        expected_op: &str,
1304        deadline: Instant,
1305    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1306        self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1307    }
1308
1309    pub(crate) fn begin_health_probe_rpc_for(
1310        &self,
1311        module_id: &str,
1312        expected_op: &str,
1313        probe_started_at: Instant,
1314        deadline: Instant,
1315    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1316        self.begin_module_control_rpc_inner(
1317            module_id,
1318            expected_op,
1319            deadline,
1320            Some(probe_started_at),
1321            false,
1322        )
1323    }
1324
1325    pub(crate) fn begin_drain_health_probe_rpc_for(
1326        &self,
1327        module_id: &str,
1328        expected_op: &str,
1329        probe_started_at: Instant,
1330        deadline: Instant,
1331    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1332        self.begin_module_control_rpc_inner(
1333            module_id,
1334            expected_op,
1335            deadline,
1336            Some(probe_started_at),
1337            true,
1338        )
1339    }
1340
1341    /// A health probe addressed to one endpoint, in whichever slot it is.
1342    ///
1343    /// Every other control RPC resolves the id's ACTIVE endpoint, which is how
1344    /// a swap candidate (not active until cutover) and a superseded incumbent
1345    /// (not active after it) are unreachable by them. A swap needs to probe
1346    /// exactly those two: the candidate before promoting it, and the incumbent
1347    /// for its busy gauges while it drains.
1348    pub(crate) fn begin_endpoint_health_probe_rpc_for(
1349        &self,
1350        endpoint: ModuleEndpointId,
1351        expected_op: &str,
1352        probe_started_at: Instant,
1353        deadline: Instant,
1354    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1355        let inner = self.write_inner()?;
1356        let module = module_connection_for_endpoint_locked(&inner, endpoint)
1357            .cloned()
1358            .ok_or(ForwardingError::NoModuleConnection)?;
1359        let module_id = inner
1360            .module_id_by_endpoint
1361            .get(&endpoint)
1362            .cloned()
1363            .unwrap_or_default();
1364        // A draining incumbent is exactly what this probes for busy gauges, so
1365        // draining is allowed, as it is for the by-id drain probe.
1366        self.begin_control_rpc_locked(
1367            inner,
1368            &module_id,
1369            module,
1370            expected_op,
1371            deadline,
1372            Some(probe_started_at),
1373            true,
1374        )
1375    }
1376
1377    fn begin_module_control_rpc_inner(
1378        &self,
1379        module_id: &str,
1380        expected_op: &str,
1381        deadline: Instant,
1382        health_probe_started_at: Option<Instant>,
1383        allow_draining: bool,
1384    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1385        let inner = self.write_inner()?;
1386        let module = inner
1387            .modules_by_id
1388            .get(module_id)
1389            .cloned()
1390            .ok_or(ForwardingError::NoModuleConnection)?;
1391        self.begin_control_rpc_locked(
1392            inner,
1393            module_id,
1394            module,
1395            expected_op,
1396            deadline,
1397            health_probe_started_at,
1398            allow_draining,
1399        )
1400    }
1401
1402    #[allow(clippy::too_many_arguments)]
1403    fn begin_control_rpc_locked(
1404        &self,
1405        mut inner: RwLockWriteGuard<'_, ForwardingInner>,
1406        module_id: &str,
1407        module: ModuleConnection,
1408        expected_op: &str,
1409        deadline: Instant,
1410        health_probe_started_at: Option<Instant>,
1411        allow_draining: bool,
1412    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1413        if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1414            return Err(ForwardingError::ModuleReloading {
1415                module_id: module_id.to_string(),
1416            });
1417        }
1418        if inner
1419            .closing_connections
1420            .contains(&module.endpoint.connection_id)
1421        {
1422            return Err(ForwardingError::ConnectionClosing {
1423                connection_id: module.endpoint.connection_id,
1424            });
1425        }
1426        if health_probe_started_at.is_some() {
1427            // Recurring health probes are serial per endpoint. Once the next one
1428            // starts, an older answer can no longer improve the current snapshot,
1429            // so retaining more than the newest unanswered probe has no value.
1430            inner
1431                .health_probe_tombstones
1432                .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1433        }
1434        let corr = match inner.allocate_control_corr(module.endpoint) {
1435            Ok(corr) => corr,
1436            Err(err) => {
1437                drop(inner);
1438                self.request_connection_close(
1439                    module.endpoint.connection_id,
1440                    CloseReason::new(
1441                        "control_correlation_exhausted",
1442                        "daemon-originated channel-0 correlation space exhausted",
1443                    ),
1444                );
1445                return Err(err);
1446            }
1447        };
1448        let (sender, receiver) = oneshot::channel();
1449        inner.pending_control_rpcs.insert(
1450            (module.endpoint, corr),
1451            PendingModuleControlRpcEntry {
1452                expected_op: expected_op.to_string(),
1453                deadline,
1454                health_probe_started_at,
1455                sender,
1456            },
1457        );
1458
1459        Ok(PendingModuleControlRpc {
1460            endpoint: module.endpoint,
1461            module_sink: module.sink,
1462            negotiated_ver: module.negotiated_ver,
1463            corr,
1464            receiver,
1465        })
1466    }
1467
1468    #[allow(clippy::too_many_arguments)]
1469    fn begin_route_bind_relay_inner(
1470        &self,
1471        client_connection_id: ConnectionId,
1472        client_sink: FrameSink,
1473        client_negotiated_ver: u8,
1474        client_corr: u64,
1475        expected_module_id: &str,
1476        principal: Principal,
1477        scope: Option<BoundScope>,
1478        project_root: Option<ProjectRootId>,
1479        harness: String,
1480        build_class: Option<String>,
1481        operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1482        deadline: Instant,
1483        client_permit: crate::router::EgressPermit,
1484    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1485        let mut inner = self.write_inner()?;
1486        // HELLO and route.open can run on different tasks. Decide the role under
1487        // the same lock that installs endpoints and reserves client channels so
1488        // neither task can turn one connection into both kinds of owner.
1489        if inner
1490            .endpoint_by_connection
1491            .contains_key(&client_connection_id)
1492        {
1493            return Err(ForwardingError::ConnectionRoleConflict {
1494                connection_id: client_connection_id,
1495            });
1496        }
1497        if inner.closing_connections.contains(&client_connection_id) {
1498            return Err(ForwardingError::ConnectionClosing {
1499                connection_id: client_connection_id,
1500            });
1501        }
1502        let module = inner
1503            .modules_by_id
1504            .get(expected_module_id)
1505            .cloned()
1506            .ok_or(ForwardingError::NoModuleConnection)?;
1507        if inner.draining_endpoints.contains_key(&module.endpoint) {
1508            return Err(ForwardingError::ModuleReloading {
1509                module_id: expected_module_id.to_string(),
1510            });
1511        }
1512        if inner
1513            .closing_connections
1514            .contains(&module.endpoint.connection_id)
1515        {
1516            return Err(ForwardingError::ConnectionClosing {
1517                connection_id: module.endpoint.connection_id,
1518            });
1519        }
1520
1521        let corr = match inner.allocate_control_corr(module.endpoint) {
1522            Ok(corr) => corr,
1523            Err(err) => {
1524                drop(inner);
1525                self.request_connection_close(
1526                    module.endpoint.connection_id,
1527                    CloseReason::new(
1528                        "control_correlation_exhausted",
1529                        "daemon-originated channel-0 correlation space exhausted",
1530                    ),
1531                );
1532                return Err(err);
1533            }
1534        };
1535        let (client_channel, client_epoch, module_channel, module_epoch) =
1536            inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1537        let client_key = ClientRouteKey {
1538            connection_id: client_connection_id,
1539            channel: client_channel,
1540        };
1541        let module_key = ModuleRouteKey {
1542            endpoint: module.endpoint,
1543            channel: module_channel,
1544        };
1545        let reservation = RouteReservation {
1546            client_key,
1547            module_key,
1548            client_epoch,
1549            module_epoch,
1550            project_root,
1551        };
1552        let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
1553            route_channel: client_channel,
1554            route_epoch: client_epoch,
1555        })
1556        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1557        let route_open_frame = Frame::build_with_version(
1558            client_negotiated_ver,
1559            FrameType::Response,
1560            Flags::new(false, Priority::Passive, false),
1561            0,
1562            0,
1563            client_corr,
1564            response_body,
1565        )
1566        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1567        let (sender, receiver) = oneshot::channel();
1568        inner.reserved_client.insert(client_key, module_key);
1569        inner.reserved_module.insert(module_key, client_key);
1570        inner.pending_relays.insert(
1571            (module.endpoint, corr),
1572            PendingRouteBindRelayEntry {
1573                reservation,
1574                client_sink,
1575                client_negotiated_ver,
1576                client_permit,
1577                route_open_frame,
1578                principal,
1579                scope,
1580                harness,
1581                build_class,
1582                operator_app,
1583                deadline,
1584                relay_enqueued: false,
1585                sender,
1586            },
1587        );
1588
1589        Ok(PendingRouteBindRelay {
1590            endpoint: module.endpoint,
1591            module_sink: module.sink,
1592            negotiated_ver: module.negotiated_ver,
1593            client_channel,
1594            client_epoch,
1595            module_channel,
1596            module_epoch,
1597            corr,
1598            receiver,
1599        })
1600    }
1601
1602    /// Called by `route.open` right after the route.bind entered the module's
1603    /// egress queue.
1604    ///
1605    /// The module reader runs concurrently with `route.open`, so a fast module
1606    /// can receive the bind, ack it, and have the reader commit the route
1607    /// before this runs. A missing entry therefore does not mean the bind was
1608    /// abandoned: only a removal recorded in `relays_released_before_enqueue`
1609    /// does. Telling the two apart is what stops `route.open` from sending a
1610    /// GOODBYE for a route that was just committed.
1611    pub(crate) fn mark_route_bind_relay_enqueued(
1612        &self,
1613        endpoint: ModuleEndpointId,
1614        corr: u64,
1615    ) -> Result<RouteBindRelayMark, ForwardingError> {
1616        let mut inner = self.write_inner()?;
1617        if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
1618            pending.relay_enqueued = true;
1619            return Ok(RouteBindRelayMark::Marked);
1620        }
1621        if inner
1622            .relays_released_before_enqueue
1623            .remove(&(endpoint, corr))
1624        {
1625            return Ok(RouteBindRelayMark::ReleasedWithoutReply);
1626        }
1627        Ok(RouteBindRelayMark::SettledByReply)
1628    }
1629
1630    pub(crate) fn release_client_route(
1631        &self,
1632        client_connection_id: ConnectionId,
1633        client_channel: u16,
1634        expected_epoch: u32,
1635    ) -> Result<RouteRelease, ForwardingError> {
1636        let mut inner = self.write_inner()?;
1637        let release = release_client_route_locked(
1638            &mut inner,
1639            ClientRouteKey {
1640                connection_id: client_connection_id,
1641                channel: client_channel,
1642            },
1643            expected_epoch,
1644        );
1645        self.record_route_release(&release);
1646        Ok(release)
1647    }
1648
1649    pub(crate) fn release_module_route(
1650        &self,
1651        module_connection_id: ConnectionId,
1652        module_channel: u16,
1653        expected_epoch: u32,
1654    ) -> Result<RouteRelease, ForwardingError> {
1655        let mut inner = self.write_inner()?;
1656        let Some(endpoint) = inner
1657            .endpoint_by_connection
1658            .get(&module_connection_id)
1659            .copied()
1660        else {
1661            return Ok(RouteRelease::Absent);
1662        };
1663        let release = release_module_route_locked(
1664            &mut inner,
1665            ModuleRouteKey {
1666                endpoint,
1667                channel: module_channel,
1668            },
1669            expected_epoch,
1670        );
1671        self.record_route_release(&release);
1672        Ok(release)
1673    }
1674
1675    pub(crate) fn abort_pending_relay(
1676        &self,
1677        endpoint: ModuleEndpointId,
1678        corr: u64,
1679        outcome: RouteBindRelayOutcome,
1680    ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
1681        let mut inner = self.write_inner()?;
1682        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1683            // If another path dropped the relay while `route.open` was still
1684            // sending the bind, and that open is now giving up without marking
1685            // it (its send failed or it was cancelled), forget the record so it
1686            // cannot outlive the open.
1687            inner
1688                .relays_released_before_enqueue
1689                .remove(&(endpoint, corr));
1690            return Ok(None);
1691        };
1692        release_reserved_route_locked(
1693            &mut inner,
1694            pending.reservation.client_key,
1695            pending.reservation.module_key,
1696        );
1697        let target = pending
1698            .relay_enqueued
1699            .then(|| abandoned_route_target(&inner, &pending.reservation));
1700        let _ = pending.sender.send(outcome);
1701        Ok(target.flatten())
1702    }
1703
1704    pub(crate) fn cancel_module_control_rpc(
1705        &self,
1706        endpoint: ModuleEndpointId,
1707        corr: u64,
1708    ) -> Result<(), ForwardingError> {
1709        self.write_inner()?
1710            .pending_control_rpcs
1711            .remove(&(endpoint, corr));
1712        Ok(())
1713    }
1714
1715    pub(crate) fn tombstone_health_probe_rpc(
1716        &self,
1717        endpoint: ModuleEndpointId,
1718        corr: u64,
1719    ) -> Result<bool, ForwardingError> {
1720        let key = (endpoint, corr);
1721        let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
1722        {
1723            let mut inner = self.write_inner()?;
1724            let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
1725                return Ok(false);
1726            };
1727            let Some(probe_started_at) = pending.health_probe_started_at else {
1728                inner.pending_control_rpcs.insert(key, pending);
1729                return Ok(false);
1730            };
1731            let module_id = inner
1732                .module_id_by_endpoint
1733                .get(&endpoint)
1734                .cloned()
1735                .unwrap_or_else(|| "unknown".to_string());
1736            inner.health_probe_tombstones.insert(
1737                key,
1738                HealthProbeTombstone {
1739                    expected_op: pending.expected_op,
1740                    module_id,
1741                    probe_started_at,
1742                    expires_at,
1743                },
1744            );
1745        }
1746        self.schedule_health_probe_tombstone_expiration(key, expires_at);
1747        Ok(true)
1748    }
1749
1750    fn schedule_health_probe_tombstone_expiration(
1751        &self,
1752        key: (ModuleEndpointId, u64),
1753        expires_at: Instant,
1754    ) {
1755        let inner = Arc::downgrade(&self.inner);
1756        tokio::spawn(async move {
1757            tokio::time::sleep_until(expires_at).await;
1758            let Some(inner) = inner.upgrade() else {
1759                return;
1760            };
1761            let Ok(mut inner) = inner.write() else {
1762                return;
1763            };
1764            let expired = inner
1765                .health_probe_tombstones
1766                .get(&key)
1767                .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
1768            if expired {
1769                inner.health_probe_tombstones.remove(&key);
1770            }
1771        });
1772    }
1773
1774    pub(crate) fn complete_pending_relay(
1775        &self,
1776        connection_id: ConnectionId,
1777        corr: u64,
1778        outcome: RouteBindRelayOutcome,
1779    ) -> Result<PendingRelayCompletion, ForwardingError> {
1780        let mut inner = self.write_inner()?;
1781        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1782            return Ok(PendingRelayCompletion {
1783                settled: false,
1784                abandoned: None,
1785            });
1786        };
1787        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1788            return Ok(PendingRelayCompletion {
1789                settled: false,
1790                abandoned: None,
1791            });
1792        };
1793
1794        if Instant::now() >= pending.deadline {
1795            release_reserved_route_locked(
1796                &mut inner,
1797                pending.reservation.client_key,
1798                pending.reservation.module_key,
1799            );
1800            let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
1801                .then(|| abandoned_route_target(&inner, &pending.reservation))
1802                .flatten();
1803            let _ = pending
1804                .sender
1805                .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody {
1806                    code: "module_timeout".to_string(),
1807                    message: "route.bind response arrived after its daemon deadline".to_string(),
1808                    detail: None,
1809                }));
1810            return Ok(PendingRelayCompletion {
1811                settled: true,
1812                abandoned,
1813            });
1814        }
1815
1816        match outcome {
1817            // Two shapes of "the client is not there to receive this route" that
1818            // must resolve identically: its egress is already closed, or it is
1819            // marked closing (its connection loop has been asked to end, but has
1820            // not drained yet, so the sink is still open).
1821            //
1822            // Only the first used to be caught here. The second fell through to
1823            // `commit_route_locked`, which refuses a closing client with
1824            // `ConnectionClosing` -- and this function is called from the MODULE
1825            // connection's frame handler, so that refusal ended the module's
1826            // connection instead of this one client's route. One dying client's
1827            // route.open then took down a connection carrying every other
1828            // client's routes to that module.
1829            //
1830            // The remedy for both is the same, which is why they share an arm:
1831            // give back the reserved handle pair, tell the waiting route.open the
1832            // route is gone, and report the module-side channel so the caller can
1833            // send a channel-scoped GOODBYE for the binding the module just
1834            // created. Nothing here touches the module connection.
1835            RouteBindRelayOutcome::Accepted
1836                if pending.client_sink.is_closed()
1837                    || inner
1838                        .closing_connections
1839                        .contains(&pending.reservation.client_key.connection_id) =>
1840            {
1841                let reason = if pending.client_sink.is_closed() {
1842                    "client egress closed before route publication"
1843                } else {
1844                    "client connection is closing before route publication"
1845                };
1846                release_reserved_route_locked(
1847                    &mut inner,
1848                    pending.reservation.client_key,
1849                    pending.reservation.module_key,
1850                );
1851                // Not gated on `relay_enqueued`: an ack proves the module
1852                // received the bind, and `route.open` sends nothing for a
1853                // relay a reply already settled, so this is the only GOODBYE.
1854                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1855                let _ = pending
1856                    .sender
1857                    .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
1858                return Ok(PendingRelayCompletion {
1859                    settled: true,
1860                    abandoned,
1861                });
1862            }
1863            // The endpoint that acked this bind is no longer accepting new
1864            // routes, for one of two reasons. Either it was the module's active
1865            // endpoint when this relay was reserved and a blue/green swap has
1866            // since promoted a candidate (the replacement process registered
1867            // beside it) into the active slot, superseding it; or a daemon
1868            // drain has gated it, and the ack arrived before the ordered module
1869            // drain settled pending relays. The module did nothing wrong: it
1870            // bound the route it was asked to bind, and the endpoint still
1871            // carries every other client's routes until it is drained. So the
1872            // ack is settled here, while the pending entry still holds the
1873            // client's sender and the reserved handle pair: release the pair,
1874            // refuse the waiting route.open with `module_reloading`, which
1875            // callers retry (after a swap the retry reserves on the promoted
1876            // endpoint), and hand back the module-side channel so the caller
1877            // sends one channel-scoped GOODBYE for the binding the module just
1878            // created. Returning an error instead would end the module
1879            // connection all those other routes share; nothing here touches it.
1880            //
1881            // An endpoint that stopped being active any other way is stale
1882            // (replaced, but not by a swap and not draining). It still falls
1883            // through to `commit_route_locked`, which refuses it with
1884            // `StaleModuleEndpoint` exactly as before swaps existed.
1885            RouteBindRelayOutcome::Accepted
1886                if inner.superseded_endpoints.contains_key(&endpoint)
1887                    || inner.draining_endpoints.contains_key(&endpoint) =>
1888            {
1889                release_reserved_route_locked(
1890                    &mut inner,
1891                    pending.reservation.client_key,
1892                    pending.reservation.module_key,
1893                );
1894                // Not gated on `relay_enqueued`: an ack proves the module
1895                // received the bind, whether or not the enqueue mark was set.
1896                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1897                let module_id = inner
1898                    .module_id_by_endpoint
1899                    .get(&endpoint)
1900                    .cloned()
1901                    .unwrap_or_else(|| "unknown".to_string());
1902                let _ = pending
1903                    .sender
1904                    .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
1905                        "module_reloading",
1906                        format!("module_id '{module_id}' is reloading"),
1907                    )));
1908                return Ok(PendingRelayCompletion {
1909                    settled: true,
1910                    abandoned,
1911                });
1912            }
1913            // The scope the open was admitted under ended or changed after
1914            // admission. The module bound a route under a stamp that is no
1915            // longer true, so it must not become routable. Settled here like
1916            // the superseded arm above, never as an error from
1917            // `commit_route_locked`, which would close the module's whole
1918            // connection and every other client's routes to it: release the
1919            // pair, answer the waiting route.open by name, and hand back the
1920            // module-side channel for one channel-scoped GOODBYE.
1921            RouteBindRelayOutcome::Accepted
1922                if pending
1923                    .scope
1924                    .as_ref()
1925                    .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
1926            {
1927                let refusal = pending
1928                    .scope
1929                    .as_ref()
1930                    .and_then(|scope| scope_refusal_locked(&inner, scope))
1931                    .expect("guard matched a refusal under the same lock");
1932                release_reserved_route_locked(
1933                    &mut inner,
1934                    pending.reservation.client_key,
1935                    pending.reservation.module_key,
1936                );
1937                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1938                let _ = pending
1939                    .sender
1940                    .send(RouteBindRelayOutcome::DaemonRejected(refusal));
1941                return Ok(PendingRelayCompletion {
1942                    settled: true,
1943                    abandoned,
1944                });
1945            }
1946            RouteBindRelayOutcome::Accepted => {
1947                let abandoned = commit_route_locked(&mut inner, pending)?;
1948                return Ok(PendingRelayCompletion {
1949                    settled: true,
1950                    abandoned,
1951                });
1952            }
1953            terminal => {
1954                release_reserved_route_locked(
1955                    &mut inner,
1956                    pending.reservation.client_key,
1957                    pending.reservation.module_key,
1958                );
1959                let _ = pending.sender.send(terminal);
1960            }
1961        }
1962        Ok(PendingRelayCompletion {
1963            settled: true,
1964            abandoned: None,
1965        })
1966    }
1967
1968    pub(crate) fn pending_module_control_op(
1969        &self,
1970        connection_id: ConnectionId,
1971        corr: u64,
1972    ) -> Result<Option<String>, ForwardingError> {
1973        let inner = self.read_inner()?;
1974        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1975            return Ok(None);
1976        };
1977        let key = (endpoint, corr);
1978        Ok(inner
1979            .pending_control_rpcs
1980            .get(&key)
1981            .map(|pending| pending.expected_op.clone())
1982            .or_else(|| {
1983                inner
1984                    .health_probe_tombstones
1985                    .get(&key)
1986                    .filter(|tombstone| tombstone.expires_at > Instant::now())
1987                    .map(|tombstone| tombstone.expected_op.clone())
1988            }))
1989    }
1990
1991    pub(crate) fn complete_module_control_rpc(
1992        &self,
1993        connection_id: ConnectionId,
1994        corr: u64,
1995        actual_op: Option<&str>,
1996        outcome: ModuleControlRpcOutcome,
1997    ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
1998        let now = Instant::now();
1999        let mut inner = self.write_inner()?;
2000        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2001            return Ok(ModuleControlRpcCompletion::Unknown);
2002        };
2003        let key = (endpoint, corr);
2004        if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2005            if now >= pending.deadline {
2006                let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
2007                    ModuleControlRpcCompletion::LateHealthAnswer {
2008                        module_id: inner
2009                            .module_id_by_endpoint
2010                            .get(&endpoint)
2011                            .cloned()
2012                            .unwrap_or_else(|| "unknown".to_string()),
2013                        latency: now.saturating_duration_since(probe_started_at),
2014                    }
2015                });
2016                let _ = pending
2017                    .sender
2018                    .send(ModuleControlRpcOutcome::DeadlineElapsed);
2019                return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
2020            }
2021            let outcome = match actual_op {
2022                Some(actual) if actual != pending.expected_op => {
2023                    ModuleControlRpcOutcome::UnexpectedOp {
2024                        expected: pending.expected_op,
2025                        actual: actual.to_string(),
2026                    }
2027                }
2028                _ => outcome,
2029            };
2030            let _ = pending.sender.send(outcome);
2031            return Ok(ModuleControlRpcCompletion::Settled);
2032        }
2033
2034        let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
2035            return Ok(ModuleControlRpcCompletion::Unknown);
2036        };
2037        if tombstone.expires_at <= now {
2038            return Ok(ModuleControlRpcCompletion::Unknown);
2039        }
2040        Ok(ModuleControlRpcCompletion::LateHealthAnswer {
2041            module_id: tombstone.module_id,
2042            latency: now.saturating_duration_since(tombstone.probe_started_at),
2043        })
2044    }
2045
2046    #[cfg(test)]
2047    pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
2048        Ok(self.read_inner()?.health_probe_tombstones.len())
2049    }
2050
2051    #[cfg(test)]
2052    pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
2053        Ok(self.read_inner()?.closing_connections.len())
2054    }
2055
2056    /// Reserved-but-uncommitted route handles, counted on both index sides, so
2057    /// a test can assert a reservation pair was actually given back.
2058    #[cfg(test)]
2059    pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
2060        let inner = self.read_inner()?;
2061        Ok((inner.reserved_client.len(), inner.reserved_module.len()))
2062    }
2063
2064    pub(crate) fn module_endpoint_for_connection(
2065        &self,
2066        connection_id: ConnectionId,
2067    ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
2068        Ok(self
2069            .read_inner()?
2070            .endpoint_by_connection
2071            .get(&connection_id)
2072            .copied())
2073    }
2074
2075    /// Looks up the module registered on a data-plane connection so route-drop
2076    /// diagnostics name the emitter instead of only reporting a daemon total.
2077    pub(crate) fn module_id_for_connection(
2078        &self,
2079        connection_id: ConnectionId,
2080    ) -> Result<Option<String>, ForwardingError> {
2081        let inner = self.read_inner()?;
2082        Ok(inner
2083            .endpoint_by_connection
2084            .get(&connection_id)
2085            .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
2086            .cloned())
2087    }
2088
2089    /// Whether the daemon ever allocated `(channel, epoch)` on this module
2090    /// connection. Epochs on a module channel are handed out as 1, 2, 3, ...
2091    /// and the last one handed out is remembered until the connection ends, so
2092    /// every epoch from 1 up to that one was a real route at some point. Used
2093    /// only on the drop path, to tell a module still sending on a route the
2094    /// daemon released from one sending on a route that never existed.
2095    pub(crate) fn module_route_epoch_was_allocated(
2096        &self,
2097        connection_id: ConnectionId,
2098        channel: u16,
2099        epoch: u32,
2100    ) -> Result<bool, ForwardingError> {
2101        let inner = self.read_inner()?;
2102        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2103            return Ok(false);
2104        };
2105        Ok(inner
2106            .module_slot_epochs
2107            .get(&ModuleRouteKey { endpoint, channel })
2108            .is_some_and(|last| epoch != 0 && epoch <= *last))
2109    }
2110
2111    pub(crate) fn has_live_module_connection(
2112        &self,
2113        module_id: &str,
2114    ) -> Result<bool, ForwardingError> {
2115        Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
2116    }
2117
2118    pub(crate) fn lookup_data_route(
2119        &self,
2120        connection_id: ConnectionId,
2121        channel: u16,
2122        epoch: u32,
2123    ) -> Result<DataRoute, ForwardingError> {
2124        let inner = self.read_inner()?;
2125        let state = if let Some(endpoint) =
2126            inner.endpoint_by_connection.get(&connection_id).copied()
2127        {
2128            let key = ModuleRouteKey { endpoint, channel };
2129            match inner.module_to_client.get(&key) {
2130                Some(route) if route.module_epoch == epoch => {
2131                    DataRouteState::Bound(Arc::clone(route))
2132                }
2133                Some(_) => DataRouteState::EpochMismatch,
2134                None if inner.reserved_module.contains_key(&key)
2135                    && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
2136                {
2137                    DataRouteState::Reserved
2138                }
2139                None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
2140                None => DataRouteState::Absent,
2141            }
2142        } else {
2143            let key = ClientRouteKey {
2144                connection_id,
2145                channel,
2146            };
2147            match inner.client_to_module.get(&key) {
2148                Some(route) if route.client_epoch == epoch => {
2149                    DataRouteState::Bound(Arc::clone(route))
2150                }
2151                Some(_) => DataRouteState::EpochMismatch,
2152                None if inner.reserved_client.contains_key(&key)
2153                    && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
2154                {
2155                    DataRouteState::Reserved
2156                }
2157                None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
2158                None => DataRouteState::Absent,
2159            }
2160        };
2161        Ok(
2162            if inner.endpoint_by_connection.contains_key(&connection_id) {
2163                DataRoute::Module(state)
2164            } else {
2165                DataRoute::Client(state)
2166            },
2167        )
2168    }
2169
2170    #[cfg(test)]
2171    pub(crate) fn inject_client_slot_epoch(
2172        &self,
2173        connection_id: ConnectionId,
2174        channel: u16,
2175        last_epoch: u32,
2176    ) {
2177        let mut inner = self.write_inner().expect("forwarding lock");
2178        inner.client_slot_epochs.insert(
2179            ClientRouteKey {
2180                connection_id,
2181                channel,
2182            },
2183            last_epoch,
2184        );
2185        inner.next_client_channel.insert(connection_id, channel);
2186    }
2187
2188    #[cfg(test)]
2189    pub(crate) fn inject_module_slot_epoch(
2190        &self,
2191        endpoint: ModuleEndpointId,
2192        channel: u16,
2193        last_epoch: u32,
2194    ) {
2195        let mut inner = self.write_inner().expect("forwarding lock");
2196        inner
2197            .module_slot_epochs
2198            .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
2199        inner.next_module_channel.insert(endpoint, channel);
2200    }
2201
2202    #[cfg(test)]
2203    pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
2204        self.write_inner()
2205            .expect("forwarding lock")
2206            .next_control_corr
2207            .insert(endpoint, next_corr);
2208    }
2209
2210    pub(crate) fn cache_status(
2211        &self,
2212        endpoint: ModuleEndpointId,
2213        module_channel: u16,
2214        module_epoch: u32,
2215        status: String,
2216    ) -> Result<bool, ForwardingError> {
2217        let mut inner = self.write_inner()?;
2218        if !inner.module_id_by_endpoint.contains_key(&endpoint) {
2219            return Err(ForwardingError::StaleModuleEndpoint);
2220        }
2221
2222        let module_key = ModuleRouteKey {
2223            endpoint,
2224            channel: module_channel,
2225        };
2226        let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
2227            (route.module_epoch == module_epoch).then_some((
2228                ClientRouteKey {
2229                    connection_id: route.client_connection_id,
2230                    channel: route.client_channel,
2231                },
2232                route.client_epoch,
2233            ))
2234        } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
2235            (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
2236                client_key,
2237                inner
2238                    .client_slot_epochs
2239                    .get(&client_key)
2240                    .copied()
2241                    .unwrap_or(0),
2242            ))
2243        } else {
2244            None
2245        };
2246
2247        if let Some(handle) = handle {
2248            inner.status.insert(handle, status);
2249            Ok(true)
2250        } else {
2251            debug!(
2252                module_channel,
2253                module_epoch,
2254                generation = endpoint.generation,
2255                connection_id = endpoint.connection_id.get(),
2256                "dropping stale status update for module route handle"
2257            );
2258            Ok(false)
2259        }
2260    }
2261
2262    pub(crate) fn route_poll_snapshot(
2263        &self,
2264        client_connection_id: ConnectionId,
2265        client_channel: u16,
2266        client_epoch: u32,
2267    ) -> Result<RoutePollSnapshot, ForwardingError> {
2268        let inner = self.read_inner()?;
2269        let client_key = ClientRouteKey {
2270            connection_id: client_connection_id,
2271            channel: client_channel,
2272        };
2273        let Some(route) = inner.client_to_module.get(&client_key) else {
2274            return Ok(RoutePollSnapshot::Absent);
2275        };
2276        if route.client_epoch != client_epoch
2277            || !inner
2278                .module_id_by_endpoint
2279                .contains_key(&route.module_endpoint)
2280        {
2281            return Ok(RoutePollSnapshot::Absent);
2282        }
2283        Ok(RoutePollSnapshot::Bound {
2284            module_id: route.module_id.clone(),
2285            status: inner.status.get(&(client_key, client_epoch)).cloned(),
2286        })
2287    }
2288
2289    pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2290        Ok(self.read_inner()?.client_to_module.len())
2291    }
2292
2293    /// How many distinct client connections hold at least one committed route,
2294    /// alongside the largest number of routes any single connection holds.
2295    ///
2296    /// `connected_clients` alone cannot distinguish many clients with a route
2297    /// each from one client accumulating hundreds, and those have opposite
2298    /// causes. Reading it required an out-of-band `lsof` during a live
2299    /// investigation, and the count of connections was mistaken for a count of
2300    /// client processes — which sent two of us after cleanup paths that were
2301    /// working correctly.
2302    pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2303        let inner = self.read_inner()?;
2304        let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2305        for key in inner.client_to_module.keys() {
2306            *per_connection.entry(key.connection_id).or_insert(0) += 1;
2307        }
2308        let max = per_connection.values().copied().max().unwrap_or(0);
2309        Ok((per_connection.len(), max))
2310    }
2311
2312    pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2313        let inner = self.read_inner()?;
2314        Ok(inner
2315            .client_to_module
2316            .keys()
2317            .any(|key| key.channel == route_channel))
2318    }
2319
2320    /// Whether daemon-wide shutdown has begun draining all providers.
2321    /// `None` means normal operation; during daemon shutdown, return the
2322    /// endpoint's measured result, or false if no completed measurement exists.
2323    pub(crate) fn daemon_drain_result_for_connection(
2324        &self,
2325        connection_id: ConnectionId,
2326    ) -> Result<Option<bool>, ForwardingError> {
2327        let inner = self.read_inner()?;
2328        Ok(inner.daemon_draining.then(|| {
2329            inner
2330                .endpoint_by_connection
2331                .get(&connection_id)
2332                .and_then(|endpoint| inner.daemon_drain_results.get(endpoint))
2333                .copied()
2334                .unwrap_or(false)
2335        }))
2336    }
2337
2338    #[cfg(unix)]
2339    pub(crate) fn record_daemon_drain_result(
2340        &self,
2341        endpoint: ModuleEndpointId,
2342        drained: bool,
2343    ) -> Result<(), ForwardingError> {
2344        let mut inner = self.write_inner()?;
2345        // Only update an endpoint captured at the start of daemon shutdown.
2346        // Teardown removes its entry, so a late wait cannot recreate the result
2347        // or attribute it to a replacement connection.
2348        if let Some(result) = inner.daemon_drain_results.get_mut(&endpoint) {
2349            *result = drained;
2350        }
2351        Ok(())
2352    }
2353
2354    /// Gate every provider atomically, including registrations racing shutdown.
2355    /// No supervisor lock is held while taking the forwarding lock.
2356    #[cfg(unix)]
2357    pub(crate) fn begin_daemon_drain(
2358        &self,
2359    ) -> Result<Vec<(String, ModuleEndpointId)>, ForwardingError> {
2360        let mut inner = self.write_inner()?;
2361        if inner.daemon_draining {
2362            return Ok(Vec::new());
2363        }
2364        inner.daemon_draining = true;
2365        let modules = inner
2366            .modules_by_id
2367            .iter()
2368            .map(|(id, module)| (id.clone(), module.endpoint))
2369            .collect::<Vec<_>>();
2370        for (_, endpoint) in &modules {
2371            inner.daemon_drain_results.insert(*endpoint, false);
2372            inner
2373                .draining_endpoints
2374                .insert(*endpoint, RouteCloseReason::Restart);
2375        }
2376        // Swap candidates and superseded incumbents are not routable, but they
2377        // are live endpoints that could still be sent module-control work, so
2378        // the daemon-wide gate covers them too. Only the active endpoints are
2379        // returned to the supervisor for measurement, one per module id.
2380        let off_slot_endpoints = inner
2381            .candidates_by_id
2382            .values()
2383            .map(|module| module.endpoint)
2384            .chain(inner.superseded_endpoints.keys().copied())
2385            .collect::<Vec<_>>();
2386        for endpoint in off_slot_endpoints {
2387            inner
2388                .draining_endpoints
2389                .insert(endpoint, RouteCloseReason::Restart);
2390        }
2391        Ok(modules)
2392    }
2393
2394    /// Begin draining whatever endpoint is ACTIVE for `module_id`.
2395    ///
2396    /// After a swap's cutover the active endpoint is the promoted candidate, so
2397    /// this must not be used to drain the incumbent; use
2398    /// [`Self::begin_endpoint_drain`] with the endpoint `cutover_candidate`
2399    /// returned.
2400    pub(crate) fn begin_module_drain(
2401        &self,
2402        module_id: &str,
2403        reason: RouteCloseReason,
2404    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2405        let mut inner = self.write_inner()?;
2406        let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2407            return Ok(None);
2408        };
2409        Ok(Some(begin_drain_locked(
2410            &mut inner, module_id, module, reason,
2411        )))
2412    }
2413
2414    /// Begin draining one specific endpoint, whichever slot it is in.
2415    ///
2416    /// This is how a swap drains its incumbent after cutover: the incumbent's
2417    /// endpoint is captured by `cutover_candidate`, and resolving it by module id
2418    /// instead would find the promoted candidate and leave neither process
2419    /// routable. Returns `Ok(None)` when the endpoint is no longer registered.
2420    pub(crate) fn begin_endpoint_drain(
2421        &self,
2422        endpoint: ModuleEndpointId,
2423        reason: RouteCloseReason,
2424    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2425        let mut inner = self.write_inner()?;
2426        let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2427            return Ok(None);
2428        };
2429        let module_id = inner
2430            .module_id_by_endpoint
2431            .get(&endpoint)
2432            .cloned()
2433            .expect("an endpoint resolved to a module connection has a module id");
2434        Ok(Some(begin_drain_locked(
2435            &mut inner, &module_id, module, reason,
2436        )))
2437    }
2438}
2439
2440/// Mark `module`'s endpoint draining and settle everything still pending on it.
2441/// Shared by the by-id and by-endpoint drain entry points, which differ only in
2442/// how they find the endpoint.
2443fn begin_drain_locked(
2444    inner: &mut ForwardingInner,
2445    module_id: &str,
2446    module: ModuleConnection,
2447    reason: RouteCloseReason,
2448) -> ModuleDrainTarget {
2449    {
2450        let endpoint = module.endpoint;
2451        inner.draining_endpoints.insert(endpoint, reason);
2452
2453        let flows = inner
2454            .client_to_module
2455            .values()
2456            .filter(|route| route.module_endpoint == endpoint)
2457            .map(|route| Arc::clone(&route.flow))
2458            .collect::<Vec<_>>();
2459        let excluded_subscriptions = flows
2460            .into_iter()
2461            .map(|flow| flow.begin_drain())
2462            .fold(0u32, u32::saturating_add);
2463
2464        let pending_keys = inner
2465            .pending_relays
2466            .keys()
2467            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2468            .copied()
2469            .collect::<Vec<_>>();
2470        let mut abandoned_bindings = Vec::new();
2471        for key in pending_keys {
2472            let Some(pending) = inner.pending_relays.remove(&key) else {
2473                continue;
2474            };
2475            release_reserved_route_locked(
2476                inner,
2477                pending.reservation.client_key,
2478                pending.reservation.module_key,
2479            );
2480            if pending.relay_enqueued {
2481                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2482                    abandoned_bindings.push(target);
2483                }
2484            } else {
2485                // The bind may still be on its way into the module's queue;
2486                // a GOODBYE sent now could overtake it. `route.open` sends
2487                // it once its send completes.
2488                inner.relays_released_before_enqueue.insert(key);
2489            }
2490            let _ = pending
2491                .sender
2492                .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
2493                    "module_reloading",
2494                    format!("module_id '{module_id}' is reloading"),
2495                )));
2496        }
2497
2498        let pending_control_keys = inner
2499            .pending_control_rpcs
2500            .keys()
2501            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2502            .copied()
2503            .collect::<Vec<_>>();
2504        for key in pending_control_keys {
2505            if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2506                let _ = pending
2507                    .sender
2508                    .send(ModuleControlRpcOutcome::ModuleGone(format!(
2509                        "module '{module_id}' began draining during module-control RPC"
2510                    )));
2511            }
2512        }
2513
2514        ModuleDrainTarget {
2515            endpoint,
2516            sink: module.sink,
2517            negotiated_ver: module.negotiated_ver,
2518            abandoned_bindings,
2519            excluded_subscriptions,
2520        }
2521    }
2522}
2523
2524/// What a drain that timed out was still waiting on, for the log line that
2525/// reports the timeout. Per-request ages are not tracked.
2526#[derive(Debug, Default, PartialEq, Eq)]
2527pub(crate) struct DrainHoldouts {
2528    /// Requests still counted against the drain (subscriptions flagged as
2529    /// such are excluded from the drain and not counted here).
2530    pub(crate) requests: usize,
2531    /// Routes holding at least one of those requests.
2532    pub(crate) routes: usize,
2533    /// Every route on the endpoint, for scale.
2534    pub(crate) total_routes: usize,
2535    /// The client connections holding the most requests, largest first, at
2536    /// most three: enough to name the consumer without listing every route.
2537    pub(crate) top_connections: Vec<(u64, usize)>,
2538    /// The held requests themselves, as the module saw them, so its own log
2539    /// can say what each one was: `(module channel, corr)`, ordered by channel
2540    /// then corr, at most [`DRAIN_HELD_REQUESTS_LISTED`]. The daemon never reads
2541    /// request bodies, so it cannot name a request's method; the module can,
2542    /// from the channel and corr. A request is released only when the module
2543    /// sends a terminal frame (Response, Error or StreamEnd) with its corr on
2544    /// its route, so every pair listed here is one the module ended, if at all,
2545    /// without sending that frame.
2546    pub(crate) held: Vec<(u16, u64)>,
2547}
2548
2549/// How many held requests the drain-timeout line lists by channel and corr.
2550pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
2551
2552impl ForwardingTable {
2553    /// Summarise the requests one endpoint's drain is still waiting on.
2554    pub(crate) fn endpoint_drain_holdouts(
2555        &self,
2556        endpoint: ModuleEndpointId,
2557    ) -> Result<DrainHoldouts, ForwardingError> {
2558        let inner = self.read_inner()?;
2559        let mut holdouts = DrainHoldouts::default();
2560        let mut by_connection: HashMap<u64, usize> = HashMap::new();
2561        for (key, route) in &inner.client_to_module {
2562            if route.module_endpoint != endpoint {
2563                continue;
2564            }
2565            holdouts.total_routes += 1;
2566            let held = route.flow.drain_in_flight();
2567            if held == 0 {
2568                continue;
2569            }
2570            holdouts.requests += held;
2571            holdouts.routes += 1;
2572            *by_connection.entry(key.connection_id.get()).or_default() += held;
2573            holdouts.held.extend(
2574                route
2575                    .flow
2576                    .drain_held_corrs()
2577                    .into_iter()
2578                    .map(|corr| (route.module_channel, corr)),
2579            );
2580        }
2581        holdouts.held.sort_unstable();
2582        holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
2583        let mut connections = by_connection.into_iter().collect::<Vec<_>>();
2584        connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
2585        connections.truncate(3);
2586        holdouts.top_connections = connections;
2587        Ok(holdouts)
2588    }
2589
2590    pub(crate) fn endpoint_in_flight_count(
2591        &self,
2592        endpoint: ModuleEndpointId,
2593    ) -> Result<usize, ForwardingError> {
2594        let inner = self.read_inner()?;
2595        Ok(inner
2596            .client_to_module
2597            .values()
2598            .filter(|route| route.module_endpoint == endpoint)
2599            .map(|route| route.flow.drain_in_flight())
2600            .sum())
2601    }
2602
2603    pub(crate) fn endpoint_is_draining(
2604        &self,
2605        endpoint: ModuleEndpointId,
2606    ) -> Result<bool, ForwardingError> {
2607        Ok(self
2608            .read_inner()?
2609            .draining_endpoints
2610            .contains_key(&endpoint))
2611    }
2612
2613    pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
2614        let inner = self.read_inner()?;
2615        Ok(inner
2616            .modules_by_id
2617            .get(module_id)
2618            .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
2619    }
2620
2621    pub(crate) fn release_module_endpoint_routes(
2622        &self,
2623        endpoint: ModuleEndpointId,
2624    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2625        let mut inner = self.write_inner()?;
2626        let routes = inner
2627            .module_to_client
2628            .iter()
2629            .filter(|(module_key, _)| module_key.endpoint == endpoint)
2630            .map(|(module_key, route)| (*module_key, route.module_epoch))
2631            .collect::<Vec<_>>();
2632        let mut released = Vec::with_capacity(routes.len());
2633        for (module_key, epoch) in routes {
2634            if let RouteRelease::Removed(target) =
2635                release_module_route_locked(&mut inner, module_key, epoch)
2636            {
2637                released.push(target);
2638            }
2639        }
2640        Ok(released)
2641    }
2642
2643    /// Enumerate one endpoint's current routes without contacting the module.
2644    ///
2645    /// The read lock makes this safe while the endpoint drains: the returned
2646    /// `draining` marker describes the same table state that owns the route,
2647    /// rather than inferring liveness from a module that may be stopping.
2648    pub(crate) fn endpoint_routes(
2649        &self,
2650        endpoint: ModuleEndpointId,
2651    ) -> Result<Vec<EndpointRoute>, ForwardingError> {
2652        let inner = self.read_inner()?;
2653        Ok(endpoint_routes_locked(&inner, endpoint))
2654    }
2655
2656    /// Snapshot all live endpoint route sets under one forwarding-table read lock.
2657    pub(crate) fn route_census(
2658        &self,
2659        module_id: Option<&str>,
2660    ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
2661        let inner = self.read_inner()?;
2662        let mut endpoints = inner
2663            .modules_by_id
2664            .iter()
2665            .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
2666            .map(|(id, module)| (id.clone(), module.endpoint))
2667            .collect::<Vec<_>>();
2668        endpoints.sort_by(|left, right| left.0.cmp(&right.0));
2669        Ok(endpoints
2670            .into_iter()
2671            .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
2672            .collect())
2673    }
2674
2675    /// Snapshot only the endpoint currently routable under this module id.
2676    pub(crate) fn live_roots(
2677        &self,
2678        module_id: &str,
2679    ) -> Result<ModuleControlResponseToModule, ForwardingError> {
2680        let inner = self.read_inner()?;
2681        let endpoint = inner
2682            .modules_by_id
2683            .get(module_id)
2684            .map(|module| module.endpoint);
2685        let mut roots = BTreeMap::new();
2686        let mut unknown_root_bindings = 0;
2687        let mut total_bindings = 0;
2688        if let Some(endpoint) = endpoint {
2689            for binding in inner
2690                .module_to_client
2691                .values()
2692                .filter(|binding| binding.module_endpoint == endpoint)
2693            {
2694                total_bindings += 1;
2695                let workdir = binding
2696                    .scope
2697                    .as_ref()
2698                    .and_then(|scope| scope.workdir.as_deref())
2699                    .map(std::path::Path::new);
2700                if let Some(root) = &binding.project_root {
2701                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2702                    entry.0 += 1;
2703                } else if workdir.is_none() {
2704                    unknown_root_bindings += 1;
2705                }
2706                if let Some(workdir) = workdir.filter(|workdir| {
2707                    binding.project_root.as_ref().map(ProjectRootId::as_path) != Some(*workdir)
2708                }) {
2709                    roots.entry(workdir.to_path_buf()).or_insert((0, 0)).0 += 1;
2710                }
2711            }
2712            for pending in inner
2713                .pending_relays
2714                .values()
2715                .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
2716            {
2717                total_bindings += 1;
2718                if let Some(root) = &pending.reservation.project_root {
2719                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2720                    entry.1 += 1;
2721                } else {
2722                    unknown_root_bindings += 1;
2723                }
2724            }
2725        }
2726        Ok(ModuleControlResponseToModule::LiveRoots {
2727            roots: roots
2728                .into_iter()
2729                .map(|(project_root, (bound, pending))| LiveRoot {
2730                    project_root,
2731                    bound,
2732                    pending,
2733                })
2734                .collect(),
2735            unknown_root_bindings,
2736            total_bindings,
2737        })
2738    }
2739
2740    /// True if this connection already owns committed or reserved CLIENT routes.
2741    /// A module registers (HELLO) before serving and never opens client routes, so
2742    /// a connection that has client routes must not also become a module endpoint
2743    /// — otherwise one connection holds both client and module state and cleanup
2744    /// only releases one side.
2745    pub(crate) fn connection_has_client_routes(
2746        &self,
2747        connection_id: ConnectionId,
2748    ) -> Result<bool, ForwardingError> {
2749        let inner = self.read_inner()?;
2750        Ok(connection_has_client_routes_locked(&inner, connection_id))
2751    }
2752
2753    pub(crate) fn cleanup_connection(
2754        &self,
2755        connection_id: ConnectionId,
2756    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2757        self.cleanup_connection_counted(connection_id)
2758            .map(|cleanup| cleanup.released)
2759    }
2760
2761    /// [`Self::cleanup_connection`], also reporting how many pending
2762    /// route.bind relays to the closed module were aborted. That count is the
2763    /// `abandoned` figure of the `route.closed` push sent for a lost module
2764    /// connection; it is always zero for a client connection.
2765    pub(crate) fn cleanup_connection_counted(
2766        &self,
2767        connection_id: ConnectionId,
2768    ) -> Result<ConnectionCleanup, ForwardingError> {
2769        self.frame_drop_warnings
2770            .lock()
2771            .unwrap_or_else(std::sync::PoisonError::into_inner)
2772            .remove(&connection_id);
2773        let mut inner = self.write_inner()?;
2774        inner.closing_connections.insert(connection_id);
2775        let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
2776            remove_module_connection_locked(&mut inner, endpoint)
2777        } else {
2778            ConnectionCleanup {
2779                released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
2780                abandoned_relays: 0,
2781            }
2782        };
2783        // The closing mark refuses new work for a connection whose teardown is
2784        // still pending. This is the latest point at which lifting it is safe:
2785        // teardown has just removed every per-connection entry above, under
2786        // this same write lock, so no lookup can still find live state for the
2787        // id; and ids come from a monotonic counter that never reissues one,
2788        // so the id can never name a future connection either. Keeping the
2789        // mark past this point only grew the set by one entry per connection
2790        // for the life of the daemon.
2791        inner.closing_connections.remove(&connection_id);
2792        Ok(cleanup)
2793    }
2794
2795    fn cleanup_client_connection_locked(
2796        inner: &mut ForwardingInner,
2797        connection_id: ConnectionId,
2798    ) -> Vec<GoodbyeTarget> {
2799        let routes = inner
2800            .client_to_module
2801            .iter()
2802            .filter(|(key, _)| key.connection_id == connection_id)
2803            .map(|(key, route)| (*key, route.client_epoch))
2804            .collect::<Vec<_>>();
2805        let mut released = Vec::with_capacity(routes.len());
2806        for (client_key, epoch) in routes {
2807            if let RouteRelease::Removed(target) =
2808                release_client_route_locked(inner, client_key, epoch)
2809            {
2810                released.push(target);
2811            }
2812        }
2813
2814        let pending_keys = inner
2815            .pending_relays
2816            .iter()
2817            .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
2818            .map(|(key, _)| *key)
2819            .collect::<Vec<_>>();
2820        for key in pending_keys {
2821            let Some(pending) = inner.pending_relays.remove(&key) else {
2822                continue;
2823            };
2824            release_reserved_route_locked(
2825                inner,
2826                pending.reservation.client_key,
2827                pending.reservation.module_key,
2828            );
2829            if pending.relay_enqueued {
2830                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2831                    released.push(target);
2832                }
2833            } else {
2834                // The bind may still be on its way into the module's queue;
2835                // a GOODBYE sent now could overtake it. `route.open` sends
2836                // it once its send completes.
2837                inner.relays_released_before_enqueue.insert(key);
2838            }
2839            let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
2840                "client connection closed during route.bind relay".to_string(),
2841            ));
2842        }
2843
2844        let orphaned = inner
2845            .reserved_client
2846            .iter()
2847            .filter(|(key, _)| key.connection_id == connection_id)
2848            .map(|(client, module)| (*client, *module))
2849            .collect::<Vec<_>>();
2850        for (client_key, module_key) in orphaned {
2851            release_reserved_route_locked(inner, client_key, module_key);
2852        }
2853        inner.next_client_channel.remove(&connection_id);
2854        inner
2855            .client_slot_epochs
2856            .retain(|key, _| key.connection_id != connection_id);
2857        inner
2858            .last_published_epoch
2859            .retain(|key, _| key.connection_id != connection_id);
2860        inner
2861            .status
2862            .retain(|(key, _), _| key.connection_id != connection_id);
2863
2864        released
2865    }
2866
2867    /// Close a client connection whose egress queue refused a frame for the
2868    /// route `(connection_id, channel)` at `expected_epoch`. The whole
2869    /// connection closes because its queue is shared by every route on it.
2870    ///
2871    /// The first request that actually closes the connection logs one WARN with
2872    /// the diagnosis: which principals the connection's routes belonged to,
2873    /// which module's frame did not fit and on which client channel, and what
2874    /// the queue held at that moment.
2875    pub(crate) fn escalate_client_delivery_failure(
2876        &self,
2877        connection_id: ConnectionId,
2878        channel: u16,
2879        expected_epoch: u32,
2880        reason: CloseReason,
2881        undelivered: UndeliveredFrame<'_>,
2882    ) -> Result<bool, ForwardingError> {
2883        self.escalate_client_delivery_failure_diagnosed(
2884            connection_id,
2885            channel,
2886            expected_epoch,
2887            reason,
2888            undelivered,
2889            None,
2890        )
2891    }
2892
2893    pub(crate) fn escalate_client_delivery_failure_diagnosed(
2894        &self,
2895        connection_id: ConnectionId,
2896        channel: u16,
2897        expected_epoch: u32,
2898        reason: CloseReason,
2899        undelivered: UndeliveredFrame<'_>,
2900        diagnosis: Option<&crate::router::ClientDeliveryDiagnosis>,
2901    ) -> Result<bool, ForwardingError> {
2902        let principals = {
2903            let mut inner = self.write_inner()?;
2904            let key = ClientRouteKey {
2905                connection_id,
2906                channel,
2907            };
2908            if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
2909                None
2910            } else {
2911                inner.closing_connections.insert(connection_id);
2912                Some(connection_principals_locked(&inner, connection_id))
2913            }
2914        };
2915        let Some(principals) = principals else {
2916            // A concurrent release can remove the epoch before escalation.
2917            // The frame was still lost, even though this route can no longer
2918            // authorize closing the shared client connection.
2919            self.warn_frame_drop(
2920                connection_id,
2921                channel,
2922                expected_epoch,
2923                "module_to_client",
2924                diagnosis
2925                    .map(|failure| failure.cause)
2926                    .unwrap_or("delivery_failed"),
2927            );
2928            return Ok(false);
2929        };
2930        let backlog = undelivered.sink.backlog();
2931        let close_reason = reason.to_string();
2932        if self.request_connection_close(connection_id, reason) {
2933            warn!(
2934                connection_id = connection_id.get(),
2935                principals = %principals,
2936                module_id = undelivered.module_id.unwrap_or("unknown"),
2937                client_channel = channel,
2938                queued_bytes = backlog.queued_bytes,
2939                queued_frames = backlog.queued_frames,
2940                frame_type = ?diagnosis.map(|failure| failure.frame_type),
2941                frame_flags = ?diagnosis.map(|failure| failure.flags.0),
2942                close_cause = diagnosis.map(|failure| failure.cause).unwrap_or("delivery_failed"),
2943                oldest_queued_ms = backlog
2944                    .oldest_age
2945                    .map(|age| age.as_millis() as u64)
2946                    .unwrap_or(0),
2947                close_reason = %close_reason,
2948                "closing client connection: its egress queue could not take a frame"
2949            );
2950        }
2951        Ok(true)
2952    }
2953
2954    /// Publish the tags a scope sync changed and close the live routes its
2955    /// drain table selects, in one critical section.
2956    ///
2957    /// The caller holds the scope table's write lock (scope table, then this
2958    /// table, always). Publishing and selecting under one lock is what closes
2959    /// the race with a bind commit: a commit before this sees the old tag and
2960    /// its route is selected here like any other; a commit after it sees the
2961    /// new tag and is refused.
2962    ///
2963    /// Every client route index entry is scanned, which includes routes on a
2964    /// swap's superseded endpoint (the old process of a module being replaced
2965    /// blue/green, which keeps its existing routes until they drain): those
2966    /// stay indexed until drained, so ending a scope reaches them too.
2967    pub(crate) fn publish_scope_changes(
2968        &self,
2969        changes: &[ScopeTagChange],
2970    ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
2971        let mut inner = self.write_inner()?;
2972        let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
2973        for change in changes {
2974            let key = (change.owner.clone(), change.scope_ref.clone());
2975            match change.after {
2976                Some(tag) => {
2977                    inner.scope_tags.insert(key, tag);
2978                }
2979                None => {
2980                    inner.scope_tags.remove(&key);
2981                }
2982            }
2983            by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
2984        }
2985        let mut selected = Vec::new();
2986        for (client_key, route) in &inner.client_to_module {
2987            let Some(scope) = &route.scope else {
2988                continue;
2989            };
2990            let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
2991            else {
2992                continue;
2993            };
2994            if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
2995                continue;
2996            }
2997            let reason = match &change.drain {
2998                ScopeDrain::Nothing => continue,
2999                ScopeDrain::All(reason) => *reason,
3000                ScopeDrain::Carriers(narrowed) => {
3001                    let owner = Principal::Reserved {
3002                        module_id: scope.owner.clone(),
3003                    };
3004                    let hit = route.principal != owner
3005                        && narrowed.iter().any(|(principal, allowed)| {
3006                            *principal == route.principal
3007                                && allowed
3008                                    .as_ref()
3009                                    .is_none_or(|targets| !targets.contains(&route.module_id))
3010                        });
3011                    if !hit {
3012                        continue;
3013                    }
3014                    RouteCloseReason::ScopeCarrierRemoved
3015                }
3016            };
3017            selected.push((*client_key, route.client_epoch, reason));
3018        }
3019        let mut drained = Vec::new();
3020        for (client_key, client_epoch, reason) in selected {
3021            let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
3022                continue;
3023            };
3024            let release = release_client_route_locked(&mut inner, client_key, client_epoch);
3025            self.record_route_release(&release);
3026            if let RouteRelease::Removed(module) = release {
3027                drained.push(ScopeDrainedRoute {
3028                    reason,
3029                    scope: route
3030                        .scope
3031                        .clone()
3032                        .expect("selected scope drain has a bound scope"),
3033                    module_id: route.module_id.clone(),
3034                    client: GoodbyeTarget {
3035                        connection_id: route.client_connection_id,
3036                        sink: route.client_sink.clone(),
3037                        negotiated_ver: route.client_negotiated_ver,
3038                        channel: route.client_channel,
3039                        epoch: route.client_epoch,
3040                        kind: GoodbyeTargetKind::Client,
3041                        module_id: Some(route.module_id.clone()),
3042                    },
3043                    module,
3044                });
3045            }
3046        }
3047        Ok(drained)
3048    }
3049
3050    /// The published tag of one scope, for tests of the commit re-check.
3051    #[cfg(test)]
3052    pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
3053        self.read_inner()
3054            .ok()?
3055            .scope_tags
3056            .get(&(owner.to_string(), scope_ref.to_string()))
3057            .copied()
3058    }
3059
3060    fn record_route_release(&self, release: &RouteRelease) {
3061        match release {
3062            RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
3063            RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
3064            RouteRelease::Absent => {}
3065        }
3066    }
3067
3068    fn read_inner(&self) -> Result<RwLockReadGuard<'_, ForwardingInner>, ForwardingError> {
3069        self.inner.read().map_err(|_| ForwardingError::Poisoned)
3070    }
3071
3072    fn write_inner(&self) -> Result<RwLockWriteGuard<'_, ForwardingInner>, ForwardingError> {
3073        self.inner.write().map_err(|_| ForwardingError::Poisoned)
3074    }
3075
3076    fn lock_close_registry(
3077        &self,
3078    ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
3079        self.close_registry
3080            .lock()
3081            .unwrap_or_else(|poisoned| poisoned.into_inner())
3082    }
3083}
3084
3085impl ForwardingInner {
3086    fn allocate_route_slots(
3087        &mut self,
3088        connection_id: ConnectionId,
3089        endpoint: ModuleEndpointId,
3090    ) -> Result<(u16, u32, u16, u32), ForwardingError> {
3091        let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
3092        let mut client_channel = client_start;
3093        let client_channel = loop {
3094            let key = ClientRouteKey {
3095                connection_id,
3096                channel: client_channel,
3097            };
3098            let eligible = !self.client_to_module.contains_key(&key)
3099                && !self.reserved_client.contains_key(&key)
3100                && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3101            if eligible {
3102                break client_channel;
3103            }
3104            client_channel = next_channel(client_channel);
3105            if client_channel == client_start {
3106                return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
3107            }
3108        };
3109
3110        let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
3111        let mut module_channel = module_start;
3112        let module_channel = loop {
3113            let key = ModuleRouteKey {
3114                endpoint,
3115                channel: module_channel,
3116            };
3117            let eligible = !self.module_to_client.contains_key(&key)
3118                && !self.reserved_module.contains_key(&key)
3119                && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3120            if eligible {
3121                break module_channel;
3122            }
3123            module_channel = next_channel(module_channel);
3124            if module_channel == module_start {
3125                return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
3126            }
3127        };
3128
3129        let client_key = ClientRouteKey {
3130            connection_id,
3131            channel: client_channel,
3132        };
3133        let module_key = ModuleRouteKey {
3134            endpoint,
3135            channel: module_channel,
3136        };
3137        let client_epoch = self
3138            .client_slot_epochs
3139            .get(&client_key)
3140            .copied()
3141            .unwrap_or(0)
3142            + 1;
3143        let module_epoch = self
3144            .module_slot_epochs
3145            .get(&module_key)
3146            .copied()
3147            .unwrap_or(0)
3148            + 1;
3149        self.client_slot_epochs.insert(client_key, client_epoch);
3150        self.module_slot_epochs.insert(module_key, module_epoch);
3151        self.next_client_channel
3152            .insert(connection_id, next_channel(client_channel));
3153        self.next_module_channel
3154            .insert(endpoint, next_channel(module_channel));
3155        Ok((client_channel, client_epoch, module_channel, module_epoch))
3156    }
3157
3158    fn allocate_control_corr(
3159        &mut self,
3160        endpoint: ModuleEndpointId,
3161    ) -> Result<u64, ForwardingError> {
3162        let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
3163        if candidate == 0 {
3164            self.closing_connections.insert(endpoint.connection_id);
3165            return Err(ForwardingError::RelayCorrelationExhausted);
3166        }
3167        self.next_control_corr.insert(
3168            endpoint,
3169            if candidate == u64::MAX {
3170                0
3171            } else {
3172                candidate + 1
3173            },
3174        );
3175        Ok(candidate)
3176    }
3177}
3178
3179fn connection_has_client_routes_locked(
3180    inner: &ForwardingInner,
3181    connection_id: ConnectionId,
3182) -> bool {
3183    inner
3184        .client_to_module
3185        .keys()
3186        .any(|key| key.connection_id == connection_id)
3187        || inner
3188            .reserved_client
3189            .keys()
3190            .any(|key| key.connection_id == connection_id)
3191}
3192
3193fn check_module_connection_role_locked(
3194    inner: &ForwardingInner,
3195    connection_id: ConnectionId,
3196) -> Result<(), ForwardingError> {
3197    if inner.endpoint_by_connection.contains_key(&connection_id)
3198        || connection_has_client_routes_locked(inner, connection_id)
3199    {
3200        return Err(ForwardingError::ConnectionRoleConflict { connection_id });
3201    }
3202    Ok(())
3203}
3204
3205fn next_channel(channel: u16) -> u16 {
3206    let next = channel.wrapping_add(1);
3207    if next == 0 {
3208        1
3209    } else {
3210        next
3211    }
3212}
3213
3214fn endpoint_routes_locked(
3215    inner: &ForwardingInner,
3216    endpoint: ModuleEndpointId,
3217) -> Vec<EndpointRoute> {
3218    let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
3219    let draining = drain_reason.is_some();
3220    let mut routes = inner
3221        .module_to_client
3222        .iter()
3223        .filter(|(module_key, _)| module_key.endpoint == endpoint)
3224        .map(|(_, route)| EndpointRoute {
3225            goodbye_target: GoodbyeTarget {
3226                connection_id: route.client_connection_id,
3227                sink: route.client_sink.clone(),
3228                negotiated_ver: route.client_negotiated_ver,
3229                channel: route.client_channel,
3230                epoch: route.client_epoch,
3231                kind: GoodbyeTargetKind::Client,
3232                module_id: Some(route.module_id.clone()),
3233            },
3234            principal: route.principal.clone(),
3235            bound_at: route.bound_at,
3236            draining,
3237            drain_reason,
3238        })
3239        .collect::<Vec<_>>();
3240    routes.sort_by_key(|route| {
3241        (
3242            route.goodbye_target.connection_id.get(),
3243            route.goodbye_target.channel,
3244            route.goodbye_target.epoch,
3245        )
3246    });
3247    routes
3248}
3249
3250/// Why a bind admitted under `scope` may no longer commit, read from the
3251/// published tags: `scope_ended` when the scope is gone or at another epoch,
3252/// `scope_changed` (retryable) when only its content moved.
3253fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
3254    let key = (scope.owner.clone(), scope.scope_ref.clone());
3255    match inner.scope_tags.get(&key) {
3256        Some(current) if *current == scope.tag => None,
3257        Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
3258            subc_protocol::error_codes::SCOPE_CHANGED,
3259            format!(
3260                "scope '{}' of {} changed while the route was being bound; re-open it",
3261                scope.scope_ref, scope.owner
3262            ),
3263        )),
3264        _ => Some(ErrorBody::new(
3265            subc_protocol::error_codes::SCOPE_ENDED,
3266            format!(
3267                "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
3268                scope.scope_ref, scope.owner, scope.tag.scope_epoch
3269            ),
3270        )),
3271    }
3272}
3273
3274fn release_reserved_route_locked(
3275    inner: &mut ForwardingInner,
3276    client_key: ClientRouteKey,
3277    module_key: ModuleRouteKey,
3278) {
3279    if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
3280        inner.reserved_client.remove(&client_key);
3281    }
3282    if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
3283        inner.reserved_module.remove(&module_key);
3284    }
3285    inner.status.retain(|(key, _), _| *key != client_key);
3286}
3287
3288fn release_client_route_locked(
3289    inner: &mut ForwardingInner,
3290    client_key: ClientRouteKey,
3291    expected_epoch: u32,
3292) -> RouteRelease {
3293    let Some(route) = inner.client_to_module.get(&client_key) else {
3294        return RouteRelease::Absent;
3295    };
3296    if route.client_epoch != expected_epoch {
3297        return RouteRelease::Stale;
3298    }
3299    let route = inner
3300        .client_to_module
3301        .remove(&client_key)
3302        .expect("route checked under the same forwarding lock");
3303    route.flow.close();
3304    if let Some(scope) = &route.scope {
3305        scope.release_bind();
3306    }
3307    inner.module_to_client.remove(&ModuleRouteKey {
3308        endpoint: route.module_endpoint,
3309        channel: route.module_channel,
3310    });
3311    inner.operator_confirms.route_closed(ModuleRouteKey {
3312        endpoint: route.module_endpoint,
3313        channel: route.module_channel,
3314    });
3315    inner.status.remove(&(client_key, expected_epoch));
3316    RouteRelease::Removed(GoodbyeTarget {
3317        connection_id: route.module_endpoint.connection_id,
3318        sink: route.module_sink.clone(),
3319        negotiated_ver: route.module_negotiated_ver,
3320        channel: route.module_channel,
3321        epoch: route.module_epoch,
3322        kind: GoodbyeTargetKind::Module,
3323        module_id: Some(route.module_id.clone()),
3324    })
3325}
3326
3327fn release_module_route_locked(
3328    inner: &mut ForwardingInner,
3329    module_key: ModuleRouteKey,
3330    expected_epoch: u32,
3331) -> RouteRelease {
3332    let Some(route) = inner.module_to_client.get(&module_key) else {
3333        return RouteRelease::Absent;
3334    };
3335    if route.module_epoch != expected_epoch {
3336        return RouteRelease::Stale;
3337    }
3338    let route = inner
3339        .module_to_client
3340        .remove(&module_key)
3341        .expect("route checked under the same forwarding lock");
3342    inner.operator_confirms.route_closed(module_key);
3343    route.flow.close();
3344    if let Some(scope) = &route.scope {
3345        scope.release_bind();
3346    }
3347    let client_key = ClientRouteKey {
3348        connection_id: route.client_connection_id,
3349        channel: route.client_channel,
3350    };
3351    inner.client_to_module.remove(&client_key);
3352    inner.status.remove(&(client_key, route.client_epoch));
3353    RouteRelease::Removed(GoodbyeTarget {
3354        connection_id: route.client_connection_id,
3355        sink: route.client_sink.clone(),
3356        negotiated_ver: route.client_negotiated_ver,
3357        channel: route.client_channel,
3358        epoch: route.client_epoch,
3359        kind: GoodbyeTargetKind::Client,
3360        module_id: Some(route.module_id.clone()),
3361    })
3362}
3363
3364fn commit_route_locked(
3365    inner: &mut ForwardingInner,
3366    pending: PendingRouteBindRelayEntry,
3367) -> Result<Option<GoodbyeTarget>, ForwardingError> {
3368    let reservation = pending.reservation;
3369    if inner
3370        .closing_connections
3371        .contains(&reservation.client_key.connection_id)
3372    {
3373        return Err(ForwardingError::ConnectionClosing {
3374            connection_id: reservation.client_key.connection_id,
3375        });
3376    }
3377    let module_id = inner
3378        .module_id_by_endpoint
3379        .get(&reservation.module_key.endpoint)
3380        .cloned()
3381        .ok_or(ForwardingError::StaleModuleEndpoint)?;
3382    if inner
3383        .draining_endpoints
3384        .contains_key(&reservation.module_key.endpoint)
3385    {
3386        return Err(ForwardingError::ModuleReloading { module_id });
3387    }
3388    if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
3389        || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
3390    {
3391        return Err(ForwardingError::UnknownReservation {
3392            client_channel: reservation.client_key.channel,
3393            module_channel: reservation.module_key.channel,
3394        });
3395    }
3396    let module = inner
3397        .modules_by_id
3398        .get(&module_id)
3399        .filter(|module| module.endpoint == reservation.module_key.endpoint)
3400        .cloned()
3401        .ok_or(ForwardingError::StaleModuleEndpoint)?;
3402    let binding = Arc::new(RouteBinding {
3403        client_connection_id: reservation.client_key.connection_id,
3404        client_sink: pending.client_sink,
3405        client_negotiated_ver: pending.client_negotiated_ver,
3406        client_channel: reservation.client_key.channel,
3407        client_epoch: reservation.client_epoch,
3408        module_id,
3409        module_endpoint: reservation.module_key.endpoint,
3410        module_sink: module.sink,
3411        module_negotiated_ver: module.negotiated_ver,
3412        module_channel: reservation.module_key.channel,
3413        module_epoch: reservation.module_epoch,
3414        principal: pending.principal,
3415        harness: pending.harness,
3416        build_class: pending.build_class,
3417        operator_app: pending.operator_app,
3418        project_root: reservation.project_root.clone(),
3419        bound_at: Instant::now(),
3420        flow: Arc::new(ChannelFlow::new(window_for(&module.concurrency))),
3421        scope: pending.scope,
3422    });
3423    inner
3424        .client_to_module
3425        .insert(reservation.client_key, Arc::clone(&binding));
3426    inner
3427        .module_to_client
3428        .insert(reservation.module_key, Arc::clone(&binding));
3429    let previous_published = inner
3430        .last_published_epoch
3431        .insert(reservation.client_key, reservation.client_epoch);
3432
3433    // This point is reachable only through an accepted route.bind acknowledgement.
3434    // Publish history before notifying the opener, so a subsequent
3435    // `scope.served_by` query sees the acknowledged bind.
3436    if let Some(scope) = &binding.scope {
3437        scope.record_served(&binding.module_id);
3438    }
3439
3440    // Sending through the reserved slot cannot fail, but it reports a receiver
3441    // that closed after reservation and before this locked publication point.
3442    // The frame is stamped and charged to the queued-byte count here, when it
3443    // actually enters the queue, not when the slot was reserved.
3444    let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
3445    if client_writer_closed {
3446        // Not gated on `relay_enqueued`: only a module ack reaches this
3447        // commit, which proves the module holds the binding, and `route.open`
3448        // sends nothing for a relay a reply already settled.
3449        let abandoned = abandoned_route_target(inner, &reservation);
3450        if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
3451            route.flow.close();
3452        }
3453        inner.module_to_client.remove(&reservation.module_key);
3454        inner
3455            .status
3456            .remove(&(reservation.client_key, reservation.client_epoch));
3457        match previous_published {
3458            Some(epoch) => {
3459                inner
3460                    .last_published_epoch
3461                    .insert(reservation.client_key, epoch);
3462            }
3463            None => {
3464                inner.last_published_epoch.remove(&reservation.client_key);
3465            }
3466        }
3467        let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3468            "client egress closed during route publication".to_string(),
3469        ));
3470        return Ok(abandoned);
3471    }
3472
3473    if let Some(scope) = &binding.scope {
3474        scope.commit_presence();
3475    }
3476    let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
3477    Ok(None)
3478}
3479
3480/// The live module connection registered under exactly `endpoint`, whichever
3481/// slot holds it: active, swap candidate, or superseded incumbent.
3482///
3483/// Resolving by endpoint rather than by module id is what keeps an incumbent
3484/// addressable after a swap promoted a candidate over its id. An endpoint that
3485/// is in none of the three (a stale one, replaced without a promotion) resolves
3486/// to nothing, as it always has.
3487fn module_connection_for_endpoint_locked(
3488    inner: &ForwardingInner,
3489    endpoint: ModuleEndpointId,
3490) -> Option<&ModuleConnection> {
3491    let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
3492    inner
3493        .modules_by_id
3494        .get(module_id)
3495        .filter(|module| module.endpoint == endpoint)
3496        .or_else(|| {
3497            inner
3498                .candidates_by_id
3499                .get(module_id)
3500                .filter(|module| module.endpoint == endpoint)
3501        })
3502        .or_else(|| inner.superseded_endpoints.get(&endpoint))
3503}
3504
3505fn abandoned_route_target(
3506    inner: &ForwardingInner,
3507    reservation: &RouteReservation,
3508) -> Option<GoodbyeTarget> {
3509    let module_id = inner
3510        .module_id_by_endpoint
3511        .get(&reservation.module_key.endpoint)?;
3512    let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
3513    (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
3514        connection_id: module.endpoint.connection_id,
3515        sink: module.sink.clone(),
3516        negotiated_ver: module.negotiated_ver,
3517        channel: reservation.module_key.channel,
3518        epoch: reservation.module_epoch,
3519        kind: GoodbyeTargetKind::Module,
3520        module_id: Some(module_id.clone()),
3521    })
3522}
3523
3524/// Queue a registering module's HELLO_ACK on its sink. Called with the
3525/// forwarding write lock held, before the endpoint is inserted, which is what
3526/// puts the ack ahead of any `route.bind` or control RPC routed to the module.
3527/// `try_send` never waits, so holding the lock across it is safe.
3528fn enqueue_hello_ack_locked(
3529    sink: &FrameSink,
3530    connection_id: ConnectionId,
3531    hello_ack: Option<Frame>,
3532) -> Result<(), ForwardingError> {
3533    let Some(hello_ack) = hello_ack else {
3534        return Ok(());
3535    };
3536    sink.try_send(hello_ack)
3537        .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
3538}
3539
3540fn remove_module_connection_locked(
3541    inner: &mut ForwardingInner,
3542    endpoint: ModuleEndpointId,
3543) -> ConnectionCleanup {
3544    // Commit the disconnect before its routes are released, so their teardown
3545    // cannot replace module_closed with route_closed.
3546    inner.operator_confirms.module_closed(endpoint);
3547    inner.draining_endpoints.remove(&endpoint);
3548    inner.daemon_drain_results.remove(&endpoint);
3549    let module_id = inner.module_id_by_endpoint.remove(&endpoint);
3550    if let Some(module_id) = module_id.as_ref() {
3551        if inner
3552            .modules_by_id
3553            .get(module_id)
3554            .is_some_and(|module| module.endpoint == endpoint)
3555        {
3556            inner.modules_by_id.remove(module_id);
3557        }
3558        if inner
3559            .candidates_by_id
3560            .get(module_id)
3561            .is_some_and(|module| module.endpoint == endpoint)
3562        {
3563            inner.candidates_by_id.remove(module_id);
3564        }
3565    }
3566    inner.superseded_endpoints.remove(&endpoint);
3567    inner.endpoint_by_connection.remove(&endpoint.connection_id);
3568    inner.next_module_channel.remove(&endpoint);
3569    inner.next_control_corr.remove(&endpoint);
3570    inner
3571        .health_probe_tombstones
3572        .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
3573    inner
3574        .module_slot_epochs
3575        .retain(|key, _| key.endpoint != endpoint);
3576    let reserved_module_keys: Vec<ModuleRouteKey> = inner
3577        .reserved_module
3578        .keys()
3579        .filter(|module_key| module_key.endpoint == endpoint)
3580        .copied()
3581        .collect();
3582    for module_key in reserved_module_keys {
3583        if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
3584            release_reserved_route_locked(inner, client_key, module_key);
3585        }
3586    }
3587
3588    let pending_keys: Vec<_> = inner
3589        .pending_relays
3590        .keys()
3591        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3592        .copied()
3593        .collect();
3594    let pending: Vec<_> = pending_keys
3595        .into_iter()
3596        .filter_map(|key| {
3597            inner
3598                .pending_relays
3599                .remove(&key)
3600                .map(|pending| (key, pending))
3601        })
3602        .collect();
3603    let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
3604    for (key, pending) in pending {
3605        // Recorded like every other removal without a reply, so a `route.open`
3606        // still sending the bind learns the relay was released rather than
3607        // settled by the module.
3608        if !pending.relay_enqueued {
3609            inner.relays_released_before_enqueue.insert(key);
3610        }
3611        let module_label = module_id.as_deref().unwrap_or("unknown");
3612        let _ = pending
3613            .sender
3614            .send(RouteBindRelayOutcome::ModuleGone(format!(
3615                "module '{module_label}' connection closed during route.bind relay"
3616            )));
3617    }
3618
3619    let pending_control_keys: Vec<_> = inner
3620        .pending_control_rpcs
3621        .keys()
3622        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3623        .copied()
3624        .collect();
3625    let pending_control: Vec<_> = pending_control_keys
3626        .into_iter()
3627        .filter_map(|key| inner.pending_control_rpcs.remove(&key))
3628        .collect();
3629    for pending in pending_control {
3630        let module_label = module_id.as_deref().unwrap_or("unknown");
3631        let _ = pending
3632            .sender
3633            .send(ModuleControlRpcOutcome::ModuleGone(format!(
3634                "module '{module_label}' connection closed during module-control RPC"
3635            )));
3636    }
3637
3638    let module_routes = inner
3639        .module_to_client
3640        .iter()
3641        .filter(|(module_key, _)| module_key.endpoint == endpoint)
3642        .map(|(module_key, route)| (*module_key, route.module_epoch))
3643        .collect::<Vec<_>>();
3644    let mut released = Vec::with_capacity(module_routes.len());
3645    for (module_key, epoch) in module_routes {
3646        if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
3647        {
3648            released.push(target);
3649        }
3650    }
3651    ConnectionCleanup {
3652        released,
3653        abandoned_relays,
3654    }
3655}
3656
3657#[derive(Debug, Clone, Copy)]
3658struct RequestCredit {
3659    subscription: bool,
3660    excluded_from_drain: bool,
3661}
3662
3663#[derive(Debug, Default)]
3664struct CreditLedger {
3665    by_corr: HashMap<u64, Vec<RequestCredit>>,
3666}
3667
3668impl CreditLedger {
3669    fn acquire(&mut self, corr: u64, subscription: bool) {
3670        self.by_corr.entry(corr).or_default().push(RequestCredit {
3671            subscription,
3672            excluded_from_drain: false,
3673        });
3674    }
3675
3676    fn release(&mut self, corr: u64) -> bool {
3677        let Some(credits) = self.by_corr.get_mut(&corr) else {
3678            return false;
3679        };
3680        let released = credits.pop().is_some();
3681        if credits.is_empty() {
3682            self.by_corr.remove(&corr);
3683        }
3684        released
3685    }
3686
3687    fn capture_subscription_exclusions(&mut self) -> u32 {
3688        let mut excluded = 0u32;
3689        for credit in self.by_corr.values_mut().flatten() {
3690            if credit.subscription && !credit.excluded_from_drain {
3691                credit.excluded_from_drain = true;
3692                excluded = excluded.saturating_add(1);
3693            }
3694        }
3695        excluded
3696    }
3697
3698    #[cfg(test)]
3699    fn in_flight(&self) -> usize {
3700        self.by_corr.values().map(Vec::len).sum()
3701    }
3702
3703    fn drain_in_flight(&self) -> usize {
3704        self.by_corr
3705            .values()
3706            .flatten()
3707            .filter(|credit| !credit.excluded_from_drain)
3708            .count()
3709    }
3710
3711    /// The correlation ids of the requests a drain still waits on, one entry
3712    /// per held credit, in ascending order.
3713    fn drain_held_corrs(&self) -> Vec<u64> {
3714        let mut corrs = self
3715            .by_corr
3716            .iter()
3717            .flat_map(|(corr, credits)| {
3718                credits
3719                    .iter()
3720                    .filter(|credit| !credit.excluded_from_drain)
3721                    .map(move |_| *corr)
3722            })
3723            .collect::<Vec<_>>();
3724        corrs.sort_unstable();
3725        corrs
3726    }
3727}
3728
3729#[derive(Debug, Default)]
3730struct ChannelFlowState {
3731    closed: bool,
3732    credits: CreditLedger,
3733}
3734
3735/// Per-channel request-credit accounting shared by the client and module route halves.
3736#[derive(Debug)]
3737pub(crate) struct ChannelFlow {
3738    sem: Semaphore,
3739    window: usize,
3740    state: Mutex<ChannelFlowState>,
3741}
3742
3743impl ChannelFlow {
3744    pub(crate) fn new(window: usize) -> Self {
3745        debug_assert!(window > 0, "flow-control window must be non-zero");
3746        Self {
3747            sem: Semaphore::new(window),
3748            window,
3749            state: Mutex::new(ChannelFlowState::default()),
3750        }
3751    }
3752
3753    #[cfg(test)]
3754    pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
3755        self.acquire_tagged(0, false).await
3756    }
3757
3758    pub(crate) async fn acquire_tagged(
3759        &self,
3760        corr: u64,
3761        subscription: bool,
3762    ) -> Result<(), ChannelFlowClosed> {
3763        let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
3764        let mut state = self
3765            .state
3766            .lock()
3767            .unwrap_or_else(|poisoned| poisoned.into_inner());
3768        if state.closed {
3769            return Err(ChannelFlowClosed);
3770        }
3771        state.credits.acquire(corr, subscription);
3772        permit.forget();
3773        Ok(())
3774    }
3775
3776    #[cfg(test)]
3777    pub(crate) fn release(&self) {
3778        self.release_corr(0);
3779    }
3780
3781    pub(crate) fn release_corr(&self, corr: u64) {
3782        let released = self
3783            .state
3784            .lock()
3785            .unwrap_or_else(|poisoned| poisoned.into_inner())
3786            .credits
3787            .release(corr);
3788        if !released {
3789            // Protocol-conforming modules emit exactly one terminal per request.
3790            // This guard is a best-effort safety net against window growth, not a
3791            // security boundary against malicious peers.
3792            warn!(
3793                window = self.window,
3794                available = self.sem.available_permits(),
3795                "flow-control over-release ignored"
3796            );
3797            return;
3798        }
3799        if !self.sem.is_closed() {
3800            self.sem.add_permits(1);
3801        }
3802    }
3803
3804    #[cfg(test)]
3805    pub(crate) fn in_flight(&self) -> usize {
3806        self.state
3807            .lock()
3808            .unwrap_or_else(|poisoned| poisoned.into_inner())
3809            .credits
3810            .in_flight()
3811    }
3812
3813    pub(crate) fn drain_in_flight(&self) -> usize {
3814        self.state
3815            .lock()
3816            .unwrap_or_else(|poisoned| poisoned.into_inner())
3817            .credits
3818            .drain_in_flight()
3819    }
3820
3821    pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
3822        self.state
3823            .lock()
3824            .unwrap_or_else(|poisoned| poisoned.into_inner())
3825            .credits
3826            .drain_held_corrs()
3827    }
3828
3829    #[cfg(test)]
3830    pub(crate) fn available_permits(&self) -> usize {
3831        self.sem.available_permits()
3832    }
3833
3834    pub(crate) fn begin_drain(&self) -> u32 {
3835        let mut state = self
3836            .state
3837            .lock()
3838            .unwrap_or_else(|poisoned| poisoned.into_inner());
3839        state.closed = true;
3840        self.sem.close();
3841        state.credits.capture_subscription_exclusions()
3842    }
3843
3844    pub(crate) fn close(&self) {
3845        self.state
3846            .lock()
3847            .unwrap_or_else(|poisoned| poisoned.into_inner())
3848            .closed = true;
3849        self.sem.close();
3850    }
3851}
3852
3853#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3854pub(crate) struct ChannelFlowClosed;
3855
3856impl fmt::Display for ChannelFlowClosed {
3857    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3858        write!(f, "flow-control window closed")
3859    }
3860}
3861
3862impl Error for ChannelFlowClosed {}
3863
3864fn window_for(concurrency: &Concurrency) -> usize {
3865    match concurrency {
3866        Concurrency::Serial => 1,
3867        Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
3868        Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
3869    }
3870}
3871
3872#[derive(Debug, Clone, PartialEq, Eq)]
3873pub enum ForwardingError {
3874    /// A connection cannot hold both client routes and a module endpoint, or
3875    /// replace a module identity while that endpoint is still registered.
3876    ConnectionRoleConflict {
3877        connection_id: ConnectionId,
3878    },
3879    NoModuleConnection,
3880    ModuleReloading {
3881        module_id: String,
3882    },
3883    StaleModuleEndpoint,
3884    UnknownReservation {
3885        client_channel: u16,
3886        module_channel: u16,
3887    },
3888    ClientRouteChannelExhausted {
3889        connection_id: ConnectionId,
3890    },
3891    ModuleRouteChannelExhausted {
3892        endpoint: ModuleEndpointId,
3893    },
3894    RelayCorrelationExhausted,
3895    ConnectionClosing {
3896        connection_id: ConnectionId,
3897    },
3898    ClientEgressClosed {
3899        connection_id: ConnectionId,
3900    },
3901    RouteOpenBuild(String),
3902    /// A swap candidate is already registered for this module id.
3903    CandidateSlotOccupied {
3904        module_id: String,
3905    },
3906    /// A registering module's HELLO_ACK could not be queued because its
3907    /// outbound queue is closed or full.
3908    ModuleEgressUnavailable {
3909        connection_id: ConnectionId,
3910    },
3911    Poisoned,
3912}
3913
3914impl fmt::Display for ForwardingError {
3915    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3916        match self {
3917            Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
3918            Self::NoModuleConnection => write!(f, "no module connection is registered"),
3919            Self::ModuleReloading { module_id } => {
3920                write!(f, "module_id '{module_id}' is reloading")
3921            }
3922            Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
3923            Self::UnknownReservation {
3924                client_channel,
3925                module_channel,
3926            } => write!(
3927                f,
3928                "route reservation client channel {client_channel} / module channel {module_channel} was not found"
3929            ),
3930            Self::ClientRouteChannelExhausted { connection_id } => write!(
3931                f,
3932                "no client route channels are available for connection {}",
3933                connection_id.get()
3934            ),
3935            Self::ModuleRouteChannelExhausted { endpoint } => write!(
3936                f,
3937                "no module route channels are available for endpoint generation {} on connection {}",
3938                endpoint.generation,
3939                endpoint.connection_id.get()
3940            ),
3941            Self::RelayCorrelationExhausted => {
3942                write!(f, "module control correlation ids are exhausted")
3943            }
3944            Self::ConnectionClosing { connection_id } => write!(
3945                f,
3946                "connection {} is closing and cannot accept route allocation",
3947                connection_id.get()
3948            ),
3949            Self::ClientEgressClosed { connection_id } => write!(
3950                f,
3951                "client connection {} egress is closed",
3952                connection_id.get()
3953            ),
3954            Self::RouteOpenBuild(message) => {
3955                write!(f, "failed to prebuild route.open response: {message}")
3956            }
3957            Self::CandidateSlotOccupied { module_id } => write!(
3958                f,
3959                "module_id '{module_id}' already has a swap candidate registered"
3960            ),
3961            Self::ModuleEgressUnavailable { connection_id } => write!(
3962                f,
3963                "module connection {} egress is unavailable; HELLO_ACK could not be queued",
3964                connection_id.get()
3965            ),
3966            Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
3967        }
3968    }
3969}
3970
3971impl Error for ForwardingError {}
3972
3973#[cfg(test)]
3974mod tests {
3975    use std::time::Duration;
3976
3977    use super::*;
3978    use tokio::sync::mpsc;
3979
3980    #[test]
3981    fn ordinary_long_running_request_is_not_excluded_from_drain() {
3982        let mut ledger = CreditLedger::default();
3983        ledger.acquire(1, false);
3984
3985        assert_eq!(ledger.capture_subscription_exclusions(), 0);
3986        assert_eq!(ledger.drain_in_flight(), 1);
3987    }
3988
3989    #[test]
3990    fn bit_set_subscription_is_excluded_and_counted() {
3991        let mut ledger = CreditLedger::default();
3992        ledger.acquire(1, true);
3993
3994        assert_eq!(ledger.capture_subscription_exclusions(), 1);
3995        assert_eq!(ledger.drain_in_flight(), 0);
3996    }
3997
3998    #[test]
3999    fn subscription_opened_after_drain_snapshot_is_not_excluded() {
4000        let mut ledger = CreditLedger::default();
4001        ledger.acquire(1, true);
4002        assert_eq!(ledger.capture_subscription_exclusions(), 1);
4003
4004        ledger.acquire(2, true);
4005
4006        assert_eq!(ledger.drain_in_flight(), 1);
4007    }
4008
4009    #[test]
4010    fn drain_with_no_subscriptions_reports_zero_excluded() {
4011        let mut ledger = CreditLedger::default();
4012        assert_eq!(ledger.capture_subscription_exclusions(), 0);
4013    }
4014
4015    fn test_hello_ack(corr: u64) -> Frame {
4016        Frame::build(
4017            FrameType::HelloAck,
4018            Flags::new(false, Priority::Passive, false),
4019            0,
4020            0,
4021            corr,
4022            Vec::new(),
4023        )
4024        .unwrap()
4025    }
4026
4027    /// An acked registration whose HELLO_ACK cannot be queued must leave no
4028    /// endpoint behind: a routable module that never got its ack would read a
4029    /// route.bind first and exit. A closed queue and a full one both refuse.
4030    #[test]
4031    fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
4032        let forwarding = ForwardingTable::default();
4033
4034        let (closed_tx, closed_rx) = mpsc::channel(8);
4035        drop(closed_rx);
4036        let closed = ConnectionId::new(1);
4037        assert_eq!(
4038            forwarding.register_module_connection_acked(
4039                closed,
4040                "closed".to_string(),
4041                2,
4042                Concurrency::ModuleManaged,
4043                FrameSink::new(closed_tx),
4044                test_hello_ack(1),
4045            ),
4046            Err(ForwardingError::ModuleEgressUnavailable {
4047                connection_id: closed
4048            })
4049        );
4050
4051        let (full_tx, _full_rx) = mpsc::channel(1);
4052        let full_sink = FrameSink::new(full_tx);
4053        full_sink.try_send(test_hello_ack(99)).unwrap();
4054        let full = ConnectionId::new(2);
4055        assert_eq!(
4056            forwarding.register_module_connection_acked(
4057                full,
4058                "full".to_string(),
4059                2,
4060                Concurrency::ModuleManaged,
4061                full_sink.clone(),
4062                test_hello_ack(2),
4063            ),
4064            Err(ForwardingError::ModuleEgressUnavailable {
4065                connection_id: full
4066            })
4067        );
4068        assert_eq!(
4069            forwarding.register_candidate_module_connection_acked(
4070                full,
4071                "full".to_string(),
4072                2,
4073                Concurrency::ModuleManaged,
4074                full_sink,
4075                test_hello_ack(3),
4076            ),
4077            Err(ForwardingError::ModuleEgressUnavailable {
4078                connection_id: full
4079            })
4080        );
4081
4082        for (connection, module_id) in [(closed, "closed"), (full, "full")] {
4083            assert_eq!(
4084                forwarding
4085                    .module_endpoint_for_connection(connection)
4086                    .unwrap(),
4087                None
4088            );
4089            let (client_tx, _client_rx) = mpsc::channel(8);
4090            assert_eq!(
4091                forwarding
4092                    .begin_route_bind_relay_for_test(
4093                        ConnectionId::new(50),
4094                        FrameSink::new(client_tx),
4095                        1,
4096                        module_id,
4097                    )
4098                    .err(),
4099                Some(ForwardingError::NoModuleConnection)
4100            );
4101        }
4102        assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
4103    }
4104
4105    /// Both acked registration forms put the HELLO_ACK on the module's queue
4106    /// by the time the endpoint can be resolved.
4107    #[test]
4108    fn acked_registration_queues_the_hello_ack_first() {
4109        let forwarding = ForwardingTable::default();
4110        let (active_tx, mut active_rx) = mpsc::channel(8);
4111        forwarding
4112            .register_module_connection_acked(
4113                ConnectionId::new(1),
4114                "acked".to_string(),
4115                2,
4116                Concurrency::ModuleManaged,
4117                FrameSink::new(active_tx),
4118                test_hello_ack(11),
4119            )
4120            .unwrap();
4121        let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
4122        forwarding
4123            .register_candidate_module_connection_acked(
4124                ConnectionId::new(2),
4125                "acked".to_string(),
4126                2,
4127                Concurrency::ModuleManaged,
4128                FrameSink::new(candidate_tx),
4129                test_hello_ack(12),
4130            )
4131            .unwrap();
4132
4133        let active_first = active_rx.try_recv().unwrap().frame;
4134        assert_eq!(active_first.header.ty, FrameType::HelloAck);
4135        assert_eq!(active_first.header.corr, 11);
4136        let candidate_first = candidate_rx.try_recv().unwrap().frame;
4137        assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
4138        assert_eq!(candidate_first.header.corr, 12);
4139    }
4140
4141    #[test]
4142    fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
4143        let forwarding = ForwardingTable::default();
4144        let module_connection = ConnectionId::new(10);
4145        let exhausted_client = ConnectionId::new(20);
4146        let second_client = ConnectionId::new(30);
4147        let (module_tx, _module_rx) = mpsc::channel(1);
4148        let endpoint = forwarding
4149            .register_module_connection(
4150                module_connection,
4151                "route-limit-provider".to_string(),
4152                1,
4153                Concurrency::ModuleManaged,
4154                FrameSink::new(module_tx),
4155            )
4156            .unwrap();
4157
4158        {
4159            let mut inner = forwarding.inner.write().unwrap();
4160            for channel in 1..=u16::MAX {
4161                inner.reserved_client.insert(
4162                    ClientRouteKey {
4163                        connection_id: exhausted_client,
4164                        channel,
4165                    },
4166                    ModuleRouteKey {
4167                        endpoint,
4168                        channel: 1,
4169                    },
4170                );
4171            }
4172        }
4173
4174        let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
4175        let err = forwarding
4176            .begin_route_bind_relay_for_test(
4177                exhausted_client,
4178                FrameSink::new(exhausted_tx),
4179                1,
4180                "route-limit-provider",
4181            )
4182            .unwrap_err();
4183        assert!(matches!(
4184            err,
4185            ForwardingError::ClientRouteChannelExhausted { connection_id }
4186                if connection_id == exhausted_client
4187        ));
4188
4189        let (second_tx, _second_rx) = mpsc::channel(1);
4190        let pending = forwarding
4191            .begin_route_bind_relay_for_test(
4192                second_client,
4193                FrameSink::new(second_tx),
4194                2,
4195                "route-limit-provider",
4196            )
4197            .unwrap();
4198        assert_eq!(pending.client_channel, 1);
4199    }
4200
4201    #[test]
4202    fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
4203        let forwarding = ForwardingTable::default();
4204        let module_connection = ConnectionId::new(40);
4205        let client = ConnectionId::new(50);
4206        let (module_tx, _module_rx) = mpsc::channel(1);
4207        forwarding
4208            .register_module_connection(
4209                module_connection,
4210                "slot-reuse-provider".to_string(),
4211                1,
4212                Concurrency::ModuleManaged,
4213                FrameSink::new(module_tx),
4214            )
4215            .unwrap();
4216
4217        let (client_tx, _client_rx) = mpsc::channel(1);
4218        let client_sink = FrameSink::new(client_tx);
4219        let mut wrapped_channel = None;
4220        for index in 0..=usize::from(u16::MAX) {
4221            let pending = forwarding
4222                .begin_route_bind_relay_for_test(
4223                    client,
4224                    client_sink.clone(),
4225                    index as u64 + 1,
4226                    "slot-reuse-provider",
4227                )
4228                .unwrap();
4229            if index == usize::from(u16::MAX) {
4230                wrapped_channel = Some(pending.module_channel);
4231            }
4232            forwarding
4233                .abort_pending_relay(
4234                    pending.endpoint,
4235                    pending.corr,
4236                    RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
4237                )
4238                .unwrap();
4239        }
4240
4241        assert_eq!(wrapped_channel, Some(1));
4242    }
4243
4244    #[test]
4245    fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
4246        let forwarding = ForwardingTable::default();
4247        let client = ConnectionId::new(60);
4248        forwarding
4249            .inner
4250            .write()
4251            .unwrap()
4252            .next_client_channel
4253            .insert(client, 41);
4254
4255        let released = forwarding.cleanup_connection(client).unwrap();
4256
4257        assert!(released.is_empty());
4258        assert!(!forwarding
4259            .inner
4260            .read()
4261            .unwrap()
4262            .next_client_channel
4263            .contains_key(&client));
4264    }
4265
4266    #[test]
4267    fn stale_module_cleanup_preserves_fast_reconnect_successor() {
4268        let forwarding = ForwardingTable::default();
4269        let module_id = "fast-reconnect-provider";
4270        let first_connection = ConnectionId::new(70);
4271        let second_connection = ConnectionId::new(80);
4272        let (first_tx, _first_rx) = mpsc::channel(1);
4273        let first_endpoint = forwarding
4274            .register_module_connection(
4275                first_connection,
4276                module_id.to_string(),
4277                1,
4278                Concurrency::ModuleManaged,
4279                FrameSink::new(first_tx),
4280            )
4281            .unwrap();
4282        let (second_tx, _second_rx) = mpsc::channel(1);
4283        let second_endpoint = forwarding
4284            .register_module_connection(
4285                second_connection,
4286                module_id.to_string(),
4287                1,
4288                Concurrency::ModuleManaged,
4289                FrameSink::new(second_tx),
4290            )
4291            .unwrap();
4292        assert_ne!(first_endpoint, second_endpoint);
4293
4294        let released = forwarding.cleanup_connection(first_connection).unwrap();
4295
4296        assert!(released.is_empty());
4297        assert_eq!(
4298            forwarding
4299                .inner
4300                .read()
4301                .unwrap()
4302                .modules_by_id
4303                .get(module_id)
4304                .map(|module| module.endpoint),
4305            Some(second_endpoint)
4306        );
4307        assert!(forwarding.has_live_module_connection(module_id).unwrap());
4308        let control_rpc = forwarding
4309            .begin_module_control_rpc_for(
4310                module_id,
4311                "health.check",
4312                Instant::now() + Duration::from_secs(1),
4313            )
4314            .unwrap();
4315        assert_eq!(control_rpc.endpoint, second_endpoint);
4316    }
4317
4318    fn route_fixture(
4319        module_id: &str,
4320    ) -> (
4321        ForwardingTable,
4322        ConnectionId,
4323        ModuleEndpointId,
4324        ConnectionId,
4325        FrameSink,
4326        mpsc::Receiver<crate::router::OutboundFrame>,
4327    ) {
4328        let forwarding = ForwardingTable::default();
4329        let module_connection = ConnectionId::new(100);
4330        let client_connection = ConnectionId::new(200);
4331        let (module_tx, _module_rx) = mpsc::channel(8);
4332        let endpoint = forwarding
4333            .register_module_connection(
4334                module_connection,
4335                module_id.to_string(),
4336                2,
4337                Concurrency::ModuleManaged,
4338                FrameSink::new(module_tx),
4339            )
4340            .unwrap();
4341        let (client_tx, client_rx) = mpsc::channel(8);
4342        (
4343            forwarding,
4344            module_connection,
4345            endpoint,
4346            client_connection,
4347            FrameSink::new(client_tx),
4348            client_rx,
4349        )
4350    }
4351
4352    #[test]
4353    #[cfg(unix)]
4354    fn daemon_drain_gates_current_and_racing_provider_registrations() {
4355        let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
4356        assert_eq!(
4357            forwarding.begin_daemon_drain().unwrap(),
4358            [("provider".to_string(), endpoint)]
4359        );
4360        assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
4361        assert!(matches!(
4362            forwarding.register_module_connection(
4363                ConnectionId::new(300),
4364                "late-provider".into(),
4365                2,
4366                Concurrency::ModuleManaged,
4367                sink,
4368            ),
4369            Err(ForwardingError::ConnectionClosing { .. })
4370        ));
4371    }
4372
4373    #[test]
4374    #[cfg(unix)]
4375    fn daemon_drain_results_are_endpoint_fenced_and_removed_on_cleanup() {
4376        let (forwarding, connection, endpoint, _, _, _) = route_fixture("provider");
4377        assert_eq!(
4378            forwarding
4379                .daemon_drain_result_for_connection(connection)
4380                .unwrap(),
4381            None
4382        );
4383        forwarding
4384            .record_daemon_drain_result(endpoint, true)
4385            .unwrap();
4386        forwarding.begin_daemon_drain().unwrap();
4387        assert_eq!(
4388            forwarding
4389                .daemon_drain_result_for_connection(connection)
4390                .unwrap(),
4391            Some(false)
4392        );
4393        let replacement = ModuleEndpointId {
4394            generation: endpoint.generation + 1,
4395            ..endpoint
4396        };
4397        forwarding
4398            .record_daemon_drain_result(replacement, true)
4399            .unwrap();
4400        assert_eq!(
4401            forwarding
4402                .daemon_drain_result_for_connection(connection)
4403                .unwrap(),
4404            Some(false)
4405        );
4406        forwarding
4407            .record_daemon_drain_result(endpoint, true)
4408            .unwrap();
4409        assert_eq!(
4410            forwarding
4411                .daemon_drain_result_for_connection(connection)
4412                .unwrap(),
4413            Some(true)
4414        );
4415        assert_eq!(
4416            forwarding
4417                .daemon_drain_result_for_connection(ConnectionId::new(300))
4418                .unwrap(),
4419            Some(false)
4420        );
4421        forwarding.cleanup_connection(connection).unwrap();
4422        forwarding
4423            .record_daemon_drain_result(endpoint, true)
4424            .unwrap();
4425        assert_eq!(
4426            forwarding
4427                .daemon_drain_result_for_connection(connection)
4428                .unwrap(),
4429            Some(false)
4430        );
4431        assert!(forwarding
4432            .read_inner()
4433            .unwrap()
4434            .daemon_drain_results
4435            .is_empty());
4436    }
4437
4438    #[test]
4439    #[cfg(unix)]
4440    fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
4441        let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
4442            route_fixture("provider");
4443        let mut pending =
4444            begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
4445        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
4446        forwarding.begin_daemon_drain().unwrap();
4447        let completion = forwarding
4448            .complete_pending_relay(
4449                module_connection,
4450                pending.corr,
4451                RouteBindRelayOutcome::Accepted,
4452            )
4453            .expect("draining admission is not a fatal module error");
4454        assert!(completion.settled);
4455        let abandoned = completion
4456            .abandoned
4457            .expect("late accepted bind needs a route GOODBYE");
4458        assert_eq!(abandoned.channel, pending.module_channel);
4459        assert!(matches!(pending.receiver.try_recv().unwrap(),
4460            RouteBindRelayOutcome::DaemonRejected(error) if error.code == "module_reloading"));
4461        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4462        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4463        assert!(
4464            rx.try_recv().is_err(),
4465            "no route.open may be published during drain"
4466        );
4467        assert_eq!(
4468            forwarding
4469                .module_endpoint_for_connection(module_connection)
4470                .unwrap(),
4471            Some(endpoint)
4472        );
4473        assert!(
4474            !forwarding
4475                .complete_pending_relay(
4476                    module_connection,
4477                    pending.corr,
4478                    RouteBindRelayOutcome::Accepted,
4479                )
4480                .unwrap()
4481                .settled
4482        );
4483    }
4484
4485    fn test_ping(corr: u64) -> Frame {
4486        Frame::build(
4487            FrameType::Ping,
4488            Flags::new(false, Priority::Passive, false),
4489            0,
4490            0,
4491            corr,
4492            Vec::new(),
4493        )
4494        .unwrap()
4495    }
4496
4497    fn begin_test_route(
4498        forwarding: &ForwardingTable,
4499        client_connection: ConnectionId,
4500        client_sink: FrameSink,
4501        corr: u64,
4502        module_id: &str,
4503    ) -> PendingRouteBindRelay {
4504        forwarding
4505            .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
4506            .unwrap()
4507    }
4508
4509    #[test]
4510    fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
4511        for candidate in [false, true] {
4512            for register_first in [false, true] {
4513                let (forwarding, module_connection, _, client_connection, sink, _rx) =
4514                    route_fixture("target");
4515                let register = || {
4516                    if candidate {
4517                        forwarding.register_candidate_module_connection(
4518                            client_connection,
4519                            "source".into(),
4520                            2,
4521                            Concurrency::ModuleManaged,
4522                            sink.clone(),
4523                        )
4524                    } else {
4525                        forwarding.register_module_connection(
4526                            client_connection,
4527                            "source".into(),
4528                            2,
4529                            Concurrency::ModuleManaged,
4530                            sink.clone(),
4531                        )
4532                    }
4533                };
4534                if register_first {
4535                    register().unwrap();
4536                    assert!(
4537                        forwarding
4538                            .begin_route_bind_relay_for_test(
4539                                client_connection,
4540                                sink.clone(),
4541                                1,
4542                                "target",
4543                            )
4544                            .is_err(),
4545                        "a deferred route.open cannot reserve after HELLO"
4546                    );
4547                    assert!(forwarding
4548                        .cleanup_connection(client_connection)
4549                        .unwrap()
4550                        .is_empty());
4551                } else {
4552                    let pending =
4553                        begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
4554                    assert!(
4555                        register().is_err(),
4556                        "HELLO cannot register after a route reservation"
4557                    );
4558                    forwarding
4559                        .complete_pending_relay(
4560                            module_connection,
4561                            pending.corr,
4562                            RouteBindRelayOutcome::Accepted,
4563                        )
4564                        .unwrap();
4565                    assert_eq!(
4566                        forwarding
4567                            .cleanup_connection(client_connection)
4568                            .unwrap()
4569                            .len(),
4570                        1
4571                    );
4572                }
4573                assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4574                assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4575            }
4576        }
4577    }
4578
4579    /// A pending route.open reserves its queue slot with `reserve_owned` and
4580    /// sends its prebuilt response through that slot when the module accepts.
4581    /// A queue already holding many data frames (far more than the old 64-frame
4582    /// queue) must not stop the open from reserving or completing, and the
4583    /// response must be charged to the queue's byte count when it is sent.
4584    #[tokio::test]
4585    async fn pending_route_open_completes_behind_queued_data_frames() {
4586        assert_eq!(
4587            crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
4588            8,
4589            "the per-connection pending route.open limit is its own constant"
4590        );
4591        let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
4592            route_fixture("open-behind-data");
4593        let (sink, mut client_rx) = crate::server::connection_egress();
4594        const DATA_FRAMES: usize = 1_000;
4595        let data = |corr: u64| {
4596            Frame::build(
4597                FrameType::StreamData,
4598                Flags::new(false, Priority::Interactive, false),
4599                9,
4600                1,
4601                corr,
4602                vec![b'x'; 200],
4603            )
4604            .unwrap()
4605        };
4606        for corr in 0..DATA_FRAMES as u64 {
4607            sink.try_send(data(corr)).unwrap();
4608        }
4609        let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
4610        assert_eq!(sink.backlog().queued_bytes, data_bytes);
4611
4612        let pending = tokio::time::timeout(
4613            Duration::from_secs(5),
4614            forwarding.begin_route_bind_relay_for(
4615                client,
4616                sink.clone(),
4617                subc_protocol::PROTOCOL_VERSION,
4618                4_242,
4619                "open-behind-data",
4620                Principal::Direct,
4621                None,
4622                None,
4623                "fixture".into(),
4624                None,
4625                None,
4626                Instant::now() + Duration::from_secs(60),
4627            ),
4628        )
4629        .await
4630        .expect("reserving the route.open slot must not wait behind data frames")
4631        .unwrap();
4632        forwarding
4633            .complete_pending_relay(
4634                module_connection,
4635                pending.corr,
4636                RouteBindRelayOutcome::Accepted,
4637            )
4638            .unwrap();
4639
4640        let backlog = sink.backlog();
4641        assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
4642        assert!(
4643            backlog.queued_bytes > data_bytes,
4644            "the route.open response must be counted in queued bytes: {backlog:?}"
4645        );
4646        for corr in 0..DATA_FRAMES as u64 {
4647            assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
4648        }
4649        let open = client_rx.recv().await.unwrap();
4650        assert_eq!(open.header.corr, 4_242);
4651        assert_eq!(open.header.ty, FrameType::Response);
4652        drop(open);
4653        assert_eq!(sink.backlog().queued_bytes, 0);
4654        assert_eq!(sink.backlog().queued_frames, 0);
4655    }
4656
4657    /// The drain-timeout line reports these numbers, so they must count the
4658    /// requests the drain is waiting on and no others: a route with nothing in
4659    /// flight is not a holdout, and a flagged subscription is excluded from the
4660    /// drain and so from the count.
4661    #[tokio::test]
4662    async fn drain_holdouts_count_held_requests_and_name_the_connection() {
4663        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4664            route_fixture("holdouts");
4665        let mut bound = |corr| {
4666            let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
4667            forwarding
4668                .complete_pending_relay(
4669                    module_connection,
4670                    route.corr,
4671                    RouteBindRelayOutcome::Accepted,
4672                )
4673                .unwrap();
4674            client_rx.try_recv().unwrap();
4675            match forwarding
4676                .lookup_data_route(client, route.client_channel, route.client_epoch)
4677                .unwrap()
4678            {
4679                DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4680                other => panic!("expected live route, got {other:?}"),
4681            }
4682        };
4683        let holding = bound(61);
4684        let _idle = bound(62);
4685        holding.flow.acquire_tagged(7, false).await.unwrap();
4686        holding.flow.acquire_tagged(2, false).await.unwrap();
4687        holding.flow.acquire_tagged(3, true).await.unwrap();
4688        forwarding
4689            .begin_module_drain("holdouts", RouteCloseReason::Restart)
4690            .unwrap();
4691
4692        let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
4693        assert_eq!(
4694            holdouts,
4695            DrainHoldouts {
4696                requests: 2,
4697                routes: 1,
4698                total_routes: 2,
4699                top_connections: vec![(client.get(), 2)],
4700                // The held requests by the module's channel and corr, ascending,
4701                // without the excluded subscription (corr 3).
4702                held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
4703            }
4704        );
4705    }
4706
4707    #[test]
4708    fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
4709        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4710            route_fixture("census");
4711        let pending = begin_test_route(&forwarding, client, sink, 1, "census");
4712        forwarding
4713            .complete_pending_relay(
4714                module_connection,
4715                pending.corr,
4716                RouteBindRelayOutcome::Accepted,
4717            )
4718            .unwrap();
4719
4720        let routes = forwarding.endpoint_routes(endpoint).unwrap();
4721        assert_eq!(routes.len(), 1);
4722        assert!(matches!(routes[0].principal, Principal::Direct));
4723        assert_eq!(routes[0].goodbye_target.connection_id, client);
4724        assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
4725        assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
4726        assert!(!routes[0].draining);
4727
4728        forwarding
4729            .begin_module_drain("census", RouteCloseReason::Restart)
4730            .unwrap();
4731        let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
4732        assert_eq!(draining_routes.len(), 1);
4733        assert!(draining_routes[0].draining);
4734    }
4735
4736    #[test]
4737    fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
4738        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
4739        let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
4740        assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
4741        forwarding
4742            .abort_pending_relay(
4743                first.endpoint,
4744                first.corr,
4745                RouteBindRelayOutcome::ModuleGone("abort".into()),
4746            )
4747            .unwrap();
4748        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4749        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4750
4751        let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
4752        assert_eq!(second.client_channel, first.client_channel);
4753        assert_eq!(second.module_channel, first.module_channel);
4754        assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
4755    }
4756
4757    #[test]
4758    fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
4759        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4760            route_fixture("epoch-release");
4761        let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
4762        forwarding
4763            .complete_pending_relay(
4764                module_connection,
4765                first.corr,
4766                RouteBindRelayOutcome::Accepted,
4767            )
4768            .unwrap();
4769        assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
4770        assert!(matches!(
4771            forwarding
4772                .release_client_route(client, first.client_channel, first.client_epoch)
4773                .unwrap(),
4774            RouteRelease::Removed(_)
4775        ));
4776        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4777        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4778
4779        let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
4780        forwarding
4781            .complete_pending_relay(
4782                module_connection,
4783                second.corr,
4784                RouteBindRelayOutcome::Accepted,
4785            )
4786            .unwrap();
4787        assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
4788        assert!(matches!(
4789            forwarding
4790                .release_client_route(client, second.client_channel, first.client_epoch)
4791                .unwrap(),
4792            RouteRelease::Stale
4793        ));
4794        assert!(!forwarding
4795            .cache_status(
4796                endpoint,
4797                second.module_channel,
4798                first.module_epoch,
4799                "stale".into(),
4800            )
4801            .unwrap());
4802        assert!(forwarding
4803            .cache_status(
4804                endpoint,
4805                second.module_channel,
4806                second.module_epoch,
4807                "current".into(),
4808            )
4809            .unwrap());
4810        match forwarding
4811            .route_poll_snapshot(client, second.client_channel, second.client_epoch)
4812            .unwrap()
4813        {
4814            RoutePollSnapshot::Bound { status, .. } => {
4815                assert_eq!(status.as_deref(), Some("current"));
4816            }
4817            RoutePollSnapshot::Absent => panic!("successor binding was removed"),
4818        }
4819        let counters = forwarding.counters().snapshot();
4820        assert_eq!(counters["route_released_epoch_fenced"], 1);
4821        assert_eq!(counters["route_release_stale_skipped"], 1);
4822    }
4823
4824    #[test]
4825    fn max_epoch_reservation_retires_only_that_slot() {
4826        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
4827        forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
4828        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
4829        let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
4830        assert_eq!(
4831            (final_use.client_channel, final_use.client_epoch),
4832            (7, u32::MAX)
4833        );
4834        assert_eq!(
4835            (final_use.module_channel, final_use.module_epoch),
4836            (9, u32::MAX)
4837        );
4838        forwarding
4839            .abort_pending_relay(
4840                endpoint,
4841                final_use.corr,
4842                RouteBindRelayOutcome::ModuleGone("abort".into()),
4843            )
4844            .unwrap();
4845        forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
4846        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
4847        let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
4848        assert_ne!(next.client_channel, 7);
4849        assert_ne!(next.module_channel, 9);
4850        assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
4851    }
4852
4853    #[test]
4854    fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
4855        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4856            route_fixture("corr-shared");
4857        let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
4858        assert_eq!(bind.corr, 1);
4859        forwarding
4860            .abort_pending_relay(
4861                endpoint,
4862                bind.corr,
4863                RouteBindRelayOutcome::ModuleGone("abort".into()),
4864            )
4865            .unwrap();
4866        let rpc = forwarding
4867            .begin_module_control_rpc_for(
4868                "corr-shared",
4869                "health.check",
4870                Instant::now() - Duration::from_millis(1),
4871            )
4872            .unwrap();
4873        assert_eq!(rpc.corr, 2);
4874        assert_eq!(
4875            forwarding
4876                .complete_module_control_rpc(
4877                    module_connection,
4878                    rpc.corr,
4879                    Some("health.check"),
4880                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
4881                        status: subc_protocol::session::HealthStatus::Ok,
4882                        detail: None,
4883                        metrics: None,
4884                    }),
4885                )
4886                .unwrap(),
4887            ModuleControlRpcCompletion::Settled
4888        );
4889        assert!(matches!(
4890            rpc.receiver.blocking_recv().unwrap(),
4891            ModuleControlRpcOutcome::DeadlineElapsed
4892        ));
4893    }
4894
4895    #[tokio::test(start_paused = true)]
4896    async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
4897        let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
4898        let probe_started_at = Instant::now();
4899        let rpc = forwarding
4900            .begin_health_probe_rpc_for(
4901                "tombstone-ttl",
4902                "health.check",
4903                probe_started_at,
4904                probe_started_at + Duration::from_secs(5),
4905            )
4906            .unwrap();
4907        assert!(forwarding
4908            .tombstone_health_probe_rpc(endpoint, rpc.corr)
4909            .unwrap());
4910        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
4911
4912        tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
4913        tokio::task::yield_now().await;
4914
4915        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
4916    }
4917
4918    #[test]
4919    fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
4920        let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
4921        let mut close = forwarding.register_connection_close(endpoint.connection_id);
4922        forwarding.inject_control_corr(endpoint, u64::MAX);
4923        let final_rpc = forwarding
4924            .begin_module_control_rpc_for(
4925                "corr-max",
4926                "health.check",
4927                Instant::now() + Duration::from_secs(1),
4928            )
4929            .unwrap();
4930        assert_eq!(final_rpc.corr, u64::MAX);
4931        forwarding
4932            .cancel_module_control_rpc(endpoint, final_rpc.corr)
4933            .unwrap();
4934        assert!(matches!(
4935            forwarding.begin_module_control_rpc_for(
4936                "corr-max",
4937                "health.check",
4938                Instant::now() + Duration::from_secs(1),
4939            ),
4940            Err(ForwardingError::RelayCorrelationExhausted)
4941        ));
4942        assert!(close.try_recv().is_ok());
4943    }
4944
4945    #[test]
4946    fn publication_epoch_controls_delivery_failure_escalation() {
4947        fn setup_successor(
4948            commit_successor: Option<bool>,
4949        ) -> (ForwardingTable, ConnectionId, u16, u32) {
4950            let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4951                route_fixture("escalation");
4952            let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
4953            forwarding
4954                .complete_pending_relay(
4955                    module_connection,
4956                    first.corr,
4957                    RouteBindRelayOutcome::Accepted,
4958                )
4959                .unwrap();
4960            client_rx.try_recv().unwrap();
4961            assert!(matches!(
4962                forwarding
4963                    .release_client_route(client, first.client_channel, first.client_epoch)
4964                    .unwrap(),
4965                RouteRelease::Removed(_)
4966            ));
4967            if let Some(commit_successor) = commit_successor {
4968                forwarding.inject_client_slot_epoch(
4969                    client,
4970                    first.client_channel,
4971                    first.client_epoch,
4972                );
4973                forwarding.inject_module_slot_epoch(
4974                    endpoint,
4975                    first.module_channel,
4976                    first.module_epoch,
4977                );
4978                let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
4979                if commit_successor {
4980                    forwarding
4981                        .complete_pending_relay(
4982                            module_connection,
4983                            successor.corr,
4984                            RouteBindRelayOutcome::Accepted,
4985                        )
4986                        .unwrap();
4987                    client_rx.try_recv().unwrap();
4988                } else {
4989                    forwarding
4990                        .abort_pending_relay(
4991                            endpoint,
4992                            successor.corr,
4993                            RouteBindRelayOutcome::ModuleGone("abort".into()),
4994                        )
4995                        .unwrap();
4996                }
4997            }
4998            (forwarding, client, first.client_channel, first.client_epoch)
4999        }
5000
5001        let probe_sink = FrameSink::new(mpsc::channel(1).0);
5002        let (no_successor, client, channel, epoch) = setup_successor(None);
5003        let mut close = no_successor.register_connection_close(client);
5004        assert!(no_successor
5005            .escalate_client_delivery_failure(
5006                client,
5007                channel,
5008                epoch,
5009                CloseReason::new("delivery", "failed"),
5010                UndeliveredFrame {
5011                    module_id: None,
5012                    sink: &probe_sink,
5013                },
5014            )
5015            .unwrap());
5016        assert!(close.try_recv().is_ok());
5017
5018        let (aborted, client, channel, epoch) = setup_successor(Some(false));
5019        let mut close = aborted.register_connection_close(client);
5020        assert!(aborted
5021            .escalate_client_delivery_failure(
5022                client,
5023                channel,
5024                epoch,
5025                CloseReason::new("delivery", "failed"),
5026                UndeliveredFrame {
5027                    module_id: None,
5028                    sink: &probe_sink,
5029                },
5030            )
5031            .unwrap());
5032        assert!(close.try_recv().is_ok());
5033
5034        let (published, client, channel, epoch) = setup_successor(Some(true));
5035        let mut close = published.register_connection_close(client);
5036        assert!(!published
5037            .escalate_client_delivery_failure(
5038                client,
5039                channel,
5040                epoch,
5041                CloseReason::new("delivery", "stale failure"),
5042                UndeliveredFrame {
5043                    module_id: None,
5044                    sink: &probe_sink,
5045                },
5046            )
5047            .unwrap());
5048        assert!(close.try_recv().is_err());
5049    }
5050
5051    #[test]
5052    fn route_concentration_separates_client_count_from_routes_per_client() {
5053        // The distinction this asserts is the one a bare connection count cannot
5054        // make: two connections holding one route each and one connection
5055        // holding two are the same total, and have opposite causes.
5056        let (forwarding, module_connection, _, client, sink, _client_rx) =
5057            route_fixture("concentration");
5058        assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
5059
5060        for corr in [70_u64, 71] {
5061            let pending =
5062                begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
5063            forwarding
5064                .complete_pending_relay(
5065                    module_connection,
5066                    pending.corr,
5067                    RouteBindRelayOutcome::Accepted,
5068                )
5069                .unwrap();
5070        }
5071
5072        // One connection, two routes — not two connections with a route each.
5073        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5074        assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
5075    }
5076
5077    #[test]
5078    fn cleanup_and_accepted_resolution_have_one_lock_winner() {
5079        let (forwarding, module_connection, _, client, sink, mut client_rx) =
5080            route_fixture("cleanup-race");
5081        let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
5082        forwarding
5083            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5084            .unwrap();
5085        let released = forwarding.cleanup_connection(client).unwrap();
5086        assert_eq!(released.len(), 1);
5087        let completion = forwarding
5088            .complete_pending_relay(
5089                module_connection,
5090                pending.corr,
5091                RouteBindRelayOutcome::Accepted,
5092            )
5093            .unwrap();
5094        assert!(!completion.settled);
5095        assert!(client_rx.try_recv().is_err());
5096        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5097
5098        let (forwarding, module_connection, _, client, sink, mut client_rx) =
5099            route_fixture("accepted-race");
5100        let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
5101        forwarding
5102            .complete_pending_relay(
5103                module_connection,
5104                pending.corr,
5105                RouteBindRelayOutcome::Accepted,
5106            )
5107            .unwrap();
5108        assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
5109        let released = forwarding.cleanup_connection(client).unwrap();
5110        assert_eq!(released.len(), 1);
5111        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5112    }
5113
5114    #[test]
5115    fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
5116        let (forwarding, module_connection, _, client, sink, mut client_rx) =
5117            route_fixture("drain-gap");
5118        let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
5119        forwarding
5120            .complete_pending_relay(
5121                module_connection,
5122                live.corr,
5123                RouteBindRelayOutcome::Accepted,
5124            )
5125            .unwrap();
5126        client_rx.try_recv().unwrap();
5127        let binding = match forwarding
5128            .lookup_data_route(client, live.client_channel, live.client_epoch)
5129            .unwrap()
5130        {
5131            DataRoute::Client(DataRouteState::Bound(binding)) => binding,
5132            other => panic!("expected live route, got {other:?}"),
5133        };
5134
5135        let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
5136        forwarding
5137            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5138            .unwrap();
5139        let control_rpc = forwarding
5140            .begin_module_control_rpc_for(
5141                "drain-gap",
5142                "health.check",
5143                Instant::now() + Duration::from_secs(1),
5144            )
5145            .unwrap();
5146        let target = forwarding
5147            .begin_module_drain("drain-gap", RouteCloseReason::Reload)
5148            .unwrap()
5149            .unwrap();
5150        assert!(matches!(
5151            control_rpc.receiver.blocking_recv().unwrap(),
5152            ModuleControlRpcOutcome::ModuleGone(_)
5153        ));
5154        assert_eq!(target.abandoned_bindings.len(), 1);
5155        assert!(binding.flow.sem.is_closed());
5156        assert!(
5157            !forwarding
5158                .complete_pending_relay(
5159                    module_connection,
5160                    pending.corr,
5161                    RouteBindRelayOutcome::Accepted,
5162                )
5163                .unwrap()
5164                .settled
5165        );
5166        assert!(matches!(
5167            forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
5168            Err(ForwardingError::ModuleReloading { .. })
5169        ));
5170        let released = forwarding
5171            .release_module_endpoint_routes(target.endpoint)
5172            .unwrap();
5173        assert_eq!(released.len(), 1);
5174        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5175    }
5176
5177    /// A client can be marked closing while its egress is still open: the daemon
5178    /// asks a connection to close (here through the production path, a module
5179    /// frame that its egress refused) and the connection loop tears down a moment
5180    /// later. A route.bind ack that lands inside that window is answered on the
5181    /// MODULE connection's frame handler, so resolving it must not produce an
5182    /// error -- an error there ends the module connection, and that connection
5183    /// carries every other client's routes to the module.
5184    #[test]
5185    fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
5186        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5187            route_fixture("closing-client");
5188
5189        // A published route on this client: escalate_client_delivery_failure only
5190        // marks a connection closing for a route it has already published.
5191        let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
5192        forwarding
5193            .complete_pending_relay(
5194                module_connection,
5195                live.corr,
5196                RouteBindRelayOutcome::Accepted,
5197            )
5198            .unwrap();
5199        client_rx.try_recv().unwrap();
5200
5201        // A second route.open from the same client, relayed and awaiting its ack.
5202        let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
5203        forwarding
5204            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5205            .unwrap();
5206
5207        // The window: closing, but the sink is still open.
5208        assert!(forwarding
5209            .escalate_client_delivery_failure(
5210                client,
5211                live.client_channel,
5212                live.client_epoch,
5213                CloseReason::new(
5214                    "module_to_client_delivery_failed",
5215                    "client egress refused a module frame",
5216                ),
5217                UndeliveredFrame {
5218                    module_id: None,
5219                    sink: &sink,
5220                },
5221            )
5222            .unwrap());
5223        assert!(!sink.is_closed());
5224
5225        let completion = forwarding
5226            .complete_pending_relay(
5227                module_connection,
5228                pending.corr,
5229                RouteBindRelayOutcome::Accepted,
5230            )
5231            .expect("a closing client must not turn a module's ack into an error");
5232
5233        assert!(completion.settled);
5234        let abandoned = completion
5235            .abandoned
5236            .expect("the module must be told to drop the binding it just created");
5237        assert_eq!(abandoned.connection_id, module_connection);
5238        assert_eq!(abandoned.channel, pending.module_channel);
5239        assert_eq!(abandoned.epoch, pending.module_epoch);
5240        assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
5241        assert!(matches!(
5242            pending.receiver.blocking_recv().unwrap(),
5243            RouteBindRelayOutcome::ModuleGone(_)
5244        ));
5245        // No route was published to a client that is on its way out, and the
5246        // reserved handle pair went back.
5247        assert!(client_rx.try_recv().is_err());
5248        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
5249
5250        // The module endpoint is untouched: still live, and still able to take a
5251        // route from another client.
5252        assert!(forwarding
5253            .has_live_module_connection("closing-client")
5254            .unwrap());
5255        let cotenant = ConnectionId::new(201);
5256        let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
5257        let cotenant_route = begin_test_route(
5258            &forwarding,
5259            cotenant,
5260            FrameSink::new(cotenant_tx),
5261            62,
5262            "closing-client",
5263        );
5264        assert_eq!(cotenant_route.endpoint, endpoint);
5265        forwarding
5266            .complete_pending_relay(
5267                module_connection,
5268                cotenant_route.corr,
5269                RouteBindRelayOutcome::Accepted,
5270            )
5271            .unwrap();
5272        assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
5273        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5274    }
5275
5276    #[test]
5277    fn pending_route_permit_is_released_on_rejection_and_abort() {
5278        let forwarding = ForwardingTable::default();
5279        let module_connection = ConnectionId::new(300);
5280        let client = ConnectionId::new(301);
5281        let (module_tx, _module_rx) = mpsc::channel(1);
5282        let endpoint = forwarding
5283            .register_module_connection(
5284                module_connection,
5285                "permit".into(),
5286                2,
5287                Concurrency::ModuleManaged,
5288                FrameSink::new(module_tx),
5289            )
5290            .unwrap();
5291        let (client_tx, mut client_rx) = mpsc::channel(1);
5292        let sink = FrameSink::new(client_tx);
5293        let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
5294        assert!(sink.try_send(test_ping(999)).is_err());
5295        forwarding
5296            .complete_pending_relay(
5297                module_connection,
5298                rejected.corr,
5299                RouteBindRelayOutcome::Rejected(ErrorBody {
5300                    code: "no".into(),
5301                    message: "rejected".into(),
5302                    detail: None,
5303                }),
5304            )
5305            .unwrap();
5306        sink.try_send(test_ping(1000)).unwrap();
5307        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
5308
5309        let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
5310        assert!(sink.try_send(test_ping(1001)).is_err());
5311        forwarding
5312            .abort_pending_relay(
5313                endpoint,
5314                aborted.corr,
5315                RouteBindRelayOutcome::ModuleGone("abort".into()),
5316            )
5317            .unwrap();
5318        sink.try_send(test_ping(1002)).unwrap();
5319        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
5320
5321        let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
5322        forwarding
5323            .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
5324            .unwrap();
5325        drop(client_rx);
5326        let completion = forwarding
5327            .complete_pending_relay(
5328                module_connection,
5329                receiver_closed.corr,
5330                RouteBindRelayOutcome::Accepted,
5331            )
5332            .unwrap();
5333        assert!(completion.abandoned.is_some());
5334        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5335    }
5336
5337    /// Every connection teardown passes through `cleanup_connection`, and
5338    /// connection ids come from a monotonic counter that never hands an id out
5339    /// twice. If the closing mark survives teardown, the set grows by one entry
5340    /// per connection for the life of the daemon -- the self-watchdog alone
5341    /// reconnects once a minute.
5342    #[test]
5343    fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
5344        let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
5345            route_fixture("closing-set-leak");
5346
5347        const CONNECTIONS: u64 = 32;
5348        for index in 0..CONNECTIONS {
5349            let client = ConnectionId::new(1000 + index);
5350            let (client_tx, _client_rx) = mpsc::channel(8);
5351            let route = begin_test_route(
5352                &forwarding,
5353                client,
5354                FrameSink::new(client_tx),
5355                index + 1,
5356                "closing-set-leak",
5357            );
5358            forwarding
5359                .complete_pending_relay(
5360                    module_connection,
5361                    route.corr,
5362                    RouteBindRelayOutcome::Accepted,
5363                )
5364                .unwrap();
5365            forwarding.cleanup_connection(client).unwrap();
5366        }
5367        forwarding.cleanup_connection(module_connection).unwrap();
5368
5369        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5370    }
5371
5372    /// The closing mark exists to refuse new work for a connection that is on
5373    /// its way out but whose teardown has not run yet: the daemon asks the
5374    /// connection loop to end, and only when the loop reacts does
5375    /// `cleanup_connection` strip the connection's state. Inside that window an
5376    /// operation for the dying connection must still be refused; only after
5377    /// cleanup completes may the mark go.
5378    #[test]
5379    fn closing_connection_is_refused_new_work_until_cleanup_completes() {
5380        let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
5381            route_fixture("closing-gate");
5382
5383        // A published route: escalate_client_delivery_failure only marks a
5384        // connection closing for a route it has already published.
5385        let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
5386        forwarding
5387            .complete_pending_relay(
5388                module_connection,
5389                live.corr,
5390                RouteBindRelayOutcome::Accepted,
5391            )
5392            .unwrap();
5393        client_rx.try_recv().unwrap();
5394
5395        // Mark the connection closing through the production path without
5396        // running teardown, pinning the window open.
5397        assert!(forwarding
5398            .escalate_client_delivery_failure(
5399                client,
5400                live.client_channel,
5401                live.client_epoch,
5402                CloseReason::new(
5403                    "module_to_client_delivery_failed",
5404                    "client egress refused a module frame",
5405                ),
5406                UndeliveredFrame {
5407                    module_id: None,
5408                    sink: &sink,
5409                },
5410            )
5411            .unwrap());
5412        assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
5413
5414        // A late route.open for the closing client is refused ...
5415        assert!(matches!(
5416            forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
5417            Err(ForwardingError::ConnectionClosing { connection_id })
5418                if connection_id == client
5419        ));
5420        // ... and so is a late attempt to register the connection as a module.
5421        let (late_tx, _late_rx) = mpsc::channel(1);
5422        assert!(matches!(
5423            forwarding.register_module_connection(
5424                client,
5425                "late-module".into(),
5426                2,
5427                Concurrency::ModuleManaged,
5428                FrameSink::new(late_tx),
5429            ),
5430            Err(ForwardingError::ConnectionClosing { connection_id })
5431                if connection_id == client
5432        ));
5433
5434        // Teardown is the point that lifts the mark: it has just removed every
5435        // per-connection entry under the same lock, so the gate has nothing
5436        // left to protect for this id.
5437        forwarding.cleanup_connection(client).unwrap();
5438        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5439    }
5440}
5441
5442/// Blue/green swap slots: a candidate registered beside the active endpoint,
5443/// promoted by `cutover_candidate`, with the old incumbent drained by endpoint.
5444#[cfg(test)]
5445mod swap_slot_tests {
5446    use std::time::Duration;
5447
5448    use super::*;
5449    use tokio::sync::mpsc;
5450
5451    const MODULE_ID: &str = "swapped";
5452
5453    struct SwapFixture {
5454        forwarding: ForwardingTable,
5455        incumbent_connection: ConnectionId,
5456        incumbent: ModuleEndpointId,
5457        candidate_connection: ConnectionId,
5458        candidate: ModuleEndpointId,
5459        _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
5460    }
5461
5462    fn swap_fixture() -> SwapFixture {
5463        let forwarding = ForwardingTable::default();
5464        let incumbent_connection = ConnectionId::new(100);
5465        let candidate_connection = ConnectionId::new(110);
5466        let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
5467        let incumbent = forwarding
5468            .register_module_connection(
5469                incumbent_connection,
5470                MODULE_ID.to_string(),
5471                2,
5472                Concurrency::ModuleManaged,
5473                FrameSink::new(incumbent_tx),
5474            )
5475            .unwrap();
5476        let (candidate_tx, candidate_rx) = mpsc::channel(8);
5477        let candidate = forwarding
5478            .register_candidate_module_connection(
5479                candidate_connection,
5480                MODULE_ID.to_string(),
5481                2,
5482                Concurrency::ModuleManaged,
5483                FrameSink::new(candidate_tx),
5484            )
5485            .unwrap();
5486        SwapFixture {
5487            forwarding,
5488            incumbent_connection,
5489            incumbent,
5490            candidate_connection,
5491            candidate,
5492            _module_rxs: vec![incumbent_rx, candidate_rx],
5493        }
5494    }
5495
5496    fn client(
5497        raw: u64,
5498    ) -> (
5499        ConnectionId,
5500        FrameSink,
5501        mpsc::Receiver<crate::router::OutboundFrame>,
5502    ) {
5503        let (tx, rx) = mpsc::channel(8);
5504        (ConnectionId::new(raw), FrameSink::new(tx), rx)
5505    }
5506
5507    fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
5508        forwarding
5509            .read_inner()
5510            .unwrap()
5511            .client_to_module
5512            .values()
5513            .map(|route| route.module_endpoint)
5514            .collect()
5515    }
5516
5517    #[test]
5518    fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
5519        let fixture = swap_fixture();
5520        let forwarding = &fixture.forwarding;
5521        assert_ne!(fixture.incumbent, fixture.candidate);
5522
5523        // Every by-id consumer still resolves the incumbent.
5524        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5525        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5526        let (client_connection, client_sink, _client_rx) = client(200);
5527        let pending = forwarding
5528            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5529            .unwrap();
5530        assert_eq!(pending.endpoint, fixture.incumbent);
5531        let rpc = forwarding
5532            .begin_module_control_rpc_for(
5533                MODULE_ID,
5534                "health.check",
5535                Instant::now() + Duration::from_secs(1),
5536            )
5537            .unwrap();
5538        assert_eq!(rpc.endpoint, fixture.incumbent);
5539        let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
5540        assert_eq!(census.len(), 1, "the census lists one endpoint per id");
5541
5542        // Connection-keyed lookups see the candidate, so its own frames resolve.
5543        assert_eq!(
5544            forwarding
5545                .module_endpoint_for_connection(fixture.candidate_connection)
5546                .unwrap(),
5547            Some(fixture.candidate)
5548        );
5549        assert_eq!(
5550            forwarding
5551                .module_id_for_connection(fixture.candidate_connection)
5552                .unwrap()
5553                .as_deref(),
5554            Some(MODULE_ID)
5555        );
5556
5557        // One candidate per id.
5558        let (other_tx, _other_rx) = mpsc::channel(1);
5559        assert_eq!(
5560            forwarding.register_candidate_module_connection(
5561                ConnectionId::new(120),
5562                MODULE_ID.to_string(),
5563                2,
5564                Concurrency::ModuleManaged,
5565                FrameSink::new(other_tx),
5566            ),
5567            Err(ForwardingError::CandidateSlotOccupied {
5568                module_id: MODULE_ID.to_string()
5569            })
5570        );
5571    }
5572
5573    /// The cutover linearization point. A relay reserved on the incumbent before
5574    /// cutover must never become a route on the incumbent, and every relay
5575    /// reserved after it must land on the promoted candidate.
5576    #[test]
5577    fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
5578        let fixture = swap_fixture();
5579        let forwarding = &fixture.forwarding;
5580        let (early_client, early_sink, _early_rx) = client(200);
5581        let mut early = forwarding
5582            .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
5583            .unwrap();
5584        assert_eq!(early.endpoint, fixture.incumbent);
5585        assert_eq!(
5586            forwarding
5587                .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
5588                .unwrap(),
5589            RouteBindRelayMark::Marked
5590        );
5591
5592        let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5593        assert_eq!(
5594            cutover,
5595            ForwardingCutover {
5596                promoted: fixture.candidate,
5597                incumbent: Some(fixture.incumbent),
5598            }
5599        );
5600
5601        // A route.open reserved after cutover goes to the promoted candidate.
5602        let (late_client, late_sink, _late_rx) = client(201);
5603        let late = forwarding
5604            .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
5605            .unwrap();
5606        assert_eq!(
5607            late.endpoint, fixture.candidate,
5608            "a route.open after cutover was reserved on the incumbent"
5609        );
5610
5611        // The incumbent acks the early relay after cutover.
5612        let completion = forwarding
5613            .complete_pending_relay(
5614                fixture.incumbent_connection,
5615                early.corr,
5616                RouteBindRelayOutcome::Accepted,
5617            )
5618            .expect("a superseded endpoint's ack is not an error on its connection");
5619        assert!(completion.settled);
5620        assert!(
5621            !committed_endpoints(forwarding).contains(&fixture.incumbent),
5622            "a relay reserved before cutover committed a route on the incumbent"
5623        );
5624        let goodbye = completion
5625            .abandoned
5626            .expect("the incumbent is told to drop the binding it just created");
5627        assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
5628        assert_eq!(goodbye.channel, early.module_channel);
5629        assert_eq!(goodbye.epoch, early.module_epoch);
5630        assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
5631        match early.receiver.try_recv() {
5632            Ok(RouteBindRelayOutcome::DaemonRejected(body)) => {
5633                assert_eq!(body.code, "module_reloading")
5634            }
5635            other => panic!("expected a retryable module_reloading answer, got {other:?}"),
5636        }
5637        assert!(matches!(
5638            forwarding
5639                .lookup_data_route(early_client, early.client_channel, early.client_epoch)
5640                .unwrap(),
5641            DataRoute::Client(DataRouteState::Absent)
5642        ));
5643
5644        // The late relay commits on the candidate; only its pair stays reserved
5645        // until then.
5646        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
5647        forwarding
5648            .complete_pending_relay(
5649                fixture.candidate_connection,
5650                late.corr,
5651                RouteBindRelayOutcome::Accepted,
5652            )
5653            .unwrap();
5654        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5655        assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
5656    }
5657
5658    #[test]
5659    fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
5660        let fixture = swap_fixture();
5661        let forwarding = &fixture.forwarding;
5662        // One bound route and one in-flight relay on the incumbent.
5663        let (bound_client, bound_sink, _bound_rx) = client(200);
5664        let bound = forwarding
5665            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5666            .unwrap();
5667        forwarding
5668            .complete_pending_relay(
5669                fixture.incumbent_connection,
5670                bound.corr,
5671                RouteBindRelayOutcome::Accepted,
5672            )
5673            .unwrap();
5674        let (pending_client, pending_sink, _pending_rx) = client(201);
5675        let mut in_flight = forwarding
5676            .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
5677            .unwrap();
5678        forwarding
5679            .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
5680            .unwrap();
5681
5682        let incumbent = forwarding
5683            .cutover_candidate(MODULE_ID)
5684            .unwrap()
5685            .unwrap()
5686            .incumbent
5687            .unwrap();
5688        let target = forwarding
5689            .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
5690            .unwrap()
5691            .expect("the superseded incumbent is still registered");
5692
5693        assert_eq!(target.endpoint, fixture.incumbent);
5694        assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
5695        assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
5696        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5697        assert_eq!(target.abandoned_bindings.len(), 1);
5698        assert_eq!(
5699            target.abandoned_bindings[0].channel,
5700            in_flight.module_channel
5701        );
5702        assert!(matches!(
5703            in_flight.receiver.try_recv(),
5704            Ok(RouteBindRelayOutcome::DaemonRejected(body)) if body.code == "module_reloading"
5705        ));
5706        assert_eq!(
5707            forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
5708            1,
5709            "the incumbent's bound route stays until its drain finishes"
5710        );
5711
5712        let (next_client, next_sink, _next_rx) = client(202);
5713        let next = forwarding
5714            .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
5715            .expect("the promoted candidate keeps accepting routes");
5716        assert_eq!(next.endpoint, fixture.candidate);
5717    }
5718
5719    /// An endpoint replaced WITHOUT a promotion (a successor registered over it
5720    /// as an ordinary active HELLO) is stale, not superseded, and its ack still
5721    /// fails the acking connection exactly as it did before swap slots existed:
5722    /// `StaleModuleEndpoint`, the reservation indexes already stripped, and the
5723    /// waiting client's sender dropped unanswered.
5724    #[test]
5725    fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
5726        let forwarding = ForwardingTable::default();
5727        let first_connection = ConnectionId::new(70);
5728        let (first_tx, _first_rx) = mpsc::channel(8);
5729        forwarding
5730            .register_module_connection(
5731                first_connection,
5732                MODULE_ID.to_string(),
5733                2,
5734                Concurrency::ModuleManaged,
5735                FrameSink::new(first_tx),
5736            )
5737            .unwrap();
5738        let (client_connection, client_sink, _client_rx) = client(200);
5739        let mut pending = forwarding
5740            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5741            .unwrap();
5742        let (second_tx, _second_rx) = mpsc::channel(8);
5743        forwarding
5744            .register_module_connection(
5745                ConnectionId::new(80),
5746                MODULE_ID.to_string(),
5747                2,
5748                Concurrency::ModuleManaged,
5749                FrameSink::new(second_tx),
5750            )
5751            .unwrap();
5752
5753        assert_eq!(
5754            forwarding
5755                .complete_pending_relay(
5756                    first_connection,
5757                    pending.corr,
5758                    RouteBindRelayOutcome::Accepted
5759                )
5760                .unwrap_err(),
5761            ForwardingError::StaleModuleEndpoint
5762        );
5763        assert!(committed_endpoints(&forwarding).is_empty());
5764        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5765        assert!(matches!(
5766            pending.receiver.try_recv(),
5767            Err(oneshot::error::TryRecvError::Closed)
5768        ));
5769    }
5770
5771    #[test]
5772    fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
5773        // A candidate whose connection drops leaves the incumbent routable.
5774        let fixture = swap_fixture();
5775        let forwarding = &fixture.forwarding;
5776        assert!(forwarding
5777            .cleanup_connection(fixture.candidate_connection)
5778            .unwrap()
5779            .is_empty());
5780        assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
5781        let (client_connection, client_sink, _client_rx) = client(200);
5782        assert_eq!(
5783            forwarding
5784                .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5785                .unwrap()
5786                .endpoint,
5787            fixture.incumbent
5788        );
5789
5790        // After a cutover, the incumbent's teardown releases its own routes and
5791        // leaves the promoted candidate in place.
5792        let fixture = swap_fixture();
5793        let forwarding = &fixture.forwarding;
5794        let (bound_client, bound_sink, _bound_rx) = client(200);
5795        let bound = forwarding
5796            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5797            .unwrap();
5798        forwarding
5799            .complete_pending_relay(
5800                fixture.incumbent_connection,
5801                bound.corr,
5802                RouteBindRelayOutcome::Accepted,
5803            )
5804            .unwrap();
5805        forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5806        let released = forwarding
5807            .cleanup_connection(fixture.incumbent_connection)
5808            .unwrap();
5809        assert_eq!(released.len(), 1);
5810        assert_eq!(released[0].connection_id, bound_client);
5811        assert!(forwarding
5812            .read_inner()
5813            .unwrap()
5814            .superseded_endpoints
5815            .is_empty());
5816        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5817        let (next_client, next_sink, _next_rx) = client(201);
5818        assert_eq!(
5819            forwarding
5820                .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
5821                .unwrap()
5822                .endpoint,
5823            fixture.candidate
5824        );
5825    }
5826}