Skip to main content

subc_daemon/
forwarding.rs

1use std::{
2    collections::{BTreeMap, HashMap, HashSet},
3    error::Error,
4    fmt,
5    sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
6    time::Duration,
7};
8
9use subc_control::{ClientControlResponse, RouteCloseReason};
10use subc_protocol::{
11    manifest::Concurrency,
12    session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
13    ErrorBody, Flags, FrameType, Principal, Priority,
14};
15use tokio::sync::{oneshot, Semaphore};
16use tokio::time::Instant;
17use tracing::{debug, info, warn};
18
19use crate::{
20    control::{RouteBindBreakers, RouteBindConcurrency},
21    observability::DaemonCounters,
22    registry::ConnectionId,
23    router::FrameSink,
24    scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
25    Frame, ProjectRootId,
26};
27
28/// Default per-channel request-credit window for modules that schedule internally.
29const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
30
31/// High per-channel cap for stateless modules; this is an OOM guard, not scheduling policy.
32const STATELESS_PARALLEL_WINDOW: usize = 1024;
33
34/// A stopped probe cycle cannot retain its last unanswered correlation forever.
35/// Active endpoints replace the tombstone on their next serial health probe;
36/// this backstop covers endpoints that stop probing altogether.
37const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
38
39/// Module connection identity used in forwarding keys.
40///
41/// The generation is bumped every time a module connection is registered so a future restart cannot
42/// accidentally reuse a stale `(connection_id, route_channel)` binding.
43#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
44pub struct ModuleEndpointId {
45    pub connection_id: ConnectionId,
46    pub generation: u64,
47}
48
49/// Client-local route key. A route channel is unique only within one client connection.
50#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
51pub(crate) struct ClientRouteKey {
52    pub connection_id: ConnectionId,
53    pub channel: u16,
54}
55
56/// Module-local route key. A route channel is unique only within one live module endpoint.
57#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
58pub(crate) struct ModuleRouteKey {
59    pub endpoint: ModuleEndpointId,
60    pub channel: u16,
61}
62
63#[derive(Debug)]
64pub(crate) struct RouteBinding {
65    pub client_connection_id: ConnectionId,
66    pub client_sink: FrameSink,
67    pub client_negotiated_ver: u8,
68    pub client_channel: u16,
69    pub client_epoch: u32,
70    pub module_id: String,
71    pub module_endpoint: ModuleEndpointId,
72    pub module_sink: FrameSink,
73    pub module_negotiated_ver: u8,
74    pub module_channel: u16,
75    pub module_epoch: u32,
76    pub principal: Principal,
77    /// Opener attributes captured at admission for daemon-side prompt policy.
78    #[allow(dead_code)]
79    pub harness: String,
80    /// Only Callosum's exact production claim is retained; never relayed.
81    #[allow(dead_code)]
82    pub build_class: Option<String>,
83    /// Captured at connect, carried through admission; not standing authority.
84    #[allow(dead_code)]
85    pub operator_app: Option<crate::operator_apps::OperatorAppStamp>,
86    pub project_root: Option<ProjectRootId>,
87    pub bound_at: Instant,
88    pub flow: Arc<ChannelFlow>,
89    /// The scope the route was admitted under, with the tag its bind was
90    /// stamped at. A scope change finds the routes to close by it.
91    pub scope: Option<BoundScope>,
92}
93
94#[derive(Debug, Clone)]
95pub(crate) enum DataRoute {
96    Client(DataRouteState),
97    Module(DataRouteState),
98}
99
100#[derive(Debug, Clone)]
101pub(crate) enum DataRouteState {
102    Bound(Arc<RouteBinding>),
103    Reserved,
104    EpochMismatch,
105    Absent,
106}
107
108/// Which kind of peer a route GOODBYE is being delivered to. This decides what
109/// happens when the GOODBYE cannot be enqueued (egress full/closed):
110/// - `Client`: escalate to closing that client connection (a socket close is a
111///   stronger teardown signal, and a full client egress means it is the slow
112///   client we would drop anyway).
113/// - `Module`: never close; deliver late instead (see
114///   [`send_module_route_goodbye`]), and drop only if the module still has no
115///   room after [`LATE_MODULE_GOODBYE_DEADLINE`]. A client-disconnect notifies the
116///   SHARED module that one client's route is gone; closing the module on its
117///   egress backpressure would tear down every co-tenant client (the exact
118///   cross-tenant blast radius this never-close rule exists to prevent — observed when a
119///   flooding dead client filled BOTH its own and the module's egress, so its
120///   route-gone GOODBYE to the module failed and closed the shared connection).
121///   subc has already removed the route from its forwarding state and drops
122///   stale module frames for the released channel (see router.rs), so subc's
123///   own routing is correct. The residual: under SUSTAINED module-egress
124///   backpressure a module-targeted route-gone notification can be lost, which a
125///   module using it for client-refcounting (e.g. AFT's session accounting)
126///   would miss. This is INTENTIONALLY ACCEPTED, not a gap. A consuming module
127///   must bound stale bindings with its own idle-activity reaper (last-touched
128///   TTL) independent of route-gone signals — AFT does exactly this, so a lost
129///   GOODBYE degrades to "the
130///   binding stays warm until its idle TTL" (bounded wasted resources), never an
131///   unbounded leak; disk-durable replay is unaffected. A dedicated reliable
132///   module control lane was evaluated and deliberately NOT
133///   built: it would add starvation-avoidance machinery to the thin core for a
134///   bounded warm-resource window that is not a correctness issue. never-close
135///   is the invariant that matters here.
136#[derive(Debug, Clone, Copy, PartialEq, Eq)]
137pub(crate) enum GoodbyeTargetKind {
138    Client,
139    Module,
140}
141
142#[derive(Debug, Clone)]
143pub(crate) struct GoodbyeTarget {
144    pub connection_id: ConnectionId,
145    pub sink: FrameSink,
146    pub negotiated_ver: u8,
147    pub channel: u16,
148    pub epoch: u32,
149    pub kind: GoodbyeTargetKind,
150    /// The module on the other end of the route: for a module-targeted relay
151    /// the receiving module (attributing a dropped relay), for a client target
152    /// the module whose route is going away (naming it when an undeliverable
153    /// relay closes the client).
154    pub module_id: Option<String>,
155}
156
157/// The frame a client connection's egress queue refused, for the diagnosis
158/// logged when that refusal closes the connection.
159#[derive(Debug, Clone, Copy)]
160pub(crate) struct UndeliveredFrame<'a> {
161    /// The module on the other end of the route the frame belonged to, when known.
162    pub module_id: Option<&'a str>,
163    /// The refusing connection's sink, read for what its queue held.
164    pub sink: &'a FrameSink,
165}
166
167/// How a route's principal appears in logs: `direct`, or `reserved:<module>`.
168pub(crate) fn principal_label(principal: &Principal) -> String {
169    match principal {
170        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
171        Principal::Direct => "direct".to_string(),
172        other => format!("{other:?}"),
173    }
174}
175
176/// The distinct principals of the routes bound on one client connection,
177/// sorted and comma-joined, or `none` when it has no bound route.
178fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
179    let labels = inner
180        .client_to_module
181        .iter()
182        .filter(|(key, _)| key.connection_id == connection_id)
183        .map(|(_, route)| principal_label(&route.principal))
184        .collect::<std::collections::BTreeSet<_>>();
185    if labels.is_empty() {
186        "none".to_string()
187    } else {
188        labels.into_iter().collect::<Vec<_>>().join(",")
189    }
190}
191
192impl GoodbyeTarget {
193    /// True only when an undeliverable GOODBYE should escalate to closing the
194    /// target connection. Never escalate for module recipients.
195    pub(crate) fn close_on_delivery_failure(&self) -> bool {
196        matches!(self.kind, GoodbyeTargetKind::Client)
197    }
198}
199
200/// How long a route GOODBYE that a module's egress queue refused keeps waiting
201/// for room before it is given up and counted as dropped.
202///
203/// A module that stops reading for a moment (a GC pause, a long synchronous
204/// handler, a reconnect herd it is still working through) should still learn
205/// that the route is gone, because a module that never hears the GOODBYE keeps
206/// the route and keeps sending on it for as long as its connection lives. The
207/// value is the default module drain timeout: the daemon already treats that as
208/// the longest it is reasonable to wait on a module that is busy but alive, and
209/// a module that cannot free 21 bytes of egress in that time is wedged rather
210/// than slow. Resolving each module's own configured drain timeout here would
211/// need a registry lookup on a path that holds only the module's sink.
212pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
213
214/// Send a route GOODBYE to a module without ever closing the module's shared
215/// connection, which would sever every other route it serves.
216///
217/// The frame is enqueued at once when the module's egress queue has room. When
218/// it does not, a detached task waits for room (bounded by
219/// [`LATE_MODULE_GOODBYE_DEADLINE`]) and sends it then. Arriving late is safe:
220/// the daemon has already released the route, and module SDKs tear a route
221/// down only when both the channel AND the epoch match the route installed on
222/// that channel, so a late GOODBYE for a channel since reused at a newer epoch
223/// is ignored. The GOODBYE is counted in `goodbye_relay_module_dropped` only
224/// when the connection is closed or the deadline passes.
225pub(crate) fn send_module_route_goodbye(
226    counters: &DaemonCounters,
227    sink: &FrameSink,
228    frame: Frame,
229    module_id: Option<&str>,
230    context: &'static str,
231) {
232    let channel = frame.header.channel;
233    let epoch = frame.header.epoch;
234    let Err(err) = sink.try_send(frame.clone()) else {
235        return;
236    };
237    // Waiting is pointless once the module's writer is gone, and impossible
238    // outside a Tokio runtime (cleanup that runs from a destructor at shutdown).
239    let runtime = match tokio::runtime::Handle::try_current() {
240        Ok(runtime) if !sink.is_closed() => runtime,
241        _ => {
242            counters.increment_goodbye_relay_module_dropped(module_id);
243            warn!(
244            module_id = module_id.unwrap_or("unknown"),
245            route_channel = channel,
246            route_epoch = epoch,
247            error = %err,
248            context,
249            "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
250            );
251            return;
252        }
253    };
254    debug!(
255        module_id = module_id.unwrap_or("unknown"),
256        route_channel = channel,
257        route_epoch = epoch,
258        error = %err,
259        context,
260        "module egress queue refused route GOODBYE; delivering it once the module frees room"
261    );
262    let counters = counters.clone();
263    let sink = sink.clone();
264    let module_id = module_id.map(str::to_string);
265    runtime.spawn(async move {
266        let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
267        let why = match outcome {
268            Ok(Ok(())) => {
269                debug!(
270                    module_id = module_id.as_deref().unwrap_or("unknown"),
271                    route_channel = channel,
272                    route_epoch = epoch,
273                    context,
274                    "late route GOODBYE delivered to module"
275                );
276                return;
277            }
278            Ok(Err(err)) => err.to_string(),
279            Err(_) => format!(
280                "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
281            ),
282        };
283        counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
284        warn!(
285            module_id = module_id.as_deref().unwrap_or("unknown"),
286            route_channel = channel,
287            route_epoch = epoch,
288            error = %why,
289            context,
290            "route GOODBYE to module dropped under backpressure; not closing shared module connection"
291        );
292    });
293}
294
295/// One route currently served by a module endpoint.
296///
297/// `goodbye_target` is deliberately retained alongside the census projection so
298/// a draining caller can address exactly the same route set that this read
299/// reports, without a second forwarding-table pass.
300#[derive(Debug, Clone)]
301pub(crate) struct EndpointRoute {
302    pub goodbye_target: GoodbyeTarget,
303    pub principal: Principal,
304    pub bound_at: Instant,
305    pub draining: bool,
306    /// WHY the endpoint is draining, when it is. Carried per-route so the
307    /// census can answer "closing because of what" without a second lookup;
308    /// `None` exactly when `draining` is false (one source: the drain map).
309    pub drain_reason: Option<RouteCloseReason>,
310}
311
312/// One live route a scope change closed: both ends to send GOODBYE to, and the
313/// reason for the client's `route.closed`.
314#[derive(Debug, Clone)]
315pub(crate) struct ScopeDrainedRoute {
316    pub reason: RouteCloseReason,
317    pub scope: BoundScope,
318    pub module_id: String,
319    pub client: GoodbyeTarget,
320    pub module: GoodbyeTarget,
321}
322
323#[derive(Debug)]
324pub(crate) struct PendingRouteBindRelay {
325    pub endpoint: ModuleEndpointId,
326    pub module_sink: FrameSink,
327    pub negotiated_ver: u8,
328    pub client_channel: u16,
329    pub client_epoch: u32,
330    pub module_channel: u16,
331    pub module_epoch: u32,
332    pub corr: u64,
333    pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
334}
335
336#[derive(Debug, Clone)]
337pub(crate) struct ModuleDrainTarget {
338    pub endpoint: ModuleEndpointId,
339    pub sink: FrameSink,
340    pub negotiated_ver: u8,
341    pub abandoned_bindings: Vec<GoodbyeTarget>,
342    pub excluded_subscriptions: u32,
343}
344
345/// One registered module connection, as [`ForwardingTable::module_connections`]
346/// reports it: enough to send it a frame and then close it.
347#[cfg(unix)]
348#[derive(Debug, Clone)]
349pub(crate) struct ModuleConnectionTarget {
350    pub module_id: String,
351    pub endpoint: ModuleEndpointId,
352    pub sink: FrameSink,
353    pub negotiated_ver: u8,
354}
355
356#[derive(Debug, Clone)]
357pub(crate) enum RouteBindRelayOutcome {
358    Accepted,
359    /// The module refused the bind with its own body. Relayed to the client
360    /// without the daemon-origin bit: the code and message are module-chosen,
361    /// so the frame must not claim daemon authorship, even when the module
362    /// picks a daemon code like `module_reloading`.
363    Rejected(ErrorBody),
364    /// The daemon refused the bind itself — a drain or superseded endpoint
365    /// (`module_reloading`), a reply that arrived after the deadline
366    /// (`module_timeout`), or a scope that ended mid-bind. Carries the
367    /// daemon-origin bit because the daemon wrote it. The module may still
368    /// have received the bind (a late reply, or a drain that began after it
369    /// accepted); the daemon then ends that binding with a GOODBYE, so no
370    /// call runs on it.
371    DaemonRejected(ErrorBody),
372    ModuleGone(String),
373}
374
375/// What [`ForwardingTable::cutover_candidate`] did.
376#[derive(Debug, Clone, Copy, PartialEq, Eq)]
377pub(crate) struct ForwardingCutover {
378    /// The endpoint now in the active slot (the former candidate).
379    pub promoted: ModuleEndpointId,
380    /// The endpoint demoted out of the active slot, to be drained by endpoint.
381    /// `None` when the incumbent's connection was already gone.
382    pub incumbent: Option<ModuleEndpointId>,
383}
384
385/// What tearing down one connection released.
386#[derive(Debug)]
387pub(crate) struct ConnectionCleanup {
388    /// Routes whose other end must be sent GOODBYE.
389    pub released: Vec<GoodbyeTarget>,
390    /// Pending route.bind relays to the closed module that were aborted: opens
391    /// that had not been bound yet. Zero when a client connection closed.
392    pub abandoned_relays: u32,
393}
394
395#[derive(Debug, Clone)]
396pub(crate) struct PendingRelayCompletion {
397    pub settled: bool,
398    pub abandoned: Option<GoodbyeTarget>,
399}
400
401#[derive(Debug)]
402pub(crate) struct PendingModuleControlRpc {
403    pub endpoint: ModuleEndpointId,
404    pub module_sink: FrameSink,
405    pub negotiated_ver: u8,
406    pub corr: u64,
407    pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
408}
409
410#[derive(Debug, Clone)]
411pub(crate) enum ModuleControlRpcOutcome {
412    Response(ModuleControlResponse),
413    Rejected(ErrorBody),
414    ModuleGone(String),
415    MalformedResponse(String),
416    UnexpectedOp { expected: String, actual: String },
417    DeadlineElapsed,
418}
419
420#[derive(Debug, Clone, PartialEq, Eq)]
421pub(crate) enum ModuleControlRpcCompletion {
422    Unknown,
423    Settled,
424    LateHealthAnswer {
425        module_id: String,
426        latency: Duration,
427    },
428}
429
430#[derive(Debug)]
431struct PendingModuleControlRpcEntry {
432    expected_op: String,
433    deadline: Instant,
434    health_probe_started_at: Option<Instant>,
435    sender: oneshot::Sender<ModuleControlRpcOutcome>,
436}
437
438#[derive(Debug)]
439struct HealthProbeTombstone {
440    expected_op: String,
441    module_id: String,
442    probe_started_at: Instant,
443    expires_at: Instant,
444}
445
446#[derive(Debug, Clone)]
447struct RouteReservation {
448    client_key: ClientRouteKey,
449    module_key: ModuleRouteKey,
450    client_epoch: u32,
451    module_epoch: u32,
452    project_root: Option<ProjectRootId>,
453}
454
455#[derive(Debug)]
456struct PendingRouteBindRelayEntry {
457    reservation: RouteReservation,
458    client_sink: FrameSink,
459    client_negotiated_ver: u8,
460    client_permit: crate::router::EgressPermit,
461    route_open_frame: Frame,
462    principal: Principal,
463    /// The scope admitted at `route.open`, with the tag captured then. Commit
464    /// compares it with the published tag.
465    scope: Option<BoundScope>,
466    harness: String,
467    build_class: Option<String>,
468    operator_app: Option<crate::operator_apps::OperatorAppStamp>,
469    deadline: Instant,
470    /// Set once `route.open` has finished putting the route.bind on the
471    /// module's egress queue. A path that drops this entry WITHOUT a module
472    /// reply (client teardown, module drain, the open's own abort) sends the
473    /// module a GOODBYE for the bind only when this is set, because only then
474    /// is the GOODBYE guaranteed to queue behind the bind. When it is not set,
475    /// the removal is recorded in `relays_released_before_enqueue` instead and
476    /// `route.open` sends that GOODBYE itself once its send has completed.
477    ///
478    /// A module reply does not consult it: an ack proves the module received
479    /// the bind, so the reply path decides about the GOODBYE on its own.
480    relay_enqueued: bool,
481    sender: oneshot::Sender<RouteBindRelayOutcome>,
482}
483
484/// What `mark_route_bind_relay_enqueued` found once `route.open` had put the
485/// route.bind on the module's queue.
486#[derive(Debug, Clone, Copy, PartialEq, Eq)]
487pub(crate) enum RouteBindRelayMark {
488    /// The relay is still pending; later removals handle their own GOODBYE.
489    Marked,
490    /// The module already answered and the reply path settled the relay:
491    /// it committed the route, or it refused it and sent any GOODBYE the
492    /// module needed. `route.open` must send nothing.
493    SettledByReply,
494    /// The relay was dropped without a module reply while the bind was still
495    /// being queued, so nothing has told the module to drop the binding it
496    /// is about to create. `route.open` must send that GOODBYE.
497    ReleasedWithoutReply,
498}
499
500#[derive(Debug, Clone)]
501pub(crate) enum RouteRelease {
502    Removed(GoodbyeTarget),
503    Stale,
504    Absent,
505}
506
507#[derive(Debug, Clone)]
508pub(crate) enum RoutePollSnapshot {
509    Bound {
510        module_id: String,
511        status: Option<String>,
512    },
513    Absent,
514}
515
516#[derive(Debug, Clone)]
517struct ModuleConnection {
518    endpoint: ModuleEndpointId,
519    sink: FrameSink,
520    negotiated_ver: u8,
521    concurrency: Concurrency,
522}
523
524#[derive(Debug, Default)]
525struct ForwardingInner {
526    operator_confirms: Arc<crate::operator_confirm::OperatorConfirms>,
527    daemon_draining: bool,
528    /// Whether forwarding reached quiescence during daemon shutdown, for each
529    /// active endpoint present when shutdown began. A pending wait is false
530    /// because it has not proved quiescence. Endpoint generations prevent a
531    /// result from describing a replacement connection with the same module id.
532    daemon_drain_results: HashMap<ModuleEndpointId, bool>,
533    /// The ACTIVE slot: the one endpoint per module id that routing resolves.
534    /// Every by-id lookup (relay reservation, drain-by-id, liveness, census,
535    /// live roots, module-control RPCs) reads this map and nothing else.
536    modules_by_id: HashMap<String, ModuleConnection>,
537    /// The CANDIDATE slot of a blue/green swap: a second process registered
538    /// under an id that already has an active endpoint. It has a full endpoint
539    /// identity (so its own connection can be looked up and torn down) but no
540    /// by-id lookup sees it, so nothing is routed to it until `cutover_candidate`
541    /// promotes it.
542    candidates_by_id: HashMap<String, ModuleConnection>,
543    /// Former active endpoints demoted by `cutover_candidate`, until their
544    /// connection is removed. Membership is what distinguishes an endpoint that
545    /// was SUPERSEDED by a promotion (still alive, still carrying bound routes,
546    /// about to be drained) from one that is merely STALE (replaced some other
547    /// way, which is treated as a fault on the acking connection).
548    superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
549    endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
550    module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
551    /// Endpoints mid-drain, keyed to the reason the drain was begun with. The
552    /// value serves the census ("draining because restart"); membership alone
553    /// still answers every admission-gate check.
554    draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
555    closing_connections: HashSet<ConnectionId>,
556    next_generation: u64,
557    reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
558    reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
559    next_client_channel: HashMap<ConnectionId, u16>,
560    next_module_channel: HashMap<ModuleEndpointId, u16>,
561    client_slot_epochs: HashMap<ClientRouteKey, u32>,
562    module_slot_epochs: HashMap<ModuleRouteKey, u32>,
563    last_published_epoch: HashMap<ClientRouteKey, u32>,
564    client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
565    module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
566    status: HashMap<(ClientRouteKey, u32), String>,
567    pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
568    /// Pending relays removed without a module reply before `route.open`
569    /// marked them enqueued. `route.open` consumes its key when it marks the
570    /// relay (and sends the GOODBYE the removal could not send safely) or when
571    /// it aborts, so an entry lives only while that open is mid-send.
572    relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
573    /// Each scope's current `(scope_epoch, version)`, keyed `(owner, ref)`,
574    /// published by a scope sync in the same step that changes the record. A
575    /// bind commit reads it here so it never takes the scope table's lock
576    /// while holding this one. Absent means the scope is not live.
577    scope_tags: HashMap<(String, String), ScopeTag>,
578    next_control_corr: HashMap<ModuleEndpointId, u64>,
579    pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
580    health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
581}
582
583#[derive(Debug, Clone)]
584pub(crate) struct CloseReason {
585    code: &'static str,
586    message: String,
587}
588
589impl CloseReason {
590    pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
591        Self {
592            code,
593            message: message.into(),
594        }
595    }
596}
597
598impl fmt::Display for CloseReason {
599    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
600        write!(f, "{}: {}", self.code, self.message)
601    }
602}
603
604pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
605
606/// Dynamic forwarding state shared by the control plane and data-plane router.
607#[derive(Debug, Default)]
608pub struct ForwardingTable {
609    inner: Arc<RwLock<ForwardingInner>>,
610    #[cfg(feature = "test-support")]
611    route_closing_hold: Mutex<Option<oneshot::Receiver<()>>>,
612    close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
613    counters: DaemonCounters,
614    /// Per-target-module bind-relay breaker state. It lives here, beside the
615    /// module connections it describes, because this is where a module
616    /// connection's identity is established and therefore where a stale
617    /// verdict has to be discarded.
618    route_bind_breakers: RouteBindBreakers,
619    /// Current route.bind relays keyed by target module. Admission is shared
620    /// across every client connection that points at the same endpoint.
621    route_bind_concurrency: RouteBindConcurrency,
622    /// Start and end of each module's `route.open` outage. Held here for the
623    /// same reason as the breakers: every handler built over this table must
624    /// see one outage per module, or each would log its own opening line.
625    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
626}
627
628impl ForwardingTable {
629    /// Install scripted operator confirmation for an in-process integration harness,
630    /// without bootstrapping a daemon process. Only available with `test-support`.
631    ///
632    /// The script has one response per line: `approve`, `decline`, `unavailable`,
633    /// `hang` (until withdrawn), or `approve when <path>` (wait for the path to
634    /// exist). Unreadable or exhausted scripts fail closed. If supplied, `events`
635    /// receives the provider's JSON-lines event log. `limits: None` uses
636    /// [`OperatorLimits::default`](crate::operator_confirm::OperatorLimits); use
637    /// `Some(OperatorLimits { queue_wait, prompt_timeout, stuck_grace })` to
638    /// override the queue wait, prompt timeout and stuck-provider grace periods.
639    ///
640    /// The current executable's file name **must start with `ckdev-`**. Otherwise
641    /// this logs a warning and keeps the OS provider, just like bootstrap.
642    /// `cortexkit-test-support`'s `ckdev_binary` copies a harness executable under
643    /// a qualifying name; run that copy to enable scripted confirmation. See
644    /// `crates/subc-client-rs/tests/real_daemon/operator_confirm.rs` for the
645    /// reference end-to-end harness.
646    ///
647    /// Configure the table shared by the harness's `ControlHandler` and `Router`
648    /// before serving any requests. This also configures privileged confirmations
649    /// that use the same table.
650    ///
651    /// # Panics
652    ///
653    /// Panics if operator-confirm processing has already started.
654    ///
655    /// # Example
656    ///
657    /// ```no_run
658    /// use std::{path::Path, sync::Arc};
659    /// use subc_daemon::{ControlHandler, ForwardingTable, Registry};
660    ///
661    /// let registry = Arc::new(Registry::default());
662    /// let forwarding = Arc::new(ForwardingTable::default());
663    /// forwarding.configure_operator_script(
664    ///     Path::new("operator.script"),
665    ///     Some(Path::new("operator.events")),
666    ///     None,
667    /// );
668    /// let handler = ControlHandler::with_forwarding(registry, forwarding);
669    /// ```
670    #[cfg(feature = "test-support")]
671    pub fn configure_operator_script(
672        &self,
673        script: &std::path::Path,
674        events: Option<&std::path::Path>,
675        limits: Option<crate::operator_confirm::OperatorLimits>,
676    ) {
677        self.operator_confirms()
678            .configure_script(script, events, limits);
679    }
680
681    /// Hold the next supervisor drain between enqueueing `module.draining` and
682    /// `route.closing`, so a test can forward a response in that window. Dropping
683    /// the returned sender also releases the hold, including on a test panic.
684    #[cfg(feature = "test-support")]
685    pub fn hold_route_closing_for_test(&self) -> oneshot::Sender<()> {
686        let (release, hold) = oneshot::channel();
687        *self.route_closing_hold.lock().unwrap() = Some(hold);
688        release
689    }
690
691    #[cfg(feature = "test-support")]
692    pub(crate) async fn wait_before_route_closing_for_test(&self) {
693        let hold = self.route_closing_hold.lock().unwrap().take();
694        if let Some(hold) = hold {
695            let _ = hold.await;
696        }
697    }
698
699    pub(crate) fn operator_confirms(&self) -> Arc<crate::operator_confirm::OperatorConfirms> {
700        Arc::clone(
701            &self
702                .inner
703                .read()
704                .unwrap_or_else(|p| p.into_inner())
705                .operator_confirms,
706        )
707    }
708
709    #[cfg(test)]
710    pub(crate) fn inject_operator_principal(&self, key: ModuleRouteKey, principal: Principal) {
711        let mut inner = self.write_inner().unwrap();
712        let old = inner.module_to_client.remove(&key).unwrap();
713        let client = ClientRouteKey {
714            connection_id: old.client_connection_id,
715            channel: old.client_channel,
716        };
717        inner.client_to_module.remove(&client);
718        let mut binding = Arc::try_unwrap(old).expect("test binding has no outstanding readers");
719        binding.principal = principal;
720        let binding = Arc::new(binding);
721        inner.client_to_module.insert(client, Arc::clone(&binding));
722        inner.module_to_client.insert(key, binding);
723    }
724
725    /// Keep the binding protected through confirmation admission. Route release
726    /// takes the write lock and then the same confirm-state lock, so it cannot
727    /// miss an admitted request or leave it waiting on a vanished route.
728    pub(crate) fn with_operator_route<T>(
729        &self,
730        connection_id: ConnectionId,
731        channel: u16,
732        epoch: u32,
733        admit: impl FnOnce(Option<&RouteBinding>) -> T,
734    ) -> Result<T, ForwardingError> {
735        let inner = self.read_inner()?;
736        let binding = inner
737            .endpoint_by_connection
738            .get(&connection_id)
739            .and_then(|endpoint| {
740                inner.module_to_client.get(&ModuleRouteKey {
741                    endpoint: *endpoint,
742                    channel,
743                })
744            })
745            .filter(|binding| binding.module_epoch == epoch);
746        Ok(admit(binding.map(Arc::as_ref)))
747    }
748
749    pub(crate) fn counters(&self) -> DaemonCounters {
750        self.counters.clone()
751    }
752
753    pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
754        self.route_bind_breakers.clone()
755    }
756
757    pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
758        self.route_bind_concurrency.clone()
759    }
760
761    pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
762        Arc::clone(&self.route_outages)
763    }
764
765    pub(crate) fn register_connection_close(
766        &self,
767        connection_id: ConnectionId,
768    ) -> ConnectionCloseReceiver {
769        let (sender, receiver) = oneshot::channel();
770        let replaced = self
771            .lock_close_registry()
772            .insert(connection_id, sender)
773            .is_some();
774        if replaced {
775            warn!(
776                connection_id = connection_id.get(),
777                "replaced existing connection close registration"
778            );
779        }
780        receiver
781    }
782
783    pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
784        self.lock_close_registry().remove(&connection_id);
785    }
786
787    /// Close every established connection, modules and clients alike, as the
788    /// last step of an announced daemon shutdown. By the time this runs each
789    /// registered module has already been sent a module GOODBYE (see
790    /// `Supervisor::end_children_for_daemon_shutdown`), so the EOF that follows
791    /// is a planned stop; EOF with no GOODBYE before it stays reserved for a
792    /// daemon that went away unannounced. Closing while the daemon is still
793    /// alive lets it wait for the modules' own teardowns before it ends
794    /// whatever is left.
795    #[cfg(unix)]
796    pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
797        let senders: Vec<_> = self.lock_close_registry().drain().collect();
798        let count = senders.len();
799        for (_, sender) in senders {
800            let _ = sender.send(reason.clone());
801        }
802        count
803    }
804
805    /// Every registered module connection, whichever slot it occupies (active,
806    /// swap candidate, or superseded incumbent), once each. Daemon shutdown
807    /// uses this to tell each of them it is a planned stop before closing it.
808    #[cfg(unix)]
809    pub(crate) fn module_connections(
810        &self,
811    ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
812        let inner = self.read_inner()?;
813        let mut seen = HashSet::new();
814        Ok(inner
815            .modules_by_id
816            .values()
817            .chain(inner.candidates_by_id.values())
818            .chain(inner.superseded_endpoints.values())
819            .filter(|module| seen.insert(module.endpoint))
820            .map(|module| ModuleConnectionTarget {
821                module_id: inner
822                    .module_id_by_endpoint
823                    .get(&module.endpoint)
824                    .cloned()
825                    .unwrap_or_default(),
826                endpoint: module.endpoint,
827                sink: module.sink.clone(),
828                negotiated_ver: module.negotiated_ver,
829            })
830            .collect())
831    }
832
833    /// Ask a registered connection to close. Returns true only for the request
834    /// that actually reached it; later requests for the same connection, and
835    /// requests for a connection that is not registered, return false.
836    pub(crate) fn request_connection_close(
837        &self,
838        connection_id: ConnectionId,
839        reason: CloseReason,
840    ) -> bool {
841        let sender = self.lock_close_registry().remove(&connection_id);
842        if let Some(sender) = sender {
843            debug!(
844                connection_id = connection_id.get(),
845                close_reason = %reason,
846                "requesting connection close"
847            );
848            let _ = sender.send(reason);
849            true
850        } else {
851            debug!(
852                connection_id = connection_id.get(),
853                close_reason = %reason,
854                "connection close request ignored for inactive connection"
855            );
856            false
857        }
858    }
859
860    pub fn register_module_connection(
861        &self,
862        connection_id: ConnectionId,
863        module_id: String,
864        negotiated_ver: u8,
865        concurrency: Concurrency,
866        sink: FrameSink,
867    ) -> Result<ModuleEndpointId, ForwardingError> {
868        self.register_module_connection_inner(
869            connection_id,
870            module_id,
871            negotiated_ver,
872            concurrency,
873            sink,
874            None,
875        )
876    }
877
878    /// Register a module connection and queue its HELLO_ACK as the first frame
879    /// on its sink, in the same write-lock critical section that makes the
880    /// endpoint visible.
881    ///
882    /// A module reads HELLO_ACK as the first frame after its HELLO and exits on
883    /// anything else. Once the endpoint is in `modules_by_id`, a `route.open`
884    /// on any other connection can queue a `route.bind` request onto this sink,
885    /// so the ack must already be queued by then. Every lookup that can queue a
886    /// frame for the module takes this lock, so nothing can get in ahead of it.
887    ///
888    /// If the ack cannot be queued (the sink is closed or full), the
889    /// registration fails with nothing inserted.
890    pub(crate) fn register_module_connection_acked(
891        &self,
892        connection_id: ConnectionId,
893        module_id: String,
894        negotiated_ver: u8,
895        concurrency: Concurrency,
896        sink: FrameSink,
897        hello_ack: Frame,
898    ) -> Result<ModuleEndpointId, ForwardingError> {
899        self.register_module_connection_inner(
900            connection_id,
901            module_id,
902            negotiated_ver,
903            concurrency,
904            sink,
905            Some(hello_ack),
906        )
907    }
908
909    fn register_module_connection_inner(
910        &self,
911        connection_id: ConnectionId,
912        module_id: String,
913        negotiated_ver: u8,
914        concurrency: Concurrency,
915        sink: FrameSink,
916        hello_ack: Option<Frame>,
917    ) -> Result<ModuleEndpointId, ForwardingError> {
918        let mut inner = self.write_inner()?;
919        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
920            return Err(ForwardingError::ConnectionClosing { connection_id });
921        }
922        check_module_connection_role_locked(&inner, connection_id)?;
923        // Every refusal check has passed and nothing has been mutated yet, so
924        // a failed enqueue leaves the table exactly as it was.
925        enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
926
927        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
928        let endpoint = ModuleEndpointId {
929            connection_id,
930            generation: inner.next_generation,
931        };
932        inner.endpoint_by_connection.insert(connection_id, endpoint);
933        inner
934            .module_id_by_endpoint
935            .insert(endpoint, module_id.clone());
936        inner.next_module_channel.insert(endpoint, 1);
937        inner.next_control_corr.insert(endpoint, 1);
938        inner.modules_by_id.insert(
939            module_id.clone(),
940            ModuleConnection {
941                endpoint,
942                sink,
943                negotiated_ver,
944                concurrency,
945            },
946        );
947        drop(inner);
948
949        // A new module connection has arrived under this id, so anything the
950        // bind-relay breaker learned was learned about a process that is no
951        // longer the one behind this name. See
952        // `RouteBindBreakers::reset_for_new_module_connection`.
953        //
954        // Keyed on ARRIVAL rather than on teardown deliberately: a connection
955        // going away is not evidence about anything, and a module whose
956        // connection drops without coming back should keep its verdict until
957        // something actually registers in its place. This also covers the
958        // unclean replacements -- a module killed mid-bind, or one whose
959        // connection was closing -- because registration is the single path by
960        // which any module connection becomes usable.
961        if let Some(discarded) = self
962            .route_bind_breakers
963            .reset_for_new_module_connection(&module_id)
964        {
965            info!(
966                module_id = %module_id,
967                discarded_consecutive_timeouts = discarded,
968                "route.bind breaker state discarded: a new module connection replaced the process it described"
969            );
970        }
971        Ok(endpoint)
972    }
973
974    /// Register a blue/green swap candidate for `module_id` into the candidate
975    /// slot, alongside whatever endpoint is active for the id.
976    ///
977    /// Unlike [`Self::register_module_connection`], this never touches the
978    /// active slot and never resets the bind-relay breaker: the incumbent is
979    /// still the process serving the id, so what the breaker learned about it
980    /// is still true. The breaker is reset at cutover instead, when the process
981    /// behind the name actually changes. A second candidate for the same id is
982    /// refused.
983    ///
984    /// Production registers candidates through
985    /// [`Self::register_candidate_module_connection_acked`]; this ack-less form
986    /// is for tests that build forwarding state directly.
987    #[cfg(test)]
988    pub(crate) fn register_candidate_module_connection(
989        &self,
990        connection_id: ConnectionId,
991        module_id: String,
992        negotiated_ver: u8,
993        concurrency: Concurrency,
994        sink: FrameSink,
995    ) -> Result<ModuleEndpointId, ForwardingError> {
996        self.register_candidate_module_connection_inner(
997            connection_id,
998            module_id,
999            negotiated_ver,
1000            concurrency,
1001            sink,
1002            None,
1003        )
1004    }
1005
1006    /// The swap-candidate counterpart of
1007    /// [`Self::register_module_connection_acked`]: the HELLO_ACK is queued on
1008    /// the candidate's sink before its endpoint is inserted. No by-id lookup
1009    /// sees a candidate, but its own connection's endpoint does become
1010    /// resolvable here, and at cutover it becomes routable; the ack has to be
1011    /// ahead of anything either can queue.
1012    pub(crate) fn register_candidate_module_connection_acked(
1013        &self,
1014        connection_id: ConnectionId,
1015        module_id: String,
1016        negotiated_ver: u8,
1017        concurrency: Concurrency,
1018        sink: FrameSink,
1019        hello_ack: Frame,
1020    ) -> Result<ModuleEndpointId, ForwardingError> {
1021        self.register_candidate_module_connection_inner(
1022            connection_id,
1023            module_id,
1024            negotiated_ver,
1025            concurrency,
1026            sink,
1027            Some(hello_ack),
1028        )
1029    }
1030
1031    fn register_candidate_module_connection_inner(
1032        &self,
1033        connection_id: ConnectionId,
1034        module_id: String,
1035        negotiated_ver: u8,
1036        concurrency: Concurrency,
1037        sink: FrameSink,
1038        hello_ack: Option<Frame>,
1039    ) -> Result<ModuleEndpointId, ForwardingError> {
1040        let mut inner = self.write_inner()?;
1041        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1042            return Err(ForwardingError::ConnectionClosing { connection_id });
1043        }
1044        check_module_connection_role_locked(&inner, connection_id)?;
1045        if inner.candidates_by_id.contains_key(&module_id) {
1046            return Err(ForwardingError::CandidateSlotOccupied { module_id });
1047        }
1048        // As in `register_module_connection_inner`: after every refusal check,
1049        // before any mutation.
1050        enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1051
1052        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1053        let endpoint = ModuleEndpointId {
1054            connection_id,
1055            generation: inner.next_generation,
1056        };
1057        inner.endpoint_by_connection.insert(connection_id, endpoint);
1058        inner
1059            .module_id_by_endpoint
1060            .insert(endpoint, module_id.clone());
1061        inner.next_module_channel.insert(endpoint, 1);
1062        inner.next_control_corr.insert(endpoint, 1);
1063        inner.candidates_by_id.insert(
1064            module_id,
1065            ModuleConnection {
1066                endpoint,
1067                sink,
1068                negotiated_ver,
1069                concurrency,
1070            },
1071        );
1072        Ok(endpoint)
1073    }
1074
1075    /// Promote `module_id`'s candidate to the active slot, in ONE forwarding
1076    /// write-lock critical section.
1077    ///
1078    /// That single section is the linearization point of a swap. Every relay
1079    /// reservation resolves the active slot under the same lock, so each one
1080    /// lands wholly before cutover (on the incumbent) or wholly after it (on the
1081    /// promoted candidate), never on a mix. A relay reserved on the incumbent
1082    /// before cutover can no longer commit: `commit_route_locked` requires the
1083    /// reservation's endpoint to be the active one, and an ack for it arriving
1084    /// later is answered by the superseded-endpoint arm of
1085    /// `complete_pending_relay` instead of ending the incumbent's connection.
1086    ///
1087    /// Both endpoints keep their identities: nothing is re-keyed, so the
1088    /// incumbent's bound routes and its pending correlation keys stay exactly
1089    /// where they are until it is drained with [`Self::begin_endpoint_drain`],
1090    /// using the incumbent endpoint this returns. Returns `Ok(None)` when there
1091    /// is no candidate for the id.
1092    pub(crate) fn cutover_candidate(
1093        &self,
1094        module_id: &str,
1095    ) -> Result<Option<ForwardingCutover>, ForwardingError> {
1096        let mut inner = self.write_inner()?;
1097        if inner.daemon_draining {
1098            return Err(ForwardingError::ModuleReloading {
1099                module_id: module_id.to_string(),
1100            });
1101        }
1102        let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
1103            return Ok(None);
1104        };
1105        let promoted = candidate.endpoint;
1106        let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
1107        let incumbent = incumbent.map(|incumbent| {
1108            let endpoint = incumbent.endpoint;
1109            inner.superseded_endpoints.insert(endpoint, incumbent);
1110            endpoint
1111        });
1112        drop(inner);
1113
1114        // A different process now answers for this id; see the matching reset
1115        // in `register_module_connection` for why a verdict about the old one
1116        // must not carry over.
1117        if let Some(discarded) = self
1118            .route_bind_breakers
1119            .reset_for_new_module_connection(module_id)
1120        {
1121            info!(
1122                module_id = %module_id,
1123                discarded_consecutive_timeouts = discarded,
1124                "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
1125            );
1126        }
1127        Ok(Some(ForwardingCutover {
1128            promoted,
1129            incumbent,
1130        }))
1131    }
1132
1133    #[allow(clippy::too_many_arguments)]
1134    pub(crate) async fn begin_route_bind_relay_for(
1135        &self,
1136        client_connection_id: ConnectionId,
1137        client_sink: FrameSink,
1138        client_negotiated_ver: u8,
1139        client_corr: u64,
1140        module_id: &str,
1141        principal: Principal,
1142        scope: Option<BoundScope>,
1143        project_root: Option<ProjectRootId>,
1144        harness: String,
1145        build_class: Option<String>,
1146        operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1147        deadline: Instant,
1148    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1149        // Reserve egress capacity before taking the forwarding lock. The permit is
1150        // held until the bind reaches one terminal state, so an accepted bind can
1151        // publish its table entry and RouteOpen response in one critical section.
1152        let client_permit =
1153            client_sink
1154                .reserve_owned()
1155                .await
1156                .map_err(|_| ForwardingError::ClientEgressClosed {
1157                    connection_id: client_connection_id,
1158                })?;
1159        self.begin_route_bind_relay_inner(
1160            client_connection_id,
1161            client_sink,
1162            client_negotiated_ver,
1163            client_corr,
1164            module_id,
1165            principal,
1166            scope,
1167            project_root,
1168            harness,
1169            build_class,
1170            operator_app,
1171            deadline,
1172            client_permit,
1173        )
1174    }
1175
1176    #[cfg(test)]
1177    pub(crate) fn begin_route_bind_relay_for_test(
1178        &self,
1179        client_connection_id: ConnectionId,
1180        client_sink: FrameSink,
1181        client_corr: u64,
1182        module_id: &str,
1183    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1184        let permit =
1185            client_sink
1186                .try_reserve_owned()
1187                .map_err(|_| ForwardingError::ClientEgressClosed {
1188                    connection_id: client_connection_id,
1189                })?;
1190        self.begin_route_bind_relay_inner(
1191            client_connection_id,
1192            client_sink,
1193            subc_protocol::PROTOCOL_VERSION,
1194            client_corr,
1195            module_id,
1196            Principal::Direct,
1197            None,
1198            None,
1199            String::new(),
1200            None,
1201            None,
1202            Instant::now() + std::time::Duration::from_secs(60),
1203            permit,
1204        )
1205    }
1206
1207    pub(crate) fn begin_module_control_rpc_for(
1208        &self,
1209        module_id: &str,
1210        expected_op: &str,
1211        deadline: Instant,
1212    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1213        self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1214    }
1215
1216    pub(crate) fn begin_health_probe_rpc_for(
1217        &self,
1218        module_id: &str,
1219        expected_op: &str,
1220        probe_started_at: Instant,
1221        deadline: Instant,
1222    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1223        self.begin_module_control_rpc_inner(
1224            module_id,
1225            expected_op,
1226            deadline,
1227            Some(probe_started_at),
1228            false,
1229        )
1230    }
1231
1232    pub(crate) fn begin_drain_health_probe_rpc_for(
1233        &self,
1234        module_id: &str,
1235        expected_op: &str,
1236        probe_started_at: Instant,
1237        deadline: Instant,
1238    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1239        self.begin_module_control_rpc_inner(
1240            module_id,
1241            expected_op,
1242            deadline,
1243            Some(probe_started_at),
1244            true,
1245        )
1246    }
1247
1248    /// A health probe addressed to one endpoint, in whichever slot it is.
1249    ///
1250    /// Every other control RPC resolves the id's ACTIVE endpoint, which is how
1251    /// a swap candidate (not active until cutover) and a superseded incumbent
1252    /// (not active after it) are unreachable by them. A swap needs to probe
1253    /// exactly those two: the candidate before promoting it, and the incumbent
1254    /// for its busy gauges while it drains.
1255    pub(crate) fn begin_endpoint_health_probe_rpc_for(
1256        &self,
1257        endpoint: ModuleEndpointId,
1258        expected_op: &str,
1259        probe_started_at: Instant,
1260        deadline: Instant,
1261    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1262        let inner = self.write_inner()?;
1263        let module = module_connection_for_endpoint_locked(&inner, endpoint)
1264            .cloned()
1265            .ok_or(ForwardingError::NoModuleConnection)?;
1266        let module_id = inner
1267            .module_id_by_endpoint
1268            .get(&endpoint)
1269            .cloned()
1270            .unwrap_or_default();
1271        // A draining incumbent is exactly what this probes for busy gauges, so
1272        // draining is allowed, as it is for the by-id drain probe.
1273        self.begin_control_rpc_locked(
1274            inner,
1275            &module_id,
1276            module,
1277            expected_op,
1278            deadline,
1279            Some(probe_started_at),
1280            true,
1281        )
1282    }
1283
1284    fn begin_module_control_rpc_inner(
1285        &self,
1286        module_id: &str,
1287        expected_op: &str,
1288        deadline: Instant,
1289        health_probe_started_at: Option<Instant>,
1290        allow_draining: bool,
1291    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1292        let inner = self.write_inner()?;
1293        let module = inner
1294            .modules_by_id
1295            .get(module_id)
1296            .cloned()
1297            .ok_or(ForwardingError::NoModuleConnection)?;
1298        self.begin_control_rpc_locked(
1299            inner,
1300            module_id,
1301            module,
1302            expected_op,
1303            deadline,
1304            health_probe_started_at,
1305            allow_draining,
1306        )
1307    }
1308
1309    #[allow(clippy::too_many_arguments)]
1310    fn begin_control_rpc_locked(
1311        &self,
1312        mut inner: RwLockWriteGuard<'_, ForwardingInner>,
1313        module_id: &str,
1314        module: ModuleConnection,
1315        expected_op: &str,
1316        deadline: Instant,
1317        health_probe_started_at: Option<Instant>,
1318        allow_draining: bool,
1319    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1320        if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1321            return Err(ForwardingError::ModuleReloading {
1322                module_id: module_id.to_string(),
1323            });
1324        }
1325        if inner
1326            .closing_connections
1327            .contains(&module.endpoint.connection_id)
1328        {
1329            return Err(ForwardingError::ConnectionClosing {
1330                connection_id: module.endpoint.connection_id,
1331            });
1332        }
1333        if health_probe_started_at.is_some() {
1334            // Recurring health probes are serial per endpoint. Once the next one
1335            // starts, an older answer can no longer improve the current snapshot,
1336            // so retaining more than the newest unanswered probe has no value.
1337            inner
1338                .health_probe_tombstones
1339                .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1340        }
1341        let corr = match inner.allocate_control_corr(module.endpoint) {
1342            Ok(corr) => corr,
1343            Err(err) => {
1344                drop(inner);
1345                self.request_connection_close(
1346                    module.endpoint.connection_id,
1347                    CloseReason::new(
1348                        "control_correlation_exhausted",
1349                        "daemon-originated channel-0 correlation space exhausted",
1350                    ),
1351                );
1352                return Err(err);
1353            }
1354        };
1355        let (sender, receiver) = oneshot::channel();
1356        inner.pending_control_rpcs.insert(
1357            (module.endpoint, corr),
1358            PendingModuleControlRpcEntry {
1359                expected_op: expected_op.to_string(),
1360                deadline,
1361                health_probe_started_at,
1362                sender,
1363            },
1364        );
1365
1366        Ok(PendingModuleControlRpc {
1367            endpoint: module.endpoint,
1368            module_sink: module.sink,
1369            negotiated_ver: module.negotiated_ver,
1370            corr,
1371            receiver,
1372        })
1373    }
1374
1375    #[allow(clippy::too_many_arguments)]
1376    fn begin_route_bind_relay_inner(
1377        &self,
1378        client_connection_id: ConnectionId,
1379        client_sink: FrameSink,
1380        client_negotiated_ver: u8,
1381        client_corr: u64,
1382        expected_module_id: &str,
1383        principal: Principal,
1384        scope: Option<BoundScope>,
1385        project_root: Option<ProjectRootId>,
1386        harness: String,
1387        build_class: Option<String>,
1388        operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1389        deadline: Instant,
1390        client_permit: crate::router::EgressPermit,
1391    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1392        let mut inner = self.write_inner()?;
1393        // HELLO and route.open can run on different tasks. Decide the role under
1394        // the same lock that installs endpoints and reserves client channels so
1395        // neither task can turn one connection into both kinds of owner.
1396        if inner
1397            .endpoint_by_connection
1398            .contains_key(&client_connection_id)
1399        {
1400            return Err(ForwardingError::ConnectionRoleConflict {
1401                connection_id: client_connection_id,
1402            });
1403        }
1404        if inner.closing_connections.contains(&client_connection_id) {
1405            return Err(ForwardingError::ConnectionClosing {
1406                connection_id: client_connection_id,
1407            });
1408        }
1409        let module = inner
1410            .modules_by_id
1411            .get(expected_module_id)
1412            .cloned()
1413            .ok_or(ForwardingError::NoModuleConnection)?;
1414        if inner.draining_endpoints.contains_key(&module.endpoint) {
1415            return Err(ForwardingError::ModuleReloading {
1416                module_id: expected_module_id.to_string(),
1417            });
1418        }
1419        if inner
1420            .closing_connections
1421            .contains(&module.endpoint.connection_id)
1422        {
1423            return Err(ForwardingError::ConnectionClosing {
1424                connection_id: module.endpoint.connection_id,
1425            });
1426        }
1427
1428        let corr = match inner.allocate_control_corr(module.endpoint) {
1429            Ok(corr) => corr,
1430            Err(err) => {
1431                drop(inner);
1432                self.request_connection_close(
1433                    module.endpoint.connection_id,
1434                    CloseReason::new(
1435                        "control_correlation_exhausted",
1436                        "daemon-originated channel-0 correlation space exhausted",
1437                    ),
1438                );
1439                return Err(err);
1440            }
1441        };
1442        let (client_channel, client_epoch, module_channel, module_epoch) =
1443            inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1444        let client_key = ClientRouteKey {
1445            connection_id: client_connection_id,
1446            channel: client_channel,
1447        };
1448        let module_key = ModuleRouteKey {
1449            endpoint: module.endpoint,
1450            channel: module_channel,
1451        };
1452        let reservation = RouteReservation {
1453            client_key,
1454            module_key,
1455            client_epoch,
1456            module_epoch,
1457            project_root,
1458        };
1459        let response_body = serde_json::to_vec(&ClientControlResponse::RouteOpen {
1460            route_channel: client_channel,
1461            route_epoch: client_epoch,
1462        })
1463        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1464        let route_open_frame = Frame::build_with_version(
1465            client_negotiated_ver,
1466            FrameType::Response,
1467            Flags::new(false, Priority::Passive, false),
1468            0,
1469            0,
1470            client_corr,
1471            response_body,
1472        )
1473        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1474        let (sender, receiver) = oneshot::channel();
1475        inner.reserved_client.insert(client_key, module_key);
1476        inner.reserved_module.insert(module_key, client_key);
1477        inner.pending_relays.insert(
1478            (module.endpoint, corr),
1479            PendingRouteBindRelayEntry {
1480                reservation,
1481                client_sink,
1482                client_negotiated_ver,
1483                client_permit,
1484                route_open_frame,
1485                principal,
1486                scope,
1487                harness,
1488                build_class,
1489                operator_app,
1490                deadline,
1491                relay_enqueued: false,
1492                sender,
1493            },
1494        );
1495
1496        Ok(PendingRouteBindRelay {
1497            endpoint: module.endpoint,
1498            module_sink: module.sink,
1499            negotiated_ver: module.negotiated_ver,
1500            client_channel,
1501            client_epoch,
1502            module_channel,
1503            module_epoch,
1504            corr,
1505            receiver,
1506        })
1507    }
1508
1509    /// Called by `route.open` right after the route.bind entered the module's
1510    /// egress queue.
1511    ///
1512    /// The module reader runs concurrently with `route.open`, so a fast module
1513    /// can receive the bind, ack it, and have the reader commit the route
1514    /// before this runs. A missing entry therefore does not mean the bind was
1515    /// abandoned: only a removal recorded in `relays_released_before_enqueue`
1516    /// does. Telling the two apart is what stops `route.open` from sending a
1517    /// GOODBYE for a route that was just committed.
1518    pub(crate) fn mark_route_bind_relay_enqueued(
1519        &self,
1520        endpoint: ModuleEndpointId,
1521        corr: u64,
1522    ) -> Result<RouteBindRelayMark, ForwardingError> {
1523        let mut inner = self.write_inner()?;
1524        if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
1525            pending.relay_enqueued = true;
1526            return Ok(RouteBindRelayMark::Marked);
1527        }
1528        if inner
1529            .relays_released_before_enqueue
1530            .remove(&(endpoint, corr))
1531        {
1532            return Ok(RouteBindRelayMark::ReleasedWithoutReply);
1533        }
1534        Ok(RouteBindRelayMark::SettledByReply)
1535    }
1536
1537    pub(crate) fn release_client_route(
1538        &self,
1539        client_connection_id: ConnectionId,
1540        client_channel: u16,
1541        expected_epoch: u32,
1542    ) -> Result<RouteRelease, ForwardingError> {
1543        let mut inner = self.write_inner()?;
1544        let release = release_client_route_locked(
1545            &mut inner,
1546            ClientRouteKey {
1547                connection_id: client_connection_id,
1548                channel: client_channel,
1549            },
1550            expected_epoch,
1551        );
1552        self.record_route_release(&release);
1553        Ok(release)
1554    }
1555
1556    pub(crate) fn release_module_route(
1557        &self,
1558        module_connection_id: ConnectionId,
1559        module_channel: u16,
1560        expected_epoch: u32,
1561    ) -> Result<RouteRelease, ForwardingError> {
1562        let mut inner = self.write_inner()?;
1563        let Some(endpoint) = inner
1564            .endpoint_by_connection
1565            .get(&module_connection_id)
1566            .copied()
1567        else {
1568            return Ok(RouteRelease::Absent);
1569        };
1570        let release = release_module_route_locked(
1571            &mut inner,
1572            ModuleRouteKey {
1573                endpoint,
1574                channel: module_channel,
1575            },
1576            expected_epoch,
1577        );
1578        self.record_route_release(&release);
1579        Ok(release)
1580    }
1581
1582    pub(crate) fn abort_pending_relay(
1583        &self,
1584        endpoint: ModuleEndpointId,
1585        corr: u64,
1586        outcome: RouteBindRelayOutcome,
1587    ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
1588        let mut inner = self.write_inner()?;
1589        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1590            // If another path dropped the relay while `route.open` was still
1591            // sending the bind, and that open is now giving up without marking
1592            // it (its send failed or it was cancelled), forget the record so it
1593            // cannot outlive the open.
1594            inner
1595                .relays_released_before_enqueue
1596                .remove(&(endpoint, corr));
1597            return Ok(None);
1598        };
1599        release_reserved_route_locked(
1600            &mut inner,
1601            pending.reservation.client_key,
1602            pending.reservation.module_key,
1603        );
1604        let target = pending
1605            .relay_enqueued
1606            .then(|| abandoned_route_target(&inner, &pending.reservation));
1607        let _ = pending.sender.send(outcome);
1608        Ok(target.flatten())
1609    }
1610
1611    pub(crate) fn cancel_module_control_rpc(
1612        &self,
1613        endpoint: ModuleEndpointId,
1614        corr: u64,
1615    ) -> Result<(), ForwardingError> {
1616        self.write_inner()?
1617            .pending_control_rpcs
1618            .remove(&(endpoint, corr));
1619        Ok(())
1620    }
1621
1622    pub(crate) fn tombstone_health_probe_rpc(
1623        &self,
1624        endpoint: ModuleEndpointId,
1625        corr: u64,
1626    ) -> Result<bool, ForwardingError> {
1627        let key = (endpoint, corr);
1628        let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
1629        {
1630            let mut inner = self.write_inner()?;
1631            let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
1632                return Ok(false);
1633            };
1634            let Some(probe_started_at) = pending.health_probe_started_at else {
1635                inner.pending_control_rpcs.insert(key, pending);
1636                return Ok(false);
1637            };
1638            let module_id = inner
1639                .module_id_by_endpoint
1640                .get(&endpoint)
1641                .cloned()
1642                .unwrap_or_else(|| "unknown".to_string());
1643            inner.health_probe_tombstones.insert(
1644                key,
1645                HealthProbeTombstone {
1646                    expected_op: pending.expected_op,
1647                    module_id,
1648                    probe_started_at,
1649                    expires_at,
1650                },
1651            );
1652        }
1653        self.schedule_health_probe_tombstone_expiration(key, expires_at);
1654        Ok(true)
1655    }
1656
1657    fn schedule_health_probe_tombstone_expiration(
1658        &self,
1659        key: (ModuleEndpointId, u64),
1660        expires_at: Instant,
1661    ) {
1662        let inner = Arc::downgrade(&self.inner);
1663        tokio::spawn(async move {
1664            tokio::time::sleep_until(expires_at).await;
1665            let Some(inner) = inner.upgrade() else {
1666                return;
1667            };
1668            let Ok(mut inner) = inner.write() else {
1669                return;
1670            };
1671            let expired = inner
1672                .health_probe_tombstones
1673                .get(&key)
1674                .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
1675            if expired {
1676                inner.health_probe_tombstones.remove(&key);
1677            }
1678        });
1679    }
1680
1681    pub(crate) fn complete_pending_relay(
1682        &self,
1683        connection_id: ConnectionId,
1684        corr: u64,
1685        outcome: RouteBindRelayOutcome,
1686    ) -> Result<PendingRelayCompletion, ForwardingError> {
1687        let mut inner = self.write_inner()?;
1688        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1689            return Ok(PendingRelayCompletion {
1690                settled: false,
1691                abandoned: None,
1692            });
1693        };
1694        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
1695            return Ok(PendingRelayCompletion {
1696                settled: false,
1697                abandoned: None,
1698            });
1699        };
1700
1701        if Instant::now() >= pending.deadline {
1702            release_reserved_route_locked(
1703                &mut inner,
1704                pending.reservation.client_key,
1705                pending.reservation.module_key,
1706            );
1707            let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
1708                .then(|| abandoned_route_target(&inner, &pending.reservation))
1709                .flatten();
1710            let _ = pending
1711                .sender
1712                .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody {
1713                    code: "module_timeout".to_string(),
1714                    message: "route.bind response arrived after its daemon deadline".to_string(),
1715                    detail: None,
1716                }));
1717            return Ok(PendingRelayCompletion {
1718                settled: true,
1719                abandoned,
1720            });
1721        }
1722
1723        match outcome {
1724            // Two shapes of "the client is not there to receive this route" that
1725            // must resolve identically: its egress is already closed, or it is
1726            // marked closing (its connection loop has been asked to end, but has
1727            // not drained yet, so the sink is still open).
1728            //
1729            // Only the first used to be caught here. The second fell through to
1730            // `commit_route_locked`, which refuses a closing client with
1731            // `ConnectionClosing` -- and this function is called from the MODULE
1732            // connection's frame handler, so that refusal ended the module's
1733            // connection instead of this one client's route. One dying client's
1734            // route.open then took down a connection carrying every other
1735            // client's routes to that module.
1736            //
1737            // The remedy for both is the same, which is why they share an arm:
1738            // give back the reserved handle pair, tell the waiting route.open the
1739            // route is gone, and report the module-side channel so the caller can
1740            // send a channel-scoped GOODBYE for the binding the module just
1741            // created. Nothing here touches the module connection.
1742            RouteBindRelayOutcome::Accepted
1743                if pending.client_sink.is_closed()
1744                    || inner
1745                        .closing_connections
1746                        .contains(&pending.reservation.client_key.connection_id) =>
1747            {
1748                let reason = if pending.client_sink.is_closed() {
1749                    "client egress closed before route publication"
1750                } else {
1751                    "client connection is closing before route publication"
1752                };
1753                release_reserved_route_locked(
1754                    &mut inner,
1755                    pending.reservation.client_key,
1756                    pending.reservation.module_key,
1757                );
1758                // Not gated on `relay_enqueued`: an ack proves the module
1759                // received the bind, and `route.open` sends nothing for a
1760                // relay a reply already settled, so this is the only GOODBYE.
1761                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1762                let _ = pending
1763                    .sender
1764                    .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
1765                return Ok(PendingRelayCompletion {
1766                    settled: true,
1767                    abandoned,
1768                });
1769            }
1770            // The endpoint that acked this bind is no longer accepting new
1771            // routes, for one of two reasons. Either it was the module's active
1772            // endpoint when this relay was reserved and a blue/green swap has
1773            // since promoted a candidate (the replacement process registered
1774            // beside it) into the active slot, superseding it; or a daemon
1775            // drain has gated it, and the ack arrived before the ordered module
1776            // drain settled pending relays. The module did nothing wrong: it
1777            // bound the route it was asked to bind, and the endpoint still
1778            // carries every other client's routes until it is drained. So the
1779            // ack is settled here, while the pending entry still holds the
1780            // client's sender and the reserved handle pair: release the pair,
1781            // refuse the waiting route.open with `module_reloading`, which
1782            // callers retry (after a swap the retry reserves on the promoted
1783            // endpoint), and hand back the module-side channel so the caller
1784            // sends one channel-scoped GOODBYE for the binding the module just
1785            // created. Returning an error instead would end the module
1786            // connection all those other routes share; nothing here touches it.
1787            //
1788            // An endpoint that stopped being active any other way is stale
1789            // (replaced, but not by a swap and not draining). It still falls
1790            // through to `commit_route_locked`, which refuses it with
1791            // `StaleModuleEndpoint` exactly as before swaps existed.
1792            RouteBindRelayOutcome::Accepted
1793                if inner.superseded_endpoints.contains_key(&endpoint)
1794                    || inner.draining_endpoints.contains_key(&endpoint) =>
1795            {
1796                release_reserved_route_locked(
1797                    &mut inner,
1798                    pending.reservation.client_key,
1799                    pending.reservation.module_key,
1800                );
1801                // Not gated on `relay_enqueued`: an ack proves the module
1802                // received the bind, whether or not the enqueue mark was set.
1803                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1804                let module_id = inner
1805                    .module_id_by_endpoint
1806                    .get(&endpoint)
1807                    .cloned()
1808                    .unwrap_or_else(|| "unknown".to_string());
1809                let _ = pending
1810                    .sender
1811                    .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
1812                        "module_reloading",
1813                        format!("module_id '{module_id}' is reloading"),
1814                    )));
1815                return Ok(PendingRelayCompletion {
1816                    settled: true,
1817                    abandoned,
1818                });
1819            }
1820            // The scope the open was admitted under ended or changed after
1821            // admission. The module bound a route under a stamp that is no
1822            // longer true, so it must not become routable. Settled here like
1823            // the superseded arm above, never as an error from
1824            // `commit_route_locked`, which would close the module's whole
1825            // connection and every other client's routes to it: release the
1826            // pair, answer the waiting route.open by name, and hand back the
1827            // module-side channel for one channel-scoped GOODBYE.
1828            RouteBindRelayOutcome::Accepted
1829                if pending
1830                    .scope
1831                    .as_ref()
1832                    .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
1833            {
1834                let refusal = pending
1835                    .scope
1836                    .as_ref()
1837                    .and_then(|scope| scope_refusal_locked(&inner, scope))
1838                    .expect("guard matched a refusal under the same lock");
1839                release_reserved_route_locked(
1840                    &mut inner,
1841                    pending.reservation.client_key,
1842                    pending.reservation.module_key,
1843                );
1844                let abandoned = abandoned_route_target(&inner, &pending.reservation);
1845                let _ = pending
1846                    .sender
1847                    .send(RouteBindRelayOutcome::DaemonRejected(refusal));
1848                return Ok(PendingRelayCompletion {
1849                    settled: true,
1850                    abandoned,
1851                });
1852            }
1853            RouteBindRelayOutcome::Accepted => {
1854                let abandoned = commit_route_locked(&mut inner, pending)?;
1855                return Ok(PendingRelayCompletion {
1856                    settled: true,
1857                    abandoned,
1858                });
1859            }
1860            terminal => {
1861                release_reserved_route_locked(
1862                    &mut inner,
1863                    pending.reservation.client_key,
1864                    pending.reservation.module_key,
1865                );
1866                let _ = pending.sender.send(terminal);
1867            }
1868        }
1869        Ok(PendingRelayCompletion {
1870            settled: true,
1871            abandoned: None,
1872        })
1873    }
1874
1875    pub(crate) fn pending_module_control_op(
1876        &self,
1877        connection_id: ConnectionId,
1878        corr: u64,
1879    ) -> Result<Option<String>, ForwardingError> {
1880        let inner = self.read_inner()?;
1881        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1882            return Ok(None);
1883        };
1884        let key = (endpoint, corr);
1885        Ok(inner
1886            .pending_control_rpcs
1887            .get(&key)
1888            .map(|pending| pending.expected_op.clone())
1889            .or_else(|| {
1890                inner
1891                    .health_probe_tombstones
1892                    .get(&key)
1893                    .filter(|tombstone| tombstone.expires_at > Instant::now())
1894                    .map(|tombstone| tombstone.expected_op.clone())
1895            }))
1896    }
1897
1898    pub(crate) fn complete_module_control_rpc(
1899        &self,
1900        connection_id: ConnectionId,
1901        corr: u64,
1902        actual_op: Option<&str>,
1903        outcome: ModuleControlRpcOutcome,
1904    ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
1905        let now = Instant::now();
1906        let mut inner = self.write_inner()?;
1907        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
1908            return Ok(ModuleControlRpcCompletion::Unknown);
1909        };
1910        let key = (endpoint, corr);
1911        if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
1912            if now >= pending.deadline {
1913                let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
1914                    ModuleControlRpcCompletion::LateHealthAnswer {
1915                        module_id: inner
1916                            .module_id_by_endpoint
1917                            .get(&endpoint)
1918                            .cloned()
1919                            .unwrap_or_else(|| "unknown".to_string()),
1920                        latency: now.saturating_duration_since(probe_started_at),
1921                    }
1922                });
1923                let _ = pending
1924                    .sender
1925                    .send(ModuleControlRpcOutcome::DeadlineElapsed);
1926                return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
1927            }
1928            let outcome = match actual_op {
1929                Some(actual) if actual != pending.expected_op => {
1930                    ModuleControlRpcOutcome::UnexpectedOp {
1931                        expected: pending.expected_op,
1932                        actual: actual.to_string(),
1933                    }
1934                }
1935                _ => outcome,
1936            };
1937            let _ = pending.sender.send(outcome);
1938            return Ok(ModuleControlRpcCompletion::Settled);
1939        }
1940
1941        let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
1942            return Ok(ModuleControlRpcCompletion::Unknown);
1943        };
1944        if tombstone.expires_at <= now {
1945            return Ok(ModuleControlRpcCompletion::Unknown);
1946        }
1947        Ok(ModuleControlRpcCompletion::LateHealthAnswer {
1948            module_id: tombstone.module_id,
1949            latency: now.saturating_duration_since(tombstone.probe_started_at),
1950        })
1951    }
1952
1953    #[cfg(test)]
1954    pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
1955        Ok(self.read_inner()?.health_probe_tombstones.len())
1956    }
1957
1958    #[cfg(test)]
1959    pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
1960        Ok(self.read_inner()?.closing_connections.len())
1961    }
1962
1963    /// Reserved-but-uncommitted route handles, counted on both index sides, so
1964    /// a test can assert a reservation pair was actually given back.
1965    #[cfg(test)]
1966    pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
1967        let inner = self.read_inner()?;
1968        Ok((inner.reserved_client.len(), inner.reserved_module.len()))
1969    }
1970
1971    pub(crate) fn module_endpoint_for_connection(
1972        &self,
1973        connection_id: ConnectionId,
1974    ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
1975        Ok(self
1976            .read_inner()?
1977            .endpoint_by_connection
1978            .get(&connection_id)
1979            .copied())
1980    }
1981
1982    /// Looks up the module registered on a data-plane connection so route-drop
1983    /// diagnostics name the emitter instead of only reporting a daemon total.
1984    pub(crate) fn module_id_for_connection(
1985        &self,
1986        connection_id: ConnectionId,
1987    ) -> Result<Option<String>, ForwardingError> {
1988        let inner = self.read_inner()?;
1989        Ok(inner
1990            .endpoint_by_connection
1991            .get(&connection_id)
1992            .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
1993            .cloned())
1994    }
1995
1996    /// Whether the daemon ever allocated `(channel, epoch)` on this module
1997    /// connection. Epochs on a module channel are handed out as 1, 2, 3, ...
1998    /// and the last one handed out is remembered until the connection ends, so
1999    /// every epoch from 1 up to that one was a real route at some point. Used
2000    /// only on the drop path, to tell a module still sending on a route the
2001    /// daemon released from one sending on a route that never existed.
2002    pub(crate) fn module_route_epoch_was_allocated(
2003        &self,
2004        connection_id: ConnectionId,
2005        channel: u16,
2006        epoch: u32,
2007    ) -> Result<bool, ForwardingError> {
2008        let inner = self.read_inner()?;
2009        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2010            return Ok(false);
2011        };
2012        Ok(inner
2013            .module_slot_epochs
2014            .get(&ModuleRouteKey { endpoint, channel })
2015            .is_some_and(|last| epoch != 0 && epoch <= *last))
2016    }
2017
2018    pub(crate) fn has_live_module_connection(
2019        &self,
2020        module_id: &str,
2021    ) -> Result<bool, ForwardingError> {
2022        Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
2023    }
2024
2025    pub(crate) fn lookup_data_route(
2026        &self,
2027        connection_id: ConnectionId,
2028        channel: u16,
2029        epoch: u32,
2030    ) -> Result<DataRoute, ForwardingError> {
2031        let inner = self.read_inner()?;
2032        let state = if let Some(endpoint) =
2033            inner.endpoint_by_connection.get(&connection_id).copied()
2034        {
2035            let key = ModuleRouteKey { endpoint, channel };
2036            match inner.module_to_client.get(&key) {
2037                Some(route) if route.module_epoch == epoch => {
2038                    DataRouteState::Bound(Arc::clone(route))
2039                }
2040                Some(_) => DataRouteState::EpochMismatch,
2041                None if inner.reserved_module.contains_key(&key)
2042                    && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
2043                {
2044                    DataRouteState::Reserved
2045                }
2046                None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
2047                None => DataRouteState::Absent,
2048            }
2049        } else {
2050            let key = ClientRouteKey {
2051                connection_id,
2052                channel,
2053            };
2054            match inner.client_to_module.get(&key) {
2055                Some(route) if route.client_epoch == epoch => {
2056                    DataRouteState::Bound(Arc::clone(route))
2057                }
2058                Some(_) => DataRouteState::EpochMismatch,
2059                None if inner.reserved_client.contains_key(&key)
2060                    && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
2061                {
2062                    DataRouteState::Reserved
2063                }
2064                None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
2065                None => DataRouteState::Absent,
2066            }
2067        };
2068        Ok(
2069            if inner.endpoint_by_connection.contains_key(&connection_id) {
2070                DataRoute::Module(state)
2071            } else {
2072                DataRoute::Client(state)
2073            },
2074        )
2075    }
2076
2077    #[cfg(test)]
2078    pub(crate) fn inject_client_slot_epoch(
2079        &self,
2080        connection_id: ConnectionId,
2081        channel: u16,
2082        last_epoch: u32,
2083    ) {
2084        let mut inner = self.write_inner().expect("forwarding lock");
2085        inner.client_slot_epochs.insert(
2086            ClientRouteKey {
2087                connection_id,
2088                channel,
2089            },
2090            last_epoch,
2091        );
2092        inner.next_client_channel.insert(connection_id, channel);
2093    }
2094
2095    #[cfg(test)]
2096    pub(crate) fn inject_module_slot_epoch(
2097        &self,
2098        endpoint: ModuleEndpointId,
2099        channel: u16,
2100        last_epoch: u32,
2101    ) {
2102        let mut inner = self.write_inner().expect("forwarding lock");
2103        inner
2104            .module_slot_epochs
2105            .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
2106        inner.next_module_channel.insert(endpoint, channel);
2107    }
2108
2109    #[cfg(test)]
2110    pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
2111        self.write_inner()
2112            .expect("forwarding lock")
2113            .next_control_corr
2114            .insert(endpoint, next_corr);
2115    }
2116
2117    pub(crate) fn cache_status(
2118        &self,
2119        endpoint: ModuleEndpointId,
2120        module_channel: u16,
2121        module_epoch: u32,
2122        status: String,
2123    ) -> Result<bool, ForwardingError> {
2124        let mut inner = self.write_inner()?;
2125        if !inner.module_id_by_endpoint.contains_key(&endpoint) {
2126            return Err(ForwardingError::StaleModuleEndpoint);
2127        }
2128
2129        let module_key = ModuleRouteKey {
2130            endpoint,
2131            channel: module_channel,
2132        };
2133        let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
2134            (route.module_epoch == module_epoch).then_some((
2135                ClientRouteKey {
2136                    connection_id: route.client_connection_id,
2137                    channel: route.client_channel,
2138                },
2139                route.client_epoch,
2140            ))
2141        } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
2142            (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
2143                client_key,
2144                inner
2145                    .client_slot_epochs
2146                    .get(&client_key)
2147                    .copied()
2148                    .unwrap_or(0),
2149            ))
2150        } else {
2151            None
2152        };
2153
2154        if let Some(handle) = handle {
2155            inner.status.insert(handle, status);
2156            Ok(true)
2157        } else {
2158            debug!(
2159                module_channel,
2160                module_epoch,
2161                generation = endpoint.generation,
2162                connection_id = endpoint.connection_id.get(),
2163                "dropping stale status update for module route handle"
2164            );
2165            Ok(false)
2166        }
2167    }
2168
2169    pub(crate) fn route_poll_snapshot(
2170        &self,
2171        client_connection_id: ConnectionId,
2172        client_channel: u16,
2173        client_epoch: u32,
2174    ) -> Result<RoutePollSnapshot, ForwardingError> {
2175        let inner = self.read_inner()?;
2176        let client_key = ClientRouteKey {
2177            connection_id: client_connection_id,
2178            channel: client_channel,
2179        };
2180        let Some(route) = inner.client_to_module.get(&client_key) else {
2181            return Ok(RoutePollSnapshot::Absent);
2182        };
2183        if route.client_epoch != client_epoch
2184            || !inner
2185                .module_id_by_endpoint
2186                .contains_key(&route.module_endpoint)
2187        {
2188            return Ok(RoutePollSnapshot::Absent);
2189        }
2190        Ok(RoutePollSnapshot::Bound {
2191            module_id: route.module_id.clone(),
2192            status: inner.status.get(&(client_key, client_epoch)).cloned(),
2193        })
2194    }
2195
2196    pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2197        Ok(self.read_inner()?.client_to_module.len())
2198    }
2199
2200    /// How many distinct client connections hold at least one committed route,
2201    /// alongside the largest number of routes any single connection holds.
2202    ///
2203    /// `connected_clients` alone cannot distinguish many clients with a route
2204    /// each from one client accumulating hundreds, and those have opposite
2205    /// causes. Reading it required an out-of-band `lsof` during a live
2206    /// investigation, and the count of connections was mistaken for a count of
2207    /// client processes — which sent two of us after cleanup paths that were
2208    /// working correctly.
2209    pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2210        let inner = self.read_inner()?;
2211        let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2212        for key in inner.client_to_module.keys() {
2213            *per_connection.entry(key.connection_id).or_insert(0) += 1;
2214        }
2215        let max = per_connection.values().copied().max().unwrap_or(0);
2216        Ok((per_connection.len(), max))
2217    }
2218
2219    pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2220        let inner = self.read_inner()?;
2221        Ok(inner
2222            .client_to_module
2223            .keys()
2224            .any(|key| key.channel == route_channel))
2225    }
2226
2227    /// Whether daemon-wide shutdown has begun draining all providers.
2228    /// `None` means normal operation; during daemon shutdown, return the
2229    /// endpoint's measured result, or false if no completed measurement exists.
2230    pub(crate) fn daemon_drain_result_for_connection(
2231        &self,
2232        connection_id: ConnectionId,
2233    ) -> Result<Option<bool>, ForwardingError> {
2234        let inner = self.read_inner()?;
2235        Ok(inner.daemon_draining.then(|| {
2236            inner
2237                .endpoint_by_connection
2238                .get(&connection_id)
2239                .and_then(|endpoint| inner.daemon_drain_results.get(endpoint))
2240                .copied()
2241                .unwrap_or(false)
2242        }))
2243    }
2244
2245    #[cfg(unix)]
2246    pub(crate) fn record_daemon_drain_result(
2247        &self,
2248        endpoint: ModuleEndpointId,
2249        drained: bool,
2250    ) -> Result<(), ForwardingError> {
2251        let mut inner = self.write_inner()?;
2252        // Only update an endpoint captured at the start of daemon shutdown.
2253        // Teardown removes its entry, so a late wait cannot recreate the result
2254        // or attribute it to a replacement connection.
2255        if let Some(result) = inner.daemon_drain_results.get_mut(&endpoint) {
2256            *result = drained;
2257        }
2258        Ok(())
2259    }
2260
2261    /// Gate every provider atomically, including registrations racing shutdown.
2262    /// No supervisor lock is held while taking the forwarding lock.
2263    #[cfg(unix)]
2264    pub(crate) fn begin_daemon_drain(
2265        &self,
2266    ) -> Result<Vec<(String, ModuleEndpointId)>, ForwardingError> {
2267        let mut inner = self.write_inner()?;
2268        if inner.daemon_draining {
2269            return Ok(Vec::new());
2270        }
2271        inner.daemon_draining = true;
2272        let modules = inner
2273            .modules_by_id
2274            .iter()
2275            .map(|(id, module)| (id.clone(), module.endpoint))
2276            .collect::<Vec<_>>();
2277        for (_, endpoint) in &modules {
2278            inner.daemon_drain_results.insert(*endpoint, false);
2279            inner
2280                .draining_endpoints
2281                .insert(*endpoint, RouteCloseReason::Restart);
2282        }
2283        // Swap candidates and superseded incumbents are not routable, but they
2284        // are live endpoints that could still be sent module-control work, so
2285        // the daemon-wide gate covers them too. Only the active endpoints are
2286        // returned to the supervisor for measurement, one per module id.
2287        let off_slot_endpoints = inner
2288            .candidates_by_id
2289            .values()
2290            .map(|module| module.endpoint)
2291            .chain(inner.superseded_endpoints.keys().copied())
2292            .collect::<Vec<_>>();
2293        for endpoint in off_slot_endpoints {
2294            inner
2295                .draining_endpoints
2296                .insert(endpoint, RouteCloseReason::Restart);
2297        }
2298        Ok(modules)
2299    }
2300
2301    /// Begin draining whatever endpoint is ACTIVE for `module_id`.
2302    ///
2303    /// After a swap's cutover the active endpoint is the promoted candidate, so
2304    /// this must not be used to drain the incumbent; use
2305    /// [`Self::begin_endpoint_drain`] with the endpoint `cutover_candidate`
2306    /// returned.
2307    pub(crate) fn begin_module_drain(
2308        &self,
2309        module_id: &str,
2310        reason: RouteCloseReason,
2311    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2312        let mut inner = self.write_inner()?;
2313        let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2314            return Ok(None);
2315        };
2316        Ok(Some(begin_drain_locked(
2317            &mut inner, module_id, module, reason,
2318        )))
2319    }
2320
2321    /// Begin draining one specific endpoint, whichever slot it is in.
2322    ///
2323    /// This is how a swap drains its incumbent after cutover: the incumbent's
2324    /// endpoint is captured by `cutover_candidate`, and resolving it by module id
2325    /// instead would find the promoted candidate and leave neither process
2326    /// routable. Returns `Ok(None)` when the endpoint is no longer registered.
2327    pub(crate) fn begin_endpoint_drain(
2328        &self,
2329        endpoint: ModuleEndpointId,
2330        reason: RouteCloseReason,
2331    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2332        let mut inner = self.write_inner()?;
2333        let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2334            return Ok(None);
2335        };
2336        let module_id = inner
2337            .module_id_by_endpoint
2338            .get(&endpoint)
2339            .cloned()
2340            .expect("an endpoint resolved to a module connection has a module id");
2341        Ok(Some(begin_drain_locked(
2342            &mut inner, &module_id, module, reason,
2343        )))
2344    }
2345}
2346
2347/// Mark `module`'s endpoint draining and settle everything still pending on it.
2348/// Shared by the by-id and by-endpoint drain entry points, which differ only in
2349/// how they find the endpoint.
2350fn begin_drain_locked(
2351    inner: &mut ForwardingInner,
2352    module_id: &str,
2353    module: ModuleConnection,
2354    reason: RouteCloseReason,
2355) -> ModuleDrainTarget {
2356    {
2357        let endpoint = module.endpoint;
2358        inner.draining_endpoints.insert(endpoint, reason);
2359
2360        let flows = inner
2361            .client_to_module
2362            .values()
2363            .filter(|route| route.module_endpoint == endpoint)
2364            .map(|route| Arc::clone(&route.flow))
2365            .collect::<Vec<_>>();
2366        let excluded_subscriptions = flows
2367            .into_iter()
2368            .map(|flow| flow.begin_drain())
2369            .fold(0u32, u32::saturating_add);
2370
2371        let pending_keys = inner
2372            .pending_relays
2373            .keys()
2374            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2375            .copied()
2376            .collect::<Vec<_>>();
2377        let mut abandoned_bindings = Vec::new();
2378        for key in pending_keys {
2379            let Some(pending) = inner.pending_relays.remove(&key) else {
2380                continue;
2381            };
2382            release_reserved_route_locked(
2383                inner,
2384                pending.reservation.client_key,
2385                pending.reservation.module_key,
2386            );
2387            if pending.relay_enqueued {
2388                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2389                    abandoned_bindings.push(target);
2390                }
2391            } else {
2392                // The bind may still be on its way into the module's queue;
2393                // a GOODBYE sent now could overtake it. `route.open` sends
2394                // it once its send completes.
2395                inner.relays_released_before_enqueue.insert(key);
2396            }
2397            let _ = pending
2398                .sender
2399                .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
2400                    "module_reloading",
2401                    format!("module_id '{module_id}' is reloading"),
2402                )));
2403        }
2404
2405        let pending_control_keys = inner
2406            .pending_control_rpcs
2407            .keys()
2408            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2409            .copied()
2410            .collect::<Vec<_>>();
2411        for key in pending_control_keys {
2412            if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2413                let _ = pending
2414                    .sender
2415                    .send(ModuleControlRpcOutcome::ModuleGone(format!(
2416                        "module '{module_id}' began draining during module-control RPC"
2417                    )));
2418            }
2419        }
2420
2421        ModuleDrainTarget {
2422            endpoint,
2423            sink: module.sink,
2424            negotiated_ver: module.negotiated_ver,
2425            abandoned_bindings,
2426            excluded_subscriptions,
2427        }
2428    }
2429}
2430
2431/// What a drain that timed out was still waiting on, for the log line that
2432/// reports the timeout. Per-request ages are not tracked.
2433#[derive(Debug, Default, PartialEq, Eq)]
2434pub(crate) struct DrainHoldouts {
2435    /// Requests still counted against the drain (subscriptions flagged as
2436    /// such are excluded from the drain and not counted here).
2437    pub(crate) requests: usize,
2438    /// Routes holding at least one of those requests.
2439    pub(crate) routes: usize,
2440    /// Every route on the endpoint, for scale.
2441    pub(crate) total_routes: usize,
2442    /// The client connections holding the most requests, largest first, at
2443    /// most three: enough to name the consumer without listing every route.
2444    pub(crate) top_connections: Vec<(u64, usize)>,
2445    /// The held requests themselves, as the module saw them, so its own log
2446    /// can say what each one was: `(module channel, corr)`, ordered by channel
2447    /// then corr, at most [`DRAIN_HELD_REQUESTS_LISTED`]. The daemon never reads
2448    /// request bodies, so it cannot name a request's method; the module can,
2449    /// from the channel and corr. A request is released only when the module
2450    /// sends a terminal frame (Response, Error or StreamEnd) with its corr on
2451    /// its route, so every pair listed here is one the module ended, if at all,
2452    /// without sending that frame.
2453    pub(crate) held: Vec<(u16, u64)>,
2454}
2455
2456/// How many held requests the drain-timeout line lists by channel and corr.
2457pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
2458
2459impl ForwardingTable {
2460    /// Summarise the requests one endpoint's drain is still waiting on.
2461    pub(crate) fn endpoint_drain_holdouts(
2462        &self,
2463        endpoint: ModuleEndpointId,
2464    ) -> Result<DrainHoldouts, ForwardingError> {
2465        let inner = self.read_inner()?;
2466        let mut holdouts = DrainHoldouts::default();
2467        let mut by_connection: HashMap<u64, usize> = HashMap::new();
2468        for (key, route) in &inner.client_to_module {
2469            if route.module_endpoint != endpoint {
2470                continue;
2471            }
2472            holdouts.total_routes += 1;
2473            let held = route.flow.drain_in_flight();
2474            if held == 0 {
2475                continue;
2476            }
2477            holdouts.requests += held;
2478            holdouts.routes += 1;
2479            *by_connection.entry(key.connection_id.get()).or_default() += held;
2480            holdouts.held.extend(
2481                route
2482                    .flow
2483                    .drain_held_corrs()
2484                    .into_iter()
2485                    .map(|corr| (route.module_channel, corr)),
2486            );
2487        }
2488        holdouts.held.sort_unstable();
2489        holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
2490        let mut connections = by_connection.into_iter().collect::<Vec<_>>();
2491        connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
2492        connections.truncate(3);
2493        holdouts.top_connections = connections;
2494        Ok(holdouts)
2495    }
2496
2497    pub(crate) fn endpoint_in_flight_count(
2498        &self,
2499        endpoint: ModuleEndpointId,
2500    ) -> Result<usize, ForwardingError> {
2501        let inner = self.read_inner()?;
2502        Ok(inner
2503            .client_to_module
2504            .values()
2505            .filter(|route| route.module_endpoint == endpoint)
2506            .map(|route| route.flow.drain_in_flight())
2507            .sum())
2508    }
2509
2510    pub(crate) fn endpoint_is_draining(
2511        &self,
2512        endpoint: ModuleEndpointId,
2513    ) -> Result<bool, ForwardingError> {
2514        Ok(self
2515            .read_inner()?
2516            .draining_endpoints
2517            .contains_key(&endpoint))
2518    }
2519
2520    pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
2521        let inner = self.read_inner()?;
2522        Ok(inner
2523            .modules_by_id
2524            .get(module_id)
2525            .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
2526    }
2527
2528    pub(crate) fn release_module_endpoint_routes(
2529        &self,
2530        endpoint: ModuleEndpointId,
2531    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2532        let mut inner = self.write_inner()?;
2533        let routes = inner
2534            .module_to_client
2535            .iter()
2536            .filter(|(module_key, _)| module_key.endpoint == endpoint)
2537            .map(|(module_key, route)| (*module_key, route.module_epoch))
2538            .collect::<Vec<_>>();
2539        let mut released = Vec::with_capacity(routes.len());
2540        for (module_key, epoch) in routes {
2541            if let RouteRelease::Removed(target) =
2542                release_module_route_locked(&mut inner, module_key, epoch)
2543            {
2544                released.push(target);
2545            }
2546        }
2547        Ok(released)
2548    }
2549
2550    /// Enumerate one endpoint's current routes without contacting the module.
2551    ///
2552    /// The read lock makes this safe while the endpoint drains: the returned
2553    /// `draining` marker describes the same table state that owns the route,
2554    /// rather than inferring liveness from a module that may be stopping.
2555    pub(crate) fn endpoint_routes(
2556        &self,
2557        endpoint: ModuleEndpointId,
2558    ) -> Result<Vec<EndpointRoute>, ForwardingError> {
2559        let inner = self.read_inner()?;
2560        Ok(endpoint_routes_locked(&inner, endpoint))
2561    }
2562
2563    /// Snapshot all live endpoint route sets under one forwarding-table read lock.
2564    pub(crate) fn route_census(
2565        &self,
2566        module_id: Option<&str>,
2567    ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
2568        let inner = self.read_inner()?;
2569        let mut endpoints = inner
2570            .modules_by_id
2571            .iter()
2572            .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
2573            .map(|(id, module)| (id.clone(), module.endpoint))
2574            .collect::<Vec<_>>();
2575        endpoints.sort_by(|left, right| left.0.cmp(&right.0));
2576        Ok(endpoints
2577            .into_iter()
2578            .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
2579            .collect())
2580    }
2581
2582    /// Snapshot only the endpoint currently routable under this module id.
2583    pub(crate) fn live_roots(
2584        &self,
2585        module_id: &str,
2586    ) -> Result<ModuleControlResponseToModule, ForwardingError> {
2587        let inner = self.read_inner()?;
2588        let endpoint = inner
2589            .modules_by_id
2590            .get(module_id)
2591            .map(|module| module.endpoint);
2592        let mut roots = BTreeMap::new();
2593        let mut unknown_root_bindings = 0;
2594        let mut total_bindings = 0;
2595        if let Some(endpoint) = endpoint {
2596            for binding in inner
2597                .module_to_client
2598                .values()
2599                .filter(|binding| binding.module_endpoint == endpoint)
2600            {
2601                total_bindings += 1;
2602                let workdir = binding
2603                    .scope
2604                    .as_ref()
2605                    .and_then(|scope| scope.workdir.as_deref())
2606                    .map(std::path::Path::new);
2607                if let Some(root) = &binding.project_root {
2608                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2609                    entry.0 += 1;
2610                } else if workdir.is_none() {
2611                    unknown_root_bindings += 1;
2612                }
2613                if let Some(workdir) = workdir.filter(|workdir| {
2614                    binding.project_root.as_ref().map(ProjectRootId::as_path) != Some(*workdir)
2615                }) {
2616                    roots.entry(workdir.to_path_buf()).or_insert((0, 0)).0 += 1;
2617                }
2618            }
2619            for pending in inner
2620                .pending_relays
2621                .values()
2622                .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
2623            {
2624                total_bindings += 1;
2625                if let Some(root) = &pending.reservation.project_root {
2626                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
2627                    entry.1 += 1;
2628                } else {
2629                    unknown_root_bindings += 1;
2630                }
2631            }
2632        }
2633        Ok(ModuleControlResponseToModule::LiveRoots {
2634            roots: roots
2635                .into_iter()
2636                .map(|(project_root, (bound, pending))| LiveRoot {
2637                    project_root,
2638                    bound,
2639                    pending,
2640                })
2641                .collect(),
2642            unknown_root_bindings,
2643            total_bindings,
2644        })
2645    }
2646
2647    /// True if this connection already owns committed or reserved CLIENT routes.
2648    /// A module registers (HELLO) before serving and never opens client routes, so
2649    /// a connection that has client routes must not also become a module endpoint
2650    /// — otherwise one connection holds both client and module state and cleanup
2651    /// only releases one side.
2652    pub(crate) fn connection_has_client_routes(
2653        &self,
2654        connection_id: ConnectionId,
2655    ) -> Result<bool, ForwardingError> {
2656        let inner = self.read_inner()?;
2657        Ok(connection_has_client_routes_locked(&inner, connection_id))
2658    }
2659
2660    pub(crate) fn cleanup_connection(
2661        &self,
2662        connection_id: ConnectionId,
2663    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
2664        self.cleanup_connection_counted(connection_id)
2665            .map(|cleanup| cleanup.released)
2666    }
2667
2668    /// [`Self::cleanup_connection`], also reporting how many pending
2669    /// route.bind relays to the closed module were aborted. That count is the
2670    /// `abandoned` figure of the `route.closed` push sent for a lost module
2671    /// connection; it is always zero for a client connection.
2672    pub(crate) fn cleanup_connection_counted(
2673        &self,
2674        connection_id: ConnectionId,
2675    ) -> Result<ConnectionCleanup, ForwardingError> {
2676        let mut inner = self.write_inner()?;
2677        inner.closing_connections.insert(connection_id);
2678        let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
2679            remove_module_connection_locked(&mut inner, endpoint)
2680        } else {
2681            ConnectionCleanup {
2682                released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
2683                abandoned_relays: 0,
2684            }
2685        };
2686        // The closing mark refuses new work for a connection whose teardown is
2687        // still pending. This is the latest point at which lifting it is safe:
2688        // teardown has just removed every per-connection entry above, under
2689        // this same write lock, so no lookup can still find live state for the
2690        // id; and ids come from a monotonic counter that never reissues one,
2691        // so the id can never name a future connection either. Keeping the
2692        // mark past this point only grew the set by one entry per connection
2693        // for the life of the daemon.
2694        inner.closing_connections.remove(&connection_id);
2695        Ok(cleanup)
2696    }
2697
2698    fn cleanup_client_connection_locked(
2699        inner: &mut ForwardingInner,
2700        connection_id: ConnectionId,
2701    ) -> Vec<GoodbyeTarget> {
2702        let routes = inner
2703            .client_to_module
2704            .iter()
2705            .filter(|(key, _)| key.connection_id == connection_id)
2706            .map(|(key, route)| (*key, route.client_epoch))
2707            .collect::<Vec<_>>();
2708        let mut released = Vec::with_capacity(routes.len());
2709        for (client_key, epoch) in routes {
2710            if let RouteRelease::Removed(target) =
2711                release_client_route_locked(inner, client_key, epoch)
2712            {
2713                released.push(target);
2714            }
2715        }
2716
2717        let pending_keys = inner
2718            .pending_relays
2719            .iter()
2720            .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
2721            .map(|(key, _)| *key)
2722            .collect::<Vec<_>>();
2723        for key in pending_keys {
2724            let Some(pending) = inner.pending_relays.remove(&key) else {
2725                continue;
2726            };
2727            release_reserved_route_locked(
2728                inner,
2729                pending.reservation.client_key,
2730                pending.reservation.module_key,
2731            );
2732            if pending.relay_enqueued {
2733                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2734                    released.push(target);
2735                }
2736            } else {
2737                // The bind may still be on its way into the module's queue;
2738                // a GOODBYE sent now could overtake it. `route.open` sends
2739                // it once its send completes.
2740                inner.relays_released_before_enqueue.insert(key);
2741            }
2742            let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
2743                "client connection closed during route.bind relay".to_string(),
2744            ));
2745        }
2746
2747        let orphaned = inner
2748            .reserved_client
2749            .iter()
2750            .filter(|(key, _)| key.connection_id == connection_id)
2751            .map(|(client, module)| (*client, *module))
2752            .collect::<Vec<_>>();
2753        for (client_key, module_key) in orphaned {
2754            release_reserved_route_locked(inner, client_key, module_key);
2755        }
2756        inner.next_client_channel.remove(&connection_id);
2757        inner
2758            .client_slot_epochs
2759            .retain(|key, _| key.connection_id != connection_id);
2760        inner
2761            .last_published_epoch
2762            .retain(|key, _| key.connection_id != connection_id);
2763        inner
2764            .status
2765            .retain(|(key, _), _| key.connection_id != connection_id);
2766
2767        released
2768    }
2769
2770    /// Close a client connection whose egress queue refused a frame for the
2771    /// route `(connection_id, channel)` at `expected_epoch`. The whole
2772    /// connection closes because its queue is shared by every route on it.
2773    ///
2774    /// The first request that actually closes the connection logs one WARN with
2775    /// the diagnosis: which principals the connection's routes belonged to,
2776    /// which module's frame did not fit and on which client channel, and what
2777    /// the queue held at that moment.
2778    pub(crate) fn escalate_client_delivery_failure(
2779        &self,
2780        connection_id: ConnectionId,
2781        channel: u16,
2782        expected_epoch: u32,
2783        reason: CloseReason,
2784        undelivered: UndeliveredFrame<'_>,
2785    ) -> Result<bool, ForwardingError> {
2786        self.escalate_client_delivery_failure_diagnosed(
2787            connection_id,
2788            channel,
2789            expected_epoch,
2790            reason,
2791            undelivered,
2792            None,
2793        )
2794    }
2795
2796    pub(crate) fn escalate_client_delivery_failure_diagnosed(
2797        &self,
2798        connection_id: ConnectionId,
2799        channel: u16,
2800        expected_epoch: u32,
2801        reason: CloseReason,
2802        undelivered: UndeliveredFrame<'_>,
2803        diagnosis: Option<&crate::router::ClientDeliveryDiagnosis>,
2804    ) -> Result<bool, ForwardingError> {
2805        let principals = {
2806            let mut inner = self.write_inner()?;
2807            let key = ClientRouteKey {
2808                connection_id,
2809                channel,
2810            };
2811            if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
2812                None
2813            } else {
2814                inner.closing_connections.insert(connection_id);
2815                Some(connection_principals_locked(&inner, connection_id))
2816            }
2817        };
2818        let Some(principals) = principals else {
2819            return Ok(false);
2820        };
2821        let backlog = undelivered.sink.backlog();
2822        let close_reason = reason.to_string();
2823        if self.request_connection_close(connection_id, reason) {
2824            warn!(
2825                connection_id = connection_id.get(),
2826                principals = %principals,
2827                module_id = undelivered.module_id.unwrap_or("unknown"),
2828                client_channel = channel,
2829                queued_bytes = backlog.queued_bytes,
2830                queued_frames = backlog.queued_frames,
2831                frame_type = ?diagnosis.map(|failure| failure.frame_type),
2832                frame_flags = ?diagnosis.map(|failure| failure.flags.0),
2833                close_cause = diagnosis.map(|failure| failure.cause).unwrap_or("delivery_failed"),
2834                oldest_queued_ms = backlog
2835                    .oldest_age
2836                    .map(|age| age.as_millis() as u64)
2837                    .unwrap_or(0),
2838                close_reason = %close_reason,
2839                "closing client connection: its egress queue could not take a frame"
2840            );
2841        }
2842        Ok(true)
2843    }
2844
2845    /// Publish the tags a scope sync changed and close the live routes its
2846    /// drain table selects, in one critical section.
2847    ///
2848    /// The caller holds the scope table's write lock (scope table, then this
2849    /// table, always). Publishing and selecting under one lock is what closes
2850    /// the race with a bind commit: a commit before this sees the old tag and
2851    /// its route is selected here like any other; a commit after it sees the
2852    /// new tag and is refused.
2853    ///
2854    /// Every client route index entry is scanned, which includes routes on a
2855    /// swap's superseded endpoint (the old process of a module being replaced
2856    /// blue/green, which keeps its existing routes until they drain): those
2857    /// stay indexed until drained, so ending a scope reaches them too.
2858    pub(crate) fn publish_scope_changes(
2859        &self,
2860        changes: &[ScopeTagChange],
2861    ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
2862        let mut inner = self.write_inner()?;
2863        let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
2864        for change in changes {
2865            let key = (change.owner.clone(), change.scope_ref.clone());
2866            match change.after {
2867                Some(tag) => {
2868                    inner.scope_tags.insert(key, tag);
2869                }
2870                None => {
2871                    inner.scope_tags.remove(&key);
2872                }
2873            }
2874            by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
2875        }
2876        let mut selected = Vec::new();
2877        for (client_key, route) in &inner.client_to_module {
2878            let Some(scope) = &route.scope else {
2879                continue;
2880            };
2881            let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
2882            else {
2883                continue;
2884            };
2885            if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
2886                continue;
2887            }
2888            let reason = match &change.drain {
2889                ScopeDrain::Nothing => continue,
2890                ScopeDrain::All(reason) => *reason,
2891                ScopeDrain::Carriers(narrowed) => {
2892                    let owner = Principal::Reserved {
2893                        module_id: scope.owner.clone(),
2894                    };
2895                    let hit = route.principal != owner
2896                        && narrowed.iter().any(|(principal, allowed)| {
2897                            *principal == route.principal
2898                                && allowed
2899                                    .as_ref()
2900                                    .is_none_or(|targets| !targets.contains(&route.module_id))
2901                        });
2902                    if !hit {
2903                        continue;
2904                    }
2905                    RouteCloseReason::ScopeCarrierRemoved
2906                }
2907            };
2908            selected.push((*client_key, route.client_epoch, reason));
2909        }
2910        let mut drained = Vec::new();
2911        for (client_key, client_epoch, reason) in selected {
2912            let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
2913                continue;
2914            };
2915            let release = release_client_route_locked(&mut inner, client_key, client_epoch);
2916            self.record_route_release(&release);
2917            if let RouteRelease::Removed(module) = release {
2918                drained.push(ScopeDrainedRoute {
2919                    reason,
2920                    scope: route
2921                        .scope
2922                        .clone()
2923                        .expect("selected scope drain has a bound scope"),
2924                    module_id: route.module_id.clone(),
2925                    client: GoodbyeTarget {
2926                        connection_id: route.client_connection_id,
2927                        sink: route.client_sink.clone(),
2928                        negotiated_ver: route.client_negotiated_ver,
2929                        channel: route.client_channel,
2930                        epoch: route.client_epoch,
2931                        kind: GoodbyeTargetKind::Client,
2932                        module_id: Some(route.module_id.clone()),
2933                    },
2934                    module,
2935                });
2936            }
2937        }
2938        Ok(drained)
2939    }
2940
2941    /// The published tag of one scope, for tests of the commit re-check.
2942    #[cfg(test)]
2943    pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
2944        self.read_inner()
2945            .ok()?
2946            .scope_tags
2947            .get(&(owner.to_string(), scope_ref.to_string()))
2948            .copied()
2949    }
2950
2951    fn record_route_release(&self, release: &RouteRelease) {
2952        match release {
2953            RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
2954            RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
2955            RouteRelease::Absent => {}
2956        }
2957    }
2958
2959    fn read_inner(&self) -> Result<RwLockReadGuard<'_, ForwardingInner>, ForwardingError> {
2960        self.inner.read().map_err(|_| ForwardingError::Poisoned)
2961    }
2962
2963    fn write_inner(&self) -> Result<RwLockWriteGuard<'_, ForwardingInner>, ForwardingError> {
2964        self.inner.write().map_err(|_| ForwardingError::Poisoned)
2965    }
2966
2967    fn lock_close_registry(
2968        &self,
2969    ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
2970        self.close_registry
2971            .lock()
2972            .unwrap_or_else(|poisoned| poisoned.into_inner())
2973    }
2974}
2975
2976impl ForwardingInner {
2977    fn allocate_route_slots(
2978        &mut self,
2979        connection_id: ConnectionId,
2980        endpoint: ModuleEndpointId,
2981    ) -> Result<(u16, u32, u16, u32), ForwardingError> {
2982        let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
2983        let mut client_channel = client_start;
2984        let client_channel = loop {
2985            let key = ClientRouteKey {
2986                connection_id,
2987                channel: client_channel,
2988            };
2989            let eligible = !self.client_to_module.contains_key(&key)
2990                && !self.reserved_client.contains_key(&key)
2991                && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
2992            if eligible {
2993                break client_channel;
2994            }
2995            client_channel = next_channel(client_channel);
2996            if client_channel == client_start {
2997                return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
2998            }
2999        };
3000
3001        let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
3002        let mut module_channel = module_start;
3003        let module_channel = loop {
3004            let key = ModuleRouteKey {
3005                endpoint,
3006                channel: module_channel,
3007            };
3008            let eligible = !self.module_to_client.contains_key(&key)
3009                && !self.reserved_module.contains_key(&key)
3010                && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3011            if eligible {
3012                break module_channel;
3013            }
3014            module_channel = next_channel(module_channel);
3015            if module_channel == module_start {
3016                return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
3017            }
3018        };
3019
3020        let client_key = ClientRouteKey {
3021            connection_id,
3022            channel: client_channel,
3023        };
3024        let module_key = ModuleRouteKey {
3025            endpoint,
3026            channel: module_channel,
3027        };
3028        let client_epoch = self
3029            .client_slot_epochs
3030            .get(&client_key)
3031            .copied()
3032            .unwrap_or(0)
3033            + 1;
3034        let module_epoch = self
3035            .module_slot_epochs
3036            .get(&module_key)
3037            .copied()
3038            .unwrap_or(0)
3039            + 1;
3040        self.client_slot_epochs.insert(client_key, client_epoch);
3041        self.module_slot_epochs.insert(module_key, module_epoch);
3042        self.next_client_channel
3043            .insert(connection_id, next_channel(client_channel));
3044        self.next_module_channel
3045            .insert(endpoint, next_channel(module_channel));
3046        Ok((client_channel, client_epoch, module_channel, module_epoch))
3047    }
3048
3049    fn allocate_control_corr(
3050        &mut self,
3051        endpoint: ModuleEndpointId,
3052    ) -> Result<u64, ForwardingError> {
3053        let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
3054        if candidate == 0 {
3055            self.closing_connections.insert(endpoint.connection_id);
3056            return Err(ForwardingError::RelayCorrelationExhausted);
3057        }
3058        self.next_control_corr.insert(
3059            endpoint,
3060            if candidate == u64::MAX {
3061                0
3062            } else {
3063                candidate + 1
3064            },
3065        );
3066        Ok(candidate)
3067    }
3068}
3069
3070fn connection_has_client_routes_locked(
3071    inner: &ForwardingInner,
3072    connection_id: ConnectionId,
3073) -> bool {
3074    inner
3075        .client_to_module
3076        .keys()
3077        .any(|key| key.connection_id == connection_id)
3078        || inner
3079            .reserved_client
3080            .keys()
3081            .any(|key| key.connection_id == connection_id)
3082}
3083
3084fn check_module_connection_role_locked(
3085    inner: &ForwardingInner,
3086    connection_id: ConnectionId,
3087) -> Result<(), ForwardingError> {
3088    if inner.endpoint_by_connection.contains_key(&connection_id)
3089        || connection_has_client_routes_locked(inner, connection_id)
3090    {
3091        return Err(ForwardingError::ConnectionRoleConflict { connection_id });
3092    }
3093    Ok(())
3094}
3095
3096fn next_channel(channel: u16) -> u16 {
3097    let next = channel.wrapping_add(1);
3098    if next == 0 {
3099        1
3100    } else {
3101        next
3102    }
3103}
3104
3105fn endpoint_routes_locked(
3106    inner: &ForwardingInner,
3107    endpoint: ModuleEndpointId,
3108) -> Vec<EndpointRoute> {
3109    let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
3110    let draining = drain_reason.is_some();
3111    let mut routes = inner
3112        .module_to_client
3113        .iter()
3114        .filter(|(module_key, _)| module_key.endpoint == endpoint)
3115        .map(|(_, route)| EndpointRoute {
3116            goodbye_target: GoodbyeTarget {
3117                connection_id: route.client_connection_id,
3118                sink: route.client_sink.clone(),
3119                negotiated_ver: route.client_negotiated_ver,
3120                channel: route.client_channel,
3121                epoch: route.client_epoch,
3122                kind: GoodbyeTargetKind::Client,
3123                module_id: Some(route.module_id.clone()),
3124            },
3125            principal: route.principal.clone(),
3126            bound_at: route.bound_at,
3127            draining,
3128            drain_reason,
3129        })
3130        .collect::<Vec<_>>();
3131    routes.sort_by_key(|route| {
3132        (
3133            route.goodbye_target.connection_id.get(),
3134            route.goodbye_target.channel,
3135            route.goodbye_target.epoch,
3136        )
3137    });
3138    routes
3139}
3140
3141/// Why a bind admitted under `scope` may no longer commit, read from the
3142/// published tags: `scope_ended` when the scope is gone or at another epoch,
3143/// `scope_changed` (retryable) when only its content moved.
3144fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
3145    let key = (scope.owner.clone(), scope.scope_ref.clone());
3146    match inner.scope_tags.get(&key) {
3147        Some(current) if *current == scope.tag => None,
3148        Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
3149            subc_protocol::error_codes::SCOPE_CHANGED,
3150            format!(
3151                "scope '{}' of {} changed while the route was being bound; re-open it",
3152                scope.scope_ref, scope.owner
3153            ),
3154        )),
3155        _ => Some(ErrorBody::new(
3156            subc_protocol::error_codes::SCOPE_ENDED,
3157            format!(
3158                "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
3159                scope.scope_ref, scope.owner, scope.tag.scope_epoch
3160            ),
3161        )),
3162    }
3163}
3164
3165fn release_reserved_route_locked(
3166    inner: &mut ForwardingInner,
3167    client_key: ClientRouteKey,
3168    module_key: ModuleRouteKey,
3169) {
3170    if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
3171        inner.reserved_client.remove(&client_key);
3172    }
3173    if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
3174        inner.reserved_module.remove(&module_key);
3175    }
3176    inner.status.retain(|(key, _), _| *key != client_key);
3177}
3178
3179fn release_client_route_locked(
3180    inner: &mut ForwardingInner,
3181    client_key: ClientRouteKey,
3182    expected_epoch: u32,
3183) -> RouteRelease {
3184    let Some(route) = inner.client_to_module.get(&client_key) else {
3185        return RouteRelease::Absent;
3186    };
3187    if route.client_epoch != expected_epoch {
3188        return RouteRelease::Stale;
3189    }
3190    let route = inner
3191        .client_to_module
3192        .remove(&client_key)
3193        .expect("route checked under the same forwarding lock");
3194    route.flow.close();
3195    if let Some(scope) = &route.scope {
3196        scope.release_bind();
3197    }
3198    inner.module_to_client.remove(&ModuleRouteKey {
3199        endpoint: route.module_endpoint,
3200        channel: route.module_channel,
3201    });
3202    inner.operator_confirms.route_closed(ModuleRouteKey {
3203        endpoint: route.module_endpoint,
3204        channel: route.module_channel,
3205    });
3206    inner.status.remove(&(client_key, expected_epoch));
3207    RouteRelease::Removed(GoodbyeTarget {
3208        connection_id: route.module_endpoint.connection_id,
3209        sink: route.module_sink.clone(),
3210        negotiated_ver: route.module_negotiated_ver,
3211        channel: route.module_channel,
3212        epoch: route.module_epoch,
3213        kind: GoodbyeTargetKind::Module,
3214        module_id: Some(route.module_id.clone()),
3215    })
3216}
3217
3218fn release_module_route_locked(
3219    inner: &mut ForwardingInner,
3220    module_key: ModuleRouteKey,
3221    expected_epoch: u32,
3222) -> RouteRelease {
3223    let Some(route) = inner.module_to_client.get(&module_key) else {
3224        return RouteRelease::Absent;
3225    };
3226    if route.module_epoch != expected_epoch {
3227        return RouteRelease::Stale;
3228    }
3229    let route = inner
3230        .module_to_client
3231        .remove(&module_key)
3232        .expect("route checked under the same forwarding lock");
3233    inner.operator_confirms.route_closed(module_key);
3234    route.flow.close();
3235    if let Some(scope) = &route.scope {
3236        scope.release_bind();
3237    }
3238    let client_key = ClientRouteKey {
3239        connection_id: route.client_connection_id,
3240        channel: route.client_channel,
3241    };
3242    inner.client_to_module.remove(&client_key);
3243    inner.status.remove(&(client_key, route.client_epoch));
3244    RouteRelease::Removed(GoodbyeTarget {
3245        connection_id: route.client_connection_id,
3246        sink: route.client_sink.clone(),
3247        negotiated_ver: route.client_negotiated_ver,
3248        channel: route.client_channel,
3249        epoch: route.client_epoch,
3250        kind: GoodbyeTargetKind::Client,
3251        module_id: Some(route.module_id.clone()),
3252    })
3253}
3254
3255fn commit_route_locked(
3256    inner: &mut ForwardingInner,
3257    pending: PendingRouteBindRelayEntry,
3258) -> Result<Option<GoodbyeTarget>, ForwardingError> {
3259    let reservation = pending.reservation;
3260    if inner
3261        .closing_connections
3262        .contains(&reservation.client_key.connection_id)
3263    {
3264        return Err(ForwardingError::ConnectionClosing {
3265            connection_id: reservation.client_key.connection_id,
3266        });
3267    }
3268    let module_id = inner
3269        .module_id_by_endpoint
3270        .get(&reservation.module_key.endpoint)
3271        .cloned()
3272        .ok_or(ForwardingError::StaleModuleEndpoint)?;
3273    if inner
3274        .draining_endpoints
3275        .contains_key(&reservation.module_key.endpoint)
3276    {
3277        return Err(ForwardingError::ModuleReloading { module_id });
3278    }
3279    if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
3280        || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
3281    {
3282        return Err(ForwardingError::UnknownReservation {
3283            client_channel: reservation.client_key.channel,
3284            module_channel: reservation.module_key.channel,
3285        });
3286    }
3287    let module = inner
3288        .modules_by_id
3289        .get(&module_id)
3290        .filter(|module| module.endpoint == reservation.module_key.endpoint)
3291        .cloned()
3292        .ok_or(ForwardingError::StaleModuleEndpoint)?;
3293    let binding = Arc::new(RouteBinding {
3294        client_connection_id: reservation.client_key.connection_id,
3295        client_sink: pending.client_sink,
3296        client_negotiated_ver: pending.client_negotiated_ver,
3297        client_channel: reservation.client_key.channel,
3298        client_epoch: reservation.client_epoch,
3299        module_id,
3300        module_endpoint: reservation.module_key.endpoint,
3301        module_sink: module.sink,
3302        module_negotiated_ver: module.negotiated_ver,
3303        module_channel: reservation.module_key.channel,
3304        module_epoch: reservation.module_epoch,
3305        principal: pending.principal,
3306        harness: pending.harness,
3307        build_class: pending.build_class,
3308        operator_app: pending.operator_app,
3309        project_root: reservation.project_root.clone(),
3310        bound_at: Instant::now(),
3311        flow: Arc::new(ChannelFlow::new(window_for(&module.concurrency))),
3312        scope: pending.scope,
3313    });
3314    inner
3315        .client_to_module
3316        .insert(reservation.client_key, Arc::clone(&binding));
3317    inner
3318        .module_to_client
3319        .insert(reservation.module_key, Arc::clone(&binding));
3320    let previous_published = inner
3321        .last_published_epoch
3322        .insert(reservation.client_key, reservation.client_epoch);
3323
3324    // This point is reachable only through an accepted route.bind acknowledgement.
3325    // Publish history before notifying the opener, so a subsequent
3326    // `scope.served_by` query sees the acknowledged bind.
3327    if let Some(scope) = &binding.scope {
3328        scope.record_served(&binding.module_id);
3329    }
3330
3331    // Sending through the reserved slot cannot fail, but it reports a receiver
3332    // that closed after reservation and before this locked publication point.
3333    // The frame is stamped and charged to the queued-byte count here, when it
3334    // actually enters the queue, not when the slot was reserved.
3335    let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
3336    if client_writer_closed {
3337        // Not gated on `relay_enqueued`: only a module ack reaches this
3338        // commit, which proves the module holds the binding, and `route.open`
3339        // sends nothing for a relay a reply already settled.
3340        let abandoned = abandoned_route_target(inner, &reservation);
3341        if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
3342            route.flow.close();
3343        }
3344        inner.module_to_client.remove(&reservation.module_key);
3345        inner
3346            .status
3347            .remove(&(reservation.client_key, reservation.client_epoch));
3348        match previous_published {
3349            Some(epoch) => {
3350                inner
3351                    .last_published_epoch
3352                    .insert(reservation.client_key, epoch);
3353            }
3354            None => {
3355                inner.last_published_epoch.remove(&reservation.client_key);
3356            }
3357        }
3358        let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3359            "client egress closed during route publication".to_string(),
3360        ));
3361        return Ok(abandoned);
3362    }
3363
3364    if let Some(scope) = &binding.scope {
3365        scope.commit_presence();
3366    }
3367    let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
3368    Ok(None)
3369}
3370
3371/// The live module connection registered under exactly `endpoint`, whichever
3372/// slot holds it: active, swap candidate, or superseded incumbent.
3373///
3374/// Resolving by endpoint rather than by module id is what keeps an incumbent
3375/// addressable after a swap promoted a candidate over its id. An endpoint that
3376/// is in none of the three (a stale one, replaced without a promotion) resolves
3377/// to nothing, as it always has.
3378fn module_connection_for_endpoint_locked(
3379    inner: &ForwardingInner,
3380    endpoint: ModuleEndpointId,
3381) -> Option<&ModuleConnection> {
3382    let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
3383    inner
3384        .modules_by_id
3385        .get(module_id)
3386        .filter(|module| module.endpoint == endpoint)
3387        .or_else(|| {
3388            inner
3389                .candidates_by_id
3390                .get(module_id)
3391                .filter(|module| module.endpoint == endpoint)
3392        })
3393        .or_else(|| inner.superseded_endpoints.get(&endpoint))
3394}
3395
3396fn abandoned_route_target(
3397    inner: &ForwardingInner,
3398    reservation: &RouteReservation,
3399) -> Option<GoodbyeTarget> {
3400    let module_id = inner
3401        .module_id_by_endpoint
3402        .get(&reservation.module_key.endpoint)?;
3403    let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
3404    (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
3405        connection_id: module.endpoint.connection_id,
3406        sink: module.sink.clone(),
3407        negotiated_ver: module.negotiated_ver,
3408        channel: reservation.module_key.channel,
3409        epoch: reservation.module_epoch,
3410        kind: GoodbyeTargetKind::Module,
3411        module_id: Some(module_id.clone()),
3412    })
3413}
3414
3415/// Queue a registering module's HELLO_ACK on its sink. Called with the
3416/// forwarding write lock held, before the endpoint is inserted, which is what
3417/// puts the ack ahead of any `route.bind` or control RPC routed to the module.
3418/// `try_send` never waits, so holding the lock across it is safe.
3419fn enqueue_hello_ack_locked(
3420    sink: &FrameSink,
3421    connection_id: ConnectionId,
3422    hello_ack: Option<Frame>,
3423) -> Result<(), ForwardingError> {
3424    let Some(hello_ack) = hello_ack else {
3425        return Ok(());
3426    };
3427    sink.try_send(hello_ack)
3428        .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
3429}
3430
3431fn remove_module_connection_locked(
3432    inner: &mut ForwardingInner,
3433    endpoint: ModuleEndpointId,
3434) -> ConnectionCleanup {
3435    // Commit the disconnect before its routes are released, so their teardown
3436    // cannot replace module_closed with route_closed.
3437    inner.operator_confirms.module_closed(endpoint);
3438    inner.draining_endpoints.remove(&endpoint);
3439    inner.daemon_drain_results.remove(&endpoint);
3440    let module_id = inner.module_id_by_endpoint.remove(&endpoint);
3441    if let Some(module_id) = module_id.as_ref() {
3442        if inner
3443            .modules_by_id
3444            .get(module_id)
3445            .is_some_and(|module| module.endpoint == endpoint)
3446        {
3447            inner.modules_by_id.remove(module_id);
3448        }
3449        if inner
3450            .candidates_by_id
3451            .get(module_id)
3452            .is_some_and(|module| module.endpoint == endpoint)
3453        {
3454            inner.candidates_by_id.remove(module_id);
3455        }
3456    }
3457    inner.superseded_endpoints.remove(&endpoint);
3458    inner.endpoint_by_connection.remove(&endpoint.connection_id);
3459    inner.next_module_channel.remove(&endpoint);
3460    inner.next_control_corr.remove(&endpoint);
3461    inner
3462        .health_probe_tombstones
3463        .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
3464    inner
3465        .module_slot_epochs
3466        .retain(|key, _| key.endpoint != endpoint);
3467    let reserved_module_keys: Vec<ModuleRouteKey> = inner
3468        .reserved_module
3469        .keys()
3470        .filter(|module_key| module_key.endpoint == endpoint)
3471        .copied()
3472        .collect();
3473    for module_key in reserved_module_keys {
3474        if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
3475            release_reserved_route_locked(inner, client_key, module_key);
3476        }
3477    }
3478
3479    let pending_keys: Vec<_> = inner
3480        .pending_relays
3481        .keys()
3482        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3483        .copied()
3484        .collect();
3485    let pending: Vec<_> = pending_keys
3486        .into_iter()
3487        .filter_map(|key| {
3488            inner
3489                .pending_relays
3490                .remove(&key)
3491                .map(|pending| (key, pending))
3492        })
3493        .collect();
3494    let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
3495    for (key, pending) in pending {
3496        // Recorded like every other removal without a reply, so a `route.open`
3497        // still sending the bind learns the relay was released rather than
3498        // settled by the module.
3499        if !pending.relay_enqueued {
3500            inner.relays_released_before_enqueue.insert(key);
3501        }
3502        let module_label = module_id.as_deref().unwrap_or("unknown");
3503        let _ = pending
3504            .sender
3505            .send(RouteBindRelayOutcome::ModuleGone(format!(
3506                "module '{module_label}' connection closed during route.bind relay"
3507            )));
3508    }
3509
3510    let pending_control_keys: Vec<_> = inner
3511        .pending_control_rpcs
3512        .keys()
3513        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
3514        .copied()
3515        .collect();
3516    let pending_control: Vec<_> = pending_control_keys
3517        .into_iter()
3518        .filter_map(|key| inner.pending_control_rpcs.remove(&key))
3519        .collect();
3520    for pending in pending_control {
3521        let module_label = module_id.as_deref().unwrap_or("unknown");
3522        let _ = pending
3523            .sender
3524            .send(ModuleControlRpcOutcome::ModuleGone(format!(
3525                "module '{module_label}' connection closed during module-control RPC"
3526            )));
3527    }
3528
3529    let module_routes = inner
3530        .module_to_client
3531        .iter()
3532        .filter(|(module_key, _)| module_key.endpoint == endpoint)
3533        .map(|(module_key, route)| (*module_key, route.module_epoch))
3534        .collect::<Vec<_>>();
3535    let mut released = Vec::with_capacity(module_routes.len());
3536    for (module_key, epoch) in module_routes {
3537        if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
3538        {
3539            released.push(target);
3540        }
3541    }
3542    ConnectionCleanup {
3543        released,
3544        abandoned_relays,
3545    }
3546}
3547
3548#[derive(Debug, Clone, Copy)]
3549struct RequestCredit {
3550    subscription: bool,
3551    excluded_from_drain: bool,
3552}
3553
3554#[derive(Debug, Default)]
3555struct CreditLedger {
3556    by_corr: HashMap<u64, Vec<RequestCredit>>,
3557}
3558
3559impl CreditLedger {
3560    fn acquire(&mut self, corr: u64, subscription: bool) {
3561        self.by_corr.entry(corr).or_default().push(RequestCredit {
3562            subscription,
3563            excluded_from_drain: false,
3564        });
3565    }
3566
3567    fn release(&mut self, corr: u64) -> bool {
3568        let Some(credits) = self.by_corr.get_mut(&corr) else {
3569            return false;
3570        };
3571        let released = credits.pop().is_some();
3572        if credits.is_empty() {
3573            self.by_corr.remove(&corr);
3574        }
3575        released
3576    }
3577
3578    fn capture_subscription_exclusions(&mut self) -> u32 {
3579        let mut excluded = 0u32;
3580        for credit in self.by_corr.values_mut().flatten() {
3581            if credit.subscription && !credit.excluded_from_drain {
3582                credit.excluded_from_drain = true;
3583                excluded = excluded.saturating_add(1);
3584            }
3585        }
3586        excluded
3587    }
3588
3589    #[cfg(test)]
3590    fn in_flight(&self) -> usize {
3591        self.by_corr.values().map(Vec::len).sum()
3592    }
3593
3594    fn drain_in_flight(&self) -> usize {
3595        self.by_corr
3596            .values()
3597            .flatten()
3598            .filter(|credit| !credit.excluded_from_drain)
3599            .count()
3600    }
3601
3602    /// The correlation ids of the requests a drain still waits on, one entry
3603    /// per held credit, in ascending order.
3604    fn drain_held_corrs(&self) -> Vec<u64> {
3605        let mut corrs = self
3606            .by_corr
3607            .iter()
3608            .flat_map(|(corr, credits)| {
3609                credits
3610                    .iter()
3611                    .filter(|credit| !credit.excluded_from_drain)
3612                    .map(move |_| *corr)
3613            })
3614            .collect::<Vec<_>>();
3615        corrs.sort_unstable();
3616        corrs
3617    }
3618}
3619
3620#[derive(Debug, Default)]
3621struct ChannelFlowState {
3622    closed: bool,
3623    credits: CreditLedger,
3624}
3625
3626/// Per-channel request-credit accounting shared by the client and module route halves.
3627#[derive(Debug)]
3628pub(crate) struct ChannelFlow {
3629    sem: Semaphore,
3630    window: usize,
3631    state: Mutex<ChannelFlowState>,
3632}
3633
3634impl ChannelFlow {
3635    pub(crate) fn new(window: usize) -> Self {
3636        debug_assert!(window > 0, "flow-control window must be non-zero");
3637        Self {
3638            sem: Semaphore::new(window),
3639            window,
3640            state: Mutex::new(ChannelFlowState::default()),
3641        }
3642    }
3643
3644    #[cfg(test)]
3645    pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
3646        self.acquire_tagged(0, false).await
3647    }
3648
3649    pub(crate) async fn acquire_tagged(
3650        &self,
3651        corr: u64,
3652        subscription: bool,
3653    ) -> Result<(), ChannelFlowClosed> {
3654        let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
3655        let mut state = self
3656            .state
3657            .lock()
3658            .unwrap_or_else(|poisoned| poisoned.into_inner());
3659        if state.closed {
3660            return Err(ChannelFlowClosed);
3661        }
3662        state.credits.acquire(corr, subscription);
3663        permit.forget();
3664        Ok(())
3665    }
3666
3667    #[cfg(test)]
3668    pub(crate) fn release(&self) {
3669        self.release_corr(0);
3670    }
3671
3672    pub(crate) fn release_corr(&self, corr: u64) {
3673        let released = self
3674            .state
3675            .lock()
3676            .unwrap_or_else(|poisoned| poisoned.into_inner())
3677            .credits
3678            .release(corr);
3679        if !released {
3680            // Protocol-conforming modules emit exactly one terminal per request.
3681            // This guard is a best-effort safety net against window growth, not a
3682            // security boundary against malicious peers.
3683            warn!(
3684                window = self.window,
3685                available = self.sem.available_permits(),
3686                "flow-control over-release ignored"
3687            );
3688            return;
3689        }
3690        if !self.sem.is_closed() {
3691            self.sem.add_permits(1);
3692        }
3693    }
3694
3695    #[cfg(test)]
3696    pub(crate) fn in_flight(&self) -> usize {
3697        self.state
3698            .lock()
3699            .unwrap_or_else(|poisoned| poisoned.into_inner())
3700            .credits
3701            .in_flight()
3702    }
3703
3704    pub(crate) fn drain_in_flight(&self) -> usize {
3705        self.state
3706            .lock()
3707            .unwrap_or_else(|poisoned| poisoned.into_inner())
3708            .credits
3709            .drain_in_flight()
3710    }
3711
3712    pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
3713        self.state
3714            .lock()
3715            .unwrap_or_else(|poisoned| poisoned.into_inner())
3716            .credits
3717            .drain_held_corrs()
3718    }
3719
3720    #[cfg(test)]
3721    pub(crate) fn available_permits(&self) -> usize {
3722        self.sem.available_permits()
3723    }
3724
3725    pub(crate) fn begin_drain(&self) -> u32 {
3726        let mut state = self
3727            .state
3728            .lock()
3729            .unwrap_or_else(|poisoned| poisoned.into_inner());
3730        state.closed = true;
3731        self.sem.close();
3732        state.credits.capture_subscription_exclusions()
3733    }
3734
3735    pub(crate) fn close(&self) {
3736        self.state
3737            .lock()
3738            .unwrap_or_else(|poisoned| poisoned.into_inner())
3739            .closed = true;
3740        self.sem.close();
3741    }
3742}
3743
3744#[derive(Debug, Clone, Copy, PartialEq, Eq)]
3745pub(crate) struct ChannelFlowClosed;
3746
3747impl fmt::Display for ChannelFlowClosed {
3748    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3749        write!(f, "flow-control window closed")
3750    }
3751}
3752
3753impl Error for ChannelFlowClosed {}
3754
3755fn window_for(concurrency: &Concurrency) -> usize {
3756    match concurrency {
3757        Concurrency::Serial => 1,
3758        Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
3759        Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
3760    }
3761}
3762
3763#[derive(Debug, Clone, PartialEq, Eq)]
3764pub enum ForwardingError {
3765    /// A connection cannot hold both client routes and a module endpoint, or
3766    /// replace a module identity while that endpoint is still registered.
3767    ConnectionRoleConflict {
3768        connection_id: ConnectionId,
3769    },
3770    NoModuleConnection,
3771    ModuleReloading {
3772        module_id: String,
3773    },
3774    StaleModuleEndpoint,
3775    UnknownReservation {
3776        client_channel: u16,
3777        module_channel: u16,
3778    },
3779    ClientRouteChannelExhausted {
3780        connection_id: ConnectionId,
3781    },
3782    ModuleRouteChannelExhausted {
3783        endpoint: ModuleEndpointId,
3784    },
3785    RelayCorrelationExhausted,
3786    ConnectionClosing {
3787        connection_id: ConnectionId,
3788    },
3789    ClientEgressClosed {
3790        connection_id: ConnectionId,
3791    },
3792    RouteOpenBuild(String),
3793    /// A swap candidate is already registered for this module id.
3794    CandidateSlotOccupied {
3795        module_id: String,
3796    },
3797    /// A registering module's HELLO_ACK could not be queued because its
3798    /// outbound queue is closed or full.
3799    ModuleEgressUnavailable {
3800        connection_id: ConnectionId,
3801    },
3802    Poisoned,
3803}
3804
3805impl fmt::Display for ForwardingError {
3806    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3807        match self {
3808            Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
3809            Self::NoModuleConnection => write!(f, "no module connection is registered"),
3810            Self::ModuleReloading { module_id } => {
3811                write!(f, "module_id '{module_id}' is reloading")
3812            }
3813            Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
3814            Self::UnknownReservation {
3815                client_channel,
3816                module_channel,
3817            } => write!(
3818                f,
3819                "route reservation client channel {client_channel} / module channel {module_channel} was not found"
3820            ),
3821            Self::ClientRouteChannelExhausted { connection_id } => write!(
3822                f,
3823                "no client route channels are available for connection {}",
3824                connection_id.get()
3825            ),
3826            Self::ModuleRouteChannelExhausted { endpoint } => write!(
3827                f,
3828                "no module route channels are available for endpoint generation {} on connection {}",
3829                endpoint.generation,
3830                endpoint.connection_id.get()
3831            ),
3832            Self::RelayCorrelationExhausted => {
3833                write!(f, "module control correlation ids are exhausted")
3834            }
3835            Self::ConnectionClosing { connection_id } => write!(
3836                f,
3837                "connection {} is closing and cannot accept route allocation",
3838                connection_id.get()
3839            ),
3840            Self::ClientEgressClosed { connection_id } => write!(
3841                f,
3842                "client connection {} egress is closed",
3843                connection_id.get()
3844            ),
3845            Self::RouteOpenBuild(message) => {
3846                write!(f, "failed to prebuild route.open response: {message}")
3847            }
3848            Self::CandidateSlotOccupied { module_id } => write!(
3849                f,
3850                "module_id '{module_id}' already has a swap candidate registered"
3851            ),
3852            Self::ModuleEgressUnavailable { connection_id } => write!(
3853                f,
3854                "module connection {} egress is unavailable; HELLO_ACK could not be queued",
3855                connection_id.get()
3856            ),
3857            Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
3858        }
3859    }
3860}
3861
3862impl Error for ForwardingError {}
3863
3864#[cfg(test)]
3865mod tests {
3866    use std::time::Duration;
3867
3868    use super::*;
3869    use tokio::sync::mpsc;
3870
3871    #[test]
3872    fn ordinary_long_running_request_is_not_excluded_from_drain() {
3873        let mut ledger = CreditLedger::default();
3874        ledger.acquire(1, false);
3875
3876        assert_eq!(ledger.capture_subscription_exclusions(), 0);
3877        assert_eq!(ledger.drain_in_flight(), 1);
3878    }
3879
3880    #[test]
3881    fn bit_set_subscription_is_excluded_and_counted() {
3882        let mut ledger = CreditLedger::default();
3883        ledger.acquire(1, true);
3884
3885        assert_eq!(ledger.capture_subscription_exclusions(), 1);
3886        assert_eq!(ledger.drain_in_flight(), 0);
3887    }
3888
3889    #[test]
3890    fn subscription_opened_after_drain_snapshot_is_not_excluded() {
3891        let mut ledger = CreditLedger::default();
3892        ledger.acquire(1, true);
3893        assert_eq!(ledger.capture_subscription_exclusions(), 1);
3894
3895        ledger.acquire(2, true);
3896
3897        assert_eq!(ledger.drain_in_flight(), 1);
3898    }
3899
3900    #[test]
3901    fn drain_with_no_subscriptions_reports_zero_excluded() {
3902        let mut ledger = CreditLedger::default();
3903        assert_eq!(ledger.capture_subscription_exclusions(), 0);
3904    }
3905
3906    fn test_hello_ack(corr: u64) -> Frame {
3907        Frame::build(
3908            FrameType::HelloAck,
3909            Flags::new(false, Priority::Passive, false),
3910            0,
3911            0,
3912            corr,
3913            Vec::new(),
3914        )
3915        .unwrap()
3916    }
3917
3918    /// An acked registration whose HELLO_ACK cannot be queued must leave no
3919    /// endpoint behind: a routable module that never got its ack would read a
3920    /// route.bind first and exit. A closed queue and a full one both refuse.
3921    #[test]
3922    fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
3923        let forwarding = ForwardingTable::default();
3924
3925        let (closed_tx, closed_rx) = mpsc::channel(8);
3926        drop(closed_rx);
3927        let closed = ConnectionId::new(1);
3928        assert_eq!(
3929            forwarding.register_module_connection_acked(
3930                closed,
3931                "closed".to_string(),
3932                2,
3933                Concurrency::ModuleManaged,
3934                FrameSink::new(closed_tx),
3935                test_hello_ack(1),
3936            ),
3937            Err(ForwardingError::ModuleEgressUnavailable {
3938                connection_id: closed
3939            })
3940        );
3941
3942        let (full_tx, _full_rx) = mpsc::channel(1);
3943        let full_sink = FrameSink::new(full_tx);
3944        full_sink.try_send(test_hello_ack(99)).unwrap();
3945        let full = ConnectionId::new(2);
3946        assert_eq!(
3947            forwarding.register_module_connection_acked(
3948                full,
3949                "full".to_string(),
3950                2,
3951                Concurrency::ModuleManaged,
3952                full_sink.clone(),
3953                test_hello_ack(2),
3954            ),
3955            Err(ForwardingError::ModuleEgressUnavailable {
3956                connection_id: full
3957            })
3958        );
3959        assert_eq!(
3960            forwarding.register_candidate_module_connection_acked(
3961                full,
3962                "full".to_string(),
3963                2,
3964                Concurrency::ModuleManaged,
3965                full_sink,
3966                test_hello_ack(3),
3967            ),
3968            Err(ForwardingError::ModuleEgressUnavailable {
3969                connection_id: full
3970            })
3971        );
3972
3973        for (connection, module_id) in [(closed, "closed"), (full, "full")] {
3974            assert_eq!(
3975                forwarding
3976                    .module_endpoint_for_connection(connection)
3977                    .unwrap(),
3978                None
3979            );
3980            let (client_tx, _client_rx) = mpsc::channel(8);
3981            assert_eq!(
3982                forwarding
3983                    .begin_route_bind_relay_for_test(
3984                        ConnectionId::new(50),
3985                        FrameSink::new(client_tx),
3986                        1,
3987                        module_id,
3988                    )
3989                    .err(),
3990                Some(ForwardingError::NoModuleConnection)
3991            );
3992        }
3993        assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
3994    }
3995
3996    /// Both acked registration forms put the HELLO_ACK on the module's queue
3997    /// by the time the endpoint can be resolved.
3998    #[test]
3999    fn acked_registration_queues_the_hello_ack_first() {
4000        let forwarding = ForwardingTable::default();
4001        let (active_tx, mut active_rx) = mpsc::channel(8);
4002        forwarding
4003            .register_module_connection_acked(
4004                ConnectionId::new(1),
4005                "acked".to_string(),
4006                2,
4007                Concurrency::ModuleManaged,
4008                FrameSink::new(active_tx),
4009                test_hello_ack(11),
4010            )
4011            .unwrap();
4012        let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
4013        forwarding
4014            .register_candidate_module_connection_acked(
4015                ConnectionId::new(2),
4016                "acked".to_string(),
4017                2,
4018                Concurrency::ModuleManaged,
4019                FrameSink::new(candidate_tx),
4020                test_hello_ack(12),
4021            )
4022            .unwrap();
4023
4024        let active_first = active_rx.try_recv().unwrap().frame;
4025        assert_eq!(active_first.header.ty, FrameType::HelloAck);
4026        assert_eq!(active_first.header.corr, 11);
4027        let candidate_first = candidate_rx.try_recv().unwrap().frame;
4028        assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
4029        assert_eq!(candidate_first.header.corr, 12);
4030    }
4031
4032    #[test]
4033    fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
4034        let forwarding = ForwardingTable::default();
4035        let module_connection = ConnectionId::new(10);
4036        let exhausted_client = ConnectionId::new(20);
4037        let second_client = ConnectionId::new(30);
4038        let (module_tx, _module_rx) = mpsc::channel(1);
4039        let endpoint = forwarding
4040            .register_module_connection(
4041                module_connection,
4042                "route-limit-provider".to_string(),
4043                1,
4044                Concurrency::ModuleManaged,
4045                FrameSink::new(module_tx),
4046            )
4047            .unwrap();
4048
4049        {
4050            let mut inner = forwarding.inner.write().unwrap();
4051            for channel in 1..=u16::MAX {
4052                inner.reserved_client.insert(
4053                    ClientRouteKey {
4054                        connection_id: exhausted_client,
4055                        channel,
4056                    },
4057                    ModuleRouteKey {
4058                        endpoint,
4059                        channel: 1,
4060                    },
4061                );
4062            }
4063        }
4064
4065        let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
4066        let err = forwarding
4067            .begin_route_bind_relay_for_test(
4068                exhausted_client,
4069                FrameSink::new(exhausted_tx),
4070                1,
4071                "route-limit-provider",
4072            )
4073            .unwrap_err();
4074        assert!(matches!(
4075            err,
4076            ForwardingError::ClientRouteChannelExhausted { connection_id }
4077                if connection_id == exhausted_client
4078        ));
4079
4080        let (second_tx, _second_rx) = mpsc::channel(1);
4081        let pending = forwarding
4082            .begin_route_bind_relay_for_test(
4083                second_client,
4084                FrameSink::new(second_tx),
4085                2,
4086                "route-limit-provider",
4087            )
4088            .unwrap();
4089        assert_eq!(pending.client_channel, 1);
4090    }
4091
4092    #[test]
4093    fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
4094        let forwarding = ForwardingTable::default();
4095        let module_connection = ConnectionId::new(40);
4096        let client = ConnectionId::new(50);
4097        let (module_tx, _module_rx) = mpsc::channel(1);
4098        forwarding
4099            .register_module_connection(
4100                module_connection,
4101                "slot-reuse-provider".to_string(),
4102                1,
4103                Concurrency::ModuleManaged,
4104                FrameSink::new(module_tx),
4105            )
4106            .unwrap();
4107
4108        let (client_tx, _client_rx) = mpsc::channel(1);
4109        let client_sink = FrameSink::new(client_tx);
4110        let mut wrapped_channel = None;
4111        for index in 0..=usize::from(u16::MAX) {
4112            let pending = forwarding
4113                .begin_route_bind_relay_for_test(
4114                    client,
4115                    client_sink.clone(),
4116                    index as u64 + 1,
4117                    "slot-reuse-provider",
4118                )
4119                .unwrap();
4120            if index == usize::from(u16::MAX) {
4121                wrapped_channel = Some(pending.module_channel);
4122            }
4123            forwarding
4124                .abort_pending_relay(
4125                    pending.endpoint,
4126                    pending.corr,
4127                    RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
4128                )
4129                .unwrap();
4130        }
4131
4132        assert_eq!(wrapped_channel, Some(1));
4133    }
4134
4135    #[test]
4136    fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
4137        let forwarding = ForwardingTable::default();
4138        let client = ConnectionId::new(60);
4139        forwarding
4140            .inner
4141            .write()
4142            .unwrap()
4143            .next_client_channel
4144            .insert(client, 41);
4145
4146        let released = forwarding.cleanup_connection(client).unwrap();
4147
4148        assert!(released.is_empty());
4149        assert!(!forwarding
4150            .inner
4151            .read()
4152            .unwrap()
4153            .next_client_channel
4154            .contains_key(&client));
4155    }
4156
4157    #[test]
4158    fn stale_module_cleanup_preserves_fast_reconnect_successor() {
4159        let forwarding = ForwardingTable::default();
4160        let module_id = "fast-reconnect-provider";
4161        let first_connection = ConnectionId::new(70);
4162        let second_connection = ConnectionId::new(80);
4163        let (first_tx, _first_rx) = mpsc::channel(1);
4164        let first_endpoint = forwarding
4165            .register_module_connection(
4166                first_connection,
4167                module_id.to_string(),
4168                1,
4169                Concurrency::ModuleManaged,
4170                FrameSink::new(first_tx),
4171            )
4172            .unwrap();
4173        let (second_tx, _second_rx) = mpsc::channel(1);
4174        let second_endpoint = forwarding
4175            .register_module_connection(
4176                second_connection,
4177                module_id.to_string(),
4178                1,
4179                Concurrency::ModuleManaged,
4180                FrameSink::new(second_tx),
4181            )
4182            .unwrap();
4183        assert_ne!(first_endpoint, second_endpoint);
4184
4185        let released = forwarding.cleanup_connection(first_connection).unwrap();
4186
4187        assert!(released.is_empty());
4188        assert_eq!(
4189            forwarding
4190                .inner
4191                .read()
4192                .unwrap()
4193                .modules_by_id
4194                .get(module_id)
4195                .map(|module| module.endpoint),
4196            Some(second_endpoint)
4197        );
4198        assert!(forwarding.has_live_module_connection(module_id).unwrap());
4199        let control_rpc = forwarding
4200            .begin_module_control_rpc_for(
4201                module_id,
4202                "health.check",
4203                Instant::now() + Duration::from_secs(1),
4204            )
4205            .unwrap();
4206        assert_eq!(control_rpc.endpoint, second_endpoint);
4207    }
4208
4209    fn route_fixture(
4210        module_id: &str,
4211    ) -> (
4212        ForwardingTable,
4213        ConnectionId,
4214        ModuleEndpointId,
4215        ConnectionId,
4216        FrameSink,
4217        mpsc::Receiver<crate::router::OutboundFrame>,
4218    ) {
4219        let forwarding = ForwardingTable::default();
4220        let module_connection = ConnectionId::new(100);
4221        let client_connection = ConnectionId::new(200);
4222        let (module_tx, _module_rx) = mpsc::channel(8);
4223        let endpoint = forwarding
4224            .register_module_connection(
4225                module_connection,
4226                module_id.to_string(),
4227                2,
4228                Concurrency::ModuleManaged,
4229                FrameSink::new(module_tx),
4230            )
4231            .unwrap();
4232        let (client_tx, client_rx) = mpsc::channel(8);
4233        (
4234            forwarding,
4235            module_connection,
4236            endpoint,
4237            client_connection,
4238            FrameSink::new(client_tx),
4239            client_rx,
4240        )
4241    }
4242
4243    #[test]
4244    #[cfg(unix)]
4245    fn daemon_drain_gates_current_and_racing_provider_registrations() {
4246        let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
4247        assert_eq!(
4248            forwarding.begin_daemon_drain().unwrap(),
4249            [("provider".to_string(), endpoint)]
4250        );
4251        assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
4252        assert!(matches!(
4253            forwarding.register_module_connection(
4254                ConnectionId::new(300),
4255                "late-provider".into(),
4256                2,
4257                Concurrency::ModuleManaged,
4258                sink,
4259            ),
4260            Err(ForwardingError::ConnectionClosing { .. })
4261        ));
4262    }
4263
4264    #[test]
4265    #[cfg(unix)]
4266    fn daemon_drain_results_are_endpoint_fenced_and_removed_on_cleanup() {
4267        let (forwarding, connection, endpoint, _, _, _) = route_fixture("provider");
4268        assert_eq!(
4269            forwarding
4270                .daemon_drain_result_for_connection(connection)
4271                .unwrap(),
4272            None
4273        );
4274        forwarding
4275            .record_daemon_drain_result(endpoint, true)
4276            .unwrap();
4277        forwarding.begin_daemon_drain().unwrap();
4278        assert_eq!(
4279            forwarding
4280                .daemon_drain_result_for_connection(connection)
4281                .unwrap(),
4282            Some(false)
4283        );
4284        let replacement = ModuleEndpointId {
4285            generation: endpoint.generation + 1,
4286            ..endpoint
4287        };
4288        forwarding
4289            .record_daemon_drain_result(replacement, true)
4290            .unwrap();
4291        assert_eq!(
4292            forwarding
4293                .daemon_drain_result_for_connection(connection)
4294                .unwrap(),
4295            Some(false)
4296        );
4297        forwarding
4298            .record_daemon_drain_result(endpoint, true)
4299            .unwrap();
4300        assert_eq!(
4301            forwarding
4302                .daemon_drain_result_for_connection(connection)
4303                .unwrap(),
4304            Some(true)
4305        );
4306        assert_eq!(
4307            forwarding
4308                .daemon_drain_result_for_connection(ConnectionId::new(300))
4309                .unwrap(),
4310            Some(false)
4311        );
4312        forwarding.cleanup_connection(connection).unwrap();
4313        forwarding
4314            .record_daemon_drain_result(endpoint, true)
4315            .unwrap();
4316        assert_eq!(
4317            forwarding
4318                .daemon_drain_result_for_connection(connection)
4319                .unwrap(),
4320            Some(false)
4321        );
4322        assert!(forwarding
4323            .read_inner()
4324            .unwrap()
4325            .daemon_drain_results
4326            .is_empty());
4327    }
4328
4329    #[test]
4330    #[cfg(unix)]
4331    fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
4332        let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
4333            route_fixture("provider");
4334        let mut pending =
4335            begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
4336        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
4337        forwarding.begin_daemon_drain().unwrap();
4338        let completion = forwarding
4339            .complete_pending_relay(
4340                module_connection,
4341                pending.corr,
4342                RouteBindRelayOutcome::Accepted,
4343            )
4344            .expect("draining admission is not a fatal module error");
4345        assert!(completion.settled);
4346        let abandoned = completion
4347            .abandoned
4348            .expect("late accepted bind needs a route GOODBYE");
4349        assert_eq!(abandoned.channel, pending.module_channel);
4350        assert!(matches!(pending.receiver.try_recv().unwrap(),
4351            RouteBindRelayOutcome::DaemonRejected(error) if error.code == "module_reloading"));
4352        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4353        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4354        assert!(
4355            rx.try_recv().is_err(),
4356            "no route.open may be published during drain"
4357        );
4358        assert_eq!(
4359            forwarding
4360                .module_endpoint_for_connection(module_connection)
4361                .unwrap(),
4362            Some(endpoint)
4363        );
4364        assert!(
4365            !forwarding
4366                .complete_pending_relay(
4367                    module_connection,
4368                    pending.corr,
4369                    RouteBindRelayOutcome::Accepted,
4370                )
4371                .unwrap()
4372                .settled
4373        );
4374    }
4375
4376    fn test_ping(corr: u64) -> Frame {
4377        Frame::build(
4378            FrameType::Ping,
4379            Flags::new(false, Priority::Passive, false),
4380            0,
4381            0,
4382            corr,
4383            Vec::new(),
4384        )
4385        .unwrap()
4386    }
4387
4388    fn begin_test_route(
4389        forwarding: &ForwardingTable,
4390        client_connection: ConnectionId,
4391        client_sink: FrameSink,
4392        corr: u64,
4393        module_id: &str,
4394    ) -> PendingRouteBindRelay {
4395        forwarding
4396            .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
4397            .unwrap()
4398    }
4399
4400    #[test]
4401    fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
4402        for candidate in [false, true] {
4403            for register_first in [false, true] {
4404                let (forwarding, module_connection, _, client_connection, sink, _rx) =
4405                    route_fixture("target");
4406                let register = || {
4407                    if candidate {
4408                        forwarding.register_candidate_module_connection(
4409                            client_connection,
4410                            "source".into(),
4411                            2,
4412                            Concurrency::ModuleManaged,
4413                            sink.clone(),
4414                        )
4415                    } else {
4416                        forwarding.register_module_connection(
4417                            client_connection,
4418                            "source".into(),
4419                            2,
4420                            Concurrency::ModuleManaged,
4421                            sink.clone(),
4422                        )
4423                    }
4424                };
4425                if register_first {
4426                    register().unwrap();
4427                    assert!(
4428                        forwarding
4429                            .begin_route_bind_relay_for_test(
4430                                client_connection,
4431                                sink.clone(),
4432                                1,
4433                                "target",
4434                            )
4435                            .is_err(),
4436                        "a deferred route.open cannot reserve after HELLO"
4437                    );
4438                    assert!(forwarding
4439                        .cleanup_connection(client_connection)
4440                        .unwrap()
4441                        .is_empty());
4442                } else {
4443                    let pending =
4444                        begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
4445                    assert!(
4446                        register().is_err(),
4447                        "HELLO cannot register after a route reservation"
4448                    );
4449                    forwarding
4450                        .complete_pending_relay(
4451                            module_connection,
4452                            pending.corr,
4453                            RouteBindRelayOutcome::Accepted,
4454                        )
4455                        .unwrap();
4456                    assert_eq!(
4457                        forwarding
4458                            .cleanup_connection(client_connection)
4459                            .unwrap()
4460                            .len(),
4461                        1
4462                    );
4463                }
4464                assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
4465                assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4466            }
4467        }
4468    }
4469
4470    /// A pending route.open reserves its queue slot with `reserve_owned` and
4471    /// sends its prebuilt response through that slot when the module accepts.
4472    /// A queue already holding many data frames (far more than the old 64-frame
4473    /// queue) must not stop the open from reserving or completing, and the
4474    /// response must be charged to the queue's byte count when it is sent.
4475    #[tokio::test]
4476    async fn pending_route_open_completes_behind_queued_data_frames() {
4477        assert_eq!(
4478            crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
4479            8,
4480            "the per-connection pending route.open limit is its own constant"
4481        );
4482        let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
4483            route_fixture("open-behind-data");
4484        let (sink, mut client_rx) = crate::server::connection_egress();
4485        const DATA_FRAMES: usize = 1_000;
4486        let data = |corr: u64| {
4487            Frame::build(
4488                FrameType::StreamData,
4489                Flags::new(false, Priority::Interactive, false),
4490                9,
4491                1,
4492                corr,
4493                vec![b'x'; 200],
4494            )
4495            .unwrap()
4496        };
4497        for corr in 0..DATA_FRAMES as u64 {
4498            sink.try_send(data(corr)).unwrap();
4499        }
4500        let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
4501        assert_eq!(sink.backlog().queued_bytes, data_bytes);
4502
4503        let pending = tokio::time::timeout(
4504            Duration::from_secs(5),
4505            forwarding.begin_route_bind_relay_for(
4506                client,
4507                sink.clone(),
4508                subc_protocol::PROTOCOL_VERSION,
4509                4_242,
4510                "open-behind-data",
4511                Principal::Direct,
4512                None,
4513                None,
4514                "fixture".into(),
4515                None,
4516                None,
4517                Instant::now() + Duration::from_secs(60),
4518            ),
4519        )
4520        .await
4521        .expect("reserving the route.open slot must not wait behind data frames")
4522        .unwrap();
4523        forwarding
4524            .complete_pending_relay(
4525                module_connection,
4526                pending.corr,
4527                RouteBindRelayOutcome::Accepted,
4528            )
4529            .unwrap();
4530
4531        let backlog = sink.backlog();
4532        assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
4533        assert!(
4534            backlog.queued_bytes > data_bytes,
4535            "the route.open response must be counted in queued bytes: {backlog:?}"
4536        );
4537        for corr in 0..DATA_FRAMES as u64 {
4538            assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
4539        }
4540        let open = client_rx.recv().await.unwrap();
4541        assert_eq!(open.header.corr, 4_242);
4542        assert_eq!(open.header.ty, FrameType::Response);
4543        drop(open);
4544        assert_eq!(sink.backlog().queued_bytes, 0);
4545        assert_eq!(sink.backlog().queued_frames, 0);
4546    }
4547
4548    /// The drain-timeout line reports these numbers, so they must count the
4549    /// requests the drain is waiting on and no others: a route with nothing in
4550    /// flight is not a holdout, and a flagged subscription is excluded from the
4551    /// drain and so from the count.
4552    #[tokio::test]
4553    async fn drain_holdouts_count_held_requests_and_name_the_connection() {
4554        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4555            route_fixture("holdouts");
4556        let mut bound = |corr| {
4557            let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
4558            forwarding
4559                .complete_pending_relay(
4560                    module_connection,
4561                    route.corr,
4562                    RouteBindRelayOutcome::Accepted,
4563                )
4564                .unwrap();
4565            client_rx.try_recv().unwrap();
4566            match forwarding
4567                .lookup_data_route(client, route.client_channel, route.client_epoch)
4568                .unwrap()
4569            {
4570                DataRoute::Client(DataRouteState::Bound(binding)) => binding,
4571                other => panic!("expected live route, got {other:?}"),
4572            }
4573        };
4574        let holding = bound(61);
4575        let _idle = bound(62);
4576        holding.flow.acquire_tagged(7, false).await.unwrap();
4577        holding.flow.acquire_tagged(2, false).await.unwrap();
4578        holding.flow.acquire_tagged(3, true).await.unwrap();
4579        forwarding
4580            .begin_module_drain("holdouts", RouteCloseReason::Restart)
4581            .unwrap();
4582
4583        let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
4584        assert_eq!(
4585            holdouts,
4586            DrainHoldouts {
4587                requests: 2,
4588                routes: 1,
4589                total_routes: 2,
4590                top_connections: vec![(client.get(), 2)],
4591                // The held requests by the module's channel and corr, ascending,
4592                // without the excluded subscription (corr 3).
4593                held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
4594            }
4595        );
4596    }
4597
4598    #[test]
4599    fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
4600        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4601            route_fixture("census");
4602        let pending = begin_test_route(&forwarding, client, sink, 1, "census");
4603        forwarding
4604            .complete_pending_relay(
4605                module_connection,
4606                pending.corr,
4607                RouteBindRelayOutcome::Accepted,
4608            )
4609            .unwrap();
4610
4611        let routes = forwarding.endpoint_routes(endpoint).unwrap();
4612        assert_eq!(routes.len(), 1);
4613        assert!(matches!(routes[0].principal, Principal::Direct));
4614        assert_eq!(routes[0].goodbye_target.connection_id, client);
4615        assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
4616        assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
4617        assert!(!routes[0].draining);
4618
4619        forwarding
4620            .begin_module_drain("census", RouteCloseReason::Restart)
4621            .unwrap();
4622        let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
4623        assert_eq!(draining_routes.len(), 1);
4624        assert!(draining_routes[0].draining);
4625    }
4626
4627    #[test]
4628    fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
4629        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
4630        let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
4631        assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
4632        forwarding
4633            .abort_pending_relay(
4634                first.endpoint,
4635                first.corr,
4636                RouteBindRelayOutcome::ModuleGone("abort".into()),
4637            )
4638            .unwrap();
4639        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4640        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4641
4642        let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
4643        assert_eq!(second.client_channel, first.client_channel);
4644        assert_eq!(second.module_channel, first.module_channel);
4645        assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
4646    }
4647
4648    #[test]
4649    fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
4650        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4651            route_fixture("epoch-release");
4652        let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
4653        forwarding
4654            .complete_pending_relay(
4655                module_connection,
4656                first.corr,
4657                RouteBindRelayOutcome::Accepted,
4658            )
4659            .unwrap();
4660        assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
4661        assert!(matches!(
4662            forwarding
4663                .release_client_route(client, first.client_channel, first.client_epoch)
4664                .unwrap(),
4665            RouteRelease::Removed(_)
4666        ));
4667        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
4668        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
4669
4670        let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
4671        forwarding
4672            .complete_pending_relay(
4673                module_connection,
4674                second.corr,
4675                RouteBindRelayOutcome::Accepted,
4676            )
4677            .unwrap();
4678        assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
4679        assert!(matches!(
4680            forwarding
4681                .release_client_route(client, second.client_channel, first.client_epoch)
4682                .unwrap(),
4683            RouteRelease::Stale
4684        ));
4685        assert!(!forwarding
4686            .cache_status(
4687                endpoint,
4688                second.module_channel,
4689                first.module_epoch,
4690                "stale".into(),
4691            )
4692            .unwrap());
4693        assert!(forwarding
4694            .cache_status(
4695                endpoint,
4696                second.module_channel,
4697                second.module_epoch,
4698                "current".into(),
4699            )
4700            .unwrap());
4701        match forwarding
4702            .route_poll_snapshot(client, second.client_channel, second.client_epoch)
4703            .unwrap()
4704        {
4705            RoutePollSnapshot::Bound { status, .. } => {
4706                assert_eq!(status.as_deref(), Some("current"));
4707            }
4708            RoutePollSnapshot::Absent => panic!("successor binding was removed"),
4709        }
4710        let counters = forwarding.counters().snapshot();
4711        assert_eq!(counters["route_released_epoch_fenced"], 1);
4712        assert_eq!(counters["route_release_stale_skipped"], 1);
4713    }
4714
4715    #[test]
4716    fn max_epoch_reservation_retires_only_that_slot() {
4717        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
4718        forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
4719        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
4720        let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
4721        assert_eq!(
4722            (final_use.client_channel, final_use.client_epoch),
4723            (7, u32::MAX)
4724        );
4725        assert_eq!(
4726            (final_use.module_channel, final_use.module_epoch),
4727            (9, u32::MAX)
4728        );
4729        forwarding
4730            .abort_pending_relay(
4731                endpoint,
4732                final_use.corr,
4733                RouteBindRelayOutcome::ModuleGone("abort".into()),
4734            )
4735            .unwrap();
4736        forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
4737        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
4738        let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
4739        assert_ne!(next.client_channel, 7);
4740        assert_ne!(next.module_channel, 9);
4741        assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
4742    }
4743
4744    #[test]
4745    fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
4746        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
4747            route_fixture("corr-shared");
4748        let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
4749        assert_eq!(bind.corr, 1);
4750        forwarding
4751            .abort_pending_relay(
4752                endpoint,
4753                bind.corr,
4754                RouteBindRelayOutcome::ModuleGone("abort".into()),
4755            )
4756            .unwrap();
4757        let rpc = forwarding
4758            .begin_module_control_rpc_for(
4759                "corr-shared",
4760                "health.check",
4761                Instant::now() - Duration::from_millis(1),
4762            )
4763            .unwrap();
4764        assert_eq!(rpc.corr, 2);
4765        assert_eq!(
4766            forwarding
4767                .complete_module_control_rpc(
4768                    module_connection,
4769                    rpc.corr,
4770                    Some("health.check"),
4771                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
4772                        status: subc_protocol::session::HealthStatus::Ok,
4773                        detail: None,
4774                        metrics: None,
4775                    }),
4776                )
4777                .unwrap(),
4778            ModuleControlRpcCompletion::Settled
4779        );
4780        assert!(matches!(
4781            rpc.receiver.blocking_recv().unwrap(),
4782            ModuleControlRpcOutcome::DeadlineElapsed
4783        ));
4784    }
4785
4786    #[tokio::test(start_paused = true)]
4787    async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
4788        let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
4789        let probe_started_at = Instant::now();
4790        let rpc = forwarding
4791            .begin_health_probe_rpc_for(
4792                "tombstone-ttl",
4793                "health.check",
4794                probe_started_at,
4795                probe_started_at + Duration::from_secs(5),
4796            )
4797            .unwrap();
4798        assert!(forwarding
4799            .tombstone_health_probe_rpc(endpoint, rpc.corr)
4800            .unwrap());
4801        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
4802
4803        tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
4804        tokio::task::yield_now().await;
4805
4806        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
4807    }
4808
4809    #[test]
4810    fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
4811        let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
4812        let mut close = forwarding.register_connection_close(endpoint.connection_id);
4813        forwarding.inject_control_corr(endpoint, u64::MAX);
4814        let final_rpc = forwarding
4815            .begin_module_control_rpc_for(
4816                "corr-max",
4817                "health.check",
4818                Instant::now() + Duration::from_secs(1),
4819            )
4820            .unwrap();
4821        assert_eq!(final_rpc.corr, u64::MAX);
4822        forwarding
4823            .cancel_module_control_rpc(endpoint, final_rpc.corr)
4824            .unwrap();
4825        assert!(matches!(
4826            forwarding.begin_module_control_rpc_for(
4827                "corr-max",
4828                "health.check",
4829                Instant::now() + Duration::from_secs(1),
4830            ),
4831            Err(ForwardingError::RelayCorrelationExhausted)
4832        ));
4833        assert!(close.try_recv().is_ok());
4834    }
4835
4836    #[test]
4837    fn publication_epoch_controls_delivery_failure_escalation() {
4838        fn setup_successor(
4839            commit_successor: Option<bool>,
4840        ) -> (ForwardingTable, ConnectionId, u16, u32) {
4841            let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
4842                route_fixture("escalation");
4843            let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
4844            forwarding
4845                .complete_pending_relay(
4846                    module_connection,
4847                    first.corr,
4848                    RouteBindRelayOutcome::Accepted,
4849                )
4850                .unwrap();
4851            client_rx.try_recv().unwrap();
4852            assert!(matches!(
4853                forwarding
4854                    .release_client_route(client, first.client_channel, first.client_epoch)
4855                    .unwrap(),
4856                RouteRelease::Removed(_)
4857            ));
4858            if let Some(commit_successor) = commit_successor {
4859                forwarding.inject_client_slot_epoch(
4860                    client,
4861                    first.client_channel,
4862                    first.client_epoch,
4863                );
4864                forwarding.inject_module_slot_epoch(
4865                    endpoint,
4866                    first.module_channel,
4867                    first.module_epoch,
4868                );
4869                let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
4870                if commit_successor {
4871                    forwarding
4872                        .complete_pending_relay(
4873                            module_connection,
4874                            successor.corr,
4875                            RouteBindRelayOutcome::Accepted,
4876                        )
4877                        .unwrap();
4878                    client_rx.try_recv().unwrap();
4879                } else {
4880                    forwarding
4881                        .abort_pending_relay(
4882                            endpoint,
4883                            successor.corr,
4884                            RouteBindRelayOutcome::ModuleGone("abort".into()),
4885                        )
4886                        .unwrap();
4887                }
4888            }
4889            (forwarding, client, first.client_channel, first.client_epoch)
4890        }
4891
4892        let probe_sink = FrameSink::new(mpsc::channel(1).0);
4893        let (no_successor, client, channel, epoch) = setup_successor(None);
4894        let mut close = no_successor.register_connection_close(client);
4895        assert!(no_successor
4896            .escalate_client_delivery_failure(
4897                client,
4898                channel,
4899                epoch,
4900                CloseReason::new("delivery", "failed"),
4901                UndeliveredFrame {
4902                    module_id: None,
4903                    sink: &probe_sink,
4904                },
4905            )
4906            .unwrap());
4907        assert!(close.try_recv().is_ok());
4908
4909        let (aborted, client, channel, epoch) = setup_successor(Some(false));
4910        let mut close = aborted.register_connection_close(client);
4911        assert!(aborted
4912            .escalate_client_delivery_failure(
4913                client,
4914                channel,
4915                epoch,
4916                CloseReason::new("delivery", "failed"),
4917                UndeliveredFrame {
4918                    module_id: None,
4919                    sink: &probe_sink,
4920                },
4921            )
4922            .unwrap());
4923        assert!(close.try_recv().is_ok());
4924
4925        let (published, client, channel, epoch) = setup_successor(Some(true));
4926        let mut close = published.register_connection_close(client);
4927        assert!(!published
4928            .escalate_client_delivery_failure(
4929                client,
4930                channel,
4931                epoch,
4932                CloseReason::new("delivery", "stale failure"),
4933                UndeliveredFrame {
4934                    module_id: None,
4935                    sink: &probe_sink,
4936                },
4937            )
4938            .unwrap());
4939        assert!(close.try_recv().is_err());
4940    }
4941
4942    #[test]
4943    fn route_concentration_separates_client_count_from_routes_per_client() {
4944        // The distinction this asserts is the one a bare connection count cannot
4945        // make: two connections holding one route each and one connection
4946        // holding two are the same total, and have opposite causes.
4947        let (forwarding, module_connection, _, client, sink, _client_rx) =
4948            route_fixture("concentration");
4949        assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
4950
4951        for corr in [70_u64, 71] {
4952            let pending =
4953                begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
4954            forwarding
4955                .complete_pending_relay(
4956                    module_connection,
4957                    pending.corr,
4958                    RouteBindRelayOutcome::Accepted,
4959                )
4960                .unwrap();
4961        }
4962
4963        // One connection, two routes — not two connections with a route each.
4964        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
4965        assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
4966    }
4967
4968    #[test]
4969    fn cleanup_and_accepted_resolution_have_one_lock_winner() {
4970        let (forwarding, module_connection, _, client, sink, mut client_rx) =
4971            route_fixture("cleanup-race");
4972        let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
4973        forwarding
4974            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
4975            .unwrap();
4976        let released = forwarding.cleanup_connection(client).unwrap();
4977        assert_eq!(released.len(), 1);
4978        let completion = forwarding
4979            .complete_pending_relay(
4980                module_connection,
4981                pending.corr,
4982                RouteBindRelayOutcome::Accepted,
4983            )
4984            .unwrap();
4985        assert!(!completion.settled);
4986        assert!(client_rx.try_recv().is_err());
4987        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
4988
4989        let (forwarding, module_connection, _, client, sink, mut client_rx) =
4990            route_fixture("accepted-race");
4991        let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
4992        forwarding
4993            .complete_pending_relay(
4994                module_connection,
4995                pending.corr,
4996                RouteBindRelayOutcome::Accepted,
4997            )
4998            .unwrap();
4999        assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
5000        let released = forwarding.cleanup_connection(client).unwrap();
5001        assert_eq!(released.len(), 1);
5002        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5003    }
5004
5005    #[test]
5006    fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
5007        let (forwarding, module_connection, _, client, sink, mut client_rx) =
5008            route_fixture("drain-gap");
5009        let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
5010        forwarding
5011            .complete_pending_relay(
5012                module_connection,
5013                live.corr,
5014                RouteBindRelayOutcome::Accepted,
5015            )
5016            .unwrap();
5017        client_rx.try_recv().unwrap();
5018        let binding = match forwarding
5019            .lookup_data_route(client, live.client_channel, live.client_epoch)
5020            .unwrap()
5021        {
5022            DataRoute::Client(DataRouteState::Bound(binding)) => binding,
5023            other => panic!("expected live route, got {other:?}"),
5024        };
5025
5026        let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
5027        forwarding
5028            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5029            .unwrap();
5030        let control_rpc = forwarding
5031            .begin_module_control_rpc_for(
5032                "drain-gap",
5033                "health.check",
5034                Instant::now() + Duration::from_secs(1),
5035            )
5036            .unwrap();
5037        let target = forwarding
5038            .begin_module_drain("drain-gap", RouteCloseReason::Reload)
5039            .unwrap()
5040            .unwrap();
5041        assert!(matches!(
5042            control_rpc.receiver.blocking_recv().unwrap(),
5043            ModuleControlRpcOutcome::ModuleGone(_)
5044        ));
5045        assert_eq!(target.abandoned_bindings.len(), 1);
5046        assert!(binding.flow.sem.is_closed());
5047        assert!(
5048            !forwarding
5049                .complete_pending_relay(
5050                    module_connection,
5051                    pending.corr,
5052                    RouteBindRelayOutcome::Accepted,
5053                )
5054                .unwrap()
5055                .settled
5056        );
5057        assert!(matches!(
5058            forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
5059            Err(ForwardingError::ModuleReloading { .. })
5060        ));
5061        let released = forwarding
5062            .release_module_endpoint_routes(target.endpoint)
5063            .unwrap();
5064        assert_eq!(released.len(), 1);
5065        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5066    }
5067
5068    /// A client can be marked closing while its egress is still open: the daemon
5069    /// asks a connection to close (here through the production path, a module
5070    /// frame that its egress refused) and the connection loop tears down a moment
5071    /// later. A route.bind ack that lands inside that window is answered on the
5072    /// MODULE connection's frame handler, so resolving it must not produce an
5073    /// error -- an error there ends the module connection, and that connection
5074    /// carries every other client's routes to the module.
5075    #[test]
5076    fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
5077        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5078            route_fixture("closing-client");
5079
5080        // A published route on this client: escalate_client_delivery_failure only
5081        // marks a connection closing for a route it has already published.
5082        let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
5083        forwarding
5084            .complete_pending_relay(
5085                module_connection,
5086                live.corr,
5087                RouteBindRelayOutcome::Accepted,
5088            )
5089            .unwrap();
5090        client_rx.try_recv().unwrap();
5091
5092        // A second route.open from the same client, relayed and awaiting its ack.
5093        let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
5094        forwarding
5095            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
5096            .unwrap();
5097
5098        // The window: closing, but the sink is still open.
5099        assert!(forwarding
5100            .escalate_client_delivery_failure(
5101                client,
5102                live.client_channel,
5103                live.client_epoch,
5104                CloseReason::new(
5105                    "module_to_client_delivery_failed",
5106                    "client egress refused a module frame",
5107                ),
5108                UndeliveredFrame {
5109                    module_id: None,
5110                    sink: &sink,
5111                },
5112            )
5113            .unwrap());
5114        assert!(!sink.is_closed());
5115
5116        let completion = forwarding
5117            .complete_pending_relay(
5118                module_connection,
5119                pending.corr,
5120                RouteBindRelayOutcome::Accepted,
5121            )
5122            .expect("a closing client must not turn a module's ack into an error");
5123
5124        assert!(completion.settled);
5125        let abandoned = completion
5126            .abandoned
5127            .expect("the module must be told to drop the binding it just created");
5128        assert_eq!(abandoned.connection_id, module_connection);
5129        assert_eq!(abandoned.channel, pending.module_channel);
5130        assert_eq!(abandoned.epoch, pending.module_epoch);
5131        assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
5132        assert!(matches!(
5133            pending.receiver.blocking_recv().unwrap(),
5134            RouteBindRelayOutcome::ModuleGone(_)
5135        ));
5136        // No route was published to a client that is on its way out, and the
5137        // reserved handle pair went back.
5138        assert!(client_rx.try_recv().is_err());
5139        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
5140
5141        // The module endpoint is untouched: still live, and still able to take a
5142        // route from another client.
5143        assert!(forwarding
5144            .has_live_module_connection("closing-client")
5145            .unwrap());
5146        let cotenant = ConnectionId::new(201);
5147        let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
5148        let cotenant_route = begin_test_route(
5149            &forwarding,
5150            cotenant,
5151            FrameSink::new(cotenant_tx),
5152            62,
5153            "closing-client",
5154        );
5155        assert_eq!(cotenant_route.endpoint, endpoint);
5156        forwarding
5157            .complete_pending_relay(
5158                module_connection,
5159                cotenant_route.corr,
5160                RouteBindRelayOutcome::Accepted,
5161            )
5162            .unwrap();
5163        assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
5164        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
5165    }
5166
5167    #[test]
5168    fn pending_route_permit_is_released_on_rejection_and_abort() {
5169        let forwarding = ForwardingTable::default();
5170        let module_connection = ConnectionId::new(300);
5171        let client = ConnectionId::new(301);
5172        let (module_tx, _module_rx) = mpsc::channel(1);
5173        let endpoint = forwarding
5174            .register_module_connection(
5175                module_connection,
5176                "permit".into(),
5177                2,
5178                Concurrency::ModuleManaged,
5179                FrameSink::new(module_tx),
5180            )
5181            .unwrap();
5182        let (client_tx, mut client_rx) = mpsc::channel(1);
5183        let sink = FrameSink::new(client_tx);
5184        let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
5185        assert!(sink.try_send(test_ping(999)).is_err());
5186        forwarding
5187            .complete_pending_relay(
5188                module_connection,
5189                rejected.corr,
5190                RouteBindRelayOutcome::Rejected(ErrorBody {
5191                    code: "no".into(),
5192                    message: "rejected".into(),
5193                    detail: None,
5194                }),
5195            )
5196            .unwrap();
5197        sink.try_send(test_ping(1000)).unwrap();
5198        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
5199
5200        let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
5201        assert!(sink.try_send(test_ping(1001)).is_err());
5202        forwarding
5203            .abort_pending_relay(
5204                endpoint,
5205                aborted.corr,
5206                RouteBindRelayOutcome::ModuleGone("abort".into()),
5207            )
5208            .unwrap();
5209        sink.try_send(test_ping(1002)).unwrap();
5210        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
5211
5212        let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
5213        forwarding
5214            .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
5215            .unwrap();
5216        drop(client_rx);
5217        let completion = forwarding
5218            .complete_pending_relay(
5219                module_connection,
5220                receiver_closed.corr,
5221                RouteBindRelayOutcome::Accepted,
5222            )
5223            .unwrap();
5224        assert!(completion.abandoned.is_some());
5225        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5226    }
5227
5228    /// Every connection teardown passes through `cleanup_connection`, and
5229    /// connection ids come from a monotonic counter that never hands an id out
5230    /// twice. If the closing mark survives teardown, the set grows by one entry
5231    /// per connection for the life of the daemon -- the self-watchdog alone
5232    /// reconnects once a minute.
5233    #[test]
5234    fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
5235        let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
5236            route_fixture("closing-set-leak");
5237
5238        const CONNECTIONS: u64 = 32;
5239        for index in 0..CONNECTIONS {
5240            let client = ConnectionId::new(1000 + index);
5241            let (client_tx, _client_rx) = mpsc::channel(8);
5242            let route = begin_test_route(
5243                &forwarding,
5244                client,
5245                FrameSink::new(client_tx),
5246                index + 1,
5247                "closing-set-leak",
5248            );
5249            forwarding
5250                .complete_pending_relay(
5251                    module_connection,
5252                    route.corr,
5253                    RouteBindRelayOutcome::Accepted,
5254                )
5255                .unwrap();
5256            forwarding.cleanup_connection(client).unwrap();
5257        }
5258        forwarding.cleanup_connection(module_connection).unwrap();
5259
5260        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5261    }
5262
5263    /// The closing mark exists to refuse new work for a connection that is on
5264    /// its way out but whose teardown has not run yet: the daemon asks the
5265    /// connection loop to end, and only when the loop reacts does
5266    /// `cleanup_connection` strip the connection's state. Inside that window an
5267    /// operation for the dying connection must still be refused; only after
5268    /// cleanup completes may the mark go.
5269    #[test]
5270    fn closing_connection_is_refused_new_work_until_cleanup_completes() {
5271        let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
5272            route_fixture("closing-gate");
5273
5274        // A published route: escalate_client_delivery_failure only marks a
5275        // connection closing for a route it has already published.
5276        let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
5277        forwarding
5278            .complete_pending_relay(
5279                module_connection,
5280                live.corr,
5281                RouteBindRelayOutcome::Accepted,
5282            )
5283            .unwrap();
5284        client_rx.try_recv().unwrap();
5285
5286        // Mark the connection closing through the production path without
5287        // running teardown, pinning the window open.
5288        assert!(forwarding
5289            .escalate_client_delivery_failure(
5290                client,
5291                live.client_channel,
5292                live.client_epoch,
5293                CloseReason::new(
5294                    "module_to_client_delivery_failed",
5295                    "client egress refused a module frame",
5296                ),
5297                UndeliveredFrame {
5298                    module_id: None,
5299                    sink: &sink,
5300                },
5301            )
5302            .unwrap());
5303        assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
5304
5305        // A late route.open for the closing client is refused ...
5306        assert!(matches!(
5307            forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
5308            Err(ForwardingError::ConnectionClosing { connection_id })
5309                if connection_id == client
5310        ));
5311        // ... and so is a late attempt to register the connection as a module.
5312        let (late_tx, _late_rx) = mpsc::channel(1);
5313        assert!(matches!(
5314            forwarding.register_module_connection(
5315                client,
5316                "late-module".into(),
5317                2,
5318                Concurrency::ModuleManaged,
5319                FrameSink::new(late_tx),
5320            ),
5321            Err(ForwardingError::ConnectionClosing { connection_id })
5322                if connection_id == client
5323        ));
5324
5325        // Teardown is the point that lifts the mark: it has just removed every
5326        // per-connection entry under the same lock, so the gate has nothing
5327        // left to protect for this id.
5328        forwarding.cleanup_connection(client).unwrap();
5329        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
5330    }
5331}
5332
5333/// Blue/green swap slots: a candidate registered beside the active endpoint,
5334/// promoted by `cutover_candidate`, with the old incumbent drained by endpoint.
5335#[cfg(test)]
5336mod swap_slot_tests {
5337    use std::time::Duration;
5338
5339    use super::*;
5340    use tokio::sync::mpsc;
5341
5342    const MODULE_ID: &str = "swapped";
5343
5344    struct SwapFixture {
5345        forwarding: ForwardingTable,
5346        incumbent_connection: ConnectionId,
5347        incumbent: ModuleEndpointId,
5348        candidate_connection: ConnectionId,
5349        candidate: ModuleEndpointId,
5350        _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
5351    }
5352
5353    fn swap_fixture() -> SwapFixture {
5354        let forwarding = ForwardingTable::default();
5355        let incumbent_connection = ConnectionId::new(100);
5356        let candidate_connection = ConnectionId::new(110);
5357        let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
5358        let incumbent = forwarding
5359            .register_module_connection(
5360                incumbent_connection,
5361                MODULE_ID.to_string(),
5362                2,
5363                Concurrency::ModuleManaged,
5364                FrameSink::new(incumbent_tx),
5365            )
5366            .unwrap();
5367        let (candidate_tx, candidate_rx) = mpsc::channel(8);
5368        let candidate = forwarding
5369            .register_candidate_module_connection(
5370                candidate_connection,
5371                MODULE_ID.to_string(),
5372                2,
5373                Concurrency::ModuleManaged,
5374                FrameSink::new(candidate_tx),
5375            )
5376            .unwrap();
5377        SwapFixture {
5378            forwarding,
5379            incumbent_connection,
5380            incumbent,
5381            candidate_connection,
5382            candidate,
5383            _module_rxs: vec![incumbent_rx, candidate_rx],
5384        }
5385    }
5386
5387    fn client(
5388        raw: u64,
5389    ) -> (
5390        ConnectionId,
5391        FrameSink,
5392        mpsc::Receiver<crate::router::OutboundFrame>,
5393    ) {
5394        let (tx, rx) = mpsc::channel(8);
5395        (ConnectionId::new(raw), FrameSink::new(tx), rx)
5396    }
5397
5398    fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
5399        forwarding
5400            .read_inner()
5401            .unwrap()
5402            .client_to_module
5403            .values()
5404            .map(|route| route.module_endpoint)
5405            .collect()
5406    }
5407
5408    #[test]
5409    fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
5410        let fixture = swap_fixture();
5411        let forwarding = &fixture.forwarding;
5412        assert_ne!(fixture.incumbent, fixture.candidate);
5413
5414        // Every by-id consumer still resolves the incumbent.
5415        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5416        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5417        let (client_connection, client_sink, _client_rx) = client(200);
5418        let pending = forwarding
5419            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5420            .unwrap();
5421        assert_eq!(pending.endpoint, fixture.incumbent);
5422        let rpc = forwarding
5423            .begin_module_control_rpc_for(
5424                MODULE_ID,
5425                "health.check",
5426                Instant::now() + Duration::from_secs(1),
5427            )
5428            .unwrap();
5429        assert_eq!(rpc.endpoint, fixture.incumbent);
5430        let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
5431        assert_eq!(census.len(), 1, "the census lists one endpoint per id");
5432
5433        // Connection-keyed lookups see the candidate, so its own frames resolve.
5434        assert_eq!(
5435            forwarding
5436                .module_endpoint_for_connection(fixture.candidate_connection)
5437                .unwrap(),
5438            Some(fixture.candidate)
5439        );
5440        assert_eq!(
5441            forwarding
5442                .module_id_for_connection(fixture.candidate_connection)
5443                .unwrap()
5444                .as_deref(),
5445            Some(MODULE_ID)
5446        );
5447
5448        // One candidate per id.
5449        let (other_tx, _other_rx) = mpsc::channel(1);
5450        assert_eq!(
5451            forwarding.register_candidate_module_connection(
5452                ConnectionId::new(120),
5453                MODULE_ID.to_string(),
5454                2,
5455                Concurrency::ModuleManaged,
5456                FrameSink::new(other_tx),
5457            ),
5458            Err(ForwardingError::CandidateSlotOccupied {
5459                module_id: MODULE_ID.to_string()
5460            })
5461        );
5462    }
5463
5464    /// The cutover linearization point. A relay reserved on the incumbent before
5465    /// cutover must never become a route on the incumbent, and every relay
5466    /// reserved after it must land on the promoted candidate.
5467    #[test]
5468    fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
5469        let fixture = swap_fixture();
5470        let forwarding = &fixture.forwarding;
5471        let (early_client, early_sink, _early_rx) = client(200);
5472        let mut early = forwarding
5473            .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
5474            .unwrap();
5475        assert_eq!(early.endpoint, fixture.incumbent);
5476        assert_eq!(
5477            forwarding
5478                .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
5479                .unwrap(),
5480            RouteBindRelayMark::Marked
5481        );
5482
5483        let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5484        assert_eq!(
5485            cutover,
5486            ForwardingCutover {
5487                promoted: fixture.candidate,
5488                incumbent: Some(fixture.incumbent),
5489            }
5490        );
5491
5492        // A route.open reserved after cutover goes to the promoted candidate.
5493        let (late_client, late_sink, _late_rx) = client(201);
5494        let late = forwarding
5495            .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
5496            .unwrap();
5497        assert_eq!(
5498            late.endpoint, fixture.candidate,
5499            "a route.open after cutover was reserved on the incumbent"
5500        );
5501
5502        // The incumbent acks the early relay after cutover.
5503        let completion = forwarding
5504            .complete_pending_relay(
5505                fixture.incumbent_connection,
5506                early.corr,
5507                RouteBindRelayOutcome::Accepted,
5508            )
5509            .expect("a superseded endpoint's ack is not an error on its connection");
5510        assert!(completion.settled);
5511        assert!(
5512            !committed_endpoints(forwarding).contains(&fixture.incumbent),
5513            "a relay reserved before cutover committed a route on the incumbent"
5514        );
5515        let goodbye = completion
5516            .abandoned
5517            .expect("the incumbent is told to drop the binding it just created");
5518        assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
5519        assert_eq!(goodbye.channel, early.module_channel);
5520        assert_eq!(goodbye.epoch, early.module_epoch);
5521        assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
5522        match early.receiver.try_recv() {
5523            Ok(RouteBindRelayOutcome::DaemonRejected(body)) => {
5524                assert_eq!(body.code, "module_reloading")
5525            }
5526            other => panic!("expected a retryable module_reloading answer, got {other:?}"),
5527        }
5528        assert!(matches!(
5529            forwarding
5530                .lookup_data_route(early_client, early.client_channel, early.client_epoch)
5531                .unwrap(),
5532            DataRoute::Client(DataRouteState::Absent)
5533        ));
5534
5535        // The late relay commits on the candidate; only its pair stays reserved
5536        // until then.
5537        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
5538        forwarding
5539            .complete_pending_relay(
5540                fixture.candidate_connection,
5541                late.corr,
5542                RouteBindRelayOutcome::Accepted,
5543            )
5544            .unwrap();
5545        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5546        assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
5547    }
5548
5549    #[test]
5550    fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
5551        let fixture = swap_fixture();
5552        let forwarding = &fixture.forwarding;
5553        // One bound route and one in-flight relay on the incumbent.
5554        let (bound_client, bound_sink, _bound_rx) = client(200);
5555        let bound = forwarding
5556            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5557            .unwrap();
5558        forwarding
5559            .complete_pending_relay(
5560                fixture.incumbent_connection,
5561                bound.corr,
5562                RouteBindRelayOutcome::Accepted,
5563            )
5564            .unwrap();
5565        let (pending_client, pending_sink, _pending_rx) = client(201);
5566        let mut in_flight = forwarding
5567            .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
5568            .unwrap();
5569        forwarding
5570            .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
5571            .unwrap();
5572
5573        let incumbent = forwarding
5574            .cutover_candidate(MODULE_ID)
5575            .unwrap()
5576            .unwrap()
5577            .incumbent
5578            .unwrap();
5579        let target = forwarding
5580            .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
5581            .unwrap()
5582            .expect("the superseded incumbent is still registered");
5583
5584        assert_eq!(target.endpoint, fixture.incumbent);
5585        assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
5586        assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
5587        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
5588        assert_eq!(target.abandoned_bindings.len(), 1);
5589        assert_eq!(
5590            target.abandoned_bindings[0].channel,
5591            in_flight.module_channel
5592        );
5593        assert!(matches!(
5594            in_flight.receiver.try_recv(),
5595            Ok(RouteBindRelayOutcome::DaemonRejected(body)) if body.code == "module_reloading"
5596        ));
5597        assert_eq!(
5598            forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
5599            1,
5600            "the incumbent's bound route stays until its drain finishes"
5601        );
5602
5603        let (next_client, next_sink, _next_rx) = client(202);
5604        let next = forwarding
5605            .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
5606            .expect("the promoted candidate keeps accepting routes");
5607        assert_eq!(next.endpoint, fixture.candidate);
5608    }
5609
5610    /// An endpoint replaced WITHOUT a promotion (a successor registered over it
5611    /// as an ordinary active HELLO) is stale, not superseded, and its ack still
5612    /// fails the acking connection exactly as it did before swap slots existed:
5613    /// `StaleModuleEndpoint`, the reservation indexes already stripped, and the
5614    /// waiting client's sender dropped unanswered.
5615    #[test]
5616    fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
5617        let forwarding = ForwardingTable::default();
5618        let first_connection = ConnectionId::new(70);
5619        let (first_tx, _first_rx) = mpsc::channel(8);
5620        forwarding
5621            .register_module_connection(
5622                first_connection,
5623                MODULE_ID.to_string(),
5624                2,
5625                Concurrency::ModuleManaged,
5626                FrameSink::new(first_tx),
5627            )
5628            .unwrap();
5629        let (client_connection, client_sink, _client_rx) = client(200);
5630        let mut pending = forwarding
5631            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5632            .unwrap();
5633        let (second_tx, _second_rx) = mpsc::channel(8);
5634        forwarding
5635            .register_module_connection(
5636                ConnectionId::new(80),
5637                MODULE_ID.to_string(),
5638                2,
5639                Concurrency::ModuleManaged,
5640                FrameSink::new(second_tx),
5641            )
5642            .unwrap();
5643
5644        assert_eq!(
5645            forwarding
5646                .complete_pending_relay(
5647                    first_connection,
5648                    pending.corr,
5649                    RouteBindRelayOutcome::Accepted
5650                )
5651                .unwrap_err(),
5652            ForwardingError::StaleModuleEndpoint
5653        );
5654        assert!(committed_endpoints(&forwarding).is_empty());
5655        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5656        assert!(matches!(
5657            pending.receiver.try_recv(),
5658            Err(oneshot::error::TryRecvError::Closed)
5659        ));
5660    }
5661
5662    #[test]
5663    fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
5664        // A candidate whose connection drops leaves the incumbent routable.
5665        let fixture = swap_fixture();
5666        let forwarding = &fixture.forwarding;
5667        assert!(forwarding
5668            .cleanup_connection(fixture.candidate_connection)
5669            .unwrap()
5670            .is_empty());
5671        assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
5672        let (client_connection, client_sink, _client_rx) = client(200);
5673        assert_eq!(
5674            forwarding
5675                .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
5676                .unwrap()
5677                .endpoint,
5678            fixture.incumbent
5679        );
5680
5681        // After a cutover, the incumbent's teardown releases its own routes and
5682        // leaves the promoted candidate in place.
5683        let fixture = swap_fixture();
5684        let forwarding = &fixture.forwarding;
5685        let (bound_client, bound_sink, _bound_rx) = client(200);
5686        let bound = forwarding
5687            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
5688            .unwrap();
5689        forwarding
5690            .complete_pending_relay(
5691                fixture.incumbent_connection,
5692                bound.corr,
5693                RouteBindRelayOutcome::Accepted,
5694            )
5695            .unwrap();
5696        forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
5697        let released = forwarding
5698            .cleanup_connection(fixture.incumbent_connection)
5699            .unwrap();
5700        assert_eq!(released.len(), 1);
5701        assert_eq!(released[0].connection_id, bound_client);
5702        assert!(forwarding
5703            .read_inner()
5704            .unwrap()
5705            .superseded_endpoints
5706            .is_empty());
5707        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
5708        let (next_client, next_sink, _next_rx) = client(201);
5709        assert_eq!(
5710            forwarding
5711                .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
5712                .unwrap()
5713                .endpoint,
5714            fixture.candidate
5715        );
5716    }
5717}