Skip to main content

subc_daemon/
forwarding.rs

1//! Forwarding snapshots and route publication. Spawn/claim publication takes
2//! the spawn transition, claim state/index, then forwarding, in that order.
3//! Bind commits read only the launch watch, never the spawn transition lock.
4//! No forwarding guard is held while awaiting a module.
5//! HELLO publication holds Registry before ForwardingTable::inner so an ACK
6//! cannot be observed before registry publication. Code holding inner must not
7//! take the registry lock, including through a callback.
8//!
9//! Spawn authority: an authority change takes the spawn's authority record
10//! state, then `inner` (see `crate::spawn_authority`). Code holding `inner`
11//! never takes a record's authority state; it reads only the record's
12//! immutable identity and the epochs and first-party marks published into
13//! `spawn_epochs` and `spawn_marks`.
14
15use std::{
16    collections::{BTreeMap, HashMap, HashSet},
17    error::Error,
18    fmt,
19    sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard},
20    time::Duration,
21};
22
23use subc_control::{
24    ClientControlResponse, RouteCloseReason, RouteOpenReplyExt, RouteOpenTarget,
25    RouteOpenTargetKind,
26};
27use subc_protocol::{
28    manifest::Concurrency,
29    session::{LiveRoot, ModuleControlResponse, ModuleControlResponseToModule},
30    ErrorBody, Flags, FrameType, Principal, Priority,
31};
32use tokio::sync::{oneshot, Semaphore};
33use tokio::time::Instant;
34use tracing::{debug, info, warn};
35
36use crate::spawn_authority::ApprovedDeclarations;
37use crate::{
38    control::{RouteBindBreakers, RouteBindConcurrency},
39    observability::DaemonCounters,
40    registry::ConnectionId,
41    router::FrameSink,
42    scopes::{BoundScope, ScopeDrain, ScopeTag, ScopeTagChange},
43    spawn_authority::{AuthorityPair, SpawnAuthorityRecord, SpawnKey},
44    Frame, ProjectRootId,
45};
46
47/// Default per-channel request-credit window for modules that schedule internally.
48const DEFAULT_MODULE_MANAGED_WINDOW: usize = 32;
49
50/// Watches leave eight request credits available for ordinary calls.
51const DEFAULT_MODULE_MANAGED_WATCH_WINDOW: usize = 24;
52
53/// High per-channel cap for stateless modules; this is an OOM guard, not scheduling policy.
54const STATELESS_PARALLEL_WINDOW: usize = 1024;
55
56/// A stopped probe cycle cannot retain its last unanswered correlation forever.
57/// Active endpoints replace the tombstone on their next serial health probe;
58/// this backstop covers endpoints that stop probing altogether.
59const HEALTH_PROBE_TOMBSTONE_TTL: Duration = Duration::from_secs(5 * 60);
60
61/// Module connection identity used in forwarding keys.
62///
63/// The generation is bumped every time a module connection is registered so a future restart cannot
64/// accidentally reuse a stale `(connection_id, route_channel)` binding.
65#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
66pub struct ModuleEndpointId {
67    pub connection_id: ConnectionId,
68    pub generation: u64,
69}
70
71/// Client-local route key. A route channel is unique only within one client connection.
72#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
73pub(crate) struct ClientRouteKey {
74    pub connection_id: ConnectionId,
75    pub channel: u16,
76}
77
78/// Module-local route key. A route channel is unique only within one live module endpoint.
79#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
80pub(crate) struct ModuleRouteKey {
81    pub endpoint: ModuleEndpointId,
82    pub channel: u16,
83}
84
85#[derive(Debug)]
86pub(crate) struct RouteBinding {
87    pub client_connection_id: ConnectionId,
88    pub client_sink: FrameSink,
89    pub client_negotiated_ver: u8,
90    pub client_channel: u16,
91    pub client_epoch: u32,
92    pub module_id: String,
93    pub module_endpoint: ModuleEndpointId,
94    pub module_sink: FrameSink,
95    pub module_negotiated_ver: u8,
96    pub module_channel: u16,
97    pub module_epoch: u32,
98    pub principal: Principal,
99    /// Opener attributes captured at admission for daemon-side prompt policy.
100    #[allow(dead_code)]
101    pub harness: String,
102    /// Only Callosum's exact production claim is retained; never relayed.
103    #[allow(dead_code)]
104    pub build_class: Option<String>,
105    /// Captured at connect, carried through admission; not standing authority.
106    #[allow(dead_code)]
107    pub operator_app: Option<crate::operator_apps::OperatorAppStamp>,
108    pub project_root: Option<ProjectRootId>,
109    pub bound_at: Instant,
110    pub flow: Arc<ChannelFlow>,
111    /// The scope the route was admitted under, with the tag its bind was
112    /// stamped at. A scope change finds the routes to close by it.
113    pub scope: Option<BoundScope>,
114    /// Close reasons this route's opener can decode, captured from route.open.
115    pub accepts_close_reasons: Option<Vec<String>>,
116    /// The opener's spawn and authority epoch when the open was attested;
117    /// `None` for an opener with no supervised spawn.
118    pub caller_authority: Option<AuthorityPair>,
119    /// The serving spawn and its authority epoch when the route was reserved;
120    /// `None` for a prefix or unsupervised target.
121    pub target_authority: Option<AuthorityPair>,
122}
123
124#[cfg(debug_assertions)]
125thread_local! {
126    /// Forwarding guards this thread holds, counted in debug builds.
127    static FORWARDING_GUARDS_HELD: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
128}
129
130/// A guard on the forwarding table's state. Debug builds count the guards
131/// each thread holds, so code that must never wait while the forwarding lock
132/// is held can check it with [`assert_forwarding_unlocked`].
133struct InnerGuard<G>(G);
134
135impl<G> InnerGuard<G> {
136    fn new(guard: G) -> Self {
137        #[cfg(debug_assertions)]
138        FORWARDING_GUARDS_HELD.with(|held| held.set(held.get() + 1));
139        Self(guard)
140    }
141}
142
143impl<G> Drop for InnerGuard<G> {
144    fn drop(&mut self) {
145        #[cfg(debug_assertions)]
146        FORWARDING_GUARDS_HELD.with(|held| held.set(held.get().saturating_sub(1)));
147    }
148}
149
150impl<G: std::ops::Deref> std::ops::Deref for InnerGuard<G> {
151    type Target = G::Target;
152    fn deref(&self) -> &Self::Target {
153        &self.0
154    }
155}
156
157impl<G: std::ops::DerefMut> std::ops::DerefMut for InnerGuard<G> {
158    fn deref_mut(&mut self) -> &mut Self::Target {
159        &mut self.0
160    }
161}
162
163/// Panic in a debug build when this thread holds the forwarding lock. The
164/// configuration owner's state and a spawn authority record's state are taken
165/// before the forwarding lock, never under it; their lock functions call this
166/// so a path that inverts the order fails in tests instead of deadlocking.
167pub(crate) fn assert_forwarding_unlocked(what: &str) {
168    #[cfg(debug_assertions)]
169    FORWARDING_GUARDS_HELD.with(|held| {
170        assert_eq!(
171            held.get(),
172            0,
173            "{what} must not be taken while this thread holds the forwarding lock"
174        );
175    });
176    #[cfg(not(debug_assertions))]
177    let _ = what;
178}
179
180/// The spawn authority facts `route.open` hands to the relay reservation.
181#[derive(Debug, Clone, Default)]
182pub(crate) struct RouteAuthority {
183    /// The daemon incarnation reported in the reply's `target`.
184    pub(crate) incarnation: String,
185    /// The opener's spawn and epoch, captured once at admission.
186    pub(crate) caller: Option<AuthorityPair>,
187}
188
189/// What the reservation found serving the route.
190#[derive(Debug, Clone, PartialEq, Eq)]
191pub(crate) enum TargetAuthority {
192    /// A reserved-prefix child, which never reports its owner's mark.
193    Prefix,
194    /// A registration with no supervised spawn behind it.
195    Unsupervised,
196    /// A supervised spawn's own registration, at the epoch and with the mark
197    /// the forwarding table had published when the endpoint was reserved.
198    Spawn(
199        AuthorityPair,
200        crate::spawn_authority::ApprovedDeclarations,
201        bool,
202    ),
203}
204
205impl TargetAuthority {
206    fn pair(&self) -> Option<&AuthorityPair> {
207        match self {
208            Self::Spawn(pair, _, _) => Some(pair),
209            Self::Prefix | Self::Unsupervised => None,
210        }
211    }
212
213    /// The `target` key of the `route.open` reply. The approved set is the
214    /// serving spawn record's value; the mark is the one published for it.
215    fn reply_target(&self, incarnation: &str) -> Result<RouteOpenTarget, ForwardingError> {
216        let (kind, spawn, epoch, first_party, digest) = match self {
217            Self::Prefix => (RouteOpenTargetKind::Prefix, None, None, false, None),
218            Self::Unsupervised => (RouteOpenTargetKind::Unsupervised, None, None, false, None),
219            Self::Spawn(pair, approved, first_party) => (
220                RouteOpenTargetKind::Spawn,
221                Some(pair.key.spawn),
222                Some(pair.epoch),
223                *first_party,
224                Some(approved.digest.clone()),
225            ),
226        };
227        RouteOpenTarget::new(incarnation, kind, spawn, epoch, first_party, digest)
228            .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))
229    }
230}
231
232/// Serialize a successful `route.open` reply with its `target` key.
233fn route_open_reply_body(
234    reply: &ClientControlResponse,
235    target: &TargetAuthority,
236    incarnation: &str,
237) -> Result<Vec<u8>, ForwardingError> {
238    let build = |err: serde_json::Error| ForwardingError::RouteOpenBuild(err.to_string());
239    let mut body = serde_json::to_value(reply).map_err(build)?;
240    RouteOpenReplyExt::new()
241        .with_target(Some(target.reply_target(incarnation)?))
242        .write_to(&mut body)
243        .map_err(build)?;
244    serde_json::to_vec(&body).map_err(build)
245}
246
247#[derive(Debug, Clone)]
248pub(crate) enum DataRoute {
249    Client(DataRouteState),
250    Module(DataRouteState),
251}
252
253#[derive(Debug, Clone)]
254pub(crate) enum DataRouteState {
255    Bound(Arc<RouteBinding>),
256    Reserved,
257    EpochMismatch,
258    Absent,
259}
260
261/// Which kind of peer a route GOODBYE is being delivered to. This decides what
262/// happens when the GOODBYE cannot be enqueued (egress full/closed):
263/// - `Client`: escalate to closing that client connection (a socket close is a
264///   stronger teardown signal, and a full client egress means it is the slow
265///   client we would drop anyway).
266/// - `Module`: never close; deliver late instead (see
267///   [`send_module_route_goodbye`]), and drop only if the module still has no
268///   room after [`LATE_MODULE_GOODBYE_DEADLINE`]. A client-disconnect notifies the
269///   SHARED module that one client's route is gone; closing the module on its
270///   egress backpressure would tear down every co-tenant client (the exact
271///   cross-tenant blast radius this never-close rule exists to prevent — observed when a
272///   flooding dead client filled BOTH its own and the module's egress, so its
273///   route-gone GOODBYE to the module failed and closed the shared connection).
274///   subc has already removed the route from its forwarding state and drops
275///   stale module frames for the released channel (see router.rs), so subc's
276///   own routing is correct. The residual: under SUSTAINED module-egress
277///   backpressure a module-targeted route-gone notification can be lost, which a
278///   module using it for client-refcounting (e.g. AFT's session accounting)
279///   would miss. This is INTENTIONALLY ACCEPTED, not a gap. A consuming module
280///   must bound stale bindings with its own idle-activity reaper (last-touched
281///   TTL) independent of route-gone signals — AFT does exactly this, so a lost
282///   GOODBYE degrades to "the
283///   binding stays warm until its idle TTL" (bounded wasted resources), never an
284///   unbounded leak; disk-durable replay is unaffected. A dedicated reliable
285///   module control lane was evaluated and deliberately NOT
286///   built: it would add starvation-avoidance machinery to the thin core for a
287///   bounded warm-resource window that is not a correctness issue. never-close
288///   is the invariant that matters here.
289#[derive(Debug, Clone, Copy, PartialEq, Eq)]
290pub(crate) enum GoodbyeTargetKind {
291    Client,
292    Module,
293}
294
295#[derive(Debug, Clone)]
296pub(crate) struct GoodbyeTarget {
297    pub connection_id: ConnectionId,
298    pub sink: FrameSink,
299    pub negotiated_ver: u8,
300    pub channel: u16,
301    pub epoch: u32,
302    pub kind: GoodbyeTargetKind,
303    /// The module on the other end of the route: for a module-targeted relay
304    /// the receiving module (attributing a dropped relay), for a client target
305    /// the module whose route is going away (naming it when an undeliverable
306    /// relay closes the client).
307    pub module_id: Option<String>,
308}
309
310/// The frame a client connection's egress queue refused, for the diagnosis
311/// logged when that refusal closes the connection.
312#[derive(Debug, Clone, Copy)]
313pub(crate) struct UndeliveredFrame<'a> {
314    /// The module on the other end of the route the frame belonged to, when known.
315    pub module_id: Option<&'a str>,
316    /// The refusing connection's sink, read for what its queue held.
317    pub sink: &'a FrameSink,
318}
319
320/// How a route's principal appears in logs: `direct`, or `reserved:<module>`.
321pub(crate) fn principal_label(principal: &Principal) -> String {
322    match principal {
323        Principal::Reserved { module_id } => format!("reserved:{module_id}"),
324        Principal::Direct => "direct".to_string(),
325        other => format!("{other:?}"),
326    }
327}
328
329/// The distinct principals of the routes bound on one client connection,
330/// sorted and comma-joined, or `none` when it has no bound route.
331fn connection_principals_locked(inner: &ForwardingInner, connection_id: ConnectionId) -> String {
332    let labels = inner
333        .client_to_module
334        .iter()
335        .filter(|(key, _)| key.connection_id == connection_id)
336        .map(|(_, route)| principal_label(&route.principal))
337        .collect::<std::collections::BTreeSet<_>>();
338    if labels.is_empty() {
339        "none".to_string()
340    } else {
341        labels.into_iter().collect::<Vec<_>>().join(",")
342    }
343}
344
345impl GoodbyeTarget {
346    /// True only when an undeliverable GOODBYE should escalate to closing the
347    /// target connection. Never escalate for module recipients.
348    pub(crate) fn close_on_delivery_failure(&self) -> bool {
349        matches!(self.kind, GoodbyeTargetKind::Client)
350    }
351}
352
353/// How long a route GOODBYE that a module's egress queue refused keeps waiting
354/// for room before it is given up and counted as dropped.
355///
356/// A module that stops reading for a moment (a GC pause, a long synchronous
357/// handler, a reconnect herd it is still working through) should still learn
358/// that the route is gone, because a module that never hears the GOODBYE keeps
359/// the route and keeps sending on it for as long as its connection lives. The
360/// value is the default module drain timeout: the daemon already treats that as
361/// the longest it is reasonable to wait on a module that is busy but alive, and
362/// a module that cannot free 21 bytes of egress in that time is wedged rather
363/// than slow. Resolving each module's own configured drain timeout here would
364/// need a registry lookup on a path that holds only the module's sink.
365pub(crate) const LATE_MODULE_GOODBYE_DEADLINE: Duration = crate::supervise::DEFAULT_DRAIN_TIMEOUT;
366
367/// Send a route GOODBYE to a module without ever closing the module's shared
368/// connection, which would sever every other route it serves.
369///
370/// The frame is enqueued at once when the module's egress queue has room. When
371/// it does not, a detached task waits for room (bounded by
372/// [`LATE_MODULE_GOODBYE_DEADLINE`]) and sends it then. Arriving late is safe:
373/// the daemon has already released the route, and module SDKs tear a route
374/// down only when both the channel AND the epoch match the route installed on
375/// that channel, so a late GOODBYE for a channel since reused at a newer epoch
376/// is ignored. The GOODBYE is counted in `goodbye_relay_module_dropped` only
377/// when the connection is closed or the deadline passes.
378pub(crate) fn send_module_route_goodbye(
379    counters: &DaemonCounters,
380    sink: &FrameSink,
381    frame: Frame,
382    module_id: Option<&str>,
383    context: &'static str,
384) {
385    let channel = frame.header.channel;
386    let epoch = frame.header.epoch;
387    let Err(err) = sink.try_send(frame.clone()) else {
388        return;
389    };
390    // Waiting is pointless once the module's writer is gone, and impossible
391    // outside a Tokio runtime (cleanup that runs from a destructor at shutdown).
392    let runtime = match tokio::runtime::Handle::try_current() {
393        Ok(runtime) if !sink.is_closed() => runtime,
394        _ => {
395            counters.increment_goodbye_relay_module_dropped(module_id);
396            warn!(
397            module_id = module_id.unwrap_or("unknown"),
398            route_channel = channel,
399            route_epoch = epoch,
400            error = %err,
401            context,
402            "route GOODBYE to module dropped: module connection is closed; not closing shared module connection"
403            );
404            return;
405        }
406    };
407    debug!(
408        module_id = module_id.unwrap_or("unknown"),
409        route_channel = channel,
410        route_epoch = epoch,
411        error = %err,
412        context,
413        "module egress queue refused route GOODBYE; delivering it once the module frees room"
414    );
415    let counters = counters.clone();
416    let sink = sink.clone();
417    let module_id = module_id.map(str::to_string);
418    runtime.spawn(async move {
419        let outcome = tokio::time::timeout(LATE_MODULE_GOODBYE_DEADLINE, sink.send(frame)).await;
420        let why = match outcome {
421            Ok(Ok(())) => {
422                debug!(
423                    module_id = module_id.as_deref().unwrap_or("unknown"),
424                    route_channel = channel,
425                    route_epoch = epoch,
426                    context,
427                    "late route GOODBYE delivered to module"
428                );
429                return;
430            }
431            Ok(Err(err)) => err.to_string(),
432            Err(_) => format!(
433                "module egress queue had no room within {LATE_MODULE_GOODBYE_DEADLINE:?}"
434            ),
435        };
436        counters.increment_goodbye_relay_module_dropped(module_id.as_deref());
437        warn!(
438            module_id = module_id.as_deref().unwrap_or("unknown"),
439            route_channel = channel,
440            route_epoch = epoch,
441            error = %why,
442            context,
443            "route GOODBYE to module dropped under backpressure; not closing shared module connection"
444        );
445    });
446}
447
448/// One route currently served by a module endpoint.
449///
450/// `goodbye_target` is deliberately retained alongside the census projection so
451/// a draining caller can address exactly the same route set that this read
452/// reports, without a second forwarding-table pass.
453#[derive(Debug, Clone)]
454pub(crate) struct EndpointRoute {
455    pub goodbye_target: GoodbyeTarget,
456    pub accepts_close_reasons: Option<Vec<String>>,
457    pub principal: Principal,
458    pub bound_at: Instant,
459    pub draining: bool,
460    /// WHY the endpoint is draining, when it is. Carried per-route so the
461    /// census can answer "closing because of what" without a second lookup;
462    /// `None` exactly when `draining` is false (one source: the drain map).
463    pub drain_reason: Option<RouteCloseReason>,
464}
465
466/// One live route a scope change closed: both ends to send GOODBYE to, and the
467/// reason for the client's `route.closed`.
468#[derive(Debug, Clone)]
469pub(crate) struct ScopeDrainedRoute {
470    pub reason: RouteCloseReason,
471    pub scope: BoundScope,
472    pub module_id: String,
473    pub client: GoodbyeTarget,
474    pub module: GoodbyeTarget,
475    pub accepts_close_reasons: Option<Vec<String>>,
476}
477
478/// One live route an authority change closed: both ends to send GOODBYE to,
479/// and the route as it was recorded, for the client's `route.closed`.
480#[derive(Debug, Clone)]
481pub(crate) struct AuthorityClosedRoute {
482    pub route: Arc<RouteBinding>,
483    pub client: GoodbyeTarget,
484    pub module: GoodbyeTarget,
485}
486
487#[derive(Debug)]
488pub(crate) struct PendingRouteBindRelay {
489    pub endpoint: ModuleEndpointId,
490    pub module_sink: FrameSink,
491    pub negotiated_ver: u8,
492    pub client_channel: u16,
493    pub client_epoch: u32,
494    pub module_channel: u16,
495    pub module_epoch: u32,
496    pub corr: u64,
497    pub receiver: oneshot::Receiver<RouteBindRelayOutcome>,
498}
499
500#[derive(Debug, Clone)]
501pub(crate) struct ModuleDrainTarget {
502    pub endpoint: ModuleEndpointId,
503    pub sink: FrameSink,
504    pub negotiated_ver: u8,
505    pub abandoned_bindings: Vec<GoodbyeTarget>,
506    pub excluded_subscriptions: u32,
507}
508
509/// One registered module connection, with its endpoint-bound output sink.
510#[cfg(unix)]
511#[derive(Debug, Clone)]
512pub(crate) struct ModuleConnectionTarget {
513    pub module_id: String,
514    pub endpoint: ModuleEndpointId,
515    pub sink: FrameSink,
516    pub negotiated_ver: u8,
517}
518
519#[derive(Debug, Clone)]
520pub(crate) enum RouteBindRelayOutcome {
521    Accepted,
522    /// The module refused the bind with its own body. Relayed to the client
523    /// without the daemon-origin bit: the code and message are module-chosen,
524    /// so the frame must not claim daemon authorship, even when the module
525    /// picks a daemon code like `module_reloading`.
526    Rejected(ErrorBody),
527    /// The daemon refused the bind itself — a drain or superseded endpoint
528    /// (`module_reloading`), a reply that arrived after the deadline
529    /// (`module_timeout`), or a scope that ended mid-bind. Carries the
530    /// daemon-origin bit because the daemon wrote it. The module may still
531    /// have received the bind (a late reply, or a drain that began after it
532    /// accepted); the daemon then ends that binding with a GOODBYE, so no
533    /// call runs on it.
534    DaemonRejected(ErrorBody),
535    ModuleGone(String),
536}
537
538/// What [`ForwardingTable::cutover_candidate`] did.
539#[derive(Debug, Clone, Copy, PartialEq, Eq)]
540pub(crate) struct ForwardingCutover {
541    /// The endpoint now in the active slot (the former candidate).
542    pub promoted: ModuleEndpointId,
543    /// The endpoint demoted out of the active slot, to be drained by endpoint.
544    /// `None` when the incumbent's connection was already gone.
545    pub incumbent: Option<ModuleEndpointId>,
546}
547
548/// What tearing down one connection released.
549#[derive(Debug)]
550pub(crate) struct ConnectionCleanup {
551    /// Routes whose other end must be sent GOODBYE.
552    pub released: Vec<GoodbyeTarget>,
553    /// Pending route.bind relays to the closed module that were aborted: opens
554    /// that had not been bound yet. Zero when a client connection closed.
555    pub abandoned_relays: u32,
556}
557
558#[derive(Debug, Clone)]
559pub(crate) struct PendingRelayCompletion {
560    pub settled: bool,
561    pub abandoned: Option<GoodbyeTarget>,
562}
563
564#[derive(Debug)]
565pub(crate) struct PendingModuleControlRpc {
566    pub endpoint: ModuleEndpointId,
567    pub module_sink: FrameSink,
568    pub negotiated_ver: u8,
569    pub corr: u64,
570    pub receiver: oneshot::Receiver<ModuleControlRpcOutcome>,
571}
572
573#[derive(Debug, Clone)]
574pub(crate) enum ModuleControlRpcOutcome {
575    RawResponse(Vec<u8>),
576    Response(ModuleControlResponse),
577    Rejected(ErrorBody),
578    ModuleGone(String),
579    MalformedResponse(String),
580    UnexpectedOp { expected: String, actual: String },
581    DeadlineElapsed,
582}
583
584#[derive(Debug, Clone, PartialEq, Eq)]
585pub(crate) enum ModuleControlRpcCompletion {
586    Unknown,
587    Settled,
588    LateHealthAnswer {
589        module_id: String,
590        latency: Duration,
591    },
592}
593
594#[derive(Debug)]
595struct PendingModuleControlRpcEntry {
596    expected_op: String,
597    deadline: Instant,
598    health_probe_started_at: Option<Instant>,
599    sender: oneshot::Sender<ModuleControlRpcOutcome>,
600}
601
602#[derive(Debug)]
603struct HealthProbeTombstone {
604    expected_op: String,
605    module_id: String,
606    probe_started_at: Instant,
607    expires_at: Instant,
608}
609
610#[derive(Debug, Clone)]
611struct RouteReservation {
612    client_key: ClientRouteKey,
613    module_key: ModuleRouteKey,
614    client_epoch: u32,
615    module_epoch: u32,
616    project_root: Option<ProjectRootId>,
617}
618
619#[derive(Debug)]
620struct PendingRouteBindRelayEntry {
621    reservation: RouteReservation,
622    client_sink: FrameSink,
623    client_negotiated_ver: u8,
624    client_permit: crate::router::EgressPermit,
625    route_open_frame: Frame,
626    busy_refusals: bool,
627    accepts_close_reasons: Option<Vec<String>>,
628    request_window: usize,
629    watch_window: Option<usize>,
630    principal: Principal,
631    /// The scope admitted at `route.open`, with the tag captured then. Commit
632    /// compares it with the published tag.
633    scope: Option<BoundScope>,
634    harness: String,
635    build_class: Option<String>,
636    operator_app: Option<crate::operator_apps::OperatorAppStamp>,
637    /// The opener's spawn and epoch, captured once at `route.open` admission
638    /// and never re-read; commit compares it with `spawn_epochs`.
639    caller_authority: Option<AuthorityPair>,
640    /// What served the reserved endpoint, read under the write lock that
641    /// reserved it; commit compares a spawn's epoch with `spawn_epochs`.
642    target_authority: TargetAuthority,
643    /// The daemon incarnation for the reply's `target` if commit rebuilds it.
644    incarnation: String,
645    deadline: Instant,
646    /// Set once `route.open` has finished putting the route.bind on the
647    /// module's egress queue. A path that drops this entry WITHOUT a module
648    /// reply (client teardown, module drain, the open's own abort) sends the
649    /// module a GOODBYE for the bind only when this is set, because only then
650    /// is the GOODBYE guaranteed to queue behind the bind. When it is not set,
651    /// the removal is recorded in `relays_released_before_enqueue` instead and
652    /// `route.open` sends that GOODBYE itself once its send has completed.
653    ///
654    /// A module reply does not consult it: an ack proves the module received
655    /// the bind, so the reply path decides about the GOODBYE on its own.
656    relay_enqueued: bool,
657    sender: oneshot::Sender<RouteBindRelayOutcome>,
658}
659
660/// What `mark_route_bind_relay_enqueued` found once `route.open` had put the
661/// route.bind on the module's queue.
662#[derive(Debug, Clone, Copy, PartialEq, Eq)]
663pub(crate) enum RouteBindRelayMark {
664    /// The relay is still pending; later removals handle their own GOODBYE.
665    Marked,
666    /// The module already answered and the reply path settled the relay:
667    /// it committed the route, or it refused it and sent any GOODBYE the
668    /// module needed. `route.open` must send nothing.
669    SettledByReply,
670    /// The relay was dropped without a module reply while the bind was still
671    /// being queued, so nothing has told the module to drop the binding it
672    /// is about to create. `route.open` must send that GOODBYE.
673    ReleasedWithoutReply,
674}
675
676#[derive(Debug, Clone)]
677pub(crate) enum RouteRelease {
678    Removed(GoodbyeTarget),
679    Stale,
680    Absent,
681}
682
683#[derive(Debug, Clone)]
684pub(crate) enum RoutePollSnapshot {
685    Bound {
686        module_id: String,
687        status: Option<String>,
688    },
689    Absent,
690}
691
692#[derive(Debug, Clone)]
693struct ModuleConnection {
694    endpoint: ModuleEndpointId,
695    sink: FrameSink,
696    negotiated_ver: u8,
697    concurrency: Concurrency,
698    // Captured at HELLO, not inferred from the id or a later prefix policy.
699    verified_prefix_owner: Option<String>,
700}
701
702pub(crate) struct ModuleHelloAck {
703    pub(crate) frame: Frame,
704    pub(crate) verified_prefix_owner: Option<String>,
705}
706
707#[derive(Debug, Default)]
708struct ForwardingInner {
709    operator_confirms: Arc<crate::operator_confirm::OperatorConfirms>,
710    daemon_draining: bool,
711    /// Whether forwarding reached quiescence during daemon shutdown, for each
712    /// active endpoint present when shutdown began. A pending wait is false
713    /// because it has not proved quiescence. Endpoint generations prevent a
714    /// result from describing a replacement connection with the same module id.
715    daemon_drain_results: HashMap<ModuleEndpointId, bool>,
716    /// The ACTIVE slot: the one endpoint per module id that routing resolves.
717    /// Every by-id lookup (relay reservation, drain-by-id, liveness, census,
718    /// live roots, module-control RPCs) reads this map and nothing else.
719    modules_by_id: HashMap<String, ModuleConnection>,
720    /// The CANDIDATE slot of a blue/green swap: a second process registered
721    /// under an id that already has an active endpoint. It has a full endpoint
722    /// identity (so its own connection can be looked up and torn down) but no
723    /// by-id lookup sees it, so nothing is routed to it until `cutover_candidate`
724    /// promotes it.
725    candidates_by_id: HashMap<String, ModuleConnection>,
726    /// Former active endpoints demoted by `cutover_candidate`, until their
727    /// connection is removed. Membership is what distinguishes an endpoint that
728    /// was SUPERSEDED by a promotion (still alive, still carrying bound routes,
729    /// about to be drained) from one that is merely STALE (replaced some other
730    /// way, which is treated as a fault on the acking connection).
731    superseded_endpoints: HashMap<ModuleEndpointId, ModuleConnection>,
732    endpoint_by_connection: HashMap<ConnectionId, ModuleEndpointId>,
733    module_id_by_endpoint: HashMap<ModuleEndpointId, String>,
734    /// Endpoints mid-drain, keyed to the reason the drain was begun with. The
735    /// value serves the census ("draining because restart"); membership alone
736    /// still answers every admission-gate check.
737    draining_endpoints: HashMap<ModuleEndpointId, RouteCloseReason>,
738    closing_connections: HashSet<ConnectionId>,
739    /// Pending binds check spawn liveness without keeping the spawn alive.
740    spawn_connections: HashMap<ConnectionId, std::sync::Weak<crate::supervise::RegistrationSpawn>>,
741    /// The spawn whose own registration each module connection is, for the
742    /// reply's `target`. A prefix child's connection never has an entry.
743    target_authority_by_connection: HashMap<ConnectionId, Arc<SpawnAuthorityRecord>>,
744    spawn_approved: HashMap<SpawnKey, ApprovedDeclarations>,
745    /// The authority epoch each spawn's last authority change published. A
746    /// spawn with no entry is at the initial epoch. A bind whose captured
747    /// caller or target epoch is older than this is refused at commit.
748    spawn_epochs: HashMap<SpawnKey, u64>,
749    /// The spawns whose first-party mark is raised, published by activation
750    /// and withdrawn by lowering or death, for the reply's `target`.
751    spawn_marks: HashSet<SpawnKey>,
752    /// Spawns published dead whose `spawn_epochs` entry is kept as a
753    /// tombstone. A module keeps the tombstone of its most recent death, plus
754    /// those a pending bind still names, so they cannot pile up.
755    dead_spawns: HashSet<SpawnKey>,
756    next_generation: u64,
757    reserved_client: HashMap<ClientRouteKey, ModuleRouteKey>,
758    reserved_module: HashMap<ModuleRouteKey, ClientRouteKey>,
759    next_client_channel: HashMap<ConnectionId, u16>,
760    next_module_channel: HashMap<ModuleEndpointId, u16>,
761    client_slot_epochs: HashMap<ClientRouteKey, u32>,
762    module_slot_epochs: HashMap<ModuleRouteKey, u32>,
763    last_published_epoch: HashMap<ClientRouteKey, u32>,
764    client_to_module: HashMap<ClientRouteKey, Arc<RouteBinding>>,
765    module_to_client: HashMap<ModuleRouteKey, Arc<RouteBinding>>,
766    status: HashMap<(ClientRouteKey, u32), String>,
767    pending_relays: HashMap<(ModuleEndpointId, u64), PendingRouteBindRelayEntry>,
768    /// Pending relays removed without a module reply before `route.open`
769    /// marked them enqueued. `route.open` consumes its key when it marks the
770    /// relay (and sends the GOODBYE the removal could not send safely) or when
771    /// it aborts, so an entry lives only while that open is mid-send.
772    relays_released_before_enqueue: HashSet<(ModuleEndpointId, u64)>,
773    /// Each scope's current `(scope_epoch, version)`, keyed `(owner, ref)`,
774    /// published by a scope sync in the same step that changes the record. A
775    /// bind commit reads it here so it never takes the scope table's lock
776    /// while holding this one. Absent means the scope is not live.
777    scope_tags: HashMap<(String, String), ScopeTag>,
778    next_control_corr: HashMap<ModuleEndpointId, u64>,
779    pending_control_rpcs: HashMap<(ModuleEndpointId, u64), PendingModuleControlRpcEntry>,
780    health_probe_tombstones: HashMap<(ModuleEndpointId, u64), HealthProbeTombstone>,
781}
782
783#[derive(Debug, Clone)]
784pub(crate) struct CloseReason {
785    code: &'static str,
786    message: String,
787}
788
789impl CloseReason {
790    pub(crate) fn new(code: &'static str, message: impl Into<String>) -> Self {
791        Self {
792            code,
793            message: message.into(),
794        }
795    }
796}
797
798impl fmt::Display for CloseReason {
799    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
800        write!(f, "{}: {}", self.code, self.message)
801    }
802}
803
804pub(crate) type ConnectionCloseReceiver = oneshot::Receiver<CloseReason>;
805
806/// Dynamic forwarding state shared by the control plane and data-plane router.
807#[derive(Debug, Default)]
808pub struct ForwardingTable {
809    #[cfg(all(test, target_os = "linux"))]
810    route_commit_pause: Mutex<Option<RouteCommitPause>>,
811    #[cfg(all(test, target_os = "linux"))]
812    cleanup_tie_observer: Mutex<Option<oneshot::Sender<(bool, usize)>>>,
813    #[cfg(all(test, target_os = "linux"))]
814    cleanup_pause: Mutex<Option<RouteCommitPause>>,
815    #[cfg(test)]
816    policy_rebind_before_write: Mutex<Option<Arc<std::sync::Barrier>>>,
817    inner: Arc<RwLock<ForwardingInner>>,
818    frame_drop_warnings: Mutex<HashMap<ConnectionId, ChannelDropWarnings>>,
819    #[cfg(feature = "test-support")]
820    route_closing_hold: Mutex<Option<oneshot::Receiver<()>>>,
821    close_registry: Mutex<HashMap<ConnectionId, oneshot::Sender<CloseReason>>>,
822    counters: DaemonCounters,
823    /// Per-target-module bind-relay breaker state. It lives here, beside the
824    /// module connections it describes, because this is where a module
825    /// connection's identity is established and therefore where a stale
826    /// verdict has to be discarded.
827    route_bind_breakers: RouteBindBreakers,
828    /// Current route.bind relays keyed by target module. Admission is shared
829    /// across every client connection that points at the same endpoint.
830    route_bind_concurrency: RouteBindConcurrency,
831    /// Start and end of each module's `route.open` outage. Held here for the
832    /// same reason as the breakers: every handler built over this table must
833    /// see one outage per module, or each would log its own opening line.
834    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
835}
836
837#[cfg(all(test, target_os = "linux"))]
838#[derive(Debug)]
839struct RouteCommitPause {
840    reached: oneshot::Sender<()>,
841    release: std::sync::mpsc::Receiver<()>,
842}
843
844pub(crate) type PolicyRevocation = (Vec<EndpointRoute>, Vec<GoodbyeTarget>);
845
846#[derive(Debug)]
847struct FrameDropWarning {
848    last_warned: tokio::time::Instant,
849    suppressed: u64,
850}
851
852type ChannelDropWarnings = HashMap<(u16, &'static str), FrameDropWarning>;
853
854impl ForwardingTable {
855    #[cfg(all(test, target_os = "linux"))]
856    pub(crate) fn pause_cleanup_for_test(
857        &self,
858    ) -> (oneshot::Receiver<()>, std::sync::mpsc::Sender<()>) {
859        let (reached, observed) = oneshot::channel();
860        let (release, paused) = std::sync::mpsc::channel();
861        *self.cleanup_pause.lock().unwrap() = Some(RouteCommitPause {
862            reached,
863            release: paused,
864        });
865        (observed, release)
866    }
867    #[cfg(all(test, target_os = "linux"))]
868    pub(crate) fn observe_cleanup_tie_for_test(&self) -> oneshot::Receiver<(bool, usize)> {
869        let (send, receive) = oneshot::channel();
870        *self.cleanup_tie_observer.lock().unwrap() = Some(send);
871        receive
872    }
873    #[cfg(all(test, target_os = "linux"))]
874    pub(crate) fn spawn_connection_recorded_for_test(&self, connection: ConnectionId) -> bool {
875        self.read_inner()
876            .unwrap()
877            .spawn_connections
878            .contains_key(&connection)
879    }
880    #[cfg(all(test, target_os = "linux"))]
881    pub(crate) fn pause_route_commit_for_test(
882        &self,
883    ) -> (oneshot::Receiver<()>, std::sync::mpsc::Sender<()>) {
884        let (reached, observed) = oneshot::channel();
885        let (release, paused) = std::sync::mpsc::channel();
886        *self.route_commit_pause.lock().unwrap() = Some(RouteCommitPause {
887            reached,
888            release: paused,
889        });
890        (observed, release)
891    }
892    pub(crate) fn bind_spawn(
893        &self,
894        connection: ConnectionId,
895        spawn: &Arc<crate::supervise::RegistrationSpawn>,
896    ) -> Result<(), ForwardingError> {
897        self.write_inner()?
898            .spawn_connections
899            .insert(connection, Arc::downgrade(spawn));
900        Ok(())
901    }
902
903    /// Record that `connection` is `record`'s spawn's own module registration,
904    /// so a route reserved on it reports `target.kind: spawn`.
905    pub(crate) fn bind_target_authority(
906        &self,
907        connection: ConnectionId,
908        record: Arc<SpawnAuthorityRecord>,
909    ) -> Result<(), ForwardingError> {
910        let approved = record.approved();
911        let mut inner = self.write_inner()?;
912        inner.spawn_approved.insert(record.key(), approved);
913        inner
914            .target_authority_by_connection
915            .insert(connection, record);
916        Ok(())
917    }
918
919    /// Undo [`Self::bind_target_authority`] for a HELLO that was refused
920    /// after it: the connection is no registration of the spawn.
921    pub(crate) fn unbind_target_authority(
922        &self,
923        connection: ConnectionId,
924    ) -> Result<(), ForwardingError> {
925        self.write_inner()?
926            .target_authority_by_connection
927            .remove(&connection);
928        Ok(())
929    }
930    /// The spawn whose own module registration `connection` is, if any.
931    pub(crate) fn target_authority_for_connection(
932        &self,
933        connection: ConnectionId,
934    ) -> Result<Option<Arc<SpawnAuthorityRecord>>, ForwardingError> {
935        Ok(self
936            .read_inner()?
937            .target_authority_by_connection
938            .get(&connection)
939            .cloned())
940    }
941
942    /// The module connection that is `key`'s spawn's own registration.
943    pub(crate) fn connection_for_spawn(
944        &self,
945        key: &SpawnKey,
946    ) -> Result<Option<ConnectionId>, ForwardingError> {
947        Ok(self
948            .read_inner()?
949            .target_authority_by_connection
950            .iter()
951            .find(|(_, record)| record.key() == *key)
952            .map(|(connection, _)| *connection))
953    }
954
955    pub(crate) fn active_module_endpoint(
956        &self,
957        module_id: &str,
958    ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
959        Ok(self
960            .read_inner()?
961            .modules_by_id
962            .get(module_id)
963            .map(|module| module.endpoint))
964    }
965
966    pub(crate) fn config_delivery_sink(
967        &self,
968        module: &str,
969        connection: ConnectionId,
970    ) -> Result<Option<FrameSink>, ForwardingError> {
971        let inner = self.read_inner()?;
972        Ok(inner
973            .modules_by_id
974            .get(module)
975            .filter(|target| {
976                target.endpoint.connection_id == connection
977                    && !inner.closing_connections.contains(&connection)
978            })
979            .map(|target| target.sink.clone()))
980    }
981    /// Keep a silent route diagnosable without letting a broken peer flood logs.
982    /// Epoch and direction describe the dropped frame, but do not split its limit.
983    pub(crate) fn warn_frame_drop(
984        &self,
985        connection_id: ConnectionId,
986        channel: u16,
987        epoch: u32,
988        direction: &'static str,
989        reason: &'static str,
990    ) {
991        let now = tokio::time::Instant::now();
992        let mut warnings = self
993            .frame_drop_warnings
994            .lock()
995            .unwrap_or_else(std::sync::PoisonError::into_inner);
996        let entry = warnings
997            .entry(connection_id)
998            .or_default()
999            .entry((channel, reason));
1000        let suppressed = match entry {
1001            std::collections::hash_map::Entry::Occupied(mut entry) => {
1002                let state = entry.get_mut();
1003                if now.saturating_duration_since(state.last_warned) < Duration::from_secs(60) {
1004                    state.suppressed = state.suppressed.saturating_add(1);
1005                    return;
1006                }
1007                state.last_warned = now;
1008                std::mem::take(&mut state.suppressed)
1009            }
1010            std::collections::hash_map::Entry::Vacant(entry) => {
1011                entry.insert(FrameDropWarning {
1012                    last_warned: now,
1013                    suppressed: 0,
1014                });
1015                0
1016            }
1017        };
1018        warn!(
1019            connection_id = connection_id.get(),
1020            channel, epoch, direction, reason, suppressed, "dropping data-plane frame"
1021        );
1022    }
1023
1024    /// Install scripted operator confirmation for an in-process integration harness,
1025    /// without bootstrapping a daemon process. Only available with `test-support`.
1026    ///
1027    /// The script has one response per line: `approve`, `decline`, `unavailable`,
1028    /// `hang` (until withdrawn), or `approve when <path>` (wait for the path to
1029    /// exist). Unreadable or exhausted scripts fail closed. If supplied, `events`
1030    /// receives the provider's JSON-lines event log. `limits: None` uses
1031    /// [`OperatorLimits::default`](crate::operator_confirm::OperatorLimits); use
1032    /// `Some(OperatorLimits { queue_wait, prompt_timeout, stuck_grace })` to
1033    /// override the queue wait, prompt timeout and stuck-provider grace periods.
1034    ///
1035    /// The current executable's file name **must start with `ckdev-`**. Otherwise
1036    /// this logs a warning and keeps the OS provider, just like bootstrap.
1037    /// `cortexkit-test-support`'s `ckdev_binary` copies a harness executable under
1038    /// a qualifying name; run that copy to enable scripted confirmation. See
1039    /// `crates/subc-client-rs/tests/real_daemon/operator_confirm.rs` for the
1040    /// reference end-to-end harness.
1041    ///
1042    /// Configure the table shared by the harness's `ControlHandler` and `Router`
1043    /// before serving any requests. This also configures privileged confirmations
1044    /// that use the same table.
1045    ///
1046    /// # Panics
1047    ///
1048    /// Panics if operator-confirm processing has already started.
1049    ///
1050    /// # Example
1051    ///
1052    /// ```no_run
1053    /// use std::{path::Path, sync::Arc};
1054    /// use subc_daemon::{ControlHandler, ForwardingTable, Registry};
1055    ///
1056    /// let registry = Arc::new(Registry::default());
1057    /// let forwarding = Arc::new(ForwardingTable::default());
1058    /// forwarding.configure_operator_script(
1059    ///     Path::new("operator.script"),
1060    ///     Some(Path::new("operator.events")),
1061    ///     None,
1062    /// );
1063    /// let handler = ControlHandler::with_forwarding(registry, forwarding);
1064    /// ```
1065    #[cfg(feature = "test-support")]
1066    pub fn configure_operator_script(
1067        &self,
1068        script: &std::path::Path,
1069        events: Option<&std::path::Path>,
1070        limits: Option<crate::operator_confirm::OperatorLimits>,
1071    ) {
1072        self.operator_confirms()
1073            .configure_script(script, events, limits);
1074    }
1075
1076    /// Hold the next supervisor drain between enqueueing `module.draining` and
1077    /// `route.closing`, so a test can forward a response in that window. Dropping
1078    /// the returned sender also releases the hold, including on a test panic.
1079    #[cfg(feature = "test-support")]
1080    pub fn hold_route_closing_for_test(&self) -> oneshot::Sender<()> {
1081        let (release, hold) = oneshot::channel();
1082        *self.route_closing_hold.lock().unwrap() = Some(hold);
1083        release
1084    }
1085
1086    #[cfg(feature = "test-support")]
1087    pub(crate) async fn wait_before_route_closing_for_test(&self) {
1088        let hold = self.route_closing_hold.lock().unwrap().take();
1089        if let Some(hold) = hold {
1090            let _ = hold.await;
1091        }
1092    }
1093
1094    pub(crate) fn operator_confirms(&self) -> Arc<crate::operator_confirm::OperatorConfirms> {
1095        Arc::clone(
1096            &self
1097                .inner
1098                .read()
1099                .unwrap_or_else(|p| p.into_inner())
1100                .operator_confirms,
1101        )
1102    }
1103
1104    #[cfg(test)]
1105    pub(crate) fn inject_operator_principal(&self, key: ModuleRouteKey, principal: Principal) {
1106        self.alter_operator_binding_for_test(key, |binding| binding.principal = principal);
1107    }
1108
1109    #[cfg(test)]
1110    pub(crate) fn alter_operator_binding_for_test(
1111        &self,
1112        key: ModuleRouteKey,
1113        alter: impl FnOnce(&mut RouteBinding),
1114    ) {
1115        let mut inner = self.write_inner().unwrap();
1116        let old = inner.module_to_client.remove(&key).unwrap();
1117        let client = ClientRouteKey {
1118            connection_id: old.client_connection_id,
1119            channel: old.client_channel,
1120        };
1121        inner.client_to_module.remove(&client);
1122        let mut binding = Arc::try_unwrap(old).expect("test binding has no outstanding readers");
1123        alter(&mut binding);
1124        let binding = Arc::new(binding);
1125        inner.client_to_module.insert(client, Arc::clone(&binding));
1126        inner.module_to_client.insert(key, binding);
1127    }
1128
1129    /// Mutate the route without delivering its withdrawal notice, so tests can
1130    /// exercise the commit-point route check rather than the withdrawal path.
1131    #[cfg(test)]
1132    pub(crate) fn invalidate_operator_route_for_test(&self, key: ModuleRouteKey, stale: bool) {
1133        let mut inner = self.write_inner().unwrap();
1134        let old = inner.module_to_client.remove(&key).unwrap();
1135        let client = ClientRouteKey {
1136            connection_id: old.client_connection_id,
1137            channel: old.client_channel,
1138        };
1139        inner.client_to_module.remove(&client);
1140        if stale {
1141            let mut binding = Arc::try_unwrap(old).expect("no binding readers at the test pause");
1142            binding.module_epoch += 1;
1143            let binding = Arc::new(binding);
1144            inner.client_to_module.insert(client, binding.clone());
1145            inner.module_to_client.insert(key, binding);
1146        }
1147    }
1148
1149    /// Keep the binding protected through confirmation admission. Route release
1150    /// takes the write lock and then the same confirm-state lock, so it cannot
1151    /// miss an admitted request or leave it waiting on a vanished route.
1152    pub(crate) fn with_operator_route<T>(
1153        &self,
1154        connection_id: ConnectionId,
1155        channel: u16,
1156        epoch: u32,
1157        admit: impl FnOnce(Option<&RouteBinding>) -> T,
1158    ) -> Result<T, ForwardingError> {
1159        self.with_operator_route_epoch(connection_id, channel, |binding| {
1160            admit(binding.filter(|binding| binding.module_epoch == epoch))
1161        })
1162    }
1163
1164    /// Like admission, but retains a mismatched epoch so a shortcut can audit
1165    /// a stale channel separately from a closed one, under the same read lock.
1166    pub(crate) fn with_operator_route_epoch<T>(
1167        &self,
1168        connection_id: ConnectionId,
1169        channel: u16,
1170        decide: impl FnOnce(Option<&RouteBinding>) -> T,
1171    ) -> Result<T, ForwardingError> {
1172        let inner = self.read_inner()?;
1173        let binding = inner
1174            .endpoint_by_connection
1175            .get(&connection_id)
1176            .and_then(|endpoint| {
1177                inner.module_to_client.get(&ModuleRouteKey {
1178                    endpoint: *endpoint,
1179                    channel,
1180                })
1181            });
1182        Ok(decide(binding.map(Arc::as_ref)))
1183    }
1184
1185    pub(crate) fn counters(&self) -> DaemonCounters {
1186        self.counters.clone()
1187    }
1188
1189    pub(crate) fn counters_ref(&self) -> &DaemonCounters {
1190        &self.counters
1191    }
1192
1193    pub(crate) fn route_bind_breakers(&self) -> RouteBindBreakers {
1194        self.route_bind_breakers.clone()
1195    }
1196
1197    pub(crate) fn route_bind_concurrency(&self) -> RouteBindConcurrency {
1198        self.route_bind_concurrency.clone()
1199    }
1200
1201    pub(crate) fn route_outages(&self) -> Arc<crate::route_outage::RouteOutageTracker> {
1202        Arc::clone(&self.route_outages)
1203    }
1204
1205    pub(crate) fn register_connection_close(
1206        &self,
1207        connection_id: ConnectionId,
1208    ) -> ConnectionCloseReceiver {
1209        let (sender, receiver) = oneshot::channel();
1210        let replaced = self
1211            .lock_close_registry()
1212            .insert(connection_id, sender)
1213            .is_some();
1214        if replaced {
1215            warn!(
1216                connection_id = connection_id.get(),
1217                "replaced existing connection close registration"
1218            );
1219        }
1220        receiver
1221    }
1222
1223    pub(crate) fn unregister_connection_close(&self, connection_id: ConnectionId) {
1224        self.lock_close_registry().remove(&connection_id);
1225        // The read loop and its tasks have ended and connection cleanup has
1226        // removed its routes and pending relays. The writer may still drain.
1227        // A channel-0 GOODBYE preserves the connection-to-launch association
1228        // until that cleanup.
1229        if let Ok(mut inner) = self.write_inner() {
1230            inner.spawn_connections.remove(&connection_id);
1231            inner.target_authority_by_connection.remove(&connection_id);
1232        }
1233    }
1234
1235    /// Close every established connection, modules and clients alike, as the
1236    /// last step of an announced daemon shutdown. By the time this runs each
1237    /// registered module has already been sent a module GOODBYE (see
1238    /// `Supervisor::end_children_for_daemon_shutdown`), so the EOF that follows
1239    /// is a planned stop; EOF with no GOODBYE before it stays reserved for a
1240    /// daemon that went away unannounced. Closing while the daemon is still
1241    /// alive lets it wait for the modules' own teardowns before it ends
1242    /// whatever is left.
1243    #[cfg(unix)]
1244    pub(crate) fn close_all_connections(&self, reason: &CloseReason) -> usize {
1245        let senders: Vec<_> = self.lock_close_registry().drain().collect();
1246        let count = senders.len();
1247        for (_, sender) in senders {
1248            let _ = sender.send(reason.clone());
1249        }
1250        count
1251    }
1252
1253    /// Every registered module connection, whichever slot it occupies (active,
1254    /// swap candidate, or superseded incumbent), once each. Daemon shutdown
1255    /// uses this to tell each of them it is a planned stop before closing it.
1256    #[cfg(unix)]
1257    pub(crate) fn module_connections(
1258        &self,
1259    ) -> Result<Vec<ModuleConnectionTarget>, ForwardingError> {
1260        let inner = self.read_inner()?;
1261        let mut seen = HashSet::new();
1262        Ok(inner
1263            .modules_by_id
1264            .values()
1265            .chain(inner.candidates_by_id.values())
1266            .chain(inner.superseded_endpoints.values())
1267            .filter(|module| seen.insert(module.endpoint))
1268            .map(|module| ModuleConnectionTarget {
1269                module_id: inner
1270                    .module_id_by_endpoint
1271                    .get(&module.endpoint)
1272                    .cloned()
1273                    .unwrap_or_default(),
1274                endpoint: module.endpoint,
1275                sink: module.sink.clone(),
1276                negotiated_ver: module.negotiated_ver,
1277            })
1278            .collect())
1279    }
1280
1281    /// Ask a registered connection to close. Returns true only for the request
1282    /// that actually reached it; later requests for the same connection, and
1283    /// requests for a connection that is not registered, return false.
1284    pub(crate) fn request_connection_close(
1285        &self,
1286        connection_id: ConnectionId,
1287        reason: CloseReason,
1288    ) -> bool {
1289        let sender = self.lock_close_registry().remove(&connection_id);
1290        if let Some(sender) = sender {
1291            debug!(
1292                connection_id = connection_id.get(),
1293                close_reason = %reason,
1294                "requesting connection close"
1295            );
1296            let _ = sender.send(reason);
1297            true
1298        } else {
1299            debug!(
1300                connection_id = connection_id.get(),
1301                close_reason = %reason,
1302                "connection close request ignored for inactive connection"
1303            );
1304            false
1305        }
1306    }
1307
1308    pub fn register_module_connection(
1309        &self,
1310        connection_id: ConnectionId,
1311        module_id: String,
1312        negotiated_ver: u8,
1313        concurrency: Concurrency,
1314        sink: FrameSink,
1315    ) -> Result<ModuleEndpointId, ForwardingError> {
1316        self.register_module_connection_inner(
1317            connection_id,
1318            module_id,
1319            negotiated_ver,
1320            concurrency,
1321            sink,
1322            None,
1323        )
1324    }
1325
1326    /// Register a module connection and queue its HELLO_ACK as the first frame
1327    /// on its sink, in the same write-lock critical section that makes the
1328    /// endpoint visible.
1329    ///
1330    /// A module reads HELLO_ACK as the first frame after its HELLO and exits on
1331    /// anything else. Once the endpoint is in `modules_by_id`, a `route.open`
1332    /// on any other connection can queue a `route.bind` request onto this sink,
1333    /// so the ack must already be queued by then. Every lookup that can queue a
1334    /// frame for the module takes this lock, so nothing can get in ahead of it.
1335    ///
1336    /// If the ack cannot be queued (the sink is closed or full), the
1337    /// registration fails with nothing inserted.
1338    pub(crate) fn register_module_connection_acked(
1339        &self,
1340        connection_id: ConnectionId,
1341        module_id: String,
1342        negotiated_ver: u8,
1343        concurrency: Concurrency,
1344        sink: FrameSink,
1345        hello_ack: ModuleHelloAck,
1346    ) -> Result<ModuleEndpointId, ForwardingError> {
1347        self.register_module_connection_inner(
1348            connection_id,
1349            module_id,
1350            negotiated_ver,
1351            concurrency,
1352            sink,
1353            Some(hello_ack),
1354        )
1355    }
1356
1357    fn register_module_connection_inner(
1358        &self,
1359        connection_id: ConnectionId,
1360        module_id: String,
1361        negotiated_ver: u8,
1362        concurrency: Concurrency,
1363        sink: FrameSink,
1364        hello_ack: Option<ModuleHelloAck>,
1365    ) -> Result<ModuleEndpointId, ForwardingError> {
1366        let mut inner = self.write_inner()?;
1367        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1368            return Err(ForwardingError::ConnectionClosing { connection_id });
1369        }
1370        check_module_connection_role_locked(&inner, connection_id)?;
1371        // Every refusal check has passed and nothing has been mutated yet, so
1372        // a failed enqueue leaves the table exactly as it was.
1373        let verified_prefix_owner = hello_ack
1374            .as_ref()
1375            .and_then(|hello| hello.verified_prefix_owner.clone());
1376        enqueue_hello_ack_locked(&sink, connection_id, hello_ack.map(|hello| hello.frame))?;
1377
1378        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1379        let endpoint = ModuleEndpointId {
1380            connection_id,
1381            generation: inner.next_generation,
1382        };
1383        inner.endpoint_by_connection.insert(connection_id, endpoint);
1384        inner
1385            .module_id_by_endpoint
1386            .insert(endpoint, module_id.clone());
1387        inner.next_module_channel.insert(endpoint, 1);
1388        inner.next_control_corr.insert(endpoint, 1);
1389        inner.modules_by_id.insert(
1390            module_id.clone(),
1391            ModuleConnection {
1392                endpoint,
1393                sink,
1394                negotiated_ver,
1395                concurrency,
1396                verified_prefix_owner,
1397            },
1398        );
1399        drop(inner);
1400
1401        // A new module connection has arrived under this id, so anything the
1402        // bind-relay breaker learned was learned about a process that is no
1403        // longer the one behind this name. See
1404        // `RouteBindBreakers::reset_for_new_module_connection`.
1405        //
1406        // Keyed on ARRIVAL rather than on teardown deliberately: a connection
1407        // going away is not evidence about anything, and a module whose
1408        // connection drops without coming back should keep its verdict until
1409        // something actually registers in its place. This also covers the
1410        // unclean replacements -- a module killed mid-bind, or one whose
1411        // connection was closing -- because registration is the single path by
1412        // which any module connection becomes usable.
1413        if let Some(discarded) = self
1414            .route_bind_breakers
1415            .reset_for_new_module_connection(&module_id)
1416        {
1417            info!(
1418                module_id = %module_id,
1419                discarded_consecutive_timeouts = discarded,
1420                "route.bind breaker state discarded: a new module connection replaced the process it described"
1421            );
1422        }
1423        Ok(endpoint)
1424    }
1425
1426    /// Register a blue/green swap candidate for `module_id` into the candidate
1427    /// slot, alongside whatever endpoint is active for the id.
1428    ///
1429    /// Unlike [`Self::register_module_connection`], this never touches the
1430    /// active slot and never resets the bind-relay breaker: the incumbent is
1431    /// still the process serving the id, so what the breaker learned about it
1432    /// is still true. The breaker is reset at cutover instead, when the process
1433    /// behind the name actually changes. A second candidate for the same id is
1434    /// refused.
1435    ///
1436    /// Production registers candidates through
1437    /// [`Self::register_candidate_module_connection_acked`]; this ack-less form
1438    /// is for tests that build forwarding state directly.
1439    #[cfg(test)]
1440    pub(crate) fn register_candidate_module_connection(
1441        &self,
1442        connection_id: ConnectionId,
1443        module_id: String,
1444        negotiated_ver: u8,
1445        concurrency: Concurrency,
1446        sink: FrameSink,
1447    ) -> Result<ModuleEndpointId, ForwardingError> {
1448        self.register_candidate_module_connection_inner(
1449            connection_id,
1450            module_id,
1451            negotiated_ver,
1452            concurrency,
1453            sink,
1454            None,
1455        )
1456    }
1457
1458    /// The swap-candidate counterpart of
1459    /// [`Self::register_module_connection_acked`]: the HELLO_ACK is queued on
1460    /// the candidate's sink before its endpoint is inserted. No by-id lookup
1461    /// sees a candidate, but its own connection's endpoint does become
1462    /// resolvable here, and at cutover it becomes routable; the ack has to be
1463    /// ahead of anything either can queue.
1464    pub(crate) fn register_candidate_module_connection_acked(
1465        &self,
1466        connection_id: ConnectionId,
1467        module_id: String,
1468        negotiated_ver: u8,
1469        concurrency: Concurrency,
1470        sink: FrameSink,
1471        hello_ack: Frame,
1472    ) -> Result<ModuleEndpointId, ForwardingError> {
1473        self.register_candidate_module_connection_inner(
1474            connection_id,
1475            module_id,
1476            negotiated_ver,
1477            concurrency,
1478            sink,
1479            Some(hello_ack),
1480        )
1481    }
1482
1483    fn register_candidate_module_connection_inner(
1484        &self,
1485        connection_id: ConnectionId,
1486        module_id: String,
1487        negotiated_ver: u8,
1488        concurrency: Concurrency,
1489        sink: FrameSink,
1490        hello_ack: Option<Frame>,
1491    ) -> Result<ModuleEndpointId, ForwardingError> {
1492        let mut inner = self.write_inner()?;
1493        if inner.daemon_draining || inner.closing_connections.contains(&connection_id) {
1494            return Err(ForwardingError::ConnectionClosing { connection_id });
1495        }
1496        check_module_connection_role_locked(&inner, connection_id)?;
1497        if inner.candidates_by_id.contains_key(&module_id) {
1498            return Err(ForwardingError::CandidateSlotOccupied { module_id });
1499        }
1500        // As in `register_module_connection_inner`: after every refusal check,
1501        // before any mutation.
1502        enqueue_hello_ack_locked(&sink, connection_id, hello_ack)?;
1503
1504        inner.next_generation = inner.next_generation.checked_add(1).unwrap_or(1);
1505        let endpoint = ModuleEndpointId {
1506            connection_id,
1507            generation: inner.next_generation,
1508        };
1509        inner.endpoint_by_connection.insert(connection_id, endpoint);
1510        inner
1511            .module_id_by_endpoint
1512            .insert(endpoint, module_id.clone());
1513        inner.next_module_channel.insert(endpoint, 1);
1514        inner.next_control_corr.insert(endpoint, 1);
1515        inner.candidates_by_id.insert(
1516            module_id,
1517            ModuleConnection {
1518                endpoint,
1519                sink,
1520                negotiated_ver,
1521                concurrency,
1522                verified_prefix_owner: None,
1523            },
1524        );
1525        Ok(endpoint)
1526    }
1527
1528    /// Promote `module_id`'s candidate to the active slot, in ONE forwarding
1529    /// write-lock critical section.
1530    ///
1531    /// That single section is the linearization point of a swap. Every relay
1532    /// reservation resolves the active slot under the same lock, so each one
1533    /// lands wholly before cutover (on the incumbent) or wholly after it (on the
1534    /// promoted candidate), never on a mix. A relay reserved on the incumbent
1535    /// before cutover can no longer commit: `commit_route_locked` requires the
1536    /// reservation's endpoint to be the active one, and an ack for it arriving
1537    /// later is answered by the superseded-endpoint arm of
1538    /// `complete_pending_relay` instead of ending the incumbent's connection.
1539    ///
1540    /// Both endpoints keep their identities: nothing is re-keyed, so the
1541    /// incumbent's bound routes and its pending correlation keys stay exactly
1542    /// where they are until it is drained with [`Self::begin_endpoint_drain`],
1543    /// using the incumbent endpoint this returns. Returns `Ok(None)` when there
1544    /// is no candidate for the id.
1545    pub(crate) fn cutover_candidate(
1546        &self,
1547        module_id: &str,
1548    ) -> Result<Option<ForwardingCutover>, ForwardingError> {
1549        let mut inner = self.write_inner()?;
1550        if inner.daemon_draining {
1551            return Err(ForwardingError::ModuleReloading {
1552                module_id: module_id.to_string(),
1553            });
1554        }
1555        let Some(candidate) = inner.candidates_by_id.remove(module_id) else {
1556            return Ok(None);
1557        };
1558        let promoted = candidate.endpoint;
1559        let incumbent = inner.modules_by_id.insert(module_id.to_string(), candidate);
1560        let incumbent = incumbent.map(|incumbent| {
1561            let endpoint = incumbent.endpoint;
1562            inner.superseded_endpoints.insert(endpoint, incumbent);
1563            endpoint
1564        });
1565        drop(inner);
1566
1567        // A different process now answers for this id; see the matching reset
1568        // in `register_module_connection` for why a verdict about the old one
1569        // must not carry over.
1570        if let Some(discarded) = self
1571            .route_bind_breakers
1572            .reset_for_new_module_connection(module_id)
1573        {
1574            info!(
1575                module_id = %module_id,
1576                discarded_consecutive_timeouts = discarded,
1577                "route.bind breaker state discarded: a swap candidate was promoted over the process it described"
1578            );
1579        }
1580        Ok(Some(ForwardingCutover {
1581            promoted,
1582            incumbent,
1583        }))
1584    }
1585
1586    #[allow(clippy::too_many_arguments)]
1587    pub(crate) async fn begin_route_bind_relay_for(
1588        &self,
1589        client_connection_id: ConnectionId,
1590        client_sink: FrameSink,
1591        client_negotiated_ver: u8,
1592        client_corr: u64,
1593        module_id: &str,
1594        principal: Principal,
1595        scope: Option<BoundScope>,
1596        project_root: Option<ProjectRootId>,
1597        harness: String,
1598        build_class: Option<String>,
1599        operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1600        authority: RouteAuthority,
1601        deadline: Instant,
1602        busy_refusals: bool,
1603        accepts_close_reasons: Option<Vec<String>>,
1604    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1605        // Reserve egress capacity before taking the forwarding lock. The permit is
1606        // held until the bind reaches one terminal state, so an accepted bind can
1607        // publish its table entry and RouteOpen response in one critical section.
1608        let client_permit =
1609            client_sink
1610                .reserve_owned()
1611                .await
1612                .map_err(|_| ForwardingError::ClientEgressClosed {
1613                    connection_id: client_connection_id,
1614                })?;
1615        self.begin_route_bind_relay_inner(
1616            client_connection_id,
1617            client_sink,
1618            client_negotiated_ver,
1619            client_corr,
1620            module_id,
1621            principal,
1622            scope,
1623            project_root,
1624            harness,
1625            build_class,
1626            operator_app,
1627            authority,
1628            deadline,
1629            client_permit,
1630            busy_refusals,
1631            accepts_close_reasons,
1632        )
1633    }
1634
1635    #[cfg(test)]
1636    pub(crate) fn begin_route_bind_relay_for_test(
1637        &self,
1638        client_connection_id: ConnectionId,
1639        client_sink: FrameSink,
1640        client_corr: u64,
1641        module_id: &str,
1642    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1643        self.begin_route_bind_relay_with_busy_for_test(
1644            client_connection_id,
1645            client_sink,
1646            client_corr,
1647            module_id,
1648            false,
1649        )
1650    }
1651
1652    #[cfg(test)]
1653    pub(crate) fn begin_route_bind_relay_with_busy_for_test(
1654        &self,
1655        client_connection_id: ConnectionId,
1656        client_sink: FrameSink,
1657        client_corr: u64,
1658        module_id: &str,
1659        busy_refusals: bool,
1660    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1661        let permit =
1662            client_sink
1663                .try_reserve_owned()
1664                .map_err(|_| ForwardingError::ClientEgressClosed {
1665                    connection_id: client_connection_id,
1666                })?;
1667        self.begin_route_bind_relay_inner(
1668            client_connection_id,
1669            client_sink,
1670            subc_protocol::PROTOCOL_VERSION,
1671            client_corr,
1672            module_id,
1673            Principal::Direct,
1674            None,
1675            None,
1676            String::new(),
1677            None,
1678            None,
1679            RouteAuthority::default(),
1680            Instant::now() + std::time::Duration::from_secs(60),
1681            permit,
1682            busy_refusals,
1683            None,
1684        )
1685    }
1686
1687    pub(crate) fn begin_module_control_rpc_for(
1688        &self,
1689        module_id: &str,
1690        expected_op: &str,
1691        deadline: Instant,
1692    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1693        self.begin_module_control_rpc_inner(module_id, expected_op, deadline, None, false)
1694    }
1695
1696    pub(crate) fn begin_health_probe_rpc_for(
1697        &self,
1698        module_id: &str,
1699        expected_op: &str,
1700        probe_started_at: Instant,
1701        deadline: Instant,
1702    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1703        self.begin_module_control_rpc_inner(
1704            module_id,
1705            expected_op,
1706            deadline,
1707            Some(probe_started_at),
1708            false,
1709        )
1710    }
1711
1712    pub(crate) fn begin_drain_health_probe_rpc_for(
1713        &self,
1714        module_id: &str,
1715        expected_op: &str,
1716        probe_started_at: Instant,
1717        deadline: Instant,
1718    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1719        self.begin_module_control_rpc_inner(
1720            module_id,
1721            expected_op,
1722            deadline,
1723            Some(probe_started_at),
1724            true,
1725        )
1726    }
1727
1728    /// A health probe addressed to one endpoint, in whichever slot it is.
1729    ///
1730    /// Every other control RPC resolves the id's ACTIVE endpoint, which is how
1731    /// a swap candidate (not active until cutover) and a superseded incumbent
1732    /// (not active after it) are unreachable by them. A swap needs to probe
1733    /// exactly those two: the candidate before promoting it, and the incumbent
1734    /// for its busy gauges while it drains.
1735    pub(crate) fn begin_endpoint_health_probe_rpc_for(
1736        &self,
1737        endpoint: ModuleEndpointId,
1738        expected_op: &str,
1739        probe_started_at: Instant,
1740        deadline: Instant,
1741    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1742        let inner = self.write_inner()?;
1743        let module = module_connection_for_endpoint_locked(&inner, endpoint)
1744            .cloned()
1745            .ok_or(ForwardingError::NoModuleConnection)?;
1746        let module_id = inner
1747            .module_id_by_endpoint
1748            .get(&endpoint)
1749            .cloned()
1750            .unwrap_or_default();
1751        // A draining incumbent is exactly what this probes for busy gauges, so
1752        // draining is allowed, as it is for the by-id drain probe.
1753        self.begin_control_rpc_locked(
1754            inner,
1755            &module_id,
1756            module,
1757            expected_op,
1758            deadline,
1759            Some(probe_started_at),
1760            true,
1761        )
1762    }
1763
1764    fn begin_module_control_rpc_inner(
1765        &self,
1766        module_id: &str,
1767        expected_op: &str,
1768        deadline: Instant,
1769        health_probe_started_at: Option<Instant>,
1770        allow_draining: bool,
1771    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1772        let inner = self.write_inner()?;
1773        let module = inner
1774            .modules_by_id
1775            .get(module_id)
1776            .cloned()
1777            .ok_or(ForwardingError::NoModuleConnection)?;
1778        self.begin_control_rpc_locked(
1779            inner,
1780            module_id,
1781            module,
1782            expected_op,
1783            deadline,
1784            health_probe_started_at,
1785            allow_draining,
1786        )
1787    }
1788
1789    #[allow(clippy::too_many_arguments)]
1790    fn begin_control_rpc_locked(
1791        &self,
1792        mut inner: InnerGuard<RwLockWriteGuard<'_, ForwardingInner>>,
1793        module_id: &str,
1794        module: ModuleConnection,
1795        expected_op: &str,
1796        deadline: Instant,
1797        health_probe_started_at: Option<Instant>,
1798        allow_draining: bool,
1799    ) -> Result<PendingModuleControlRpc, ForwardingError> {
1800        if !allow_draining && inner.draining_endpoints.contains_key(&module.endpoint) {
1801            return Err(ForwardingError::ModuleReloading {
1802                module_id: module_id.to_string(),
1803            });
1804        }
1805        if inner
1806            .closing_connections
1807            .contains(&module.endpoint.connection_id)
1808        {
1809            return Err(ForwardingError::ConnectionClosing {
1810                connection_id: module.endpoint.connection_id,
1811            });
1812        }
1813        if health_probe_started_at.is_some() {
1814            // Recurring health probes are serial per endpoint. Once the next one
1815            // starts, an older answer can no longer improve the current snapshot,
1816            // so retaining more than the newest unanswered probe has no value.
1817            inner
1818                .health_probe_tombstones
1819                .retain(|(endpoint, _), _| *endpoint != module.endpoint);
1820        }
1821        let corr = match inner.allocate_control_corr(module.endpoint) {
1822            Ok(corr) => corr,
1823            Err(err) => {
1824                drop(inner);
1825                self.request_connection_close(
1826                    module.endpoint.connection_id,
1827                    CloseReason::new(
1828                        "control_correlation_exhausted",
1829                        "daemon-originated channel-0 correlation space exhausted",
1830                    ),
1831                );
1832                return Err(err);
1833            }
1834        };
1835        let (sender, receiver) = oneshot::channel();
1836        inner.pending_control_rpcs.insert(
1837            (module.endpoint, corr),
1838            PendingModuleControlRpcEntry {
1839                expected_op: expected_op.to_string(),
1840                deadline,
1841                health_probe_started_at,
1842                sender,
1843            },
1844        );
1845
1846        Ok(PendingModuleControlRpc {
1847            endpoint: module.endpoint,
1848            module_sink: module.sink,
1849            negotiated_ver: module.negotiated_ver,
1850            corr,
1851            receiver,
1852        })
1853    }
1854
1855    #[allow(clippy::too_many_arguments)]
1856    fn begin_route_bind_relay_inner(
1857        &self,
1858        client_connection_id: ConnectionId,
1859        client_sink: FrameSink,
1860        client_negotiated_ver: u8,
1861        client_corr: u64,
1862        expected_module_id: &str,
1863        principal: Principal,
1864        scope: Option<BoundScope>,
1865        project_root: Option<ProjectRootId>,
1866        harness: String,
1867        build_class: Option<String>,
1868        operator_app: Option<crate::operator_apps::OperatorAppStamp>,
1869        authority: RouteAuthority,
1870        deadline: Instant,
1871        client_permit: crate::router::EgressPermit,
1872        busy_refusals: bool,
1873        accepts_close_reasons: Option<Vec<String>>,
1874    ) -> Result<PendingRouteBindRelay, ForwardingError> {
1875        let mut inner = self.write_inner()?;
1876        // HELLO and route.open can run on different tasks. Decide the role under
1877        // the same lock that installs endpoints and reserves client channels so
1878        // neither task can turn one connection into both kinds of owner.
1879        if inner
1880            .endpoint_by_connection
1881            .contains_key(&client_connection_id)
1882        {
1883            return Err(ForwardingError::ConnectionRoleConflict {
1884                connection_id: client_connection_id,
1885            });
1886        }
1887        if inner.closing_connections.contains(&client_connection_id) {
1888            return Err(ForwardingError::ConnectionClosing {
1889                connection_id: client_connection_id,
1890            });
1891        }
1892        let module = inner
1893            .modules_by_id
1894            .get(expected_module_id)
1895            .cloned()
1896            .ok_or(ForwardingError::NoModuleConnection)?;
1897        if inner.draining_endpoints.contains_key(&module.endpoint) {
1898            return Err(ForwardingError::ModuleReloading {
1899                module_id: expected_module_id.to_string(),
1900            });
1901        }
1902        if inner
1903            .closing_connections
1904            .contains(&module.endpoint.connection_id)
1905        {
1906            return Err(ForwardingError::ConnectionClosing {
1907                connection_id: module.endpoint.connection_id,
1908            });
1909        }
1910
1911        let corr = match inner.allocate_control_corr(module.endpoint) {
1912            Ok(corr) => corr,
1913            Err(err) => {
1914                drop(inner);
1915                self.request_connection_close(
1916                    module.endpoint.connection_id,
1917                    CloseReason::new(
1918                        "control_correlation_exhausted",
1919                        "daemon-originated channel-0 correlation space exhausted",
1920                    ),
1921                );
1922                return Err(err);
1923            }
1924        };
1925        let (client_channel, client_epoch, module_channel, module_epoch) =
1926            inner.allocate_route_slots(client_connection_id, module.endpoint)?;
1927        let client_key = ClientRouteKey {
1928            connection_id: client_connection_id,
1929            channel: client_channel,
1930        };
1931        let module_key = ModuleRouteKey {
1932            endpoint: module.endpoint,
1933            channel: module_channel,
1934        };
1935        let reservation = RouteReservation {
1936            client_key,
1937            module_key,
1938            client_epoch,
1939            module_epoch,
1940            project_root,
1941        };
1942        // The target is the endpoint reserved under this lock, never the
1943        // by-id registration route.open read before it waited for egress.
1944        let target_authority = target_authority_locked(&inner, &module);
1945        let response_body = route_open_reply_body(
1946            &ClientControlResponse::RouteOpen {
1947                request_window: None,
1948                watch_window: None,
1949                route_channel: client_channel,
1950                route_epoch: client_epoch,
1951            },
1952            &target_authority,
1953            &authority.incarnation,
1954        )?;
1955        let route_open_frame = Frame::build_with_version(
1956            client_negotiated_ver,
1957            FrameType::Response,
1958            Flags::new(false, Priority::Passive, false),
1959            0,
1960            0,
1961            client_corr,
1962            response_body,
1963        )
1964        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
1965        let (sender, receiver) = oneshot::channel();
1966        inner.reserved_client.insert(client_key, module_key);
1967        inner.reserved_module.insert(module_key, client_key);
1968        inner.pending_relays.insert(
1969            (module.endpoint, corr),
1970            PendingRouteBindRelayEntry {
1971                reservation,
1972                client_sink,
1973                client_negotiated_ver,
1974                client_permit,
1975                route_open_frame,
1976                busy_refusals,
1977                accepts_close_reasons,
1978                request_window: window_for(&module.concurrency),
1979                watch_window: (busy_refusals && module.concurrency == Concurrency::ModuleManaged)
1980                    .then_some(DEFAULT_MODULE_MANAGED_WATCH_WINDOW),
1981                principal,
1982                scope,
1983                harness,
1984                build_class,
1985                operator_app,
1986                caller_authority: authority.caller,
1987                target_authority,
1988                incarnation: authority.incarnation,
1989                deadline,
1990                relay_enqueued: false,
1991                sender,
1992            },
1993        );
1994
1995        Ok(PendingRouteBindRelay {
1996            endpoint: module.endpoint,
1997            module_sink: module.sink,
1998            negotiated_ver: module.negotiated_ver,
1999            client_channel,
2000            client_epoch,
2001            module_channel,
2002            module_epoch,
2003            corr,
2004            receiver,
2005        })
2006    }
2007
2008    /// Called by `route.open` right after the route.bind entered the module's
2009    /// egress queue.
2010    ///
2011    /// The module reader runs concurrently with `route.open`, so a fast module
2012    /// can receive the bind, ack it, and have the reader commit the route
2013    /// before this runs. A missing entry therefore does not mean the bind was
2014    /// abandoned: only a removal recorded in `relays_released_before_enqueue`
2015    /// does. Telling the two apart is what stops `route.open` from sending a
2016    /// GOODBYE for a route that was just committed.
2017    pub(crate) fn mark_route_bind_relay_enqueued(
2018        &self,
2019        endpoint: ModuleEndpointId,
2020        corr: u64,
2021    ) -> Result<RouteBindRelayMark, ForwardingError> {
2022        let mut inner = self.write_inner()?;
2023        if let Some(pending) = inner.pending_relays.get_mut(&(endpoint, corr)) {
2024            pending.relay_enqueued = true;
2025            return Ok(RouteBindRelayMark::Marked);
2026        }
2027        if inner
2028            .relays_released_before_enqueue
2029            .remove(&(endpoint, corr))
2030        {
2031            return Ok(RouteBindRelayMark::ReleasedWithoutReply);
2032        }
2033        Ok(RouteBindRelayMark::SettledByReply)
2034    }
2035
2036    pub(crate) fn release_client_route(
2037        &self,
2038        client_connection_id: ConnectionId,
2039        client_channel: u16,
2040        expected_epoch: u32,
2041    ) -> Result<RouteRelease, ForwardingError> {
2042        let mut inner = self.write_inner()?;
2043        let release = release_client_route_locked(
2044            &mut inner,
2045            ClientRouteKey {
2046                connection_id: client_connection_id,
2047                channel: client_channel,
2048            },
2049            expected_epoch,
2050        );
2051        self.record_route_release(&release);
2052        Ok(release)
2053    }
2054
2055    pub(crate) fn release_module_route(
2056        &self,
2057        module_connection_id: ConnectionId,
2058        module_channel: u16,
2059        expected_epoch: u32,
2060    ) -> Result<RouteRelease, ForwardingError> {
2061        let mut inner = self.write_inner()?;
2062        let Some(endpoint) = inner
2063            .endpoint_by_connection
2064            .get(&module_connection_id)
2065            .copied()
2066        else {
2067            return Ok(RouteRelease::Absent);
2068        };
2069        let release = release_module_route_locked(
2070            &mut inner,
2071            ModuleRouteKey {
2072                endpoint,
2073                channel: module_channel,
2074            },
2075            expected_epoch,
2076        );
2077        self.record_route_release(&release);
2078        Ok(release)
2079    }
2080
2081    pub(crate) fn abort_pending_relay(
2082        &self,
2083        endpoint: ModuleEndpointId,
2084        corr: u64,
2085        outcome: RouteBindRelayOutcome,
2086    ) -> Result<Option<GoodbyeTarget>, ForwardingError> {
2087        let mut inner = self.write_inner()?;
2088        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
2089            // If another path dropped the relay while `route.open` was still
2090            // sending the bind, and that open is now giving up without marking
2091            // it (its send failed or it was cancelled), forget the record so it
2092            // cannot outlive the open.
2093            inner
2094                .relays_released_before_enqueue
2095                .remove(&(endpoint, corr));
2096            return Ok(None);
2097        };
2098        release_reserved_route_locked(
2099            &mut inner,
2100            pending.reservation.client_key,
2101            pending.reservation.module_key,
2102        );
2103        let target = pending
2104            .relay_enqueued
2105            .then(|| abandoned_route_target(&inner, &pending.reservation));
2106        let _ = pending.sender.send(outcome);
2107        Ok(target.flatten())
2108    }
2109
2110    pub(crate) fn cancel_module_control_rpc(
2111        &self,
2112        endpoint: ModuleEndpointId,
2113        corr: u64,
2114    ) -> Result<(), ForwardingError> {
2115        self.write_inner()?
2116            .pending_control_rpcs
2117            .remove(&(endpoint, corr));
2118        Ok(())
2119    }
2120
2121    pub(crate) fn tombstone_health_probe_rpc(
2122        &self,
2123        endpoint: ModuleEndpointId,
2124        corr: u64,
2125    ) -> Result<bool, ForwardingError> {
2126        let key = (endpoint, corr);
2127        let expires_at = Instant::now() + HEALTH_PROBE_TOMBSTONE_TTL;
2128        {
2129            let mut inner = self.write_inner()?;
2130            let Some(pending) = inner.pending_control_rpcs.remove(&key) else {
2131                return Ok(false);
2132            };
2133            let Some(probe_started_at) = pending.health_probe_started_at else {
2134                inner.pending_control_rpcs.insert(key, pending);
2135                return Ok(false);
2136            };
2137            let module_id = inner
2138                .module_id_by_endpoint
2139                .get(&endpoint)
2140                .cloned()
2141                .unwrap_or_else(|| "unknown".to_string());
2142            inner.health_probe_tombstones.insert(
2143                key,
2144                HealthProbeTombstone {
2145                    expected_op: pending.expected_op,
2146                    module_id,
2147                    probe_started_at,
2148                    expires_at,
2149                },
2150            );
2151        }
2152        self.schedule_health_probe_tombstone_expiration(key, expires_at);
2153        Ok(true)
2154    }
2155
2156    fn schedule_health_probe_tombstone_expiration(
2157        &self,
2158        key: (ModuleEndpointId, u64),
2159        expires_at: Instant,
2160    ) {
2161        let inner = Arc::downgrade(&self.inner);
2162        tokio::spawn(async move {
2163            tokio::time::sleep_until(expires_at).await;
2164            let Some(inner) = inner.upgrade() else {
2165                return;
2166            };
2167            let Ok(mut inner) = inner.write() else {
2168                return;
2169            };
2170            let expired = inner
2171                .health_probe_tombstones
2172                .get(&key)
2173                .is_some_and(|tombstone| tombstone.expires_at <= Instant::now());
2174            if expired {
2175                inner.health_probe_tombstones.remove(&key);
2176            }
2177        });
2178    }
2179
2180    pub(crate) fn complete_pending_relay(
2181        &self,
2182        connection_id: ConnectionId,
2183        corr: u64,
2184        outcome: RouteBindRelayOutcome,
2185    ) -> Result<PendingRelayCompletion, ForwardingError> {
2186        let mut inner = self.write_inner()?;
2187        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2188            return Ok(PendingRelayCompletion {
2189                settled: false,
2190                abandoned: None,
2191            });
2192        };
2193        let Some(pending) = inner.pending_relays.remove(&(endpoint, corr)) else {
2194            return Ok(PendingRelayCompletion {
2195                settled: false,
2196                abandoned: None,
2197            });
2198        };
2199
2200        if Instant::now() >= pending.deadline {
2201            release_reserved_route_locked(
2202                &mut inner,
2203                pending.reservation.client_key,
2204                pending.reservation.module_key,
2205            );
2206            let abandoned = matches!(outcome, RouteBindRelayOutcome::Accepted)
2207                .then(|| abandoned_route_target(&inner, &pending.reservation))
2208                .flatten();
2209            let _ = pending
2210                .sender
2211                .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody {
2212                    code: "module_timeout".to_string(),
2213                    message: "route.bind response arrived after its daemon deadline".to_string(),
2214                    detail: None,
2215                }));
2216            return Ok(PendingRelayCompletion {
2217                settled: true,
2218                abandoned,
2219            });
2220        }
2221
2222        // These readiness reads are not atomic with publication: a route may
2223        // commit after Dead. Taking a spawn transition lock here would invert
2224        // the publication lock order. Retirement, in one critical section,
2225        // stops every connection bound by the launch's claims and its own
2226        // registration channel from reading or sending frames, and shuts that
2227        // channel.
2228        // Once that step completes, no new request can cross the transient route
2229        // to or from the retired launch; connection teardown removes the route.
2230        match outcome {
2231            // Three shapes of "the client is not there to receive this route" that
2232            // must resolve identically: its egress is already closed, or it is
2233            // marked closing (its connection loop has been asked to end, but has
2234            // not drained yet, so the sink is still open), or its spawn is dead
2235            // before connection-close notifications have reached that loop.
2236            //
2237            // Only the first used to be caught here. The second fell through to
2238            // `commit_route_locked`, which refuses a closing client with
2239            // `ConnectionClosing` -- and this function is called from the MODULE
2240            // connection's frame handler, so that refusal ended the module's
2241            // connection instead of this one client's route. One dying client's
2242            // route.open then took down a connection carrying every other
2243            // client's routes to that module.
2244            //
2245            // The remedy for both is the same, which is why they share an arm:
2246            // give back the reserved handle pair, tell the waiting route.open the
2247            // route is gone, and report the module-side channel so the caller can
2248            // send a channel-scoped GOODBYE for the binding the module just
2249            // created. Nothing here touches the module connection.
2250            RouteBindRelayOutcome::Accepted
2251                if pending.client_sink.is_closed()
2252                    || spawn_connection_dead(
2253                        &inner,
2254                        pending.reservation.client_key.connection_id,
2255                    )
2256                    || inner
2257                        .closing_connections
2258                        .contains(&pending.reservation.client_key.connection_id) =>
2259            {
2260                let reason = if pending.client_sink.is_closed() {
2261                    "client egress closed before route publication"
2262                } else {
2263                    "client connection is closing before route publication"
2264                };
2265                release_reserved_route_locked(
2266                    &mut inner,
2267                    pending.reservation.client_key,
2268                    pending.reservation.module_key,
2269                );
2270                // Not gated on `relay_enqueued`: an ack proves the module
2271                // received the bind, and `route.open` sends nothing for a
2272                // relay a reply already settled, so this is the only GOODBYE.
2273                let abandoned = abandoned_route_target(&inner, &pending.reservation);
2274                let _ = pending
2275                    .sender
2276                    .send(RouteBindRelayOutcome::ModuleGone(reason.to_string()));
2277                return Ok(PendingRelayCompletion {
2278                    settled: true,
2279                    abandoned,
2280                });
2281            }
2282            // The endpoint that acked this bind is no longer accepting new
2283            // routes, for one of two reasons. Either it was the module's active
2284            // endpoint when this relay was reserved and a blue/green swap has
2285            // since promoted a candidate (the replacement process registered
2286            // beside it) into the active slot, superseding it; or a daemon
2287            // drain has gated it, and the ack arrived before the ordered module
2288            // drain settled pending relays. The module did nothing wrong: it
2289            // bound the route it was asked to bind, and the endpoint still
2290            // carries every other client's routes until it is drained. So the
2291            // ack is settled here, while the pending entry still holds the
2292            // client's sender and the reserved handle pair: release the pair,
2293            // refuse the waiting route.open with `module_reloading`, which
2294            // callers retry (after a swap the retry reserves on the promoted
2295            // endpoint), and hand back the module-side channel so the caller
2296            // sends one channel-scoped GOODBYE for the binding the module just
2297            // created. Returning an error instead would end the module
2298            // connection all those other routes share; nothing here touches it.
2299            //
2300            // An endpoint that stopped being active any other way is stale
2301            // (replaced, but not by a swap and not draining). It still falls
2302            // through to `commit_route_locked`, which refuses it with
2303            // `StaleModuleEndpoint` exactly as before swaps existed.
2304            RouteBindRelayOutcome::Accepted
2305                if inner.superseded_endpoints.contains_key(&endpoint)
2306                    || spawn_connection_dead(&inner, connection_id)
2307                    || inner.draining_endpoints.contains_key(&endpoint) =>
2308            {
2309                release_reserved_route_locked(
2310                    &mut inner,
2311                    pending.reservation.client_key,
2312                    pending.reservation.module_key,
2313                );
2314                // Not gated on `relay_enqueued`: an ack proves the module
2315                // received the bind, whether or not the enqueue mark was set.
2316                let abandoned = abandoned_route_target(&inner, &pending.reservation);
2317                let module_id = inner
2318                    .module_id_by_endpoint
2319                    .get(&endpoint)
2320                    .cloned()
2321                    .unwrap_or_else(|| "unknown".to_string());
2322                let _ = pending
2323                    .sender
2324                    .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
2325                        "module_reloading",
2326                        format!("module_id '{module_id}' is reloading"),
2327                    )));
2328                return Ok(PendingRelayCompletion {
2329                    settled: true,
2330                    abandoned,
2331                });
2332            }
2333            // The scope the open was admitted under ended or changed after
2334            // admission. The module bound a route under a stamp that is no
2335            // longer true, so it must not become routable. Settled here like
2336            // the superseded arm above, never as an error from
2337            // `commit_route_locked`, which would close the module's whole
2338            // connection and every other client's routes to it: release the
2339            // pair, answer the waiting route.open by name, and hand back the
2340            // module-side channel for one channel-scoped GOODBYE.
2341            RouteBindRelayOutcome::Accepted
2342                if pending
2343                    .scope
2344                    .as_ref()
2345                    .is_some_and(|scope| scope_refusal_locked(&inner, scope).is_some()) =>
2346            {
2347                let refusal = pending
2348                    .scope
2349                    .as_ref()
2350                    .and_then(|scope| scope_refusal_locked(&inner, scope))
2351                    .expect("guard matched a refusal under the same lock");
2352                release_reserved_route_locked(
2353                    &mut inner,
2354                    pending.reservation.client_key,
2355                    pending.reservation.module_key,
2356                );
2357                let abandoned = abandoned_route_target(&inner, &pending.reservation);
2358                let _ = pending
2359                    .sender
2360                    .send(RouteBindRelayOutcome::DaemonRejected(refusal));
2361                return Ok(PendingRelayCompletion {
2362                    settled: true,
2363                    abandoned,
2364                });
2365            }
2366            // The caller's or the target's authority changed after the open
2367            // was attested, so the attestation the module bound on is no
2368            // longer true. Settled like the scope arm, never as an error from
2369            // `commit_route_locked`, which would close the module's shared
2370            // connection: release the pair, refuse the open by name, and hand
2371            // back the module-side channel for one channel-scoped GOODBYE.
2372            RouteBindRelayOutcome::Accepted
2373                if authority_stale_locked(&inner, pending.caller_authority.as_ref())
2374                    || authority_stale_locked(&inner, pending.target_authority.pair()) =>
2375            {
2376                release_reserved_route_locked(
2377                    &mut inner,
2378                    pending.reservation.client_key,
2379                    pending.reservation.module_key,
2380                );
2381                let abandoned = abandoned_route_target(&inner, &pending.reservation);
2382                let _ = pending
2383                    .sender
2384                    .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
2385                        subc_protocol::error_codes::STALE_AUTHORITY_EPOCH,
2386                        "the caller's or target's authority changed while the route was binding",
2387                    )));
2388                return Ok(PendingRelayCompletion {
2389                    settled: true,
2390                    abandoned,
2391                });
2392            }
2393            RouteBindRelayOutcome::Accepted => {
2394                #[cfg(all(test, target_os = "linux"))]
2395                if let Some(pause) = self.route_commit_pause.lock().unwrap().take() {
2396                    let _ = pause.reached.send(());
2397                    pause
2398                        .release
2399                        .recv_timeout(Duration::from_secs(5))
2400                        .expect("test must release bind publication");
2401                }
2402                let abandoned = commit_route_locked(&mut inner, pending)?;
2403                return Ok(PendingRelayCompletion {
2404                    settled: true,
2405                    abandoned,
2406                });
2407            }
2408            terminal => {
2409                release_reserved_route_locked(
2410                    &mut inner,
2411                    pending.reservation.client_key,
2412                    pending.reservation.module_key,
2413                );
2414                let _ = pending.sender.send(terminal);
2415            }
2416        }
2417        Ok(PendingRelayCompletion {
2418            settled: true,
2419            abandoned: None,
2420        })
2421    }
2422
2423    pub(crate) fn pending_module_control_op(
2424        &self,
2425        connection_id: ConnectionId,
2426        corr: u64,
2427    ) -> Result<Option<String>, ForwardingError> {
2428        let inner = self.read_inner()?;
2429        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2430            return Ok(None);
2431        };
2432        let key = (endpoint, corr);
2433        Ok(inner
2434            .pending_control_rpcs
2435            .get(&key)
2436            .map(|pending| pending.expected_op.clone())
2437            .or_else(|| {
2438                inner
2439                    .health_probe_tombstones
2440                    .get(&key)
2441                    .filter(|tombstone| tombstone.expires_at > Instant::now())
2442                    .map(|tombstone| tombstone.expected_op.clone())
2443            }))
2444    }
2445
2446    pub(crate) fn complete_module_control_rpc(
2447        &self,
2448        connection_id: ConnectionId,
2449        corr: u64,
2450        actual_op: Option<&str>,
2451        outcome: ModuleControlRpcOutcome,
2452    ) -> Result<ModuleControlRpcCompletion, ForwardingError> {
2453        let now = Instant::now();
2454        let mut inner = self.write_inner()?;
2455        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2456            return Ok(ModuleControlRpcCompletion::Unknown);
2457        };
2458        let key = (endpoint, corr);
2459        if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2460            if now >= pending.deadline {
2461                let late_health_answer = pending.health_probe_started_at.map(|probe_started_at| {
2462                    ModuleControlRpcCompletion::LateHealthAnswer {
2463                        module_id: inner
2464                            .module_id_by_endpoint
2465                            .get(&endpoint)
2466                            .cloned()
2467                            .unwrap_or_else(|| "unknown".to_string()),
2468                        latency: now.saturating_duration_since(probe_started_at),
2469                    }
2470                });
2471                let _ = pending
2472                    .sender
2473                    .send(ModuleControlRpcOutcome::DeadlineElapsed);
2474                return Ok(late_health_answer.unwrap_or(ModuleControlRpcCompletion::Settled));
2475            }
2476            let outcome = match actual_op {
2477                Some(actual) if actual != pending.expected_op => {
2478                    ModuleControlRpcOutcome::UnexpectedOp {
2479                        expected: pending.expected_op,
2480                        actual: actual.to_string(),
2481                    }
2482                }
2483                _ => outcome,
2484            };
2485            let _ = pending.sender.send(outcome);
2486            return Ok(ModuleControlRpcCompletion::Settled);
2487        }
2488
2489        let Some(tombstone) = inner.health_probe_tombstones.remove(&key) else {
2490            return Ok(ModuleControlRpcCompletion::Unknown);
2491        };
2492        if tombstone.expires_at <= now {
2493            return Ok(ModuleControlRpcCompletion::Unknown);
2494        }
2495        Ok(ModuleControlRpcCompletion::LateHealthAnswer {
2496            module_id: tombstone.module_id,
2497            latency: now.saturating_duration_since(tombstone.probe_started_at),
2498        })
2499    }
2500
2501    #[cfg(test)]
2502    pub(crate) fn health_probe_tombstone_count(&self) -> Result<usize, ForwardingError> {
2503        Ok(self.read_inner()?.health_probe_tombstones.len())
2504    }
2505
2506    #[cfg(test)]
2507    pub(crate) fn closing_connection_count(&self) -> Result<usize, ForwardingError> {
2508        Ok(self.read_inner()?.closing_connections.len())
2509    }
2510
2511    /// Reserved-but-uncommitted route handles, counted on both index sides, so
2512    /// a test can assert a reservation pair was actually given back.
2513    #[cfg(test)]
2514    pub(crate) fn reserved_route_count(&self) -> Result<(usize, usize), ForwardingError> {
2515        let inner = self.read_inner()?;
2516        Ok((inner.reserved_client.len(), inner.reserved_module.len()))
2517    }
2518
2519    pub(crate) fn module_endpoint_for_connection(
2520        &self,
2521        connection_id: ConnectionId,
2522    ) -> Result<Option<ModuleEndpointId>, ForwardingError> {
2523        Ok(self
2524            .read_inner()?
2525            .endpoint_by_connection
2526            .get(&connection_id)
2527            .copied())
2528    }
2529
2530    /// Looks up the module registered on a data-plane connection so route-drop
2531    /// diagnostics name the emitter instead of only reporting a daemon total.
2532    pub(crate) fn module_id_for_connection(
2533        &self,
2534        connection_id: ConnectionId,
2535    ) -> Result<Option<String>, ForwardingError> {
2536        let inner = self.read_inner()?;
2537        Ok(inner
2538            .endpoint_by_connection
2539            .get(&connection_id)
2540            .and_then(|endpoint| inner.module_id_by_endpoint.get(endpoint))
2541            .cloned())
2542    }
2543
2544    /// Whether the daemon ever allocated `(channel, epoch)` on this module
2545    /// connection. Epochs on a module channel are handed out as 1, 2, 3, ...
2546    /// and the last one handed out is remembered until the connection ends, so
2547    /// every epoch from 1 up to that one was a real route at some point. Used
2548    /// only on the drop path, to tell a module still sending on a route the
2549    /// daemon released from one sending on a route that never existed.
2550    pub(crate) fn module_route_epoch_was_allocated(
2551        &self,
2552        connection_id: ConnectionId,
2553        channel: u16,
2554        epoch: u32,
2555    ) -> Result<bool, ForwardingError> {
2556        let inner = self.read_inner()?;
2557        let Some(endpoint) = inner.endpoint_by_connection.get(&connection_id).copied() else {
2558            return Ok(false);
2559        };
2560        Ok(inner
2561            .module_slot_epochs
2562            .get(&ModuleRouteKey { endpoint, channel })
2563            .is_some_and(|last| epoch != 0 && epoch <= *last))
2564    }
2565
2566    pub(crate) fn has_live_module_connection(
2567        &self,
2568        module_id: &str,
2569    ) -> Result<bool, ForwardingError> {
2570        Ok(self.read_inner()?.modules_by_id.contains_key(module_id))
2571    }
2572
2573    /// Admission and route teardown share the forwarding-then-flow lock order.
2574    /// No asynchronous work may split validation from recording the credits.
2575    pub(crate) fn try_admit_request(
2576        &self,
2577        route: &Arc<RouteBinding>,
2578        epoch: u32,
2579        corr: u64,
2580        subscription: bool,
2581    ) -> Result<RequestAdmission, ForwardingError> {
2582        let inner = self.read_inner()?;
2583        let key = ClientRouteKey {
2584            connection_id: route.client_connection_id,
2585            channel: route.client_channel,
2586        };
2587        if epoch != route.client_epoch
2588            || !inner
2589                .client_to_module
2590                .get(&key)
2591                .is_some_and(|current| Arc::ptr_eq(current, route))
2592        {
2593            return Ok(RequestAdmission::Stale);
2594        }
2595        let admission = route.flow.try_acquire_tagged(corr, subscription);
2596        Ok(match admission {
2597            RequestAdmission::Stale
2598                if inner
2599                    .draining_endpoints
2600                    .contains_key(&route.module_endpoint) =>
2601            {
2602                RequestAdmission::ModuleReloading
2603            }
2604            admission => admission,
2605        })
2606    }
2607
2608    pub(crate) fn lookup_data_route(
2609        &self,
2610        connection_id: ConnectionId,
2611        channel: u16,
2612        epoch: u32,
2613    ) -> Result<DataRoute, ForwardingError> {
2614        let inner = self.read_inner()?;
2615        let state = if let Some(endpoint) =
2616            inner.endpoint_by_connection.get(&connection_id).copied()
2617        {
2618            let key = ModuleRouteKey { endpoint, channel };
2619            match inner.module_to_client.get(&key) {
2620                Some(route) if route.module_epoch == epoch => {
2621                    DataRouteState::Bound(Arc::clone(route))
2622                }
2623                Some(_) => DataRouteState::EpochMismatch,
2624                None if inner.reserved_module.contains_key(&key)
2625                    && inner.module_slot_epochs.get(&key).copied() == Some(epoch) =>
2626                {
2627                    DataRouteState::Reserved
2628                }
2629                None if inner.reserved_module.contains_key(&key) => DataRouteState::EpochMismatch,
2630                None => DataRouteState::Absent,
2631            }
2632        } else {
2633            let key = ClientRouteKey {
2634                connection_id,
2635                channel,
2636            };
2637            match inner.client_to_module.get(&key) {
2638                Some(route) if route.client_epoch == epoch => {
2639                    DataRouteState::Bound(Arc::clone(route))
2640                }
2641                Some(_) => DataRouteState::EpochMismatch,
2642                None if inner.reserved_client.contains_key(&key)
2643                    && inner.client_slot_epochs.get(&key).copied() == Some(epoch) =>
2644                {
2645                    DataRouteState::Reserved
2646                }
2647                None if inner.reserved_client.contains_key(&key) => DataRouteState::EpochMismatch,
2648                None => DataRouteState::Absent,
2649            }
2650        };
2651        Ok(
2652            if inner.endpoint_by_connection.contains_key(&connection_id) {
2653                DataRoute::Module(state)
2654            } else {
2655                DataRoute::Client(state)
2656            },
2657        )
2658    }
2659
2660    #[cfg(test)]
2661    pub(crate) fn inject_client_slot_epoch(
2662        &self,
2663        connection_id: ConnectionId,
2664        channel: u16,
2665        last_epoch: u32,
2666    ) {
2667        let mut inner = self.write_inner().expect("forwarding lock");
2668        inner.client_slot_epochs.insert(
2669            ClientRouteKey {
2670                connection_id,
2671                channel,
2672            },
2673            last_epoch,
2674        );
2675        inner.next_client_channel.insert(connection_id, channel);
2676    }
2677
2678    #[cfg(test)]
2679    pub(crate) fn inject_module_slot_epoch(
2680        &self,
2681        endpoint: ModuleEndpointId,
2682        channel: u16,
2683        last_epoch: u32,
2684    ) {
2685        let mut inner = self.write_inner().expect("forwarding lock");
2686        inner
2687            .module_slot_epochs
2688            .insert(ModuleRouteKey { endpoint, channel }, last_epoch);
2689        inner.next_module_channel.insert(endpoint, channel);
2690    }
2691
2692    #[cfg(test)]
2693    pub(crate) fn inject_control_corr(&self, endpoint: ModuleEndpointId, next_corr: u64) {
2694        self.write_inner()
2695            .expect("forwarding lock")
2696            .next_control_corr
2697            .insert(endpoint, next_corr);
2698    }
2699
2700    pub(crate) fn cache_status(
2701        &self,
2702        endpoint: ModuleEndpointId,
2703        module_channel: u16,
2704        module_epoch: u32,
2705        status: String,
2706    ) -> Result<bool, ForwardingError> {
2707        let mut inner = self.write_inner()?;
2708        if !inner.module_id_by_endpoint.contains_key(&endpoint) {
2709            return Err(ForwardingError::StaleModuleEndpoint);
2710        }
2711
2712        let module_key = ModuleRouteKey {
2713            endpoint,
2714            channel: module_channel,
2715        };
2716        let handle = if let Some(route) = inner.module_to_client.get(&module_key) {
2717            (route.module_epoch == module_epoch).then_some((
2718                ClientRouteKey {
2719                    connection_id: route.client_connection_id,
2720                    channel: route.client_channel,
2721                },
2722                route.client_epoch,
2723            ))
2724        } else if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
2725            (inner.module_slot_epochs.get(&module_key).copied() == Some(module_epoch)).then_some((
2726                client_key,
2727                inner
2728                    .client_slot_epochs
2729                    .get(&client_key)
2730                    .copied()
2731                    .unwrap_or(0),
2732            ))
2733        } else {
2734            None
2735        };
2736
2737        if let Some(handle) = handle {
2738            inner.status.insert(handle, status);
2739            Ok(true)
2740        } else {
2741            debug!(
2742                module_channel,
2743                module_epoch,
2744                generation = endpoint.generation,
2745                connection_id = endpoint.connection_id.get(),
2746                "dropping stale status update for module route handle"
2747            );
2748            Ok(false)
2749        }
2750    }
2751
2752    pub(crate) fn route_poll_snapshot(
2753        &self,
2754        client_connection_id: ConnectionId,
2755        client_channel: u16,
2756        client_epoch: u32,
2757    ) -> Result<RoutePollSnapshot, ForwardingError> {
2758        let inner = self.read_inner()?;
2759        let client_key = ClientRouteKey {
2760            connection_id: client_connection_id,
2761            channel: client_channel,
2762        };
2763        let Some(route) = inner.client_to_module.get(&client_key) else {
2764            return Ok(RoutePollSnapshot::Absent);
2765        };
2766        if route.client_epoch != client_epoch
2767            || !inner
2768                .module_id_by_endpoint
2769                .contains_key(&route.module_endpoint)
2770        {
2771            return Ok(RoutePollSnapshot::Absent);
2772        }
2773        Ok(RoutePollSnapshot::Bound {
2774            module_id: route.module_id.clone(),
2775            status: inner.status.get(&(client_key, client_epoch)).cloned(),
2776        })
2777    }
2778
2779    pub fn active_binding_count(&self) -> Result<usize, ForwardingError> {
2780        Ok(self.read_inner()?.client_to_module.len())
2781    }
2782
2783    /// How many distinct client connections hold at least one committed route,
2784    /// alongside the largest number of routes any single connection holds.
2785    ///
2786    /// `connected_clients` alone cannot distinguish many clients with a route
2787    /// each from one client accumulating hundreds, and those have opposite
2788    /// causes. Reading it required an out-of-band `lsof` during a live
2789    /// investigation, and the count of connections was mistaken for a count of
2790    /// client processes — which sent two of us after cleanup paths that were
2791    /// working correctly.
2792    pub fn client_route_concentration(&self) -> Result<(usize, usize), ForwardingError> {
2793        let inner = self.read_inner()?;
2794        let mut per_connection: HashMap<ConnectionId, usize> = HashMap::new();
2795        for key in inner.client_to_module.keys() {
2796            *per_connection.entry(key.connection_id).or_insert(0) += 1;
2797        }
2798        let max = per_connection.values().copied().max().unwrap_or(0);
2799        Ok((per_connection.len(), max))
2800    }
2801
2802    pub fn has_route_channel(&self, route_channel: u16) -> Result<bool, ForwardingError> {
2803        let inner = self.read_inner()?;
2804        Ok(inner
2805            .client_to_module
2806            .keys()
2807            .any(|key| key.channel == route_channel))
2808    }
2809
2810    /// Whether daemon-wide shutdown has begun draining all providers.
2811    /// `None` means normal operation; during daemon shutdown, return the
2812    /// endpoint's measured result, or false if no completed measurement exists.
2813    pub(crate) fn daemon_drain_result_for_connection(
2814        &self,
2815        connection_id: ConnectionId,
2816    ) -> Result<Option<bool>, ForwardingError> {
2817        let inner = self.read_inner()?;
2818        Ok(inner.daemon_draining.then(|| {
2819            inner
2820                .endpoint_by_connection
2821                .get(&connection_id)
2822                .and_then(|endpoint| inner.daemon_drain_results.get(endpoint))
2823                .copied()
2824                .unwrap_or(false)
2825        }))
2826    }
2827
2828    #[cfg(unix)]
2829    pub(crate) fn record_daemon_drain_result(
2830        &self,
2831        endpoint: ModuleEndpointId,
2832        drained: bool,
2833    ) -> Result<(), ForwardingError> {
2834        let mut inner = self.write_inner()?;
2835        // Only update an endpoint captured at the start of daemon shutdown.
2836        // Teardown removes its entry, so a late wait cannot recreate the result
2837        // or attribute it to a replacement connection.
2838        if let Some(result) = inner.daemon_drain_results.get_mut(&endpoint) {
2839            *result = drained;
2840        }
2841        Ok(())
2842    }
2843
2844    /// Gate every provider atomically, including registrations racing shutdown.
2845    /// No supervisor lock is held while taking the forwarding lock.
2846    #[cfg(unix)]
2847    pub(crate) fn begin_daemon_drain(
2848        &self,
2849    ) -> Result<Vec<(String, ModuleEndpointId)>, ForwardingError> {
2850        let mut inner = self.write_inner()?;
2851        if inner.daemon_draining {
2852            return Ok(Vec::new());
2853        }
2854        inner.daemon_draining = true;
2855        let modules = inner
2856            .modules_by_id
2857            .iter()
2858            .map(|(id, module)| (id.clone(), module.endpoint))
2859            .collect::<Vec<_>>();
2860        for (_, endpoint) in &modules {
2861            inner.daemon_drain_results.insert(*endpoint, false);
2862            inner
2863                .draining_endpoints
2864                .insert(*endpoint, RouteCloseReason::Restart);
2865        }
2866        // Swap candidates and superseded incumbents are not routable, but they
2867        // are live endpoints that could still be sent module-control work, so
2868        // the daemon-wide gate covers them too. Only the active endpoints are
2869        // returned to the supervisor for measurement, one per module id.
2870        let off_slot_endpoints = inner
2871            .candidates_by_id
2872            .values()
2873            .map(|module| module.endpoint)
2874            .chain(inner.superseded_endpoints.keys().copied())
2875            .collect::<Vec<_>>();
2876        for endpoint in off_slot_endpoints {
2877            inner
2878                .draining_endpoints
2879                .insert(endpoint, RouteCloseReason::Restart);
2880        }
2881        Ok(modules)
2882    }
2883
2884    /// Begin draining whatever endpoint is ACTIVE for `module_id`.
2885    ///
2886    /// After a swap's cutover the active endpoint is the promoted candidate, so
2887    /// this must not be used to drain the incumbent; use
2888    /// [`Self::begin_endpoint_drain`] with the endpoint `cutover_candidate`
2889    /// returned.
2890    pub(crate) fn begin_module_drain(
2891        &self,
2892        module_id: &str,
2893        reason: RouteCloseReason,
2894    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2895        let mut inner = self.write_inner()?;
2896        let Some(module) = inner.modules_by_id.get(module_id).cloned() else {
2897            return Ok(None);
2898        };
2899        Ok(Some(begin_drain_locked(
2900            &mut inner, module_id, module, reason,
2901        )))
2902    }
2903
2904    /// Begin draining one specific endpoint, whichever slot it is in.
2905    ///
2906    /// This is how a swap drains its incumbent after cutover: the incumbent's
2907    /// endpoint is captured by `cutover_candidate`, and resolving it by module id
2908    /// instead would find the promoted candidate and leave neither process
2909    /// routable. Returns `Ok(None)` when the endpoint is no longer registered.
2910    pub(crate) fn begin_endpoint_drain(
2911        &self,
2912        endpoint: ModuleEndpointId,
2913        reason: RouteCloseReason,
2914    ) -> Result<Option<ModuleDrainTarget>, ForwardingError> {
2915        let mut inner = self.write_inner()?;
2916        let Some(module) = module_connection_for_endpoint_locked(&inner, endpoint).cloned() else {
2917            return Ok(None);
2918        };
2919        let module_id = inner
2920            .module_id_by_endpoint
2921            .get(&endpoint)
2922            .cloned()
2923            .expect("an endpoint resolved to a module connection has a module id");
2924        Ok(Some(begin_drain_locked(
2925            &mut inner, &module_id, module, reason,
2926        )))
2927    }
2928}
2929
2930/// Mark `module`'s endpoint draining and settle everything still pending on it.
2931/// Shared by the by-id and by-endpoint drain entry points, which differ only in
2932/// how they find the endpoint.
2933fn begin_drain_locked(
2934    inner: &mut ForwardingInner,
2935    module_id: &str,
2936    module: ModuleConnection,
2937    reason: RouteCloseReason,
2938) -> ModuleDrainTarget {
2939    {
2940        let endpoint = module.endpoint;
2941        inner.draining_endpoints.insert(endpoint, reason);
2942
2943        let flows = inner
2944            .client_to_module
2945            .values()
2946            .filter(|route| route.module_endpoint == endpoint)
2947            .map(|route| Arc::clone(&route.flow))
2948            .collect::<Vec<_>>();
2949        let excluded_subscriptions = flows
2950            .into_iter()
2951            .map(|flow| flow.begin_drain())
2952            .fold(0u32, u32::saturating_add);
2953
2954        let pending_keys = inner
2955            .pending_relays
2956            .keys()
2957            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2958            .copied()
2959            .collect::<Vec<_>>();
2960        let mut abandoned_bindings = Vec::new();
2961        for key in pending_keys {
2962            let Some(pending) = inner.pending_relays.remove(&key) else {
2963                continue;
2964            };
2965            release_reserved_route_locked(
2966                inner,
2967                pending.reservation.client_key,
2968                pending.reservation.module_key,
2969            );
2970            if pending.relay_enqueued {
2971                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
2972                    abandoned_bindings.push(target);
2973                }
2974            } else {
2975                // The bind may still be on its way into the module's queue;
2976                // a GOODBYE sent now could overtake it. `route.open` sends
2977                // it once its send completes.
2978                inner.relays_released_before_enqueue.insert(key);
2979            }
2980            let _ = pending
2981                .sender
2982                .send(RouteBindRelayOutcome::DaemonRejected(ErrorBody::new(
2983                    "module_reloading",
2984                    format!("module_id '{module_id}' is reloading"),
2985                )));
2986        }
2987
2988        let pending_control_keys = inner
2989            .pending_control_rpcs
2990            .keys()
2991            .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
2992            .copied()
2993            .collect::<Vec<_>>();
2994        for key in pending_control_keys {
2995            if let Some(pending) = inner.pending_control_rpcs.remove(&key) {
2996                let _ = pending
2997                    .sender
2998                    .send(ModuleControlRpcOutcome::ModuleGone(format!(
2999                        "module '{module_id}' began draining during module-control RPC"
3000                    )));
3001            }
3002        }
3003
3004        ModuleDrainTarget {
3005            endpoint,
3006            sink: module.sink,
3007            negotiated_ver: module.negotiated_ver,
3008            abandoned_bindings,
3009            excluded_subscriptions,
3010        }
3011    }
3012}
3013
3014/// What a drain that timed out was still waiting on, for the log line that
3015/// reports the timeout. Per-request ages are not tracked.
3016#[derive(Debug, Default, PartialEq, Eq)]
3017pub(crate) struct DrainHoldouts {
3018    /// Requests still counted against the drain (subscriptions flagged as
3019    /// such are excluded from the drain and not counted here).
3020    pub(crate) requests: usize,
3021    /// Routes holding at least one of those requests.
3022    pub(crate) routes: usize,
3023    /// Every route on the endpoint, for scale.
3024    pub(crate) total_routes: usize,
3025    /// The client connections holding the most requests, largest first, at
3026    /// most three: enough to name the consumer without listing every route.
3027    pub(crate) top_connections: Vec<(u64, usize)>,
3028    /// The held requests themselves, as the module saw them, so its own log
3029    /// can say what each one was: `(module channel, corr)`, ordered by channel
3030    /// then corr, at most [`DRAIN_HELD_REQUESTS_LISTED`]. The daemon never reads
3031    /// request bodies, so it cannot name a request's method; the module can,
3032    /// from the channel and corr. A request is released only when the module
3033    /// sends a terminal frame (Response, Error or StreamEnd) with its corr on
3034    /// its route, so every pair listed here is one the module ended, if at all,
3035    /// without sending that frame.
3036    pub(crate) held: Vec<(u16, u64)>,
3037}
3038
3039/// How many held requests the drain-timeout line lists by channel and corr.
3040pub(crate) const DRAIN_HELD_REQUESTS_LISTED: usize = 32;
3041
3042impl ForwardingTable {
3043    /// Summarise the requests one endpoint's drain is still waiting on.
3044    pub(crate) fn endpoint_drain_holdouts(
3045        &self,
3046        endpoint: ModuleEndpointId,
3047    ) -> Result<DrainHoldouts, ForwardingError> {
3048        let inner = self.read_inner()?;
3049        let mut holdouts = DrainHoldouts::default();
3050        let mut by_connection: HashMap<u64, usize> = HashMap::new();
3051        for (key, route) in &inner.client_to_module {
3052            if route.module_endpoint != endpoint {
3053                continue;
3054            }
3055            holdouts.total_routes += 1;
3056            let held = route.flow.drain_in_flight();
3057            if held == 0 {
3058                continue;
3059            }
3060            holdouts.requests += held;
3061            holdouts.routes += 1;
3062            *by_connection.entry(key.connection_id.get()).or_default() += held;
3063            holdouts.held.extend(
3064                route
3065                    .flow
3066                    .drain_held_corrs()
3067                    .into_iter()
3068                    .map(|corr| (route.module_channel, corr)),
3069            );
3070        }
3071        holdouts.held.sort_unstable();
3072        holdouts.held.truncate(DRAIN_HELD_REQUESTS_LISTED);
3073        let mut connections = by_connection.into_iter().collect::<Vec<_>>();
3074        connections.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
3075        connections.truncate(3);
3076        holdouts.top_connections = connections;
3077        Ok(holdouts)
3078    }
3079
3080    pub(crate) fn endpoint_in_flight_count(
3081        &self,
3082        endpoint: ModuleEndpointId,
3083    ) -> Result<usize, ForwardingError> {
3084        let inner = self.read_inner()?;
3085        Ok(inner
3086            .client_to_module
3087            .values()
3088            .filter(|route| route.module_endpoint == endpoint)
3089            .map(|route| route.flow.drain_in_flight())
3090            .sum())
3091    }
3092
3093    pub(crate) fn endpoint_is_draining(
3094        &self,
3095        endpoint: ModuleEndpointId,
3096    ) -> Result<bool, ForwardingError> {
3097        Ok(self
3098            .read_inner()?
3099            .draining_endpoints
3100            .contains_key(&endpoint))
3101    }
3102
3103    pub(crate) fn module_is_draining(&self, module_id: &str) -> Result<bool, ForwardingError> {
3104        let inner = self.read_inner()?;
3105        Ok(inner
3106            .modules_by_id
3107            .get(module_id)
3108            .is_some_and(|module| inner.draining_endpoints.contains_key(&module.endpoint)))
3109    }
3110
3111    pub(crate) fn release_module_endpoint_routes(
3112        &self,
3113        endpoint: ModuleEndpointId,
3114    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
3115        let mut inner = self.write_inner()?;
3116        let routes = inner
3117            .module_to_client
3118            .iter()
3119            .filter(|(module_key, _)| module_key.endpoint == endpoint)
3120            .map(|(module_key, route)| (*module_key, route.module_epoch))
3121            .collect::<Vec<_>>();
3122        let mut released = Vec::with_capacity(routes.len());
3123        for (module_key, epoch) in routes {
3124            if let RouteRelease::Removed(target) =
3125                release_module_route_locked(&mut inner, module_key, epoch)
3126            {
3127                released.push(target);
3128            }
3129        }
3130        Ok(released)
3131    }
3132
3133    /// Revoke old bind decisions atomically, including binds still in flight.
3134    /// The provider stays available: its acknowledged config governs new binds.
3135    pub(crate) fn rebind_module_policy(
3136        &self,
3137        module_id: &str,
3138        expected_connection: Option<ConnectionId>,
3139        authorized: impl FnOnce() -> bool,
3140    ) -> Result<Option<PolicyRevocation>, ForwardingError> {
3141        #[cfg(test)]
3142        if let Some(hold) = self.policy_rebind_before_write.lock().unwrap().take() {
3143            hold.wait();
3144            hold.wait();
3145        }
3146        let mut inner = self.write_inner()?;
3147        let module = inner
3148            .modules_by_id
3149            .get(module_id)
3150            .cloned()
3151            .ok_or(ForwardingError::NoModuleConnection)?;
3152        if expected_connection.is_some_and(|connection| module.endpoint.connection_id != connection)
3153            || !authorized()
3154        {
3155            return Ok(None);
3156        }
3157        let endpoint = module.endpoint;
3158        if inner.draining_endpoints.contains_key(&endpoint) {
3159            return Err(ForwardingError::ModuleReloading {
3160                module_id: module_id.to_owned(),
3161            });
3162        }
3163        let target = begin_drain_locked(&mut inner, module_id, module, RouteCloseReason::Reload);
3164        let routes = endpoint_routes_locked(&inner, endpoint);
3165        let mut goodbyes = target.abandoned_bindings;
3166        for route in &routes {
3167            let target = &route.goodbye_target;
3168            if let RouteRelease::Removed(module) = release_client_route_locked(
3169                &mut inner,
3170                ClientRouteKey {
3171                    connection_id: target.connection_id,
3172                    channel: target.channel,
3173                },
3174                target.epoch,
3175            ) {
3176                goodbyes.push(module);
3177                goodbyes.push(target.clone());
3178            }
3179        }
3180        inner.draining_endpoints.remove(&endpoint);
3181        Ok(Some((routes, goodbyes)))
3182    }
3183
3184    #[cfg(test)]
3185    pub(crate) fn pause_policy_rebind_before_write(&self, hold: Arc<std::sync::Barrier>) {
3186        *self.policy_rebind_before_write.lock().unwrap() = Some(hold);
3187    }
3188
3189    #[cfg(test)]
3190    pub(crate) fn with_forwarding_write_lock_for_test(&self, work: impl FnOnce()) {
3191        let _guard = self.write_inner().unwrap();
3192        work();
3193    }
3194
3195    /// Enumerate one endpoint's current routes without contacting the module.
3196    ///
3197    /// The read lock makes this safe while the endpoint drains: the returned
3198    /// `draining` marker describes the same table state that owns the route,
3199    /// rather than inferring liveness from a module that may be stopping.
3200    pub(crate) fn endpoint_routes(
3201        &self,
3202        endpoint: ModuleEndpointId,
3203    ) -> Result<Vec<EndpointRoute>, ForwardingError> {
3204        let inner = self.read_inner()?;
3205        Ok(endpoint_routes_locked(&inner, endpoint))
3206    }
3207
3208    /// Snapshot all live endpoint route sets under one forwarding-table read lock.
3209    pub(crate) fn route_census(
3210        &self,
3211        module_id: Option<&str>,
3212    ) -> Result<Vec<(String, Vec<EndpointRoute>)>, ForwardingError> {
3213        let inner = self.read_inner()?;
3214        let mut endpoints = inner
3215            .modules_by_id
3216            .iter()
3217            .filter(|(id, _)| module_id.is_none_or(|requested| requested == id.as_str()))
3218            .map(|(id, module)| (id.clone(), module.endpoint))
3219            .collect::<Vec<_>>();
3220        endpoints.sort_by(|left, right| left.0.cmp(&right.0));
3221        Ok(endpoints
3222            .into_iter()
3223            .map(|(id, endpoint)| (id, endpoint_routes_locked(&inner, endpoint)))
3224            .collect())
3225    }
3226
3227    /// Snapshot only the endpoint currently routable under this module id.
3228    pub(crate) fn live_roots(
3229        &self,
3230        module_id: &str,
3231    ) -> Result<ModuleControlResponseToModule, ForwardingError> {
3232        let inner = self.read_inner()?;
3233        let endpoint = inner
3234            .modules_by_id
3235            .get(module_id)
3236            .map(|module| module.endpoint);
3237        let mut roots = BTreeMap::new();
3238        let mut unknown_root_bindings = 0;
3239        let mut total_bindings = 0;
3240        if let Some(endpoint) = endpoint {
3241            for binding in inner
3242                .module_to_client
3243                .values()
3244                .filter(|binding| binding.module_endpoint == endpoint)
3245            {
3246                total_bindings += 1;
3247                let workdir = binding
3248                    .scope
3249                    .as_ref()
3250                    .and_then(|scope| scope.workdir.as_deref())
3251                    .map(std::path::Path::new);
3252                if let Some(root) = &binding.project_root {
3253                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
3254                    entry.0 += 1;
3255                } else if workdir.is_none() {
3256                    unknown_root_bindings += 1;
3257                }
3258                if let Some(workdir) = workdir.filter(|workdir| {
3259                    binding.project_root.as_ref().map(ProjectRootId::as_path) != Some(*workdir)
3260                }) {
3261                    roots.entry(workdir.to_path_buf()).or_insert((0, 0)).0 += 1;
3262                }
3263            }
3264            for pending in inner
3265                .pending_relays
3266                .values()
3267                .filter(|pending| pending.reservation.module_key.endpoint == endpoint)
3268            {
3269                total_bindings += 1;
3270                if let Some(root) = &pending.reservation.project_root {
3271                    let entry = roots.entry(root.as_path().to_path_buf()).or_insert((0, 0));
3272                    entry.1 += 1;
3273                } else {
3274                    unknown_root_bindings += 1;
3275                }
3276            }
3277        }
3278        Ok(ModuleControlResponseToModule::LiveRoots {
3279            roots: roots
3280                .into_iter()
3281                .map(|(project_root, (bound, pending))| LiveRoot {
3282                    project_root,
3283                    bound,
3284                    pending,
3285                })
3286                .collect(),
3287            unknown_root_bindings,
3288            total_bindings,
3289        })
3290    }
3291
3292    /// True if this connection already owns committed or reserved CLIENT routes.
3293    /// A module registers (HELLO) before serving and never opens client routes, so
3294    /// a connection that has client routes must not also become a module endpoint
3295    /// — otherwise one connection holds both client and module state and cleanup
3296    /// only releases one side.
3297    pub(crate) fn connection_has_client_routes(
3298        &self,
3299        connection_id: ConnectionId,
3300    ) -> Result<bool, ForwardingError> {
3301        let inner = self.read_inner()?;
3302        Ok(connection_has_client_routes_locked(&inner, connection_id))
3303    }
3304
3305    pub(crate) fn cleanup_connection(
3306        &self,
3307        connection_id: ConnectionId,
3308    ) -> Result<Vec<GoodbyeTarget>, ForwardingError> {
3309        self.cleanup_connection_counted(connection_id)
3310            .map(|cleanup| cleanup.released)
3311    }
3312
3313    /// [`Self::cleanup_connection`], also reporting how many pending
3314    /// route.bind relays to the closed module were aborted. That count is the
3315    /// `abandoned` figure of the `route.closed` push sent for a lost module
3316    /// connection; it is always zero for a client connection.
3317    pub(crate) fn cleanup_connection_counted(
3318        &self,
3319        connection_id: ConnectionId,
3320    ) -> Result<ConnectionCleanup, ForwardingError> {
3321        self.frame_drop_warnings
3322            .lock()
3323            .unwrap_or_else(std::sync::PoisonError::into_inner)
3324            .remove(&connection_id);
3325        let mut inner = self.write_inner()?;
3326        inner.closing_connections.insert(connection_id);
3327        #[cfg(all(test, target_os = "linux"))]
3328        if let Some(pause) = self.cleanup_pause.lock().unwrap().take() {
3329            let _ = pause.reached.send(());
3330            pause
3331                .release
3332                .recv_timeout(Duration::from_secs(5))
3333                .expect("test must release connection cleanup");
3334        }
3335        #[cfg(all(test, target_os = "linux"))]
3336        if let Some(observe) = self.cleanup_tie_observer.lock().unwrap().take() {
3337            let pending = inner
3338                .pending_relays
3339                .values()
3340                .filter(|relay| relay.reservation.client_key.connection_id == connection_id)
3341                .count();
3342            let _ = observe.send((
3343                inner.spawn_connections.contains_key(&connection_id),
3344                pending,
3345            ));
3346        }
3347        let cleanup = if let Some(endpoint) = inner.endpoint_by_connection.remove(&connection_id) {
3348            remove_module_connection_locked(&mut inner, endpoint)
3349        } else {
3350            ConnectionCleanup {
3351                released: Self::cleanup_client_connection_locked(&mut inner, connection_id),
3352                abandoned_relays: 0,
3353            }
3354        };
3355        // The closing mark refuses new work for a connection whose teardown is
3356        // still pending. This is the latest point at which lifting it is safe:
3357        // teardown has just removed every per-connection entry above, under
3358        // this same write lock, so no lookup can still find live state for the
3359        // id; and ids come from a monotonic counter that never reissues one,
3360        // so the id can never name a future connection either. Keeping the
3361        // mark past this point only grew the set by one entry per connection
3362        // for the life of the daemon.
3363        inner.closing_connections.remove(&connection_id);
3364        Ok(cleanup)
3365    }
3366
3367    fn cleanup_client_connection_locked(
3368        inner: &mut ForwardingInner,
3369        connection_id: ConnectionId,
3370    ) -> Vec<GoodbyeTarget> {
3371        let routes = inner
3372            .client_to_module
3373            .iter()
3374            .filter(|(key, _)| key.connection_id == connection_id)
3375            .map(|(key, route)| (*key, route.client_epoch))
3376            .collect::<Vec<_>>();
3377        let mut released = Vec::with_capacity(routes.len());
3378        for (client_key, epoch) in routes {
3379            if let RouteRelease::Removed(target) =
3380                release_client_route_locked(inner, client_key, epoch)
3381            {
3382                released.push(target);
3383            }
3384        }
3385
3386        let pending_keys = inner
3387            .pending_relays
3388            .iter()
3389            .filter(|(_, pending)| pending.reservation.client_key.connection_id == connection_id)
3390            .map(|(key, _)| *key)
3391            .collect::<Vec<_>>();
3392        for key in pending_keys {
3393            let Some(pending) = inner.pending_relays.remove(&key) else {
3394                continue;
3395            };
3396            release_reserved_route_locked(
3397                inner,
3398                pending.reservation.client_key,
3399                pending.reservation.module_key,
3400            );
3401            if pending.relay_enqueued {
3402                if let Some(target) = abandoned_route_target(inner, &pending.reservation) {
3403                    released.push(target);
3404                }
3405            } else {
3406                // The bind may still be on its way into the module's queue;
3407                // a GOODBYE sent now could overtake it. `route.open` sends
3408                // it once its send completes.
3409                inner.relays_released_before_enqueue.insert(key);
3410            }
3411            let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
3412                "client connection closed during route.bind relay".to_string(),
3413            ));
3414        }
3415
3416        let orphaned = inner
3417            .reserved_client
3418            .iter()
3419            .filter(|(key, _)| key.connection_id == connection_id)
3420            .map(|(client, module)| (*client, *module))
3421            .collect::<Vec<_>>();
3422        for (client_key, module_key) in orphaned {
3423            release_reserved_route_locked(inner, client_key, module_key);
3424        }
3425        inner.next_client_channel.remove(&connection_id);
3426        inner
3427            .client_slot_epochs
3428            .retain(|key, _| key.connection_id != connection_id);
3429        inner
3430            .last_published_epoch
3431            .retain(|key, _| key.connection_id != connection_id);
3432        inner
3433            .status
3434            .retain(|(key, _), _| key.connection_id != connection_id);
3435
3436        released
3437    }
3438
3439    /// Close a client connection whose egress queue refused a frame for the
3440    /// route `(connection_id, channel)` at `expected_epoch`. The whole
3441    /// connection closes because its queue is shared by every route on it.
3442    ///
3443    /// The first request that actually closes the connection logs one WARN with
3444    /// the diagnosis: which principals the connection's routes belonged to,
3445    /// which module's frame did not fit and on which client channel, and what
3446    /// the queue held at that moment.
3447    pub(crate) fn escalate_client_delivery_failure(
3448        &self,
3449        connection_id: ConnectionId,
3450        channel: u16,
3451        expected_epoch: u32,
3452        reason: CloseReason,
3453        undelivered: UndeliveredFrame<'_>,
3454    ) -> Result<bool, ForwardingError> {
3455        self.escalate_client_delivery_failure_diagnosed(
3456            connection_id,
3457            channel,
3458            expected_epoch,
3459            reason,
3460            undelivered,
3461            None,
3462        )
3463    }
3464
3465    pub(crate) fn escalate_client_delivery_failure_diagnosed(
3466        &self,
3467        connection_id: ConnectionId,
3468        channel: u16,
3469        expected_epoch: u32,
3470        reason: CloseReason,
3471        undelivered: UndeliveredFrame<'_>,
3472        diagnosis: Option<&crate::router::ClientDeliveryDiagnosis>,
3473    ) -> Result<bool, ForwardingError> {
3474        let principals = {
3475            let mut inner = self.write_inner()?;
3476            let key = ClientRouteKey {
3477                connection_id,
3478                channel,
3479            };
3480            if inner.last_published_epoch.get(&key).copied() != Some(expected_epoch) {
3481                None
3482            } else {
3483                inner.closing_connections.insert(connection_id);
3484                Some(connection_principals_locked(&inner, connection_id))
3485            }
3486        };
3487        let Some(principals) = principals else {
3488            // A concurrent release can remove the epoch before escalation.
3489            // The frame was still lost, even though this route can no longer
3490            // authorize closing the shared client connection.
3491            self.warn_frame_drop(
3492                connection_id,
3493                channel,
3494                expected_epoch,
3495                "module_to_client",
3496                diagnosis
3497                    .map(|failure| failure.cause)
3498                    .unwrap_or("delivery_failed"),
3499            );
3500            return Ok(false);
3501        };
3502        let backlog = undelivered.sink.backlog();
3503        let close_reason = reason.to_string();
3504        if self.request_connection_close(connection_id, reason) {
3505            warn!(
3506                connection_id = connection_id.get(),
3507                principals = %principals,
3508                module_id = undelivered.module_id.unwrap_or("unknown"),
3509                client_channel = channel,
3510                queued_bytes = backlog.queued_bytes,
3511                queued_frames = backlog.queued_frames,
3512                frame_type = ?diagnosis.map(|failure| failure.frame_type),
3513                frame_flags = ?diagnosis.map(|failure| failure.flags.0),
3514                close_cause = diagnosis.map(|failure| failure.cause).unwrap_or("delivery_failed"),
3515                oldest_queued_ms = backlog
3516                    .oldest_age
3517                    .map(|age| age.as_millis() as u64)
3518                    .unwrap_or(0),
3519                close_reason = %close_reason,
3520                "closing client connection: its egress queue could not take a frame"
3521            );
3522        }
3523        Ok(true)
3524    }
3525
3526    /// Publish the tags a scope sync changed and close the live routes its
3527    /// drain table selects, in one critical section.
3528    ///
3529    /// The caller holds the scope table's write lock (scope table, then this
3530    /// table, always). Publishing and selecting under one lock is what closes
3531    /// the race with a bind commit: a commit before this sees the old tag and
3532    /// its route is selected here like any other; a commit after it sees the
3533    /// new tag and is refused.
3534    ///
3535    /// Every client route index entry is scanned, which includes routes on a
3536    /// swap's superseded endpoint (the old process of a module being replaced
3537    /// blue/green, which keeps its existing routes until they drain): those
3538    /// stay indexed until drained, so ending a scope reaches them too.
3539    pub(crate) fn publish_scope_changes(
3540        &self,
3541        changes: &[ScopeTagChange],
3542    ) -> Result<Vec<ScopeDrainedRoute>, ForwardingError> {
3543        let mut inner = self.write_inner()?;
3544        let mut by_scope: HashMap<(&str, &str), &ScopeTagChange> = HashMap::new();
3545        for change in changes {
3546            let key = (change.owner.clone(), change.scope_ref.clone());
3547            match change.after {
3548                Some(tag) => {
3549                    inner.scope_tags.insert(key, tag);
3550                }
3551                None => {
3552                    inner.scope_tags.remove(&key);
3553                }
3554            }
3555            by_scope.insert((change.owner.as_str(), change.scope_ref.as_str()), change);
3556        }
3557        let mut selected = Vec::new();
3558        for (client_key, route) in &inner.client_to_module {
3559            let Some(scope) = &route.scope else {
3560                continue;
3561            };
3562            let Some(change) = by_scope.get(&(scope.owner.as_str(), scope.scope_ref.as_str()))
3563            else {
3564                continue;
3565            };
3566            if change.before.map(|tag| tag.scope_epoch) != Some(scope.tag.scope_epoch) {
3567                continue;
3568            }
3569            let reason = match &change.drain {
3570                ScopeDrain::Nothing => continue,
3571                ScopeDrain::All(reason) => *reason,
3572                ScopeDrain::Carriers(narrowed) => {
3573                    let owner = Principal::Reserved {
3574                        module_id: scope.owner.clone(),
3575                    };
3576                    let hit = route.principal != owner
3577                        && narrowed.iter().any(|(principal, allowed)| {
3578                            *principal == route.principal
3579                                && allowed
3580                                    .as_ref()
3581                                    .is_none_or(|targets| !targets.contains(&route.module_id))
3582                        });
3583                    if !hit {
3584                        continue;
3585                    }
3586                    RouteCloseReason::ScopeCarrierRemoved
3587                }
3588            };
3589            selected.push((*client_key, route.client_epoch, reason));
3590        }
3591        let mut drained = Vec::new();
3592        for (client_key, client_epoch, reason) in selected {
3593            let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
3594                continue;
3595            };
3596            let release = release_client_route_locked(&mut inner, client_key, client_epoch);
3597            self.record_route_release(&release);
3598            if let RouteRelease::Removed(module) = release {
3599                drained.push(ScopeDrainedRoute {
3600                    reason,
3601                    scope: route
3602                        .scope
3603                        .clone()
3604                        .expect("selected scope drain has a bound scope"),
3605                    module_id: route.module_id.clone(),
3606                    client: GoodbyeTarget {
3607                        connection_id: route.client_connection_id,
3608                        sink: route.client_sink.clone(),
3609                        negotiated_ver: route.client_negotiated_ver,
3610                        channel: route.client_channel,
3611                        epoch: route.client_epoch,
3612                        kind: GoodbyeTargetKind::Client,
3613                        module_id: Some(route.module_id.clone()),
3614                    },
3615                    module,
3616                    accepts_close_reasons: route.accepts_close_reasons.clone(),
3617                });
3618            }
3619        }
3620        Ok(drained)
3621    }
3622
3623    /// Publish a spawn's new authority epoch and release every live route
3624    /// whose recorded caller or target is that spawn at an older epoch, in one
3625    /// critical section.
3626    ///
3627    /// The caller holds the spawn's authority record state (record, then this
3628    /// table, always; see `SpawnAuthorityRecord::change_authority`). A bind
3629    /// that commits before this is selected here like any other route; one
3630    /// that commits after it sees the new epoch and is refused as stale.
3631    ///
3632    /// Unlike [`Self::publish_scope_changes`], every client route is scanned,
3633    /// scoped or not. Once a spawn is dead (`alive` false), its own mark and
3634    /// approved set are dropped and its epoch stays as a tombstone, so a bind
3635    /// still naming it is refused as stale. Another spawn of the same module
3636    /// is never touched: a blue/green swap candidate takes a higher spawn
3637    /// number than the incumbent that keeps serving beside it, so spawn order
3638    /// says nothing about which of them is still alive. `first_party` is the
3639    /// spawn's mark after the change; a lowering withdraws it here, in the
3640    /// same critical section as the epoch.
3641    pub(crate) fn publish_authority_change(
3642        &self,
3643        key: &SpawnKey,
3644        epoch: u64,
3645        alive: bool,
3646        first_party: bool,
3647    ) -> Result<Vec<AuthorityClosedRoute>, ForwardingError> {
3648        self.publish_approved_authority_change(key, epoch, alive, Some(first_party), None)
3649    }
3650
3651    /// Publish changed mark or approval facts with the new epoch. `None`
3652    /// leaves that fact unchanged; a dead spawn always loses its mark.
3653    pub(crate) fn publish_approved_authority_change(
3654        &self,
3655        key: &SpawnKey,
3656        epoch: u64,
3657        alive: bool,
3658        first_party: Option<bool>,
3659        approved: Option<&ApprovedDeclarations>,
3660    ) -> Result<Vec<AuthorityClosedRoute>, ForwardingError> {
3661        let mut inner = self.write_inner()?;
3662        if !alive {
3663            inner.spawn_approved.remove(key);
3664            inner.dead_spawns.insert(key.clone());
3665            prune_dead_spawn_tombstones_locked(&mut inner, key);
3666        } else if let Some(approved) = approved {
3667            inner.spawn_approved.insert(key.clone(), approved.clone());
3668        }
3669        // A record's epoch only rises; taking the larger keeps a published
3670        // epoch from ever moving backwards, whatever order publications land.
3671        let published = inner.spawn_epochs.entry(key.clone()).or_insert(epoch);
3672        *published = (*published).max(epoch);
3673        if !alive || first_party == Some(false) {
3674            inner.spawn_marks.remove(key);
3675        } else if first_party == Some(true) {
3676            inner.spawn_marks.insert(key.clone());
3677        }
3678        let older = |pair: &Option<AuthorityPair>| {
3679            pair.as_ref()
3680                .is_some_and(|pair| pair.key == *key && pair.epoch < epoch)
3681        };
3682        let selected = inner
3683            .client_to_module
3684            .iter()
3685            .filter(|(_, route)| older(&route.caller_authority) || older(&route.target_authority))
3686            .map(|(client_key, route)| (*client_key, route.client_epoch))
3687            .collect::<Vec<_>>();
3688        let mut closed = Vec::new();
3689        for (client_key, client_epoch) in selected {
3690            let Some(route) = inner.client_to_module.get(&client_key).cloned() else {
3691                continue;
3692            };
3693            let release = release_client_route_locked(&mut inner, client_key, client_epoch);
3694            self.record_route_release(&release);
3695            if let RouteRelease::Removed(module) = release {
3696                closed.push(AuthorityClosedRoute {
3697                    client: GoodbyeTarget {
3698                        connection_id: route.client_connection_id,
3699                        sink: route.client_sink.clone(),
3700                        negotiated_ver: route.client_negotiated_ver,
3701                        channel: route.client_channel,
3702                        epoch: route.client_epoch,
3703                        kind: GoodbyeTargetKind::Client,
3704                        module_id: Some(route.module_id.clone()),
3705                    },
3706                    module,
3707                    route,
3708                });
3709            }
3710        }
3711        Ok(closed)
3712    }
3713
3714    /// Publish a spawn's activated first-party mark, so `route.open` replies
3715    /// reserved from now on report it. The caller holds the spawn's authority
3716    /// record state. The epoch is unchanged and no route is closed: routes
3717    /// bound before activation keep the `first_party: false` they were
3718    /// attested with.
3719    pub(crate) fn publish_spawn_mark(&self, key: &SpawnKey) -> Result<(), ForwardingError> {
3720        self.write_inner()?.spawn_marks.insert(key.clone());
3721        Ok(())
3722    }
3723
3724    /// The epoch published for a spawn, for tests of the commit re-check.
3725    #[cfg(all(test, unix, feature = "test-support"))]
3726    pub(crate) fn published_spawn_epoch(&self, key: &SpawnKey) -> u64 {
3727        published_epoch_locked(&self.read_inner().unwrap(), key)
3728    }
3729
3730    /// The caller and target authority each live route opened by `client`
3731    /// recorded at commit.
3732    #[cfg(all(test, unix, feature = "test-support"))]
3733    pub(crate) fn route_authority_for_test(
3734        &self,
3735        client: ConnectionId,
3736    ) -> Vec<(Option<AuthorityPair>, Option<AuthorityPair>)> {
3737        self.read_inner()
3738            .unwrap()
3739            .client_to_module
3740            .iter()
3741            .filter(|(key, _)| key.connection_id == client)
3742            .map(|(_, route)| {
3743                (
3744                    route.caller_authority.clone(),
3745                    route.target_authority.clone(),
3746                )
3747            })
3748            .collect()
3749    }
3750
3751    /// The published tag of one scope, for tests of the commit re-check.
3752    #[cfg(test)]
3753    pub(crate) fn published_scope_tag(&self, owner: &str, scope_ref: &str) -> Option<ScopeTag> {
3754        self.read_inner()
3755            .ok()?
3756            .scope_tags
3757            .get(&(owner.to_string(), scope_ref.to_string()))
3758            .copied()
3759    }
3760
3761    fn record_route_release(&self, release: &RouteRelease) {
3762        match release {
3763            RouteRelease::Removed(_) => self.counters.increment_route_released_epoch_fenced(),
3764            RouteRelease::Stale => self.counters.increment_route_release_stale_skipped(),
3765            RouteRelease::Absent => {}
3766        }
3767    }
3768
3769    fn read_inner(
3770        &self,
3771    ) -> Result<InnerGuard<RwLockReadGuard<'_, ForwardingInner>>, ForwardingError> {
3772        self.inner
3773            .read()
3774            .map(InnerGuard::new)
3775            .map_err(|_| ForwardingError::Poisoned)
3776    }
3777
3778    fn write_inner(
3779        &self,
3780    ) -> Result<InnerGuard<RwLockWriteGuard<'_, ForwardingInner>>, ForwardingError> {
3781        self.inner
3782            .write()
3783            .map(InnerGuard::new)
3784            .map_err(|_| ForwardingError::Poisoned)
3785    }
3786
3787    fn lock_close_registry(
3788        &self,
3789    ) -> MutexGuard<'_, HashMap<ConnectionId, oneshot::Sender<CloseReason>>> {
3790        self.close_registry
3791            .lock()
3792            .unwrap_or_else(|poisoned| poisoned.into_inner())
3793    }
3794}
3795
3796impl ForwardingInner {
3797    fn allocate_route_slots(
3798        &mut self,
3799        connection_id: ConnectionId,
3800        endpoint: ModuleEndpointId,
3801    ) -> Result<(u16, u32, u16, u32), ForwardingError> {
3802        let client_start = *self.next_client_channel.entry(connection_id).or_insert(1);
3803        let mut client_channel = client_start;
3804        let client_channel = loop {
3805            let key = ClientRouteKey {
3806                connection_id,
3807                channel: client_channel,
3808            };
3809            let eligible = !self.client_to_module.contains_key(&key)
3810                && !self.reserved_client.contains_key(&key)
3811                && self.client_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3812            if eligible {
3813                break client_channel;
3814            }
3815            client_channel = next_channel(client_channel);
3816            if client_channel == client_start {
3817                return Err(ForwardingError::ClientRouteChannelExhausted { connection_id });
3818            }
3819        };
3820
3821        let module_start = *self.next_module_channel.entry(endpoint).or_insert(1);
3822        let mut module_channel = module_start;
3823        let module_channel = loop {
3824            let key = ModuleRouteKey {
3825                endpoint,
3826                channel: module_channel,
3827            };
3828            let eligible = !self.module_to_client.contains_key(&key)
3829                && !self.reserved_module.contains_key(&key)
3830                && self.module_slot_epochs.get(&key).copied().unwrap_or(0) < u32::MAX;
3831            if eligible {
3832                break module_channel;
3833            }
3834            module_channel = next_channel(module_channel);
3835            if module_channel == module_start {
3836                return Err(ForwardingError::ModuleRouteChannelExhausted { endpoint });
3837            }
3838        };
3839
3840        let client_key = ClientRouteKey {
3841            connection_id,
3842            channel: client_channel,
3843        };
3844        let module_key = ModuleRouteKey {
3845            endpoint,
3846            channel: module_channel,
3847        };
3848        let client_epoch = self
3849            .client_slot_epochs
3850            .get(&client_key)
3851            .copied()
3852            .unwrap_or(0)
3853            + 1;
3854        let module_epoch = self
3855            .module_slot_epochs
3856            .get(&module_key)
3857            .copied()
3858            .unwrap_or(0)
3859            + 1;
3860        self.client_slot_epochs.insert(client_key, client_epoch);
3861        self.module_slot_epochs.insert(module_key, module_epoch);
3862        self.next_client_channel
3863            .insert(connection_id, next_channel(client_channel));
3864        self.next_module_channel
3865            .insert(endpoint, next_channel(module_channel));
3866        Ok((client_channel, client_epoch, module_channel, module_epoch))
3867    }
3868
3869    fn allocate_control_corr(
3870        &mut self,
3871        endpoint: ModuleEndpointId,
3872    ) -> Result<u64, ForwardingError> {
3873        let candidate = self.next_control_corr.get(&endpoint).copied().unwrap_or(1);
3874        if candidate == 0 {
3875            self.closing_connections.insert(endpoint.connection_id);
3876            return Err(ForwardingError::RelayCorrelationExhausted);
3877        }
3878        self.next_control_corr.insert(
3879            endpoint,
3880            if candidate == u64::MAX {
3881                0
3882            } else {
3883                candidate + 1
3884            },
3885        );
3886        Ok(candidate)
3887    }
3888}
3889
3890fn connection_has_client_routes_locked(
3891    inner: &ForwardingInner,
3892    connection_id: ConnectionId,
3893) -> bool {
3894    inner
3895        .client_to_module
3896        .keys()
3897        .any(|key| key.connection_id == connection_id)
3898        || inner
3899            .reserved_client
3900            .keys()
3901            .any(|key| key.connection_id == connection_id)
3902}
3903
3904fn check_module_connection_role_locked(
3905    inner: &ForwardingInner,
3906    connection_id: ConnectionId,
3907) -> Result<(), ForwardingError> {
3908    if inner.endpoint_by_connection.contains_key(&connection_id)
3909        || connection_has_client_routes_locked(inner, connection_id)
3910    {
3911        return Err(ForwardingError::ConnectionRoleConflict { connection_id });
3912    }
3913    Ok(())
3914}
3915
3916fn next_channel(channel: u16) -> u16 {
3917    let next = channel.wrapping_add(1);
3918    if next == 0 {
3919        1
3920    } else {
3921        next
3922    }
3923}
3924
3925fn endpoint_routes_locked(
3926    inner: &ForwardingInner,
3927    endpoint: ModuleEndpointId,
3928) -> Vec<EndpointRoute> {
3929    let drain_reason = inner.draining_endpoints.get(&endpoint).copied();
3930    let draining = drain_reason.is_some();
3931    let mut routes = inner
3932        .module_to_client
3933        .iter()
3934        .filter(|(module_key, _)| module_key.endpoint == endpoint)
3935        .map(|(_, route)| EndpointRoute {
3936            goodbye_target: GoodbyeTarget {
3937                connection_id: route.client_connection_id,
3938                sink: route.client_sink.clone(),
3939                negotiated_ver: route.client_negotiated_ver,
3940                channel: route.client_channel,
3941                epoch: route.client_epoch,
3942                kind: GoodbyeTargetKind::Client,
3943                module_id: Some(route.module_id.clone()),
3944            },
3945            principal: route.principal.clone(),
3946            accepts_close_reasons: route.accepts_close_reasons.clone(),
3947            bound_at: route.bound_at,
3948            draining,
3949            drain_reason,
3950        })
3951        .collect::<Vec<_>>();
3952    routes.sort_by_key(|route| {
3953        (
3954            route.goodbye_target.connection_id.get(),
3955            route.goodbye_target.channel,
3956            route.goodbye_target.epoch,
3957        )
3958    });
3959    routes
3960}
3961
3962/// Why a bind admitted under `scope` may no longer commit, read from the
3963/// published tags: `scope_ended` when the scope is gone or at another epoch,
3964/// `scope_changed` (retryable) when only its content moved.
3965fn scope_refusal_locked(inner: &ForwardingInner, scope: &BoundScope) -> Option<ErrorBody> {
3966    let key = (scope.owner.clone(), scope.scope_ref.clone());
3967    match inner.scope_tags.get(&key) {
3968        Some(current) if *current == scope.tag => None,
3969        Some(current) if current.scope_epoch == scope.tag.scope_epoch => Some(ErrorBody::new(
3970            subc_protocol::error_codes::SCOPE_CHANGED,
3971            format!(
3972                "scope '{}' of {} changed while the route was being bound; re-open it",
3973                scope.scope_ref, scope.owner
3974            ),
3975        )),
3976        _ => Some(ErrorBody::new(
3977            subc_protocol::error_codes::SCOPE_ENDED,
3978            format!(
3979                "scope '{}' of {} at scope_epoch {} ended while the route was being bound",
3980                scope.scope_ref, scope.owner, scope.tag.scope_epoch
3981            ),
3982        )),
3983    }
3984}
3985
3986fn release_reserved_route_locked(
3987    inner: &mut ForwardingInner,
3988    client_key: ClientRouteKey,
3989    module_key: ModuleRouteKey,
3990) {
3991    if inner.reserved_client.get(&client_key).copied() == Some(module_key) {
3992        inner.reserved_client.remove(&client_key);
3993    }
3994    if inner.reserved_module.get(&module_key).copied() == Some(client_key) {
3995        inner.reserved_module.remove(&module_key);
3996    }
3997    inner.status.retain(|(key, _), _| *key != client_key);
3998}
3999
4000fn release_client_route_locked(
4001    inner: &mut ForwardingInner,
4002    client_key: ClientRouteKey,
4003    expected_epoch: u32,
4004) -> RouteRelease {
4005    let Some(route) = inner.client_to_module.get(&client_key) else {
4006        return RouteRelease::Absent;
4007    };
4008    if route.client_epoch != expected_epoch {
4009        return RouteRelease::Stale;
4010    }
4011    let route = inner
4012        .client_to_module
4013        .remove(&client_key)
4014        .expect("route checked under the same forwarding lock");
4015    route.flow.close();
4016    if let Some(scope) = &route.scope {
4017        scope.release_bind();
4018    }
4019    inner.module_to_client.remove(&ModuleRouteKey {
4020        endpoint: route.module_endpoint,
4021        channel: route.module_channel,
4022    });
4023    inner.operator_confirms.route_closed(ModuleRouteKey {
4024        endpoint: route.module_endpoint,
4025        channel: route.module_channel,
4026    });
4027    inner.status.remove(&(client_key, expected_epoch));
4028    RouteRelease::Removed(GoodbyeTarget {
4029        connection_id: route.module_endpoint.connection_id,
4030        sink: route.module_sink.clone(),
4031        negotiated_ver: route.module_negotiated_ver,
4032        channel: route.module_channel,
4033        epoch: route.module_epoch,
4034        kind: GoodbyeTargetKind::Module,
4035        module_id: Some(route.module_id.clone()),
4036    })
4037}
4038
4039fn release_module_route_locked(
4040    inner: &mut ForwardingInner,
4041    module_key: ModuleRouteKey,
4042    expected_epoch: u32,
4043) -> RouteRelease {
4044    let Some(route) = inner.module_to_client.get(&module_key) else {
4045        return RouteRelease::Absent;
4046    };
4047    if route.module_epoch != expected_epoch {
4048        return RouteRelease::Stale;
4049    }
4050    let route = inner
4051        .module_to_client
4052        .remove(&module_key)
4053        .expect("route checked under the same forwarding lock");
4054    inner.operator_confirms.route_closed(module_key);
4055    route.flow.close();
4056    if let Some(scope) = &route.scope {
4057        scope.release_bind();
4058    }
4059    let client_key = ClientRouteKey {
4060        connection_id: route.client_connection_id,
4061        channel: route.client_channel,
4062    };
4063    inner.client_to_module.remove(&client_key);
4064    inner.status.remove(&(client_key, route.client_epoch));
4065    RouteRelease::Removed(GoodbyeTarget {
4066        connection_id: route.client_connection_id,
4067        sink: route.client_sink.clone(),
4068        negotiated_ver: route.client_negotiated_ver,
4069        channel: route.client_channel,
4070        epoch: route.client_epoch,
4071        kind: GoodbyeTargetKind::Client,
4072        module_id: Some(route.module_id.clone()),
4073    })
4074}
4075
4076fn commit_route_locked(
4077    inner: &mut ForwardingInner,
4078    mut pending: PendingRouteBindRelayEntry,
4079) -> Result<Option<GoodbyeTarget>, ForwardingError> {
4080    let reservation = pending.reservation;
4081    if inner
4082        .closing_connections
4083        .contains(&reservation.client_key.connection_id)
4084    {
4085        return Err(ForwardingError::ConnectionClosing {
4086            connection_id: reservation.client_key.connection_id,
4087        });
4088    }
4089    let module_id = inner
4090        .module_id_by_endpoint
4091        .get(&reservation.module_key.endpoint)
4092        .cloned()
4093        .ok_or(ForwardingError::StaleModuleEndpoint)?;
4094    if inner
4095        .draining_endpoints
4096        .contains_key(&reservation.module_key.endpoint)
4097    {
4098        return Err(ForwardingError::ModuleReloading { module_id });
4099    }
4100    if pending.busy_refusals {
4101        let body = route_open_reply_body(
4102            &ClientControlResponse::RouteOpen {
4103                route_channel: reservation.client_key.channel,
4104                route_epoch: reservation.client_epoch,
4105                request_window: Some(pending.request_window),
4106                watch_window: pending.watch_window,
4107            },
4108            &pending.target_authority,
4109            &pending.incarnation,
4110        )?;
4111        pending.route_open_frame = Frame::build_with_version(
4112            pending.client_negotiated_ver,
4113            FrameType::Response,
4114            pending.route_open_frame.header.flags,
4115            0,
4116            0,
4117            pending.route_open_frame.header.corr,
4118            body,
4119        )
4120        .map_err(|err| ForwardingError::RouteOpenBuild(err.to_string()))?;
4121    }
4122    if inner.reserved_client.remove(&reservation.client_key) != Some(reservation.module_key)
4123        || inner.reserved_module.remove(&reservation.module_key) != Some(reservation.client_key)
4124    {
4125        return Err(ForwardingError::UnknownReservation {
4126            client_channel: reservation.client_key.channel,
4127            module_channel: reservation.module_key.channel,
4128        });
4129    }
4130    let module = inner
4131        .modules_by_id
4132        .get(&module_id)
4133        .filter(|module| module.endpoint == reservation.module_key.endpoint)
4134        .cloned()
4135        .ok_or(ForwardingError::StaleModuleEndpoint)?;
4136    let binding = Arc::new(RouteBinding {
4137        client_connection_id: reservation.client_key.connection_id,
4138        client_sink: pending.client_sink,
4139        client_negotiated_ver: pending.client_negotiated_ver,
4140        client_channel: reservation.client_key.channel,
4141        client_epoch: reservation.client_epoch,
4142        module_id,
4143        module_endpoint: reservation.module_key.endpoint,
4144        module_sink: module.sink,
4145        module_negotiated_ver: module.negotiated_ver,
4146        module_channel: reservation.module_key.channel,
4147        module_epoch: reservation.module_epoch,
4148        principal: pending.principal,
4149        harness: pending.harness,
4150        build_class: pending.build_class,
4151        operator_app: pending.operator_app,
4152        project_root: reservation.project_root.clone(),
4153        bound_at: Instant::now(),
4154        flow: Arc::new(if pending.busy_refusals {
4155            ChannelFlow::with_admission(pending.request_window, true, pending.watch_window)
4156        } else {
4157            ChannelFlow::new(pending.request_window)
4158        }),
4159        scope: pending.scope,
4160        caller_authority: pending.caller_authority,
4161        target_authority: pending.target_authority.pair().cloned(),
4162        accepts_close_reasons: pending.accepts_close_reasons,
4163    });
4164    inner
4165        .client_to_module
4166        .insert(reservation.client_key, Arc::clone(&binding));
4167    inner
4168        .module_to_client
4169        .insert(reservation.module_key, Arc::clone(&binding));
4170    let previous_published = inner
4171        .last_published_epoch
4172        .insert(reservation.client_key, reservation.client_epoch);
4173
4174    // This point is reachable only through an accepted route.bind acknowledgement.
4175    // Publish history before notifying the opener, so a subsequent
4176    // `scope.served_by` query sees the acknowledged bind.
4177    if let Some(scope) = &binding.scope {
4178        scope.record_served(&binding.module_id, module.verified_prefix_owner.as_deref());
4179    }
4180
4181    // Sending through the reserved slot cannot fail, but it reports a receiver
4182    // that closed after reservation and before this locked publication point.
4183    // The frame is stamped and charged to the queued-byte count here, when it
4184    // actually enters the queue, not when the slot was reserved.
4185    let client_writer_closed = pending.client_permit.send(pending.route_open_frame);
4186    if client_writer_closed {
4187        // Not gated on `relay_enqueued`: only a module ack reaches this
4188        // commit, which proves the module holds the binding, and `route.open`
4189        // sends nothing for a relay a reply already settled.
4190        let abandoned = abandoned_route_target(inner, &reservation);
4191        if let Some(route) = inner.client_to_module.remove(&reservation.client_key) {
4192            route.flow.close();
4193        }
4194        inner.module_to_client.remove(&reservation.module_key);
4195        inner
4196            .status
4197            .remove(&(reservation.client_key, reservation.client_epoch));
4198        match previous_published {
4199            Some(epoch) => {
4200                inner
4201                    .last_published_epoch
4202                    .insert(reservation.client_key, epoch);
4203            }
4204            None => {
4205                inner.last_published_epoch.remove(&reservation.client_key);
4206            }
4207        }
4208        let _ = pending.sender.send(RouteBindRelayOutcome::ModuleGone(
4209            "client egress closed during route publication".to_string(),
4210        ));
4211        return Ok(abandoned);
4212    }
4213
4214    if let Some(scope) = &binding.scope {
4215        scope.commit_presence();
4216    }
4217    let _ = pending.sender.send(RouteBindRelayOutcome::Accepted);
4218    Ok(None)
4219}
4220
4221/// What serves `module`'s endpoint: a prefix child, a supervised spawn's own
4222/// registration at its published epoch, or an unsupervised registration.
4223fn target_authority_locked(inner: &ForwardingInner, module: &ModuleConnection) -> TargetAuthority {
4224    if module.verified_prefix_owner.is_some() {
4225        return TargetAuthority::Prefix;
4226    }
4227    match inner
4228        .target_authority_by_connection
4229        .get(&module.endpoint.connection_id)
4230    {
4231        Some(record) => {
4232            let key = record.key();
4233            TargetAuthority::Spawn(
4234                AuthorityPair {
4235                    epoch: published_epoch_locked(inner, &key),
4236                    key: key.clone(),
4237                },
4238                inner
4239                    .spawn_approved
4240                    .get(&record.key())
4241                    .cloned()
4242                    .unwrap_or_else(ApprovedDeclarations::none_approved),
4243                inner.spawn_marks.contains(&key),
4244            )
4245        }
4246        None => TargetAuthority::Unsupervised,
4247    }
4248}
4249
4250fn published_epoch_locked(inner: &ForwardingInner, key: &SpawnKey) -> u64 {
4251    inner
4252        .spawn_epochs
4253        .get(key)
4254        .copied()
4255        .unwrap_or(crate::spawn_authority::INITIAL_AUTHORITY_EPOCH)
4256}
4257
4258/// Drop the epoch tombstones of `newest`'s module's earlier dead spawns that
4259/// no pending bind names any more. A tombstone a pending bind still names is
4260/// kept, so that bind is refused as stale at commit rather than compared with
4261/// the initial epoch an absent entry reads as. Live spawns are never in
4262/// `dead_spawns`, so their entries are never touched here.
4263fn prune_dead_spawn_tombstones_locked(inner: &mut ForwardingInner, newest: &SpawnKey) {
4264    let named = inner
4265        .pending_relays
4266        .values()
4267        .flat_map(|pending| {
4268            [
4269                pending.caller_authority.as_ref(),
4270                pending.target_authority.pair(),
4271            ]
4272        })
4273        .flatten()
4274        .map(|pair| pair.key.clone())
4275        .collect::<HashSet<_>>();
4276    let expired = inner
4277        .dead_spawns
4278        .iter()
4279        .filter(|dead| {
4280            dead.module_id == newest.module_id && *dead != newest && !named.contains(*dead)
4281        })
4282        .cloned()
4283        .collect::<Vec<_>>();
4284    for dead in expired {
4285        inner.dead_spawns.remove(&dead);
4286        inner.spawn_epochs.remove(&dead);
4287    }
4288}
4289
4290fn authority_stale_locked(inner: &ForwardingInner, pair: Option<&AuthorityPair>) -> bool {
4291    pair.is_some_and(|pair| published_epoch_locked(inner, &pair.key) > pair.epoch)
4292}
4293
4294fn spawn_connection_dead(inner: &ForwardingInner, connection: ConnectionId) -> bool {
4295    inner
4296        .spawn_connections
4297        .get(&connection)
4298        .is_some_and(|spawn| spawn.upgrade().is_none_or(|spawn| !spawn.launch_ready()))
4299}
4300
4301/// The live module connection registered under exactly `endpoint`, whichever
4302/// slot holds it: active, swap candidate, or superseded incumbent.
4303///
4304/// Resolving by endpoint rather than by module id is what keeps an incumbent
4305/// addressable after a swap promoted a candidate over its id. An endpoint that
4306/// is in none of the three (a stale one, replaced without a promotion) resolves
4307/// to nothing, as it always has.
4308fn module_connection_for_endpoint_locked(
4309    inner: &ForwardingInner,
4310    endpoint: ModuleEndpointId,
4311) -> Option<&ModuleConnection> {
4312    let module_id = inner.module_id_by_endpoint.get(&endpoint)?;
4313    inner
4314        .modules_by_id
4315        .get(module_id)
4316        .filter(|module| module.endpoint == endpoint)
4317        .or_else(|| {
4318            inner
4319                .candidates_by_id
4320                .get(module_id)
4321                .filter(|module| module.endpoint == endpoint)
4322        })
4323        .or_else(|| inner.superseded_endpoints.get(&endpoint))
4324}
4325
4326fn abandoned_route_target(
4327    inner: &ForwardingInner,
4328    reservation: &RouteReservation,
4329) -> Option<GoodbyeTarget> {
4330    let module_id = inner
4331        .module_id_by_endpoint
4332        .get(&reservation.module_key.endpoint)?;
4333    let module = module_connection_for_endpoint_locked(inner, reservation.module_key.endpoint)?;
4334    (module.endpoint == reservation.module_key.endpoint).then(|| GoodbyeTarget {
4335        connection_id: module.endpoint.connection_id,
4336        sink: module.sink.clone(),
4337        negotiated_ver: module.negotiated_ver,
4338        channel: reservation.module_key.channel,
4339        epoch: reservation.module_epoch,
4340        kind: GoodbyeTargetKind::Module,
4341        module_id: Some(module_id.clone()),
4342    })
4343}
4344
4345/// Queue a registering module's HELLO_ACK on its sink. Called with the
4346/// forwarding write lock held, before the endpoint is inserted, which is what
4347/// puts the ack ahead of any `route.bind` or control RPC routed to the module.
4348/// `try_send` never waits, so holding the lock across it is safe.
4349fn enqueue_hello_ack_locked(
4350    sink: &FrameSink,
4351    connection_id: ConnectionId,
4352    hello_ack: Option<Frame>,
4353) -> Result<(), ForwardingError> {
4354    let Some(hello_ack) = hello_ack else {
4355        return Ok(());
4356    };
4357    sink.try_send(hello_ack)
4358        .map_err(|_| ForwardingError::ModuleEgressUnavailable { connection_id })
4359}
4360
4361fn remove_module_connection_locked(
4362    inner: &mut ForwardingInner,
4363    endpoint: ModuleEndpointId,
4364) -> ConnectionCleanup {
4365    // Commit the disconnect before its routes are released, so their teardown
4366    // cannot replace module_closed with route_closed.
4367    inner.operator_confirms.module_closed(endpoint);
4368    inner.draining_endpoints.remove(&endpoint);
4369    inner.daemon_drain_results.remove(&endpoint);
4370    let module_id = inner.module_id_by_endpoint.remove(&endpoint);
4371    if let Some(module_id) = module_id.as_ref() {
4372        if inner
4373            .modules_by_id
4374            .get(module_id)
4375            .is_some_and(|module| module.endpoint == endpoint)
4376        {
4377            inner.modules_by_id.remove(module_id);
4378        }
4379        if inner
4380            .candidates_by_id
4381            .get(module_id)
4382            .is_some_and(|module| module.endpoint == endpoint)
4383        {
4384            inner.candidates_by_id.remove(module_id);
4385        }
4386    }
4387    inner.superseded_endpoints.remove(&endpoint);
4388    inner.endpoint_by_connection.remove(&endpoint.connection_id);
4389    inner.next_module_channel.remove(&endpoint);
4390    inner.next_control_corr.remove(&endpoint);
4391    inner
4392        .health_probe_tombstones
4393        .retain(|(pending_endpoint, _), _| *pending_endpoint != endpoint);
4394    inner
4395        .module_slot_epochs
4396        .retain(|key, _| key.endpoint != endpoint);
4397    let reserved_module_keys: Vec<ModuleRouteKey> = inner
4398        .reserved_module
4399        .keys()
4400        .filter(|module_key| module_key.endpoint == endpoint)
4401        .copied()
4402        .collect();
4403    for module_key in reserved_module_keys {
4404        if let Some(client_key) = inner.reserved_module.get(&module_key).copied() {
4405            release_reserved_route_locked(inner, client_key, module_key);
4406        }
4407    }
4408
4409    let pending_keys: Vec<_> = inner
4410        .pending_relays
4411        .keys()
4412        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
4413        .copied()
4414        .collect();
4415    let pending: Vec<_> = pending_keys
4416        .into_iter()
4417        .filter_map(|key| {
4418            inner
4419                .pending_relays
4420                .remove(&key)
4421                .map(|pending| (key, pending))
4422        })
4423        .collect();
4424    let abandoned_relays = u32::try_from(pending.len()).unwrap_or(u32::MAX);
4425    for (key, pending) in pending {
4426        // Recorded like every other removal without a reply, so a `route.open`
4427        // still sending the bind learns the relay was released rather than
4428        // settled by the module.
4429        if !pending.relay_enqueued {
4430            inner.relays_released_before_enqueue.insert(key);
4431        }
4432        let module_label = module_id.as_deref().unwrap_or("unknown");
4433        let _ = pending
4434            .sender
4435            .send(RouteBindRelayOutcome::ModuleGone(format!(
4436                "module '{module_label}' connection closed during route.bind relay"
4437            )));
4438    }
4439
4440    let pending_control_keys: Vec<_> = inner
4441        .pending_control_rpcs
4442        .keys()
4443        .filter(|(pending_endpoint, _)| *pending_endpoint == endpoint)
4444        .copied()
4445        .collect();
4446    let pending_control: Vec<_> = pending_control_keys
4447        .into_iter()
4448        .filter_map(|key| inner.pending_control_rpcs.remove(&key))
4449        .collect();
4450    for pending in pending_control {
4451        let module_label = module_id.as_deref().unwrap_or("unknown");
4452        let _ = pending
4453            .sender
4454            .send(ModuleControlRpcOutcome::ModuleGone(format!(
4455                "module '{module_label}' connection closed during module-control RPC"
4456            )));
4457    }
4458
4459    let module_routes = inner
4460        .module_to_client
4461        .iter()
4462        .filter(|(module_key, _)| module_key.endpoint == endpoint)
4463        .map(|(module_key, route)| (*module_key, route.module_epoch))
4464        .collect::<Vec<_>>();
4465    let mut released = Vec::with_capacity(module_routes.len());
4466    for (module_key, epoch) in module_routes {
4467        if let RouteRelease::Removed(target) = release_module_route_locked(inner, module_key, epoch)
4468        {
4469            released.push(target);
4470        }
4471    }
4472    ConnectionCleanup {
4473        released,
4474        abandoned_relays,
4475    }
4476}
4477
4478#[derive(Debug, Clone, Copy)]
4479struct RequestCredit {
4480    subscription: bool,
4481    watch_credit: bool,
4482    excluded_from_drain: bool,
4483}
4484
4485#[derive(Debug, Default)]
4486struct CreditLedger {
4487    by_corr: HashMap<u64, Vec<RequestCredit>>,
4488}
4489
4490impl CreditLedger {
4491    fn acquire(&mut self, corr: u64, subscription: bool, watch_credit: bool) {
4492        self.by_corr.entry(corr).or_default().push(RequestCredit {
4493            subscription,
4494            watch_credit,
4495            excluded_from_drain: false,
4496        });
4497    }
4498
4499    fn release(&mut self, corr: u64) -> Option<RequestCredit> {
4500        let credits = self.by_corr.get_mut(&corr)?;
4501        let released = credits.pop();
4502        if credits.is_empty() {
4503            self.by_corr.remove(&corr);
4504        }
4505        released
4506    }
4507
4508    fn capture_subscription_exclusions(&mut self) -> u32 {
4509        let mut excluded = 0u32;
4510        for credit in self.by_corr.values_mut().flatten() {
4511            if credit.subscription && !credit.excluded_from_drain {
4512                credit.excluded_from_drain = true;
4513                excluded = excluded.saturating_add(1);
4514            }
4515        }
4516        excluded
4517    }
4518
4519    fn in_flight(&self) -> usize {
4520        self.by_corr.values().map(Vec::len).sum()
4521    }
4522
4523    fn drain_in_flight(&self) -> usize {
4524        self.by_corr
4525            .values()
4526            .flatten()
4527            .filter(|credit| !credit.excluded_from_drain)
4528            .count()
4529    }
4530
4531    /// The correlation ids of the requests a drain still waits on, one entry
4532    /// per held credit, in ascending order.
4533    fn drain_held_corrs(&self) -> Vec<u64> {
4534        let mut corrs = self
4535            .by_corr
4536            .iter()
4537            .flat_map(|(corr, credits)| {
4538                credits
4539                    .iter()
4540                    .filter(|credit| !credit.excluded_from_drain)
4541                    .map(move |_| *corr)
4542            })
4543            .collect::<Vec<_>>();
4544        corrs.sort_unstable();
4545        corrs
4546    }
4547}
4548
4549#[derive(Debug, Default)]
4550struct ChannelFlowState {
4551    closed: bool,
4552    credits: CreditLedger,
4553    last_busy_warn: Option<tokio::time::Instant>,
4554}
4555
4556/// Per-channel request-credit accounting shared by the client and module route halves.
4557#[derive(Debug)]
4558pub(crate) struct ChannelFlow {
4559    sem: Semaphore,
4560    window: usize,
4561    busy_refusals: bool,
4562    watch_sem: Option<Semaphore>,
4563    watch_window: Option<usize>,
4564    state: Mutex<ChannelFlowState>,
4565}
4566
4567impl ChannelFlow {
4568    pub(crate) fn new(window: usize) -> Self {
4569        Self::with_admission(window, false, None)
4570    }
4571
4572    fn with_admission(window: usize, busy_refusals: bool, watch_window: Option<usize>) -> Self {
4573        debug_assert!(window > 0, "flow-control window must be non-zero");
4574        Self {
4575            sem: Semaphore::new(window),
4576            window,
4577            busy_refusals,
4578            watch_sem: watch_window.map(Semaphore::new),
4579            watch_window,
4580            state: Mutex::new(ChannelFlowState::default()),
4581        }
4582    }
4583
4584    pub(crate) fn busy_refusals(&self) -> bool {
4585        self.busy_refusals
4586    }
4587
4588    fn try_acquire_tagged(&self, corr: u64, subscription: bool) -> RequestAdmission {
4589        let mut state = self
4590            .state
4591            .lock()
4592            .unwrap_or_else(|poisoned| poisoned.into_inner());
4593        if state.closed {
4594            return RequestAdmission::Stale;
4595        }
4596        let busy = |state: &ChannelFlowState| {
4597            RequestAdmission::Busy(subc_protocol::RouteBusyDetail {
4598                reason: "no_credit".into(),
4599                request_in_flight: state.credits.in_flight(),
4600                watch_in_flight: state
4601                    .credits
4602                    .by_corr
4603                    .values()
4604                    .flatten()
4605                    .filter(|credit| credit.subscription)
4606                    .count(),
4607                request_window: self.window,
4608                watch_window: self.watch_window,
4609            })
4610        };
4611        let Ok(total) = self.sem.try_acquire() else {
4612            return busy(&state);
4613        };
4614        // A watch takes the total credit first. Dropping it on watch-lane
4615        // exhaustion rolls back before any ledger entry becomes visible.
4616        let watch = if subscription {
4617            if let Some(sem) = &self.watch_sem {
4618                match sem.try_acquire() {
4619                    Ok(permit) => Some(permit),
4620                    Err(_) => return busy(&state),
4621                }
4622            } else {
4623                None
4624            }
4625        } else {
4626            None
4627        };
4628        state.credits.acquire(corr, subscription, watch.is_some());
4629        total.forget();
4630        if let Some(watch) = watch {
4631            watch.forget();
4632        }
4633        RequestAdmission::Admitted
4634    }
4635
4636    pub(crate) fn should_warn_busy(&self) -> bool {
4637        let mut state = self
4638            .state
4639            .lock()
4640            .unwrap_or_else(|poisoned| poisoned.into_inner());
4641        let now = tokio::time::Instant::now();
4642        if state
4643            .last_busy_warn
4644            .is_some_and(|last| now.duration_since(last) < std::time::Duration::from_secs(60))
4645        {
4646            return false;
4647        }
4648        state.last_busy_warn = Some(now);
4649        true
4650    }
4651
4652    #[cfg(test)]
4653    pub(crate) async fn acquire(&self) -> Result<(), ChannelFlowClosed> {
4654        self.acquire_tagged(0, false).await
4655    }
4656
4657    pub(crate) async fn acquire_tagged(
4658        &self,
4659        corr: u64,
4660        subscription: bool,
4661    ) -> Result<(), ChannelFlowClosed> {
4662        let permit = self.sem.acquire().await.map_err(|_| ChannelFlowClosed)?;
4663        let mut state = self
4664            .state
4665            .lock()
4666            .unwrap_or_else(|poisoned| poisoned.into_inner());
4667        if state.closed {
4668            return Err(ChannelFlowClosed);
4669        }
4670        state.credits.acquire(corr, subscription, false);
4671        permit.forget();
4672        Ok(())
4673    }
4674
4675    #[cfg(test)]
4676    pub(crate) fn release(&self) {
4677        self.release_corr(0);
4678    }
4679
4680    pub(crate) fn release_corr(&self, corr: u64) {
4681        let mut state = self
4682            .state
4683            .lock()
4684            .unwrap_or_else(|poisoned| poisoned.into_inner());
4685        let Some(released) = state.credits.release(corr) else {
4686            // Protocol-conforming modules emit exactly one terminal per request.
4687            // This guard is a best-effort safety net against window growth, not a
4688            // security boundary against malicious peers.
4689            warn!(
4690                window = self.window,
4691                available = self.sem.available_permits(),
4692                "flow-control over-release ignored"
4693            );
4694            return;
4695        };
4696        if !self.sem.is_closed() {
4697            self.sem.add_permits(1);
4698        }
4699        if released.watch_credit {
4700            if let Some(watch_sem) = self.watch_sem.as_ref().filter(|sem| !sem.is_closed()) {
4701                watch_sem.add_permits(1);
4702            }
4703        }
4704    }
4705
4706    #[cfg(test)]
4707    pub(crate) fn in_flight(&self) -> usize {
4708        self.state
4709            .lock()
4710            .unwrap_or_else(|poisoned| poisoned.into_inner())
4711            .credits
4712            .in_flight()
4713    }
4714
4715    pub(crate) fn drain_in_flight(&self) -> usize {
4716        self.state
4717            .lock()
4718            .unwrap_or_else(|poisoned| poisoned.into_inner())
4719            .credits
4720            .drain_in_flight()
4721    }
4722
4723    pub(crate) fn drain_held_corrs(&self) -> Vec<u64> {
4724        self.state
4725            .lock()
4726            .unwrap_or_else(|poisoned| poisoned.into_inner())
4727            .credits
4728            .drain_held_corrs()
4729    }
4730
4731    #[cfg(test)]
4732    pub(crate) fn available_permits(&self) -> usize {
4733        self.sem.available_permits()
4734    }
4735
4736    pub(crate) fn begin_drain(&self) -> u32 {
4737        let mut state = self
4738            .state
4739            .lock()
4740            .unwrap_or_else(|poisoned| poisoned.into_inner());
4741        state.closed = true;
4742        self.sem.close();
4743        if let Some(sem) = &self.watch_sem {
4744            sem.close();
4745        }
4746        state.credits.capture_subscription_exclusions()
4747    }
4748
4749    pub(crate) fn close(&self) {
4750        let mut state = self
4751            .state
4752            .lock()
4753            .unwrap_or_else(|poisoned| poisoned.into_inner());
4754        state.closed = true;
4755        self.sem.close();
4756        if let Some(sem) = &self.watch_sem {
4757            sem.close();
4758        }
4759    }
4760}
4761
4762#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4763pub(crate) struct ChannelFlowClosed;
4764
4765#[derive(Debug, Clone, PartialEq, Eq)]
4766pub(crate) enum RequestAdmission {
4767    Admitted,
4768    Busy(subc_protocol::RouteBusyDetail),
4769    ModuleReloading,
4770    Stale,
4771}
4772
4773impl fmt::Display for ChannelFlowClosed {
4774    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4775        write!(f, "flow-control window closed")
4776    }
4777}
4778
4779impl Error for ChannelFlowClosed {}
4780
4781#[cfg(test)]
4782#[path = "route_busy_flow_tests.rs"]
4783mod route_busy_flow_tests;
4784
4785fn window_for(concurrency: &Concurrency) -> usize {
4786    match concurrency {
4787        Concurrency::Serial => 1,
4788        Concurrency::ModuleManaged => DEFAULT_MODULE_MANAGED_WINDOW,
4789        Concurrency::StatelessParallel => STATELESS_PARALLEL_WINDOW,
4790    }
4791}
4792
4793#[derive(Debug, Clone, PartialEq, Eq)]
4794pub enum ForwardingError {
4795    /// A connection cannot hold both client routes and a module endpoint, or
4796    /// replace a module identity while that endpoint is still registered.
4797    ConnectionRoleConflict {
4798        connection_id: ConnectionId,
4799    },
4800    NoModuleConnection,
4801    ModuleReloading {
4802        module_id: String,
4803    },
4804    StaleModuleEndpoint,
4805    UnknownReservation {
4806        client_channel: u16,
4807        module_channel: u16,
4808    },
4809    ClientRouteChannelExhausted {
4810        connection_id: ConnectionId,
4811    },
4812    ModuleRouteChannelExhausted {
4813        endpoint: ModuleEndpointId,
4814    },
4815    RelayCorrelationExhausted,
4816    ConnectionClosing {
4817        connection_id: ConnectionId,
4818    },
4819    ClientEgressClosed {
4820        connection_id: ConnectionId,
4821    },
4822    RouteOpenBuild(String),
4823    /// A swap candidate is already registered for this module id.
4824    CandidateSlotOccupied {
4825        module_id: String,
4826    },
4827    /// A registering module's HELLO_ACK could not be queued because its
4828    /// outbound queue is closed or full.
4829    ModuleEgressUnavailable {
4830        connection_id: ConnectionId,
4831    },
4832    Poisoned,
4833}
4834
4835impl fmt::Display for ForwardingError {
4836    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4837        match self {
4838            Self::ConnectionRoleConflict { connection_id } => write!(f, "connection {} already holds an incompatible role", connection_id.get()),
4839            Self::NoModuleConnection => write!(f, "no module connection is registered"),
4840            Self::ModuleReloading { module_id } => {
4841                write!(f, "module_id '{module_id}' is reloading")
4842            }
4843            Self::StaleModuleEndpoint => write!(f, "module connection generation is stale"),
4844            Self::UnknownReservation {
4845                client_channel,
4846                module_channel,
4847            } => write!(
4848                f,
4849                "route reservation client channel {client_channel} / module channel {module_channel} was not found"
4850            ),
4851            Self::ClientRouteChannelExhausted { connection_id } => write!(
4852                f,
4853                "no client route channels are available for connection {}",
4854                connection_id.get()
4855            ),
4856            Self::ModuleRouteChannelExhausted { endpoint } => write!(
4857                f,
4858                "no module route channels are available for endpoint generation {} on connection {}",
4859                endpoint.generation,
4860                endpoint.connection_id.get()
4861            ),
4862            Self::RelayCorrelationExhausted => {
4863                write!(f, "module control correlation ids are exhausted")
4864            }
4865            Self::ConnectionClosing { connection_id } => write!(
4866                f,
4867                "connection {} is closing and cannot accept route allocation",
4868                connection_id.get()
4869            ),
4870            Self::ClientEgressClosed { connection_id } => write!(
4871                f,
4872                "client connection {} egress is closed",
4873                connection_id.get()
4874            ),
4875            Self::RouteOpenBuild(message) => {
4876                write!(f, "failed to prebuild route.open response: {message}")
4877            }
4878            Self::CandidateSlotOccupied { module_id } => write!(
4879                f,
4880                "module_id '{module_id}' already has a swap candidate registered"
4881            ),
4882            Self::ModuleEgressUnavailable { connection_id } => write!(
4883                f,
4884                "module connection {} egress is unavailable; HELLO_ACK could not be queued",
4885                connection_id.get()
4886            ),
4887            Self::Poisoned => write!(f, "forwarding table lock was poisoned"),
4888        }
4889    }
4890}
4891
4892impl Error for ForwardingError {}
4893
4894#[cfg(test)]
4895mod tests {
4896    use std::time::Duration;
4897
4898    use super::*;
4899    use tokio::sync::mpsc;
4900
4901    #[test]
4902    fn ordinary_long_running_request_is_not_excluded_from_drain() {
4903        let mut ledger = CreditLedger::default();
4904        ledger.acquire(1, false, false);
4905
4906        assert_eq!(ledger.capture_subscription_exclusions(), 0);
4907        assert_eq!(ledger.drain_in_flight(), 1);
4908    }
4909
4910    #[test]
4911    fn bit_set_subscription_is_excluded_and_counted() {
4912        let mut ledger = CreditLedger::default();
4913        ledger.acquire(1, true, false);
4914
4915        assert_eq!(ledger.capture_subscription_exclusions(), 1);
4916        assert_eq!(ledger.drain_in_flight(), 0);
4917    }
4918
4919    #[test]
4920    fn subscription_opened_after_drain_snapshot_is_not_excluded() {
4921        let mut ledger = CreditLedger::default();
4922        ledger.acquire(1, true, false);
4923        assert_eq!(ledger.capture_subscription_exclusions(), 1);
4924
4925        ledger.acquire(2, true, false);
4926
4927        assert_eq!(ledger.drain_in_flight(), 1);
4928    }
4929
4930    #[test]
4931    fn drain_with_no_subscriptions_reports_zero_excluded() {
4932        let mut ledger = CreditLedger::default();
4933        assert_eq!(ledger.capture_subscription_exclusions(), 0);
4934    }
4935
4936    fn test_hello_ack(corr: u64) -> Frame {
4937        Frame::build(
4938            FrameType::HelloAck,
4939            Flags::new(false, Priority::Passive, false),
4940            0,
4941            0,
4942            corr,
4943            Vec::new(),
4944        )
4945        .unwrap()
4946    }
4947
4948    /// An acked registration whose HELLO_ACK cannot be queued must leave no
4949    /// endpoint behind: a routable module that never got its ack would read a
4950    /// route.bind first and exit. A closed queue and a full one both refuse.
4951    #[test]
4952    fn acked_registration_that_cannot_queue_its_hello_ack_inserts_nothing() {
4953        let forwarding = ForwardingTable::default();
4954
4955        let (closed_tx, closed_rx) = mpsc::channel(8);
4956        drop(closed_rx);
4957        let closed = ConnectionId::new(1);
4958        assert_eq!(
4959            forwarding.register_module_connection_acked(
4960                closed,
4961                "closed".to_string(),
4962                2,
4963                Concurrency::ModuleManaged,
4964                FrameSink::new(closed_tx),
4965                ModuleHelloAck {
4966                    frame: test_hello_ack(1),
4967                    verified_prefix_owner: None
4968                },
4969            ),
4970            Err(ForwardingError::ModuleEgressUnavailable {
4971                connection_id: closed
4972            })
4973        );
4974
4975        let (full_tx, _full_rx) = mpsc::channel(1);
4976        let full_sink = FrameSink::new(full_tx);
4977        full_sink.try_send(test_hello_ack(99)).unwrap();
4978        let full = ConnectionId::new(2);
4979        assert_eq!(
4980            forwarding.register_module_connection_acked(
4981                full,
4982                "full".to_string(),
4983                2,
4984                Concurrency::ModuleManaged,
4985                full_sink.clone(),
4986                ModuleHelloAck {
4987                    frame: test_hello_ack(2),
4988                    verified_prefix_owner: None
4989                },
4990            ),
4991            Err(ForwardingError::ModuleEgressUnavailable {
4992                connection_id: full
4993            })
4994        );
4995        assert_eq!(
4996            forwarding.register_candidate_module_connection_acked(
4997                full,
4998                "full".to_string(),
4999                2,
5000                Concurrency::ModuleManaged,
5001                full_sink,
5002                test_hello_ack(3),
5003            ),
5004            Err(ForwardingError::ModuleEgressUnavailable {
5005                connection_id: full
5006            })
5007        );
5008
5009        for (connection, module_id) in [(closed, "closed"), (full, "full")] {
5010            assert_eq!(
5011                forwarding
5012                    .module_endpoint_for_connection(connection)
5013                    .unwrap(),
5014                None
5015            );
5016            let (client_tx, _client_rx) = mpsc::channel(8);
5017            assert_eq!(
5018                forwarding
5019                    .begin_route_bind_relay_for_test(
5020                        ConnectionId::new(50),
5021                        FrameSink::new(client_tx),
5022                        1,
5023                        module_id,
5024                    )
5025                    .err(),
5026                Some(ForwardingError::NoModuleConnection)
5027            );
5028        }
5029        assert!(forwarding.read_inner().unwrap().candidates_by_id.is_empty());
5030    }
5031
5032    /// Both acked registration forms put the HELLO_ACK on the module's queue
5033    /// by the time the endpoint can be resolved.
5034    #[test]
5035    fn acked_registration_queues_the_hello_ack_first() {
5036        let forwarding = ForwardingTable::default();
5037        let (active_tx, mut active_rx) = mpsc::channel(8);
5038        forwarding
5039            .register_module_connection_acked(
5040                ConnectionId::new(1),
5041                "acked".to_string(),
5042                2,
5043                Concurrency::ModuleManaged,
5044                FrameSink::new(active_tx),
5045                ModuleHelloAck {
5046                    frame: test_hello_ack(11),
5047                    verified_prefix_owner: None,
5048                },
5049            )
5050            .unwrap();
5051        let (candidate_tx, mut candidate_rx) = mpsc::channel(8);
5052        forwarding
5053            .register_candidate_module_connection_acked(
5054                ConnectionId::new(2),
5055                "acked".to_string(),
5056                2,
5057                Concurrency::ModuleManaged,
5058                FrameSink::new(candidate_tx),
5059                test_hello_ack(12),
5060            )
5061            .unwrap();
5062
5063        let active_first = active_rx.try_recv().unwrap().frame;
5064        assert_eq!(active_first.header.ty, FrameType::HelloAck);
5065        assert_eq!(active_first.header.corr, 11);
5066        let candidate_first = candidate_rx.try_recv().unwrap().frame;
5067        assert_eq!(candidate_first.header.ty, FrameType::HelloAck);
5068        assert_eq!(candidate_first.header.corr, 12);
5069    }
5070
5071    #[test]
5072    fn multi_provider_route_limit_reports_per_client_exhaustion_without_affecting_second_client() {
5073        let forwarding = ForwardingTable::default();
5074        let module_connection = ConnectionId::new(10);
5075        let exhausted_client = ConnectionId::new(20);
5076        let second_client = ConnectionId::new(30);
5077        let (module_tx, _module_rx) = mpsc::channel(1);
5078        let endpoint = forwarding
5079            .register_module_connection(
5080                module_connection,
5081                "route-limit-provider".to_string(),
5082                1,
5083                Concurrency::ModuleManaged,
5084                FrameSink::new(module_tx),
5085            )
5086            .unwrap();
5087
5088        {
5089            let mut inner = forwarding.inner.write().unwrap();
5090            for channel in 1..=u16::MAX {
5091                inner.reserved_client.insert(
5092                    ClientRouteKey {
5093                        connection_id: exhausted_client,
5094                        channel,
5095                    },
5096                    ModuleRouteKey {
5097                        endpoint,
5098                        channel: 1,
5099                    },
5100                );
5101            }
5102        }
5103
5104        let (exhausted_tx, _exhausted_rx) = mpsc::channel(1);
5105        let err = forwarding
5106            .begin_route_bind_relay_for_test(
5107                exhausted_client,
5108                FrameSink::new(exhausted_tx),
5109                1,
5110                "route-limit-provider",
5111            )
5112            .unwrap_err();
5113        assert!(matches!(
5114            err,
5115            ForwardingError::ClientRouteChannelExhausted { connection_id }
5116                if connection_id == exhausted_client
5117        ));
5118
5119        let (second_tx, _second_rx) = mpsc::channel(1);
5120        let pending = forwarding
5121            .begin_route_bind_relay_for_test(
5122                second_client,
5123                FrameSink::new(second_tx),
5124                2,
5125                "route-limit-provider",
5126            )
5127            .unwrap();
5128        assert_eq!(pending.client_channel, 1);
5129    }
5130
5131    #[test]
5132    fn released_module_channels_are_reused_after_wrap_without_slot_leak() {
5133        let forwarding = ForwardingTable::default();
5134        let module_connection = ConnectionId::new(40);
5135        let client = ConnectionId::new(50);
5136        let (module_tx, _module_rx) = mpsc::channel(1);
5137        forwarding
5138            .register_module_connection(
5139                module_connection,
5140                "slot-reuse-provider".to_string(),
5141                1,
5142                Concurrency::ModuleManaged,
5143                FrameSink::new(module_tx),
5144            )
5145            .unwrap();
5146
5147        let (client_tx, _client_rx) = mpsc::channel(1);
5148        let client_sink = FrameSink::new(client_tx);
5149        let mut wrapped_channel = None;
5150        for index in 0..=usize::from(u16::MAX) {
5151            let pending = forwarding
5152                .begin_route_bind_relay_for_test(
5153                    client,
5154                    client_sink.clone(),
5155                    index as u64 + 1,
5156                    "slot-reuse-provider",
5157                )
5158                .unwrap();
5159            if index == usize::from(u16::MAX) {
5160                wrapped_channel = Some(pending.module_channel);
5161            }
5162            forwarding
5163                .abort_pending_relay(
5164                    pending.endpoint,
5165                    pending.corr,
5166                    RouteBindRelayOutcome::ModuleGone("test abort".to_string()),
5167                )
5168                .unwrap();
5169        }
5170
5171        assert_eq!(wrapped_channel, Some(1));
5172    }
5173
5174    #[test]
5175    fn cleanup_connection_prunes_stale_next_client_channel_cursor() {
5176        let forwarding = ForwardingTable::default();
5177        let client = ConnectionId::new(60);
5178        forwarding
5179            .inner
5180            .write()
5181            .unwrap()
5182            .next_client_channel
5183            .insert(client, 41);
5184
5185        let released = forwarding.cleanup_connection(client).unwrap();
5186
5187        assert!(released.is_empty());
5188        assert!(!forwarding
5189            .inner
5190            .read()
5191            .unwrap()
5192            .next_client_channel
5193            .contains_key(&client));
5194    }
5195
5196    #[test]
5197    fn stale_module_cleanup_preserves_fast_reconnect_successor() {
5198        let forwarding = ForwardingTable::default();
5199        let module_id = "fast-reconnect-provider";
5200        let first_connection = ConnectionId::new(70);
5201        let second_connection = ConnectionId::new(80);
5202        let (first_tx, _first_rx) = mpsc::channel(1);
5203        let first_endpoint = forwarding
5204            .register_module_connection(
5205                first_connection,
5206                module_id.to_string(),
5207                1,
5208                Concurrency::ModuleManaged,
5209                FrameSink::new(first_tx),
5210            )
5211            .unwrap();
5212        let (second_tx, _second_rx) = mpsc::channel(1);
5213        let second_endpoint = forwarding
5214            .register_module_connection(
5215                second_connection,
5216                module_id.to_string(),
5217                1,
5218                Concurrency::ModuleManaged,
5219                FrameSink::new(second_tx),
5220            )
5221            .unwrap();
5222        assert_ne!(first_endpoint, second_endpoint);
5223
5224        let released = forwarding.cleanup_connection(first_connection).unwrap();
5225
5226        assert!(released.is_empty());
5227        assert_eq!(
5228            forwarding
5229                .inner
5230                .read()
5231                .unwrap()
5232                .modules_by_id
5233                .get(module_id)
5234                .map(|module| module.endpoint),
5235            Some(second_endpoint)
5236        );
5237        assert!(forwarding.has_live_module_connection(module_id).unwrap());
5238        let control_rpc = forwarding
5239            .begin_module_control_rpc_for(
5240                module_id,
5241                "health.check",
5242                Instant::now() + Duration::from_secs(1),
5243            )
5244            .unwrap();
5245        assert_eq!(control_rpc.endpoint, second_endpoint);
5246    }
5247
5248    fn route_fixture(
5249        module_id: &str,
5250    ) -> (
5251        ForwardingTable,
5252        ConnectionId,
5253        ModuleEndpointId,
5254        ConnectionId,
5255        FrameSink,
5256        mpsc::Receiver<crate::router::OutboundFrame>,
5257    ) {
5258        let forwarding = ForwardingTable::default();
5259        let module_connection = ConnectionId::new(100);
5260        let client_connection = ConnectionId::new(200);
5261        let (module_tx, _module_rx) = mpsc::channel(8);
5262        let endpoint = forwarding
5263            .register_module_connection(
5264                module_connection,
5265                module_id.to_string(),
5266                2,
5267                Concurrency::ModuleManaged,
5268                FrameSink::new(module_tx),
5269            )
5270            .unwrap();
5271        let (client_tx, client_rx) = mpsc::channel(8);
5272        (
5273            forwarding,
5274            module_connection,
5275            endpoint,
5276            client_connection,
5277            FrameSink::new(client_tx),
5278            client_rx,
5279        )
5280    }
5281
5282    #[test]
5283    #[cfg(unix)]
5284    fn daemon_drain_gates_current_and_racing_provider_registrations() {
5285        let (forwarding, _, endpoint, _, sink, _) = route_fixture("provider");
5286        assert_eq!(
5287            forwarding.begin_daemon_drain().unwrap(),
5288            [("provider".to_string(), endpoint)]
5289        );
5290        assert!(forwarding.endpoint_is_draining(endpoint).unwrap());
5291        assert!(matches!(
5292            forwarding.register_module_connection(
5293                ConnectionId::new(300),
5294                "late-provider".into(),
5295                2,
5296                Concurrency::ModuleManaged,
5297                sink,
5298            ),
5299            Err(ForwardingError::ConnectionClosing { .. })
5300        ));
5301    }
5302
5303    #[test]
5304    #[cfg(unix)]
5305    fn daemon_drain_results_are_endpoint_fenced_and_removed_on_cleanup() {
5306        let (forwarding, connection, endpoint, _, _, _) = route_fixture("provider");
5307        assert_eq!(
5308            forwarding
5309                .daemon_drain_result_for_connection(connection)
5310                .unwrap(),
5311            None
5312        );
5313        forwarding
5314            .record_daemon_drain_result(endpoint, true)
5315            .unwrap();
5316        forwarding.begin_daemon_drain().unwrap();
5317        assert_eq!(
5318            forwarding
5319                .daemon_drain_result_for_connection(connection)
5320                .unwrap(),
5321            Some(false)
5322        );
5323        let replacement = ModuleEndpointId {
5324            generation: endpoint.generation + 1,
5325            ..endpoint
5326        };
5327        forwarding
5328            .record_daemon_drain_result(replacement, true)
5329            .unwrap();
5330        assert_eq!(
5331            forwarding
5332                .daemon_drain_result_for_connection(connection)
5333                .unwrap(),
5334            Some(false)
5335        );
5336        forwarding
5337            .record_daemon_drain_result(endpoint, true)
5338            .unwrap();
5339        assert_eq!(
5340            forwarding
5341                .daemon_drain_result_for_connection(connection)
5342                .unwrap(),
5343            Some(true)
5344        );
5345        assert_eq!(
5346            forwarding
5347                .daemon_drain_result_for_connection(ConnectionId::new(300))
5348                .unwrap(),
5349            Some(false)
5350        );
5351        forwarding.cleanup_connection(connection).unwrap();
5352        forwarding
5353            .record_daemon_drain_result(endpoint, true)
5354            .unwrap();
5355        assert_eq!(
5356            forwarding
5357                .daemon_drain_result_for_connection(connection)
5358                .unwrap(),
5359            Some(false)
5360        );
5361        assert!(forwarding
5362            .read_inner()
5363            .unwrap()
5364            .daemon_drain_results
5365            .is_empty());
5366    }
5367
5368    #[test]
5369    #[cfg(unix)]
5370    fn late_bind_ack_during_daemon_drain_settles_without_leaking_or_closing_module() {
5371        let (forwarding, module_connection, endpoint, client_connection, sink, mut rx) =
5372            route_fixture("provider");
5373        let mut pending =
5374            begin_test_route(&forwarding, client_connection, sink.clone(), 1, "provider");
5375        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
5376        forwarding.begin_daemon_drain().unwrap();
5377        let completion = forwarding
5378            .complete_pending_relay(
5379                module_connection,
5380                pending.corr,
5381                RouteBindRelayOutcome::Accepted,
5382            )
5383            .expect("draining admission is not a fatal module error");
5384        assert!(completion.settled);
5385        let abandoned = completion
5386            .abandoned
5387            .expect("late accepted bind needs a route GOODBYE");
5388        assert_eq!(abandoned.channel, pending.module_channel);
5389        assert!(matches!(pending.receiver.try_recv().unwrap(),
5390            RouteBindRelayOutcome::DaemonRejected(error) if error.code == "module_reloading"));
5391        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5392        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5393        assert!(
5394            rx.try_recv().is_err(),
5395            "no route.open may be published during drain"
5396        );
5397        assert_eq!(
5398            forwarding
5399                .module_endpoint_for_connection(module_connection)
5400                .unwrap(),
5401            Some(endpoint)
5402        );
5403        assert!(
5404            !forwarding
5405                .complete_pending_relay(
5406                    module_connection,
5407                    pending.corr,
5408                    RouteBindRelayOutcome::Accepted,
5409                )
5410                .unwrap()
5411                .settled
5412        );
5413    }
5414
5415    fn test_ping(corr: u64) -> Frame {
5416        Frame::build(
5417            FrameType::Ping,
5418            Flags::new(false, Priority::Passive, false),
5419            0,
5420            0,
5421            corr,
5422            Vec::new(),
5423        )
5424        .unwrap()
5425    }
5426
5427    fn begin_test_route(
5428        forwarding: &ForwardingTable,
5429        client_connection: ConnectionId,
5430        client_sink: FrameSink,
5431        corr: u64,
5432        module_id: &str,
5433    ) -> PendingRouteBindRelay {
5434        forwarding
5435            .begin_route_bind_relay_for_test(client_connection, client_sink, corr, module_id)
5436            .unwrap()
5437    }
5438
5439    #[test]
5440    fn module_registration_and_client_reservation_are_mutually_exclusive_in_both_orders() {
5441        for candidate in [false, true] {
5442            for register_first in [false, true] {
5443                let (forwarding, module_connection, _, client_connection, sink, _rx) =
5444                    route_fixture("target");
5445                let register = || {
5446                    if candidate {
5447                        forwarding.register_candidate_module_connection(
5448                            client_connection,
5449                            "source".into(),
5450                            2,
5451                            Concurrency::ModuleManaged,
5452                            sink.clone(),
5453                        )
5454                    } else {
5455                        forwarding.register_module_connection(
5456                            client_connection,
5457                            "source".into(),
5458                            2,
5459                            Concurrency::ModuleManaged,
5460                            sink.clone(),
5461                        )
5462                    }
5463                };
5464                if register_first {
5465                    register().unwrap();
5466                    assert!(
5467                        forwarding
5468                            .begin_route_bind_relay_for_test(
5469                                client_connection,
5470                                sink.clone(),
5471                                1,
5472                                "target",
5473                            )
5474                            .is_err(),
5475                        "a deferred route.open cannot reserve after HELLO"
5476                    );
5477                    assert!(forwarding
5478                        .cleanup_connection(client_connection)
5479                        .unwrap()
5480                        .is_empty());
5481                } else {
5482                    let pending =
5483                        begin_test_route(&forwarding, client_connection, sink.clone(), 1, "target");
5484                    assert!(
5485                        register().is_err(),
5486                        "HELLO cannot register after a route reservation"
5487                    );
5488                    forwarding
5489                        .complete_pending_relay(
5490                            module_connection,
5491                            pending.corr,
5492                            RouteBindRelayOutcome::Accepted,
5493                        )
5494                        .unwrap();
5495                    assert_eq!(
5496                        forwarding
5497                            .cleanup_connection(client_connection)
5498                            .unwrap()
5499                            .len(),
5500                        1
5501                    );
5502                }
5503                assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
5504                assert_eq!(forwarding.active_binding_count().unwrap(), 0);
5505            }
5506        }
5507    }
5508
5509    /// A pending route.open reserves its queue slot with `reserve_owned` and
5510    /// sends its prebuilt response through that slot when the module accepts.
5511    /// A queue already holding many data frames (far more than the old 64-frame
5512    /// queue) must not stop the open from reserving or completing, and the
5513    /// response must be charged to the queue's byte count when it is sent.
5514    #[tokio::test]
5515    async fn pending_route_open_completes_behind_queued_data_frames() {
5516        assert_eq!(
5517            crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
5518            8,
5519            "the per-connection pending route.open limit is its own constant"
5520        );
5521        let (forwarding, module_connection, _endpoint, client, _unused_sink, _unused_rx) =
5522            route_fixture("open-behind-data");
5523        let (sink, mut client_rx) = crate::server::connection_egress();
5524        const DATA_FRAMES: usize = 1_000;
5525        let data = |corr: u64| {
5526            Frame::build(
5527                FrameType::StreamData,
5528                Flags::new(false, Priority::Interactive, false),
5529                9,
5530                1,
5531                corr,
5532                vec![b'x'; 200],
5533            )
5534            .unwrap()
5535        };
5536        for corr in 0..DATA_FRAMES as u64 {
5537            sink.try_send(data(corr)).unwrap();
5538        }
5539        let data_bytes = DATA_FRAMES * (subc_protocol::HEADER_LEN + 200);
5540        assert_eq!(sink.backlog().queued_bytes, data_bytes);
5541
5542        let pending = tokio::time::timeout(
5543            Duration::from_secs(5),
5544            forwarding.begin_route_bind_relay_for(
5545                client,
5546                sink.clone(),
5547                subc_protocol::PROTOCOL_VERSION,
5548                4_242,
5549                "open-behind-data",
5550                Principal::Direct,
5551                None,
5552                None,
5553                "fixture".into(),
5554                None,
5555                None,
5556                RouteAuthority::default(),
5557                Instant::now() + Duration::from_secs(60),
5558                false,
5559                None,
5560            ),
5561        )
5562        .await
5563        .expect("reserving the route.open slot must not wait behind data frames")
5564        .unwrap();
5565        forwarding
5566            .complete_pending_relay(
5567                module_connection,
5568                pending.corr,
5569                RouteBindRelayOutcome::Accepted,
5570            )
5571            .unwrap();
5572
5573        let backlog = sink.backlog();
5574        assert_eq!(backlog.queued_frames, DATA_FRAMES + 1);
5575        assert!(
5576            backlog.queued_bytes > data_bytes,
5577            "the route.open response must be counted in queued bytes: {backlog:?}"
5578        );
5579        for corr in 0..DATA_FRAMES as u64 {
5580            assert_eq!(client_rx.recv().await.unwrap().header.corr, corr);
5581        }
5582        let open = client_rx.recv().await.unwrap();
5583        assert_eq!(open.header.corr, 4_242);
5584        assert_eq!(open.header.ty, FrameType::Response);
5585        drop(open);
5586        assert_eq!(sink.backlog().queued_bytes, 0);
5587        assert_eq!(sink.backlog().queued_frames, 0);
5588    }
5589
5590    /// The drain-timeout line reports these numbers, so they must count the
5591    /// requests the drain is waiting on and no others: a route with nothing in
5592    /// flight is not a holdout, and a flagged subscription is excluded from the
5593    /// drain and so from the count.
5594    #[tokio::test]
5595    async fn drain_holdouts_count_held_requests_and_name_the_connection() {
5596        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5597            route_fixture("holdouts");
5598        let mut bound = |corr| {
5599            let route = begin_test_route(&forwarding, client, sink.clone(), corr, "holdouts");
5600            forwarding
5601                .complete_pending_relay(
5602                    module_connection,
5603                    route.corr,
5604                    RouteBindRelayOutcome::Accepted,
5605                )
5606                .unwrap();
5607            client_rx.try_recv().unwrap();
5608            match forwarding
5609                .lookup_data_route(client, route.client_channel, route.client_epoch)
5610                .unwrap()
5611            {
5612                DataRoute::Client(DataRouteState::Bound(binding)) => binding,
5613                other => panic!("expected live route, got {other:?}"),
5614            }
5615        };
5616        let holding = bound(61);
5617        let _idle = bound(62);
5618        holding.flow.acquire_tagged(7, false).await.unwrap();
5619        holding.flow.acquire_tagged(2, false).await.unwrap();
5620        holding.flow.acquire_tagged(3, true).await.unwrap();
5621        forwarding
5622            .begin_module_drain("holdouts", RouteCloseReason::Restart)
5623            .unwrap();
5624
5625        let holdouts = forwarding.endpoint_drain_holdouts(endpoint).unwrap();
5626        assert_eq!(
5627            holdouts,
5628            DrainHoldouts {
5629                requests: 2,
5630                routes: 1,
5631                total_routes: 2,
5632                top_connections: vec![(client.get(), 2)],
5633                // The held requests by the module's channel and corr, ascending,
5634                // without the excluded subscription (corr 3).
5635                held: vec![(holding.module_channel, 2), (holding.module_channel, 7)],
5636            }
5637        );
5638    }
5639
5640    #[test]
5641    fn endpoint_routes_keep_goodbye_targets_and_mark_draining_routes() {
5642        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
5643            route_fixture("census");
5644        let pending = begin_test_route(&forwarding, client, sink, 1, "census");
5645        forwarding
5646            .complete_pending_relay(
5647                module_connection,
5648                pending.corr,
5649                RouteBindRelayOutcome::Accepted,
5650            )
5651            .unwrap();
5652
5653        let routes = forwarding.endpoint_routes(endpoint).unwrap();
5654        assert_eq!(routes.len(), 1);
5655        assert!(matches!(routes[0].principal, Principal::Direct));
5656        assert_eq!(routes[0].goodbye_target.connection_id, client);
5657        assert_eq!(routes[0].goodbye_target.channel, pending.client_channel);
5658        assert_eq!(routes[0].goodbye_target.epoch, pending.client_epoch);
5659        assert!(!routes[0].draining);
5660
5661        forwarding
5662            .begin_module_drain("census", RouteCloseReason::Restart)
5663            .unwrap();
5664        let draining_routes = forwarding.endpoint_routes(endpoint).unwrap();
5665        assert_eq!(draining_routes.len(), 1);
5666        assert!(draining_routes[0].draining);
5667    }
5668
5669    #[test]
5670    fn aborted_reservation_consumes_both_epochs_and_reuse_advances_them() {
5671        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-abort");
5672        let first = begin_test_route(&forwarding, client, sink.clone(), 1, "epoch-abort");
5673        assert_eq!((first.client_epoch, first.module_epoch), (1, 1));
5674        forwarding
5675            .abort_pending_relay(
5676                first.endpoint,
5677                first.corr,
5678                RouteBindRelayOutcome::ModuleGone("abort".into()),
5679            )
5680            .unwrap();
5681        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
5682        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
5683
5684        let second = begin_test_route(&forwarding, client, sink, 2, "epoch-abort");
5685        assert_eq!(second.client_channel, first.client_channel);
5686        assert_eq!(second.module_channel, first.module_channel);
5687        assert_eq!((second.client_epoch, second.module_epoch), (2, 2));
5688    }
5689
5690    #[test]
5691    fn stale_release_cannot_remove_reused_successor_and_status_is_epoch_fenced() {
5692        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5693            route_fixture("epoch-release");
5694        let first = begin_test_route(&forwarding, client, sink.clone(), 10, "epoch-release");
5695        forwarding
5696            .complete_pending_relay(
5697                module_connection,
5698                first.corr,
5699                RouteBindRelayOutcome::Accepted,
5700            )
5701            .unwrap();
5702        assert_eq!(client_rx.try_recv().unwrap().header.corr, 10);
5703        assert!(matches!(
5704            forwarding
5705                .release_client_route(client, first.client_channel, first.client_epoch)
5706                .unwrap(),
5707            RouteRelease::Removed(_)
5708        ));
5709        forwarding.inject_client_slot_epoch(client, first.client_channel, first.client_epoch);
5710        forwarding.inject_module_slot_epoch(endpoint, first.module_channel, first.module_epoch);
5711
5712        let second = begin_test_route(&forwarding, client, sink, 11, "epoch-release");
5713        forwarding
5714            .complete_pending_relay(
5715                module_connection,
5716                second.corr,
5717                RouteBindRelayOutcome::Accepted,
5718            )
5719            .unwrap();
5720        assert_eq!(client_rx.try_recv().unwrap().header.corr, 11);
5721        assert!(matches!(
5722            forwarding
5723                .release_client_route(client, second.client_channel, first.client_epoch)
5724                .unwrap(),
5725            RouteRelease::Stale
5726        ));
5727        assert!(!forwarding
5728            .cache_status(
5729                endpoint,
5730                second.module_channel,
5731                first.module_epoch,
5732                "stale".into(),
5733            )
5734            .unwrap());
5735        assert!(forwarding
5736            .cache_status(
5737                endpoint,
5738                second.module_channel,
5739                second.module_epoch,
5740                "current".into(),
5741            )
5742            .unwrap());
5743        match forwarding
5744            .route_poll_snapshot(client, second.client_channel, second.client_epoch)
5745            .unwrap()
5746        {
5747            RoutePollSnapshot::Bound { status, .. } => {
5748                assert_eq!(status.as_deref(), Some("current"));
5749            }
5750            RoutePollSnapshot::Absent => panic!("successor binding was removed"),
5751        }
5752        let counters = forwarding.counters().snapshot();
5753        assert_eq!(counters["route_released_epoch_fenced"], 1);
5754        assert_eq!(counters["route_release_stale_skipped"], 1);
5755    }
5756
5757    #[test]
5758    fn max_epoch_reservation_retires_only_that_slot() {
5759        let (forwarding, _, endpoint, client, sink, _client_rx) = route_fixture("epoch-max");
5760        forwarding.inject_client_slot_epoch(client, 7, u32::MAX - 1);
5761        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX - 1);
5762        let final_use = begin_test_route(&forwarding, client, sink.clone(), 20, "epoch-max");
5763        assert_eq!(
5764            (final_use.client_channel, final_use.client_epoch),
5765            (7, u32::MAX)
5766        );
5767        assert_eq!(
5768            (final_use.module_channel, final_use.module_epoch),
5769            (9, u32::MAX)
5770        );
5771        forwarding
5772            .abort_pending_relay(
5773                endpoint,
5774                final_use.corr,
5775                RouteBindRelayOutcome::ModuleGone("abort".into()),
5776            )
5777            .unwrap();
5778        forwarding.inject_client_slot_epoch(client, 7, u32::MAX);
5779        forwarding.inject_module_slot_epoch(endpoint, 9, u32::MAX);
5780        let next = begin_test_route(&forwarding, client, sink, 21, "epoch-max");
5781        assert_ne!(next.client_channel, 7);
5782        assert_ne!(next.module_channel, 9);
5783        assert_eq!((next.client_epoch, next.module_epoch), (1, 1));
5784    }
5785
5786    #[test]
5787    fn bind_and_module_control_share_monotonic_corr_and_deadline_arbitration() {
5788        let (forwarding, module_connection, endpoint, client, sink, _client_rx) =
5789            route_fixture("corr-shared");
5790        let bind = begin_test_route(&forwarding, client, sink, 30, "corr-shared");
5791        assert_eq!(bind.corr, 1);
5792        forwarding
5793            .abort_pending_relay(
5794                endpoint,
5795                bind.corr,
5796                RouteBindRelayOutcome::ModuleGone("abort".into()),
5797            )
5798            .unwrap();
5799        let rpc = forwarding
5800            .begin_module_control_rpc_for(
5801                "corr-shared",
5802                "health.check",
5803                Instant::now() - Duration::from_millis(1),
5804            )
5805            .unwrap();
5806        assert_eq!(rpc.corr, 2);
5807        assert_eq!(
5808            forwarding
5809                .complete_module_control_rpc(
5810                    module_connection,
5811                    rpc.corr,
5812                    Some("health.check"),
5813                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
5814                        status: subc_protocol::session::HealthStatus::Ok,
5815                        detail: None,
5816                        metrics: None,
5817                    }),
5818                )
5819                .unwrap(),
5820            ModuleControlRpcCompletion::Settled
5821        );
5822        assert!(matches!(
5823            rpc.receiver.blocking_recv().unwrap(),
5824            ModuleControlRpcOutcome::DeadlineElapsed
5825        ));
5826    }
5827
5828    #[tokio::test(start_paused = true)]
5829    async fn health_probe_tombstone_ttl_removes_an_endpoint_that_stops_probing() {
5830        let (forwarding, _, endpoint, _, _, _) = route_fixture("tombstone-ttl");
5831        let probe_started_at = Instant::now();
5832        let rpc = forwarding
5833            .begin_health_probe_rpc_for(
5834                "tombstone-ttl",
5835                "health.check",
5836                probe_started_at,
5837                probe_started_at + Duration::from_secs(5),
5838            )
5839            .unwrap();
5840        assert!(forwarding
5841            .tombstone_health_probe_rpc(endpoint, rpc.corr)
5842            .unwrap());
5843        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 1);
5844
5845        tokio::time::advance(HEALTH_PROBE_TOMBSTONE_TTL).await;
5846        tokio::task::yield_now().await;
5847
5848        assert_eq!(forwarding.health_probe_tombstone_count().unwrap(), 0);
5849    }
5850
5851    #[test]
5852    fn correlation_exhaustion_emits_max_once_then_closes_endpoint() {
5853        let (forwarding, _, endpoint, _, _, _) = route_fixture("corr-max");
5854        let mut close = forwarding.register_connection_close(endpoint.connection_id);
5855        forwarding.inject_control_corr(endpoint, u64::MAX);
5856        let final_rpc = forwarding
5857            .begin_module_control_rpc_for(
5858                "corr-max",
5859                "health.check",
5860                Instant::now() + Duration::from_secs(1),
5861            )
5862            .unwrap();
5863        assert_eq!(final_rpc.corr, u64::MAX);
5864        forwarding
5865            .cancel_module_control_rpc(endpoint, final_rpc.corr)
5866            .unwrap();
5867        assert!(matches!(
5868            forwarding.begin_module_control_rpc_for(
5869                "corr-max",
5870                "health.check",
5871                Instant::now() + Duration::from_secs(1),
5872            ),
5873            Err(ForwardingError::RelayCorrelationExhausted)
5874        ));
5875        assert!(close.try_recv().is_ok());
5876    }
5877
5878    #[test]
5879    fn publication_epoch_controls_delivery_failure_escalation() {
5880        fn setup_successor(
5881            commit_successor: Option<bool>,
5882        ) -> (ForwardingTable, ConnectionId, u16, u32) {
5883            let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
5884                route_fixture("escalation");
5885            let first = begin_test_route(&forwarding, client, sink.clone(), 40, "escalation");
5886            forwarding
5887                .complete_pending_relay(
5888                    module_connection,
5889                    first.corr,
5890                    RouteBindRelayOutcome::Accepted,
5891                )
5892                .unwrap();
5893            client_rx.try_recv().unwrap();
5894            assert!(matches!(
5895                forwarding
5896                    .release_client_route(client, first.client_channel, first.client_epoch)
5897                    .unwrap(),
5898                RouteRelease::Removed(_)
5899            ));
5900            if let Some(commit_successor) = commit_successor {
5901                forwarding.inject_client_slot_epoch(
5902                    client,
5903                    first.client_channel,
5904                    first.client_epoch,
5905                );
5906                forwarding.inject_module_slot_epoch(
5907                    endpoint,
5908                    first.module_channel,
5909                    first.module_epoch,
5910                );
5911                let successor = begin_test_route(&forwarding, client, sink, 41, "escalation");
5912                if commit_successor {
5913                    forwarding
5914                        .complete_pending_relay(
5915                            module_connection,
5916                            successor.corr,
5917                            RouteBindRelayOutcome::Accepted,
5918                        )
5919                        .unwrap();
5920                    client_rx.try_recv().unwrap();
5921                } else {
5922                    forwarding
5923                        .abort_pending_relay(
5924                            endpoint,
5925                            successor.corr,
5926                            RouteBindRelayOutcome::ModuleGone("abort".into()),
5927                        )
5928                        .unwrap();
5929                }
5930            }
5931            (forwarding, client, first.client_channel, first.client_epoch)
5932        }
5933
5934        let probe_sink = FrameSink::new(mpsc::channel(1).0);
5935        let (no_successor, client, channel, epoch) = setup_successor(None);
5936        let mut close = no_successor.register_connection_close(client);
5937        assert!(no_successor
5938            .escalate_client_delivery_failure(
5939                client,
5940                channel,
5941                epoch,
5942                CloseReason::new("delivery", "failed"),
5943                UndeliveredFrame {
5944                    module_id: None,
5945                    sink: &probe_sink,
5946                },
5947            )
5948            .unwrap());
5949        assert!(close.try_recv().is_ok());
5950
5951        let (aborted, client, channel, epoch) = setup_successor(Some(false));
5952        let mut close = aborted.register_connection_close(client);
5953        assert!(aborted
5954            .escalate_client_delivery_failure(
5955                client,
5956                channel,
5957                epoch,
5958                CloseReason::new("delivery", "failed"),
5959                UndeliveredFrame {
5960                    module_id: None,
5961                    sink: &probe_sink,
5962                },
5963            )
5964            .unwrap());
5965        assert!(close.try_recv().is_ok());
5966
5967        let (published, client, channel, epoch) = setup_successor(Some(true));
5968        let mut close = published.register_connection_close(client);
5969        assert!(!published
5970            .escalate_client_delivery_failure(
5971                client,
5972                channel,
5973                epoch,
5974                CloseReason::new("delivery", "stale failure"),
5975                UndeliveredFrame {
5976                    module_id: None,
5977                    sink: &probe_sink,
5978                },
5979            )
5980            .unwrap());
5981        assert!(close.try_recv().is_err());
5982    }
5983
5984    #[test]
5985    fn route_concentration_separates_client_count_from_routes_per_client() {
5986        // The distinction this asserts is the one a bare connection count cannot
5987        // make: two connections holding one route each and one connection
5988        // holding two are the same total, and have opposite causes.
5989        let (forwarding, module_connection, _, client, sink, _client_rx) =
5990            route_fixture("concentration");
5991        assert_eq!(forwarding.client_route_concentration().unwrap(), (0, 0));
5992
5993        for corr in [70_u64, 71] {
5994            let pending =
5995                begin_test_route(&forwarding, client, sink.clone(), corr, "concentration");
5996            forwarding
5997                .complete_pending_relay(
5998                    module_connection,
5999                    pending.corr,
6000                    RouteBindRelayOutcome::Accepted,
6001                )
6002                .unwrap();
6003        }
6004
6005        // One connection, two routes — not two connections with a route each.
6006        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
6007        assert_eq!(forwarding.client_route_concentration().unwrap(), (1, 2));
6008    }
6009
6010    #[test]
6011    fn cleanup_and_accepted_resolution_have_one_lock_winner() {
6012        let (forwarding, module_connection, _, client, sink, mut client_rx) =
6013            route_fixture("cleanup-race");
6014        let pending = begin_test_route(&forwarding, client, sink, 45, "cleanup-race");
6015        forwarding
6016            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
6017            .unwrap();
6018        let released = forwarding.cleanup_connection(client).unwrap();
6019        assert_eq!(released.len(), 1);
6020        let completion = forwarding
6021            .complete_pending_relay(
6022                module_connection,
6023                pending.corr,
6024                RouteBindRelayOutcome::Accepted,
6025            )
6026            .unwrap();
6027        assert!(!completion.settled);
6028        assert!(client_rx.try_recv().is_err());
6029        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
6030
6031        let (forwarding, module_connection, _, client, sink, mut client_rx) =
6032            route_fixture("accepted-race");
6033        let pending = begin_test_route(&forwarding, client, sink, 46, "accepted-race");
6034        forwarding
6035            .complete_pending_relay(
6036                module_connection,
6037                pending.corr,
6038                RouteBindRelayOutcome::Accepted,
6039            )
6040            .unwrap();
6041        assert_eq!(client_rx.try_recv().unwrap().header.corr, 46);
6042        let released = forwarding.cleanup_connection(client).unwrap();
6043        assert_eq!(released.len(), 1);
6044        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
6045    }
6046
6047    #[test]
6048    fn drain_marks_block_reservation_commit_and_live_request_admission_until_phase_two() {
6049        let (forwarding, module_connection, _, client, sink, mut client_rx) =
6050            route_fixture("drain-gap");
6051        let live = begin_test_route(&forwarding, client, sink.clone(), 47, "drain-gap");
6052        forwarding
6053            .complete_pending_relay(
6054                module_connection,
6055                live.corr,
6056                RouteBindRelayOutcome::Accepted,
6057            )
6058            .unwrap();
6059        client_rx.try_recv().unwrap();
6060        let binding = match forwarding
6061            .lookup_data_route(client, live.client_channel, live.client_epoch)
6062            .unwrap()
6063        {
6064            DataRoute::Client(DataRouteState::Bound(binding)) => binding,
6065            other => panic!("expected live route, got {other:?}"),
6066        };
6067
6068        let pending = begin_test_route(&forwarding, client, sink.clone(), 48, "drain-gap");
6069        forwarding
6070            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
6071            .unwrap();
6072        let control_rpc = forwarding
6073            .begin_module_control_rpc_for(
6074                "drain-gap",
6075                "health.check",
6076                Instant::now() + Duration::from_secs(1),
6077            )
6078            .unwrap();
6079        let target = forwarding
6080            .begin_module_drain("drain-gap", RouteCloseReason::Reload)
6081            .unwrap()
6082            .unwrap();
6083        assert!(matches!(
6084            control_rpc.receiver.blocking_recv().unwrap(),
6085            ModuleControlRpcOutcome::ModuleGone(_)
6086        ));
6087        assert_eq!(target.abandoned_bindings.len(), 1);
6088        assert!(binding.flow.sem.is_closed());
6089        assert!(
6090            !forwarding
6091                .complete_pending_relay(
6092                    module_connection,
6093                    pending.corr,
6094                    RouteBindRelayOutcome::Accepted,
6095                )
6096                .unwrap()
6097                .settled
6098        );
6099        assert!(matches!(
6100            forwarding.begin_route_bind_relay_for_test(client, sink, 49, "drain-gap"),
6101            Err(ForwardingError::ModuleReloading { .. })
6102        ));
6103        let released = forwarding
6104            .release_module_endpoint_routes(target.endpoint)
6105            .unwrap();
6106        assert_eq!(released.len(), 1);
6107        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
6108    }
6109
6110    /// A client can be marked closing while its egress is still open: the daemon
6111    /// asks a connection to close (here through the production path, a module
6112    /// frame that its egress refused) and the connection loop tears down a moment
6113    /// later. A route.bind ack that lands inside that window is answered on the
6114    /// MODULE connection's frame handler, so resolving it must not produce an
6115    /// error -- an error there ends the module connection, and that connection
6116    /// carries every other client's routes to the module.
6117    #[test]
6118    fn accepted_bind_for_a_closing_client_releases_the_route_instead_of_failing_the_module() {
6119        let (forwarding, module_connection, endpoint, client, sink, mut client_rx) =
6120            route_fixture("closing-client");
6121
6122        // A published route on this client: escalate_client_delivery_failure only
6123        // marks a connection closing for a route it has already published.
6124        let live = begin_test_route(&forwarding, client, sink.clone(), 60, "closing-client");
6125        forwarding
6126            .complete_pending_relay(
6127                module_connection,
6128                live.corr,
6129                RouteBindRelayOutcome::Accepted,
6130            )
6131            .unwrap();
6132        client_rx.try_recv().unwrap();
6133
6134        // A second route.open from the same client, relayed and awaiting its ack.
6135        let pending = begin_test_route(&forwarding, client, sink.clone(), 61, "closing-client");
6136        forwarding
6137            .mark_route_bind_relay_enqueued(pending.endpoint, pending.corr)
6138            .unwrap();
6139
6140        // The window: closing, but the sink is still open.
6141        assert!(forwarding
6142            .escalate_client_delivery_failure(
6143                client,
6144                live.client_channel,
6145                live.client_epoch,
6146                CloseReason::new(
6147                    "module_to_client_delivery_failed",
6148                    "client egress refused a module frame",
6149                ),
6150                UndeliveredFrame {
6151                    module_id: None,
6152                    sink: &sink,
6153                },
6154            )
6155            .unwrap());
6156        assert!(!sink.is_closed());
6157
6158        let completion = forwarding
6159            .complete_pending_relay(
6160                module_connection,
6161                pending.corr,
6162                RouteBindRelayOutcome::Accepted,
6163            )
6164            .expect("a closing client must not turn a module's ack into an error");
6165
6166        assert!(completion.settled);
6167        let abandoned = completion
6168            .abandoned
6169            .expect("the module must be told to drop the binding it just created");
6170        assert_eq!(abandoned.connection_id, module_connection);
6171        assert_eq!(abandoned.channel, pending.module_channel);
6172        assert_eq!(abandoned.epoch, pending.module_epoch);
6173        assert!(matches!(abandoned.kind, GoodbyeTargetKind::Module));
6174        assert!(matches!(
6175            pending.receiver.blocking_recv().unwrap(),
6176            RouteBindRelayOutcome::ModuleGone(_)
6177        ));
6178        // No route was published to a client that is on its way out, and the
6179        // reserved handle pair went back.
6180        assert!(client_rx.try_recv().is_err());
6181        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
6182
6183        // The module endpoint is untouched: still live, and still able to take a
6184        // route from another client.
6185        assert!(forwarding
6186            .has_live_module_connection("closing-client")
6187            .unwrap());
6188        let cotenant = ConnectionId::new(201);
6189        let (cotenant_tx, mut cotenant_rx) = mpsc::channel(8);
6190        let cotenant_route = begin_test_route(
6191            &forwarding,
6192            cotenant,
6193            FrameSink::new(cotenant_tx),
6194            62,
6195            "closing-client",
6196        );
6197        assert_eq!(cotenant_route.endpoint, endpoint);
6198        forwarding
6199            .complete_pending_relay(
6200                module_connection,
6201                cotenant_route.corr,
6202                RouteBindRelayOutcome::Accepted,
6203            )
6204            .unwrap();
6205        assert_eq!(cotenant_rx.try_recv().unwrap().header.corr, 62);
6206        assert_eq!(forwarding.active_binding_count().unwrap(), 2);
6207    }
6208
6209    #[test]
6210    fn pending_route_permit_is_released_on_rejection_and_abort() {
6211        let forwarding = ForwardingTable::default();
6212        let module_connection = ConnectionId::new(300);
6213        let client = ConnectionId::new(301);
6214        let (module_tx, _module_rx) = mpsc::channel(1);
6215        let endpoint = forwarding
6216            .register_module_connection(
6217                module_connection,
6218                "permit".into(),
6219                2,
6220                Concurrency::ModuleManaged,
6221                FrameSink::new(module_tx),
6222            )
6223            .unwrap();
6224        let (client_tx, mut client_rx) = mpsc::channel(1);
6225        let sink = FrameSink::new(client_tx);
6226        let rejected = begin_test_route(&forwarding, client, sink.clone(), 50, "permit");
6227        assert!(sink.try_send(test_ping(999)).is_err());
6228        forwarding
6229            .complete_pending_relay(
6230                module_connection,
6231                rejected.corr,
6232                RouteBindRelayOutcome::Rejected(ErrorBody {
6233                    code: "no".into(),
6234                    message: "rejected".into(),
6235                    detail: None,
6236                }),
6237            )
6238            .unwrap();
6239        sink.try_send(test_ping(1000)).unwrap();
6240        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1000);
6241
6242        let aborted = begin_test_route(&forwarding, client, sink.clone(), 51, "permit");
6243        assert!(sink.try_send(test_ping(1001)).is_err());
6244        forwarding
6245            .abort_pending_relay(
6246                endpoint,
6247                aborted.corr,
6248                RouteBindRelayOutcome::ModuleGone("abort".into()),
6249            )
6250            .unwrap();
6251        sink.try_send(test_ping(1002)).unwrap();
6252        assert_eq!(client_rx.try_recv().unwrap().header.corr, 1002);
6253
6254        let receiver_closed = begin_test_route(&forwarding, client, sink, 52, "permit");
6255        forwarding
6256            .mark_route_bind_relay_enqueued(endpoint, receiver_closed.corr)
6257            .unwrap();
6258        drop(client_rx);
6259        let completion = forwarding
6260            .complete_pending_relay(
6261                module_connection,
6262                receiver_closed.corr,
6263                RouteBindRelayOutcome::Accepted,
6264            )
6265            .unwrap();
6266        assert!(completion.abandoned.is_some());
6267        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
6268    }
6269
6270    /// Every connection teardown passes through `cleanup_connection`, and
6271    /// connection ids come from a monotonic counter that never hands an id out
6272    /// twice. If the closing mark survives teardown, the set grows by one entry
6273    /// per connection for the life of the daemon -- the self-watchdog alone
6274    /// reconnects once a minute.
6275    #[test]
6276    fn cleaned_up_connections_do_not_stay_in_the_closing_set() {
6277        let (forwarding, module_connection, _endpoint, _fixture_client, _sink, _rx) =
6278            route_fixture("closing-set-leak");
6279
6280        const CONNECTIONS: u64 = 32;
6281        for index in 0..CONNECTIONS {
6282            let client = ConnectionId::new(1000 + index);
6283            let (client_tx, _client_rx) = mpsc::channel(8);
6284            let route = begin_test_route(
6285                &forwarding,
6286                client,
6287                FrameSink::new(client_tx),
6288                index + 1,
6289                "closing-set-leak",
6290            );
6291            forwarding
6292                .complete_pending_relay(
6293                    module_connection,
6294                    route.corr,
6295                    RouteBindRelayOutcome::Accepted,
6296                )
6297                .unwrap();
6298            forwarding.cleanup_connection(client).unwrap();
6299        }
6300        forwarding.cleanup_connection(module_connection).unwrap();
6301
6302        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
6303    }
6304
6305    /// The closing mark exists to refuse new work for a connection that is on
6306    /// its way out but whose teardown has not run yet: the daemon asks the
6307    /// connection loop to end, and only when the loop reacts does
6308    /// `cleanup_connection` strip the connection's state. Inside that window an
6309    /// operation for the dying connection must still be refused; only after
6310    /// cleanup completes may the mark go.
6311    #[test]
6312    fn closing_connection_is_refused_new_work_until_cleanup_completes() {
6313        let (forwarding, module_connection, _endpoint, client, sink, mut client_rx) =
6314            route_fixture("closing-gate");
6315
6316        // A published route: escalate_client_delivery_failure only marks a
6317        // connection closing for a route it has already published.
6318        let live = begin_test_route(&forwarding, client, sink.clone(), 80, "closing-gate");
6319        forwarding
6320            .complete_pending_relay(
6321                module_connection,
6322                live.corr,
6323                RouteBindRelayOutcome::Accepted,
6324            )
6325            .unwrap();
6326        client_rx.try_recv().unwrap();
6327
6328        // Mark the connection closing through the production path without
6329        // running teardown, pinning the window open.
6330        assert!(forwarding
6331            .escalate_client_delivery_failure(
6332                client,
6333                live.client_channel,
6334                live.client_epoch,
6335                CloseReason::new(
6336                    "module_to_client_delivery_failed",
6337                    "client egress refused a module frame",
6338                ),
6339                UndeliveredFrame {
6340                    module_id: None,
6341                    sink: &sink,
6342                },
6343            )
6344            .unwrap());
6345        assert_eq!(forwarding.closing_connection_count().unwrap(), 1);
6346
6347        // A late route.open for the closing client is refused ...
6348        assert!(matches!(
6349            forwarding.begin_route_bind_relay_for_test(client, sink, 81, "closing-gate"),
6350            Err(ForwardingError::ConnectionClosing { connection_id })
6351                if connection_id == client
6352        ));
6353        // ... and so is a late attempt to register the connection as a module.
6354        let (late_tx, _late_rx) = mpsc::channel(1);
6355        assert!(matches!(
6356            forwarding.register_module_connection(
6357                client,
6358                "late-module".into(),
6359                2,
6360                Concurrency::ModuleManaged,
6361                FrameSink::new(late_tx),
6362            ),
6363            Err(ForwardingError::ConnectionClosing { connection_id })
6364                if connection_id == client
6365        ));
6366
6367        // Teardown is the point that lifts the mark: it has just removed every
6368        // per-connection entry under the same lock, so the gate has nothing
6369        // left to protect for this id.
6370        forwarding.cleanup_connection(client).unwrap();
6371        assert_eq!(forwarding.closing_connection_count().unwrap(), 0);
6372    }
6373}
6374
6375/// Blue/green swap slots: a candidate registered beside the active endpoint,
6376/// promoted by `cutover_candidate`, with the old incumbent drained by endpoint.
6377#[cfg(test)]
6378mod swap_slot_tests {
6379    use std::time::Duration;
6380
6381    use super::*;
6382    use tokio::sync::mpsc;
6383
6384    const MODULE_ID: &str = "swapped";
6385
6386    struct SwapFixture {
6387        forwarding: ForwardingTable,
6388        incumbent_connection: ConnectionId,
6389        incumbent: ModuleEndpointId,
6390        candidate_connection: ConnectionId,
6391        candidate: ModuleEndpointId,
6392        _module_rxs: Vec<mpsc::Receiver<crate::router::OutboundFrame>>,
6393    }
6394
6395    fn swap_fixture() -> SwapFixture {
6396        let forwarding = ForwardingTable::default();
6397        let incumbent_connection = ConnectionId::new(100);
6398        let candidate_connection = ConnectionId::new(110);
6399        let (incumbent_tx, incumbent_rx) = mpsc::channel(8);
6400        let incumbent = forwarding
6401            .register_module_connection(
6402                incumbent_connection,
6403                MODULE_ID.to_string(),
6404                2,
6405                Concurrency::ModuleManaged,
6406                FrameSink::new(incumbent_tx),
6407            )
6408            .unwrap();
6409        let (candidate_tx, candidate_rx) = mpsc::channel(8);
6410        let candidate = forwarding
6411            .register_candidate_module_connection(
6412                candidate_connection,
6413                MODULE_ID.to_string(),
6414                2,
6415                Concurrency::ModuleManaged,
6416                FrameSink::new(candidate_tx),
6417            )
6418            .unwrap();
6419        SwapFixture {
6420            forwarding,
6421            incumbent_connection,
6422            incumbent,
6423            candidate_connection,
6424            candidate,
6425            _module_rxs: vec![incumbent_rx, candidate_rx],
6426        }
6427    }
6428
6429    fn client(
6430        raw: u64,
6431    ) -> (
6432        ConnectionId,
6433        FrameSink,
6434        mpsc::Receiver<crate::router::OutboundFrame>,
6435    ) {
6436        let (tx, rx) = mpsc::channel(8);
6437        (ConnectionId::new(raw), FrameSink::new(tx), rx)
6438    }
6439
6440    fn committed_endpoints(forwarding: &ForwardingTable) -> Vec<ModuleEndpointId> {
6441        forwarding
6442            .read_inner()
6443            .unwrap()
6444            .client_to_module
6445            .values()
6446            .map(|route| route.module_endpoint)
6447            .collect()
6448    }
6449
6450    #[test]
6451    fn candidate_is_unroutable_until_cutover_and_by_id_lookups_resolve_the_active_slot() {
6452        let fixture = swap_fixture();
6453        let forwarding = &fixture.forwarding;
6454        assert_ne!(fixture.incumbent, fixture.candidate);
6455
6456        // Every by-id consumer still resolves the incumbent.
6457        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
6458        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
6459        let (client_connection, client_sink, _client_rx) = client(200);
6460        let pending = forwarding
6461            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
6462            .unwrap();
6463        assert_eq!(pending.endpoint, fixture.incumbent);
6464        let rpc = forwarding
6465            .begin_module_control_rpc_for(
6466                MODULE_ID,
6467                "health.check",
6468                Instant::now() + Duration::from_secs(1),
6469            )
6470            .unwrap();
6471        assert_eq!(rpc.endpoint, fixture.incumbent);
6472        let census = forwarding.route_census(Some(MODULE_ID)).unwrap();
6473        assert_eq!(census.len(), 1, "the census lists one endpoint per id");
6474
6475        // Connection-keyed lookups see the candidate, so its own frames resolve.
6476        assert_eq!(
6477            forwarding
6478                .module_endpoint_for_connection(fixture.candidate_connection)
6479                .unwrap(),
6480            Some(fixture.candidate)
6481        );
6482        assert_eq!(
6483            forwarding
6484                .module_id_for_connection(fixture.candidate_connection)
6485                .unwrap()
6486                .as_deref(),
6487            Some(MODULE_ID)
6488        );
6489
6490        // One candidate per id.
6491        let (other_tx, _other_rx) = mpsc::channel(1);
6492        assert_eq!(
6493            forwarding.register_candidate_module_connection(
6494                ConnectionId::new(120),
6495                MODULE_ID.to_string(),
6496                2,
6497                Concurrency::ModuleManaged,
6498                FrameSink::new(other_tx),
6499            ),
6500            Err(ForwardingError::CandidateSlotOccupied {
6501                module_id: MODULE_ID.to_string()
6502            })
6503        );
6504    }
6505
6506    /// The cutover linearization point. A relay reserved on the incumbent before
6507    /// cutover must never become a route on the incumbent, and every relay
6508    /// reserved after it must land on the promoted candidate.
6509    #[test]
6510    fn relay_reserved_before_cutover_never_commits_and_later_relays_land_on_the_candidate() {
6511        let fixture = swap_fixture();
6512        let forwarding = &fixture.forwarding;
6513        let (early_client, early_sink, _early_rx) = client(200);
6514        let mut early = forwarding
6515            .begin_route_bind_relay_for_test(early_client, early_sink, 1, MODULE_ID)
6516            .unwrap();
6517        assert_eq!(early.endpoint, fixture.incumbent);
6518        assert_eq!(
6519            forwarding
6520                .mark_route_bind_relay_enqueued(early.endpoint, early.corr)
6521                .unwrap(),
6522            RouteBindRelayMark::Marked
6523        );
6524
6525        let cutover = forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
6526        assert_eq!(
6527            cutover,
6528            ForwardingCutover {
6529                promoted: fixture.candidate,
6530                incumbent: Some(fixture.incumbent),
6531            }
6532        );
6533
6534        // A route.open reserved after cutover goes to the promoted candidate.
6535        let (late_client, late_sink, _late_rx) = client(201);
6536        let late = forwarding
6537            .begin_route_bind_relay_for_test(late_client, late_sink, 2, MODULE_ID)
6538            .unwrap();
6539        assert_eq!(
6540            late.endpoint, fixture.candidate,
6541            "a route.open after cutover was reserved on the incumbent"
6542        );
6543
6544        // The incumbent acks the early relay after cutover.
6545        let completion = forwarding
6546            .complete_pending_relay(
6547                fixture.incumbent_connection,
6548                early.corr,
6549                RouteBindRelayOutcome::Accepted,
6550            )
6551            .expect("a superseded endpoint's ack is not an error on its connection");
6552        assert!(completion.settled);
6553        assert!(
6554            !committed_endpoints(forwarding).contains(&fixture.incumbent),
6555            "a relay reserved before cutover committed a route on the incumbent"
6556        );
6557        let goodbye = completion
6558            .abandoned
6559            .expect("the incumbent is told to drop the binding it just created");
6560        assert_eq!(goodbye.connection_id, fixture.incumbent_connection);
6561        assert_eq!(goodbye.channel, early.module_channel);
6562        assert_eq!(goodbye.epoch, early.module_epoch);
6563        assert_eq!(goodbye.kind, GoodbyeTargetKind::Module);
6564        match early.receiver.try_recv() {
6565            Ok(RouteBindRelayOutcome::DaemonRejected(body)) => {
6566                assert_eq!(body.code, "module_reloading")
6567            }
6568            other => panic!("expected a retryable module_reloading answer, got {other:?}"),
6569        }
6570        assert!(matches!(
6571            forwarding
6572                .lookup_data_route(early_client, early.client_channel, early.client_epoch)
6573                .unwrap(),
6574            DataRoute::Client(DataRouteState::Absent)
6575        ));
6576
6577        // The late relay commits on the candidate; only its pair stays reserved
6578        // until then.
6579        assert_eq!(forwarding.reserved_route_count().unwrap(), (1, 1));
6580        forwarding
6581            .complete_pending_relay(
6582                fixture.candidate_connection,
6583                late.corr,
6584                RouteBindRelayOutcome::Accepted,
6585            )
6586            .unwrap();
6587        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
6588        assert_eq!(committed_endpoints(forwarding), vec![fixture.candidate]);
6589    }
6590
6591    #[test]
6592    fn endpoint_drain_after_cutover_drains_the_incumbent_not_the_promoted_candidate() {
6593        let fixture = swap_fixture();
6594        let forwarding = &fixture.forwarding;
6595        // One bound route and one in-flight relay on the incumbent.
6596        let (bound_client, bound_sink, _bound_rx) = client(200);
6597        let bound = forwarding
6598            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
6599            .unwrap();
6600        forwarding
6601            .complete_pending_relay(
6602                fixture.incumbent_connection,
6603                bound.corr,
6604                RouteBindRelayOutcome::Accepted,
6605            )
6606            .unwrap();
6607        let (pending_client, pending_sink, _pending_rx) = client(201);
6608        let mut in_flight = forwarding
6609            .begin_route_bind_relay_for_test(pending_client, pending_sink, 2, MODULE_ID)
6610            .unwrap();
6611        forwarding
6612            .mark_route_bind_relay_enqueued(in_flight.endpoint, in_flight.corr)
6613            .unwrap();
6614
6615        let incumbent = forwarding
6616            .cutover_candidate(MODULE_ID)
6617            .unwrap()
6618            .unwrap()
6619            .incumbent
6620            .unwrap();
6621        let target = forwarding
6622            .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
6623            .unwrap()
6624            .expect("the superseded incumbent is still registered");
6625
6626        assert_eq!(target.endpoint, fixture.incumbent);
6627        assert!(forwarding.endpoint_is_draining(fixture.incumbent).unwrap());
6628        assert!(!forwarding.endpoint_is_draining(fixture.candidate).unwrap());
6629        assert!(!forwarding.module_is_draining(MODULE_ID).unwrap());
6630        assert_eq!(target.abandoned_bindings.len(), 1);
6631        assert_eq!(
6632            target.abandoned_bindings[0].channel,
6633            in_flight.module_channel
6634        );
6635        assert!(matches!(
6636            in_flight.receiver.try_recv(),
6637            Ok(RouteBindRelayOutcome::DaemonRejected(body)) if body.code == "module_reloading"
6638        ));
6639        assert_eq!(
6640            forwarding.endpoint_routes(fixture.incumbent).unwrap().len(),
6641            1,
6642            "the incumbent's bound route stays until its drain finishes"
6643        );
6644
6645        let (next_client, next_sink, _next_rx) = client(202);
6646        let next = forwarding
6647            .begin_route_bind_relay_for_test(next_client, next_sink, 3, MODULE_ID)
6648            .expect("the promoted candidate keeps accepting routes");
6649        assert_eq!(next.endpoint, fixture.candidate);
6650    }
6651
6652    /// An endpoint replaced WITHOUT a promotion (a successor registered over it
6653    /// as an ordinary active HELLO) is stale, not superseded, and its ack still
6654    /// fails the acking connection exactly as it did before swap slots existed:
6655    /// `StaleModuleEndpoint`, the reservation indexes already stripped, and the
6656    /// waiting client's sender dropped unanswered.
6657    #[test]
6658    fn stale_endpoint_ack_without_a_promotion_still_fails_as_before() {
6659        let forwarding = ForwardingTable::default();
6660        let first_connection = ConnectionId::new(70);
6661        let (first_tx, _first_rx) = mpsc::channel(8);
6662        forwarding
6663            .register_module_connection(
6664                first_connection,
6665                MODULE_ID.to_string(),
6666                2,
6667                Concurrency::ModuleManaged,
6668                FrameSink::new(first_tx),
6669            )
6670            .unwrap();
6671        let (client_connection, client_sink, _client_rx) = client(200);
6672        let mut pending = forwarding
6673            .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
6674            .unwrap();
6675        let (second_tx, _second_rx) = mpsc::channel(8);
6676        forwarding
6677            .register_module_connection(
6678                ConnectionId::new(80),
6679                MODULE_ID.to_string(),
6680                2,
6681                Concurrency::ModuleManaged,
6682                FrameSink::new(second_tx),
6683            )
6684            .unwrap();
6685
6686        assert_eq!(
6687            forwarding
6688                .complete_pending_relay(
6689                    first_connection,
6690                    pending.corr,
6691                    RouteBindRelayOutcome::Accepted
6692                )
6693                .unwrap_err(),
6694            ForwardingError::StaleModuleEndpoint
6695        );
6696        assert!(committed_endpoints(&forwarding).is_empty());
6697        assert_eq!(forwarding.reserved_route_count().unwrap(), (0, 0));
6698        assert!(matches!(
6699            pending.receiver.try_recv(),
6700            Err(oneshot::error::TryRecvError::Closed)
6701        ));
6702    }
6703
6704    #[test]
6705    fn cleanup_releases_candidate_and_superseded_slots_without_touching_the_active_one() {
6706        // A candidate whose connection drops leaves the incumbent routable.
6707        let fixture = swap_fixture();
6708        let forwarding = &fixture.forwarding;
6709        assert!(forwarding
6710            .cleanup_connection(fixture.candidate_connection)
6711            .unwrap()
6712            .is_empty());
6713        assert_eq!(forwarding.cutover_candidate(MODULE_ID).unwrap(), None);
6714        let (client_connection, client_sink, _client_rx) = client(200);
6715        assert_eq!(
6716            forwarding
6717                .begin_route_bind_relay_for_test(client_connection, client_sink, 1, MODULE_ID)
6718                .unwrap()
6719                .endpoint,
6720            fixture.incumbent
6721        );
6722
6723        // After a cutover, the incumbent's teardown releases its own routes and
6724        // leaves the promoted candidate in place.
6725        let fixture = swap_fixture();
6726        let forwarding = &fixture.forwarding;
6727        let (bound_client, bound_sink, _bound_rx) = client(200);
6728        let bound = forwarding
6729            .begin_route_bind_relay_for_test(bound_client, bound_sink, 1, MODULE_ID)
6730            .unwrap();
6731        forwarding
6732            .complete_pending_relay(
6733                fixture.incumbent_connection,
6734                bound.corr,
6735                RouteBindRelayOutcome::Accepted,
6736            )
6737            .unwrap();
6738        forwarding.cutover_candidate(MODULE_ID).unwrap().unwrap();
6739        let released = forwarding
6740            .cleanup_connection(fixture.incumbent_connection)
6741            .unwrap();
6742        assert_eq!(released.len(), 1);
6743        assert_eq!(released[0].connection_id, bound_client);
6744        assert!(forwarding
6745            .read_inner()
6746            .unwrap()
6747            .superseded_endpoints
6748            .is_empty());
6749        assert!(forwarding.has_live_module_connection(MODULE_ID).unwrap());
6750        let (next_client, next_sink, _next_rx) = client(201);
6751        assert_eq!(
6752            forwarding
6753                .begin_route_bind_relay_for_test(next_client, next_sink, 2, MODULE_ID)
6754                .unwrap()
6755                .endpoint,
6756            fixture.candidate
6757        );
6758    }
6759}