Skip to main content

subc_daemon/
router.rs

1use std::{
2    collections::HashMap,
3    error::Error,
4    fmt,
5    sync::{
6        atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
7        Arc, Mutex,
8    },
9    time::{Duration, Instant},
10};
11
12use subc_protocol::{error_codes, ErrorBody, Flags, FrameType, Priority};
13use tokio::sync::{mpsc, Notify};
14use tracing::{debug, warn};
15
16use crate::{
17    control::ControlHandler,
18    forwarding::{
19        principal_label, CloseReason, ConnectionCloseReceiver, DataRoute, DataRouteState,
20        ForwardingError, ForwardingTable, RouteBinding, RouteRelease, UndeliveredFrame,
21    },
22    registry::ConnectionId,
23    DaemonCounters, Frame, FrameBuildError,
24};
25
26/// One queued outbound frame plus the instant it entered the writer queue.
27///
28/// The stamp exists for the reply-path half of slow-control diagnosis: a
29/// handler can finish in microseconds while the reply sits in this queue
30/// waiting for the writer task to be scheduled, and without a per-item stamp
31/// that wait is invisible to every other timing point (the client's round
32/// trip is the only witness, and it cannot say which side ate the time).
33/// Constructed exclusively inside [`FrameSink`] so no caller can forget it.
34#[derive(Debug)]
35pub struct OutboundFrame {
36    pub frame: Frame,
37    pub enqueued_at: std::time::Instant,
38    pub(crate) flushed: Option<tokio::sync::oneshot::Sender<()>>,
39    /// This frame's share of the connection's queued-byte count. Dropping the
40    /// frame (after the writer has written it, or when the queue itself is
41    /// dropped with the frame still in it) gives the bytes back. `None` only for
42    /// frames built outside a [`FrameSink`], which exist in tests alone.
43    pub(crate) charge: Option<EgressCharge>,
44}
45
46impl OutboundFrame {
47    fn charged(frame: Frame, charge: EgressCharge) -> Self {
48        Self {
49            frame,
50            enqueued_at: charge.enqueued_at,
51            flushed: None,
52            charge: Some(charge),
53        }
54    }
55}
56
57/// Bytes a frame occupies in the connection's egress queue for budget
58/// purposes: the fixed envelope header plus the body.
59fn queued_frame_bytes(frame: &Frame) -> usize {
60    subc_protocol::HEADER_LEN + frame.body.len()
61}
62
63/// A point-in-time view of one connection's egress queue, for diagnosing why a
64/// frame did not fit.
65#[derive(Debug, Clone, Copy, PartialEq, Eq)]
66pub(crate) struct EgressBacklog {
67    pub queued_bytes: usize,
68    pub queued_frames: usize,
69    /// Now minus the enqueue time of the frame the writer most recently took
70    /// off the queue (the frame it is writing, or has just written), or, if
71    /// the writer has taken nothing since the queue was last empty, of the
72    /// frame that made the queue non-empty. While the writer is blocked on a
73    /// frame this is exactly the oldest frame's age; between frames it can be
74    /// one frame older than the true head. `None` when nothing is queued.
75    pub oldest_age: Option<Duration>,
76}
77
78/// Queued-byte accounting shared by every clone of one connection's
79/// [`FrameSink`] and by every frame that sink has admitted. Its per-frame
80/// accounting uses atomics: this is on the path of every frame a module sends
81/// to a client, so it takes no lock per frame.
82#[derive(Debug)]
83struct EgressAccounting {
84    // Shared by the connection's sink clones so even an invalid data frame
85    // spends the one opportunity to present a claim after authentication.
86    ingress_seen: AtomicBool,
87    ingress_terminal: AtomicBool,
88    egress_terminal: AtomicBool,
89    egress_stopped: Notify,
90    byte_budget: usize,
91    queued_bytes: AtomicUsize,
92    queued_frames: AtomicUsize,
93    /// Origin for the nanosecond timestamps in `oldest_enqueued_nanos`.
94    time_base: Instant,
95    /// Enqueue time, as nanoseconds after `time_base` plus one, of the frame
96    /// described by [`EgressBacklog::oldest_age`]; 0 means none.
97    oldest_enqueued_nanos: AtomicU64,
98    /// Awaited senders currently parked on `freed`. A release only pays for a
99    /// notification when this is non-zero.
100    waiters: AtomicUsize,
101    /// Woken when a charged frame releases its bytes while a sender waits, so
102    /// the awaited send can re-check for room.
103    freed: Notify,
104    client_delivery_closed: AtomicBool,
105    client_delivery_stopped: Notify,
106    #[cfg(test)]
107    retire_after_reserve: AtomicBool,
108}
109
110impl EgressAccounting {
111    fn new(byte_budget: usize) -> Self {
112        Self {
113            ingress_seen: AtomicBool::new(false),
114            ingress_terminal: AtomicBool::new(false),
115            egress_terminal: AtomicBool::new(false),
116            egress_stopped: Notify::new(),
117            byte_budget,
118            queued_bytes: AtomicUsize::new(0),
119            queued_frames: AtomicUsize::new(0),
120            time_base: tokio::time::Instant::now().into_std(),
121            oldest_enqueued_nanos: AtomicU64::new(0),
122            waiters: AtomicUsize::new(0),
123            freed: Notify::new(),
124            client_delivery_closed: AtomicBool::new(false),
125            client_delivery_stopped: Notify::new(),
126            #[cfg(test)]
127            retire_after_reserve: AtomicBool::new(false),
128        }
129    }
130
131    fn stamp(&self, at: Instant) -> u64 {
132        (at.saturating_duration_since(self.time_base).as_nanos() as u64).saturating_add(1)
133    }
134
135    /// Charge `bytes` if they fit in the budget. A frame larger than the whole
136    /// budget (bodies may be up to 64 MiB) is still admitted into an EMPTY
137    /// queue, since it could otherwise never be sent at all; it simply has the
138    /// queue to itself until it is written.
139    fn try_charge(self: &Arc<Self>, bytes: usize) -> Option<EgressCharge> {
140        // SeqCst pairs with `release`: either this load sees bytes a release
141        // just freed, or that release sees this sender counted in `waiters`.
142        let mut current = self.queued_bytes.load(Ordering::SeqCst);
143        loop {
144            if current != 0 && current.saturating_add(bytes) > self.byte_budget {
145                return None;
146            }
147            match self.queued_bytes.compare_exchange_weak(
148                current,
149                current + bytes,
150                Ordering::SeqCst,
151                Ordering::SeqCst,
152            ) {
153                Ok(_) => return Some(self.record(bytes)),
154                Err(actual) => current = actual,
155            }
156        }
157    }
158
159    /// Charge `bytes` regardless of the budget. Used only for a frame whose
160    /// queue slot was reserved in advance, which must be sendable.
161    fn charge_unconditionally(self: &Arc<Self>, bytes: usize) -> EgressCharge {
162        self.queued_bytes.fetch_add(bytes, Ordering::SeqCst);
163        self.record(bytes)
164    }
165
166    fn record(self: &Arc<Self>, bytes: usize) -> EgressCharge {
167        let enqueued_at = tokio::time::Instant::now().into_std();
168        let stamp = self.stamp(enqueued_at);
169        if self.queued_frames.fetch_add(1, Ordering::AcqRel) == 0 {
170            // This frame made the queue non-empty, so it is the head until the
171            // writer takes something.
172            self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
173        }
174        EgressCharge {
175            accounting: Arc::clone(self),
176            bytes,
177            stamp,
178            enqueued_at,
179        }
180    }
181
182    /// The writer has taken the frame stamped `stamp` off the queue.
183    fn taken_by_writer(&self, stamp: u64) {
184        self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
185    }
186
187    fn release(&self, bytes: usize, stamp: u64) {
188        self.queued_bytes.fetch_sub(bytes, Ordering::SeqCst);
189        if self.queued_frames.fetch_sub(1, Ordering::AcqRel) == 1 {
190            // The queue drained. Clear the marker only if it still names this
191            // frame, so a frame admitted in the meantime keeps its stamp.
192            let _ = self.oldest_enqueued_nanos.compare_exchange(
193                stamp,
194                0,
195                Ordering::AcqRel,
196                Ordering::Acquire,
197            );
198        }
199        if self.waiters.load(Ordering::SeqCst) != 0 {
200            self.freed.notify_waiters();
201        }
202    }
203
204    fn backlog(&self) -> EgressBacklog {
205        let queued_frames = self.queued_frames.load(Ordering::Acquire);
206        let stamp = self.oldest_enqueued_nanos.load(Ordering::Acquire);
207        let oldest_age = (queued_frames != 0 && stamp != 0).then(|| {
208            let enqueued = self.time_base + Duration::from_nanos(stamp - 1);
209            tokio::time::Instant::from_std(enqueued).elapsed()
210        });
211        EgressBacklog {
212            queued_bytes: self.queued_bytes.load(Ordering::Acquire),
213            queued_frames,
214            oldest_age,
215        }
216    }
217}
218
219/// One admitted frame's claim on its connection's egress byte budget,
220/// returned when the frame is dropped.
221#[derive(Debug)]
222pub(crate) struct EgressCharge {
223    accounting: Arc<EgressAccounting>,
224    bytes: usize,
225    stamp: u64,
226    enqueued_at: Instant,
227}
228
229impl EgressCharge {
230    /// Called by the connection writer when it takes this frame off the queue,
231    /// so the backlog's oldest-age figure follows the writer.
232    pub(crate) fn taken_by_writer(&self) {
233        self.accounting.taken_by_writer(self.stamp);
234    }
235}
236
237impl Drop for EgressCharge {
238    fn drop(&mut self) {
239        self.accounting.release(self.bytes, self.stamp);
240    }
241}
242
243/// A queue slot reserved ahead of time (a pending `route.open` holds one until
244/// its module answers). Sending through it never waits and never fails for
245/// lack of room: the slot is already held, and its frame is charged to the
246/// byte count outside the budget check. At most
247/// `MAX_PENDING_ROUTE_OPENS_PER_CONNECTION` such frames exist per connection.
248#[derive(Debug)]
249pub(crate) struct EgressPermit {
250    permit: mpsc::OwnedPermit<OutboundFrame>,
251    accounting: Arc<EgressAccounting>,
252}
253
254impl EgressPermit {
255    /// Enqueue `frame` in the reserved slot. Returns true when the connection's
256    /// writer had already gone away or egress was retired, so the frame will
257    /// never be written.
258    pub(crate) fn send(self, frame: Frame) -> bool {
259        if self.accounting.egress_terminal.load(Ordering::Acquire) {
260            return true;
261        }
262        let charge = self
263            .accounting
264            .charge_unconditionally(queued_frame_bytes(&frame));
265        let sender = self.permit.send(OutboundFrame::charged(frame, charge));
266        sender.is_closed()
267    }
268}
269
270/// An `OutboundFrame` is a stamped `Frame`; deref keeps every existing frame
271/// read (headers, bodies, assertions) working on queued items unchanged.
272impl std::ops::Deref for OutboundFrame {
273    type Target = Frame;
274
275    fn deref(&self) -> &Frame {
276        &self.frame
277    }
278}
279
280/// Shared tracing-capture helpers for timing-observability tests across
281/// modules (router dispatch, server reply path). Test-only.
282#[cfg(test)]
283pub(crate) mod test_log {
284    use std::{
285        io::Write,
286        sync::{Arc, Mutex, OnceLock},
287    };
288
289    #[derive(Clone)]
290    struct TestLogWriter(Arc<Mutex<Vec<u8>>>);
291
292    impl Write for TestLogWriter {
293        fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
294            self.0
295                .lock()
296                .expect("test log capture is not poisoned")
297                .extend(buffer);
298            Ok(buffer.len())
299        }
300
301        fn flush(&mut self) -> std::io::Result<()> {
302            Ok(())
303        }
304    }
305
306    pub(crate) fn log_capture(
307        level: tracing::Level,
308    ) -> (Arc<Mutex<Vec<u8>>>, tracing::dispatcher::DefaultGuard) {
309        // tracing-core's single-dispatch fast path registers callsites using
310        // the emitting thread's default subscriber. A parallel test with no
311        // subscriber can therefore cache `never` for a callsite this capture
312        // needs. Keep a second dispatch alive so registration considers all
313        // live subscribers instead. This registry is never installed as a
314        // default: each test still records only into its own thread's writer.
315        static INTEREST_ANCHOR: OnceLock<tracing::Dispatch> = OnceLock::new();
316        INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
317        let output = Arc::new(Mutex::new(Vec::new()));
318        let writer = Arc::clone(&output);
319        let subscriber = tracing_subscriber::fmt()
320            .with_max_level(level)
321            .with_ansi(false)
322            .without_time()
323            .with_target(false)
324            .with_writer(move || TestLogWriter(Arc::clone(&writer)))
325            .finish();
326        let guard = tracing::subscriber::set_default(subscriber);
327        (output, guard)
328    }
329
330    pub(crate) fn captured_logs(output: &Arc<Mutex<Vec<u8>>>) -> String {
331        String::from_utf8(
332            output
333                .lock()
334                .expect("test log capture is not poisoned")
335                .clone(),
336        )
337        .expect("tracing output is UTF-8")
338    }
339
340    #[cfg(test)]
341    mod tests {
342        use super::*;
343
344        fn emit_capture_probe(source: &str) {
345            tracing::error!(source, "parallel capture probe");
346        }
347
348        #[test]
349        fn parallel_callsite_registration_keeps_the_test_capture_enabled() {
350            let (output, _guard) = log_capture(tracing::Level::ERROR);
351            // First register the very same callsite on a thread without a
352            // default subscriber, while the test's capture remains live.
353            std::thread::spawn(|| emit_capture_probe("other-thread"))
354                .join()
355                .expect("unsubscribed thread exits");
356            emit_capture_probe("captured-thread");
357
358            let logs = captured_logs(&output);
359            assert!(
360                logs.contains("captured-thread"),
361                "capture was disabled: {logs}"
362            );
363            assert!(!logs.contains("other-thread"), "capture leaked: {logs}");
364        }
365    }
366}
367
368/// Cheaply cloneable handle to one connection's bounded outbound frame queue.
369///
370/// Backends emit responses, streaming frames, and future PUSH frames through this
371/// single path. The queue is bounded twice: by queued BYTES (the budget that
372/// matters for memory and for how long a slow reader may pause), and by the
373/// `mpsc` channel's frame count (a backstop). The socket layer owns the sole
374/// receiver/writer.
375#[derive(Debug, Clone)]
376pub struct FrameSink {
377    tx: mpsc::Sender<OutboundFrame>,
378    accounting: Arc<EgressAccounting>,
379}
380
381impl FrameSink {
382    pub(crate) fn terminate_ingress(&self) {
383        self.accounting
384            .ingress_terminal
385            .store(true, Ordering::Release);
386    }
387
388    pub(crate) fn ingress_terminal(&self) -> bool {
389        self.accounting.ingress_terminal.load(Ordering::Acquire)
390    }
391
392    // Retirement fences every sink clone and reserved permit synchronously,
393    // including senders already waiting for bytes or a queue slot.
394    pub(crate) fn terminate_egress(&self) {
395        self.accounting
396            .egress_terminal
397            .store(true, Ordering::Release);
398        self.accounting.egress_stopped.notify_waiters();
399        self.accounting.freed.notify_waiters();
400    }
401
402    pub(crate) fn egress_terminal(&self) -> bool {
403        self.accounting.egress_terminal.load(Ordering::Acquire)
404    }
405
406    pub(crate) fn first_ingress_frame(&self) -> bool {
407        !self.accounting.ingress_seen.swap(true, Ordering::Relaxed)
408    }
409    /// A sink with the standard per-connection byte budget
410    /// ([`crate::server::CONNECTION_EGRESS_BYTE_BUDGET`]); the frame-count bound
411    /// is whatever capacity `tx`'s channel was created with.
412    pub fn new(tx: mpsc::Sender<OutboundFrame>) -> Self {
413        Self::with_byte_budget(tx, crate::server::CONNECTION_EGRESS_BYTE_BUDGET)
414    }
415
416    pub(crate) fn with_byte_budget(tx: mpsc::Sender<OutboundFrame>, byte_budget: usize) -> Self {
417        Self {
418            tx,
419            accounting: Arc::new(EgressAccounting::new(byte_budget)),
420        }
421    }
422
423    /// Wait until the frame's bytes fit the budget, then charge them. Awaited
424    /// senders (control replies, error replies, shutdown notices) are never
425    /// refused by the byte budget: they wait for the writer to free bytes, just
426    /// as they wait for a free slot, which is the same backpressure they had
427    /// when the queue was bounded by frame count alone. Returns `None` if the
428    /// writer goes away or egress is retired while waiting.
429    async fn charge_waiting(&self, bytes: usize) -> Option<EgressCharge> {
430        if self.egress_terminal() {
431            return None;
432        }
433        if let Some(charge) = self.accounting.try_charge(bytes) {
434            return Some(charge);
435        }
436        // Counted as a waiter for as long as this future is parked, including
437        // when it is cancelled, so releases notify only while someone waits.
438        struct Waiting<'a>(&'a AtomicUsize);
439        impl Drop for Waiting<'_> {
440            fn drop(&mut self) {
441                self.0.fetch_sub(1, Ordering::SeqCst);
442            }
443        }
444        self.accounting.waiters.fetch_add(1, Ordering::SeqCst);
445        let _waiting = Waiting(&self.accounting.waiters);
446        loop {
447            let freed = self.accounting.freed.notified();
448            tokio::pin!(freed);
449            // Register interest before checking, so a release that happens
450            // between the check and the await still wakes this sender.
451            freed.as_mut().enable();
452            if self.egress_terminal() {
453                return None;
454            }
455            if let Some(charge) = self.accounting.try_charge(bytes) {
456                return Some(charge);
457            }
458            tokio::select! {
459                _ = &mut freed => {}
460                _ = self.tx.closed() => return None,
461            }
462        }
463    }
464
465    // Queue admission must recheck retirement after waiting for a frame slot,
466    // including when the bytes were charged before retirement began. Ordinary
467    // sends silently discard retired traffic; flush callers must not report it
468    // as queued or written.
469    async fn send_outbound(&self, outbound: OutboundFrame) -> Result<bool, RouterError> {
470        let stopped = self.accounting.egress_stopped.notified();
471        tokio::pin!(stopped);
472        stopped.as_mut().enable();
473        if self.egress_terminal() {
474            return Ok(false);
475        }
476        let header = &outbound.frame.header;
477        let closed = || {
478            RouterError::backend_with_epoch(
479                header.channel,
480                header.epoch,
481                header.corr,
482                "connection writer closed",
483            )
484        };
485        let permit = tokio::select! {
486            biased;
487            _ = &mut stopped => return Ok(false),
488            permit = self.tx.reserve() => permit.map_err(|_| closed())?,
489        };
490        if self.egress_terminal() {
491            return Ok(false);
492        }
493        permit.send(outbound);
494        Ok(true)
495    }
496
497    pub async fn send(&self, frame: Frame) -> Result<(), RouterError> {
498        if self.egress_terminal() {
499            return Ok(());
500        }
501        let channel = frame.header.channel;
502        let epoch = frame.header.epoch;
503        let corr = frame.header.corr;
504        let closed =
505            || RouterError::backend_with_epoch(channel, epoch, corr, "connection writer closed");
506        let Some(charge) = self.charge_waiting(queued_frame_bytes(&frame)).await else {
507            return if self.egress_terminal() {
508                Ok(())
509            } else {
510                Err(closed())
511            };
512        };
513        self.send_outbound(OutboundFrame::charged(frame, charge))
514            .await
515            .map(|_| ())
516    }
517
518    /// Shutdown notices must leave the socket writer before an idle daemon exits.
519    /// Queue admission alone does not prove this; the writer acknowledges flush.
520    ///
521    /// The local endpoint uses it the same way for its clients-only refusal: the
522    /// connection closes right after the refusal is sent, so the writer must
523    /// flush it first.
524    pub async fn send_flushed(&self, frame: Frame) -> Result<(), RouterError> {
525        self.send_flushed_notifying_enqueue(frame, || {}).await
526    }
527
528    /// Notify the caller after queue admission, before waiting for the writer's
529    /// flush, so cancellation withdraws only requests that could have arrived.
530    pub(crate) async fn send_flushed_notifying_enqueue(
531        &self,
532        frame: Frame,
533        queued: impl FnOnce(),
534    ) -> Result<(), RouterError> {
535        let (tx, rx) = tokio::sync::oneshot::channel();
536        let charge = self
537            .charge_waiting(queued_frame_bytes(&frame))
538            .await
539            .ok_or_else(|| RouterError::backend(0, 0, "connection writer closed"))?;
540        let mut outbound = OutboundFrame::charged(frame, charge);
541        outbound.flushed = Some(tx);
542        if !self.send_outbound(outbound).await? {
543            return Err(RouterError::backend(0, 0, "connection egress retired"));
544        }
545        queued();
546        rx.await
547            .map_err(|_| RouterError::backend(0, 0, "connection flush failed"))
548    }
549
550    pub(crate) async fn reserve_owned(&self) -> Result<EgressPermit, RouterError> {
551        let stopped = self.accounting.egress_stopped.notified();
552        tokio::pin!(stopped);
553        stopped.as_mut().enable();
554        if self.egress_terminal() {
555            return Err(RouterError::backend(0, 0, "connection egress retired"));
556        }
557        let permit = tokio::select! {
558            biased;
559            _ = &mut stopped => return Err(RouterError::backend(0, 0, "connection egress retired")),
560            permit = self.tx.clone().reserve_owned() => permit
561                .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?,
562        };
563        #[cfg(test)]
564        if self
565            .accounting
566            .retire_after_reserve
567            .swap(false, Ordering::Relaxed)
568        {
569            self.terminate_egress();
570        }
571        if self.egress_terminal() {
572            return Err(RouterError::backend(0, 0, "connection egress retired"));
573        }
574        Ok(EgressPermit {
575            permit,
576            accounting: Arc::clone(&self.accounting),
577        })
578    }
579
580    #[cfg(test)]
581    pub(crate) fn try_reserve_owned(&self) -> Result<EgressPermit, RouterError> {
582        if self.egress_terminal() {
583            return Err(RouterError::backend(0, 0, "connection egress retired"));
584        }
585        let permit = self
586            .tx
587            .clone()
588            .try_reserve_owned()
589            .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
590        Ok(EgressPermit {
591            permit,
592            accounting: Arc::clone(&self.accounting),
593        })
594    }
595
596    pub(crate) fn is_closed(&self) -> bool {
597        self.tx.is_closed()
598    }
599
600    /// What the connection's egress queue holds right now.
601    pub(crate) fn backlog(&self) -> EgressBacklog {
602        self.accounting.backlog()
603    }
604
605    /// Stop module deliveries as soon as the client reader exits, without
606    /// waiting for the socket writer's separate teardown drain grace.
607    pub(crate) fn close_client_delivery(&self) {
608        self.accounting
609            .client_delivery_closed
610            .store(true, Ordering::Release);
611        self.accounting.client_delivery_stopped.notify_waiters();
612    }
613
614    /// A full queue can be a short fanout burst rather than a stopped reader.
615    /// Spend only the grace the atomic backlog age has not already used.
616    async fn send_to_client(&self, frame: Frame) -> Result<(), ClientDeliveryFailure> {
617        if self.egress_terminal() {
618            return Ok(());
619        }
620        let frame_type = frame.header.ty;
621        let flags = frame.header.flags;
622        let stopped = self.accounting.client_delivery_stopped.notified();
623        tokio::pin!(stopped);
624        stopped.as_mut().enable();
625        let failure = |cause, error| ClientDeliveryFailure {
626            diagnosis: ClientDeliveryDiagnosis {
627                frame_type,
628                flags,
629                cause,
630            },
631            error,
632        };
633        let closed = || {
634            RouterError::backend_with_epoch(
635                frame.header.channel,
636                frame.header.epoch,
637                frame.header.corr,
638                "client connection closed",
639            )
640        };
641        if self
642            .accounting
643            .client_delivery_closed
644            .load(Ordering::Acquire)
645        {
646            return Err(failure("client_closed", closed()));
647        }
648        let (frame, error) = match self.try_send_owned(frame) {
649            Ok(()) => return Ok(()),
650            Err(refused) => refused,
651        };
652        if self.is_closed() {
653            return Err(failure("client_closed", error));
654        }
655        let grace = crate::server::CONNECTION_EGRESS_STALL_GRACE;
656        let age = self.backlog().oldest_age.unwrap_or_default();
657        if age > grace {
658            return Err(failure("oldest_frame_older_than_250_ms", error));
659        }
660        let result = tokio::select! {
661            biased;
662            _ = &mut stopped => return Err(failure("client_closed", error)),
663            result = tokio::time::timeout(grace.saturating_sub(age), self.send(frame)) => result,
664        };
665        match result {
666            Ok(Ok(())) => Ok(()),
667            Ok(Err(error)) => Err(failure("client_closed", error)),
668            Err(_) => Err(failure("waited_250_ms_without_room", error)),
669        }
670    }
671
672    /// Enqueue without waiting. Fails when the frame's bytes would take the
673    /// queue past its byte budget, when the channel's frame-count backstop is
674    /// full, or when the writer is gone.
675    pub(crate) fn try_send(&self, frame: Frame) -> Result<(), RouterError> {
676        self.try_send_owned(frame).map_err(|(_, error)| error)
677    }
678
679    fn try_send_owned(&self, frame: Frame) -> Result<(), (Frame, RouterError)> {
680        if self.egress_terminal() {
681            return Ok(());
682        }
683        let channel = frame.header.channel;
684        let epoch = frame.header.epoch;
685        let corr = frame.header.corr;
686        let unavailable = |why: String| {
687            RouterError::backend_with_epoch(
688                channel,
689                epoch,
690                corr,
691                format!("connection writer unavailable: {why}"),
692            )
693        };
694        let bytes = queued_frame_bytes(&frame);
695        let Some(charge) = self.accounting.try_charge(bytes) else {
696            if self.tx.is_closed() {
697                return Err((frame, unavailable("channel closed".to_string())));
698            }
699            return Err((
700                frame,
701                unavailable(format!(
702                    "egress byte budget exhausted ({} queued bytes, frame of {bytes} bytes, budget {})",
703                    self.accounting.queued_bytes.load(Ordering::Acquire),
704                    self.accounting.byte_budget
705                )),
706            ));
707        };
708        // Return the refused frame for a bounded retry; dropping the rest of
709        // the outbound wrapper returns its charge.
710        if self.egress_terminal() {
711            return Ok(());
712        }
713        self.tx
714            .try_send(OutboundFrame::charged(frame, charge))
715            .map_err(|err| {
716                let error = unavailable(err.to_string());
717                (err.into_inner().frame, error)
718            })
719    }
720}
721
722#[derive(Debug)]
723struct ClientDeliveryFailure {
724    diagnosis: ClientDeliveryDiagnosis,
725    error: RouterError,
726}
727
728/// Header facts only: payloads can contain secrets and must never enter logs.
729#[derive(Debug)]
730pub(crate) struct ClientDeliveryDiagnosis {
731    pub frame_type: FrameType,
732    pub flags: Flags,
733    pub cause: &'static str,
734}
735
736impl fmt::Display for ClientDeliveryFailure {
737    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
738        self.error.fmt(f)
739    }
740}
741
742/// Minimum time between two route GOODBYEs the daemon sends one module
743/// connection for one channel in answer to the module's frames on a (channel,
744/// epoch) the daemon holds no route for.
745///
746/// A module that missed a GOODBYE keeps sending on that route (a streaming
747/// module, many frames a second), and every one of those frames lands here. One
748/// answer is enough when it arrives, so answering each frame would only flood
749/// the module's egress queue at the moment it is already behind. Five seconds
750/// is long enough for an answer queued behind a deep backlog to reach the module
751/// and take effect before a second is sent, and short enough that an answer
752/// the queue refused is retried by the next orphan frame well within a minute.
753const ORPHAN_ROUTE_GOODBYE_INTERVAL: Duration = Duration::from_secs(5);
754
755/// Once one connection has this many remembered channels, entries older than
756/// [`ORPHAN_ROUTE_GOODBYE_INTERVAL`] are pruned before another is added. An
757/// expired entry has no effect on rate limiting, so pruning never changes a
758/// decision; it keeps a module that orphaned many channels long ago from
759/// pinning memory for all of them. The map is bounded in any case by the
760/// 16-bit channel space and is dropped whole when the connection ends.
761const ORPHAN_ROUTE_GOODBYE_PRUNE_AT: usize = 256;
762
763/// When each module connection was last sent an orphan-route GOODBYE, per
764/// channel. Shared by the [`Router`] (which consults it) and every
765/// [`RouterConnection`] (which removes its own entry when the connection ends).
766#[derive(Debug, Default)]
767struct OrphanGoodbyeLimiter {
768    last_sent: Mutex<HashMap<ConnectionId, HashMap<u16, tokio::time::Instant>>>,
769}
770
771impl OrphanGoodbyeLimiter {
772    /// True, and the send recorded, when `channel` on `connection_id` has not
773    /// been answered within the interval.
774    fn claim(&self, connection_id: ConnectionId, channel: u16) -> bool {
775        let now = tokio::time::Instant::now();
776        let mut last_sent = self
777            .last_sent
778            .lock()
779            .unwrap_or_else(std::sync::PoisonError::into_inner);
780        let channels = last_sent.entry(connection_id).or_default();
781        if channels.get(&channel).is_some_and(|sent| {
782            now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
783        }) {
784            return false;
785        }
786        if channels.len() >= ORPHAN_ROUTE_GOODBYE_PRUNE_AT {
787            channels.retain(|_, sent| {
788                now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
789            });
790        }
791        channels.insert(channel, now);
792        true
793    }
794
795    fn forget_connection(&self, connection_id: ConnectionId) {
796        self.last_sent
797            .lock()
798            .unwrap_or_else(std::sync::PoisonError::into_inner)
799            .remove(&connection_id);
800    }
801}
802
803/// Per-route context shared with backends besides the frame itself.
804#[derive(Debug, Clone)]
805pub struct RouteCtx {
806    pub connection_id: ConnectionId,
807    pub egress: FrameSink,
808}
809
810/// Closed set of data-plane backends for non-zero channels.
811///
812/// Channel 0 is structurally special and remains `Router::control`, not an
813/// enum variant. Static/test backends live in `Router::backends`; the forwarding
814/// backend is selected dynamically only when a per-connection forwarding binding exists.
815#[derive(Debug, Clone)]
816pub enum Backend {
817    Echo(EchoBackend),
818    Forward(ForwardBackend),
819}
820
821impl From<EchoBackend> for Backend {
822    fn from(backend: EchoBackend) -> Self {
823        Self::Echo(backend)
824    }
825}
826
827impl From<ForwardBackend> for Backend {
828    fn from(backend: ForwardBackend) -> Self {
829        Self::Forward(backend)
830    }
831}
832
833impl Backend {
834    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
835        match self {
836            Self::Echo(backend) => backend.handle(ctx, frame).await,
837            Self::Forward(backend) => backend.handle(ctx, frame).await,
838        }
839    }
840}
841
842/// I/O-agnostic splice router keyed by envelope `channel`.
843///
844/// Channel 0 is reserved for subc itself and is always dispatched to the
845/// dedicated control handler. Other channels must be explicitly registered.
846/// Unknown client-originated non-zero channels are translated to canonical JSON `ERROR` frames on
847/// the connection sink so the peer can continue using the same socket. Module-originated frames for
848/// released route channels are logged and dropped as the channel-gone race backstop.
849///
850/// DATA-PLANE BODIES ARE NEVER DECODED HERE, and the consequence is worth stating
851/// because it looks like a guarantee and is not. Additive fields in a request or
852/// response body reach the far side untouched — not because anything permits them,
853/// but because routing reads only the 21-byte header and treats the body as opaque
854/// bytes. That is a performance property, so **nothing prevents it from changing**:
855/// a future reason to inspect a body would convert a wire-transparent path into a
856/// filtering one, and nobody would think of it as a contract change.
857///
858/// So when a peer asks whether subc sees an additive field, the answer is per-path
859/// and this path's zero means WE NEVER LOOK rather than WE LOOK AT EVERYTHING. The
860/// control plane (typed enums at the frame boundary) and the MCP gateway (envelope
861/// unwrap plus a named struct) both narrow; only this one does not.
862pub struct Router {
863    backends: HashMap<u16, Backend>,
864    control: Arc<ControlHandler>,
865    forwarding: Arc<ForwardingTable>,
866    forward_backend: ForwardBackend,
867    counters: DaemonCounters,
868    next_connection_id: AtomicU64,
869    orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
870}
871
872impl Router {
873    pub fn with_control_handler(control: Arc<ControlHandler>) -> Self {
874        // The handler the router serves is the one a swap must tell when it
875        // promotes a candidate; this is where it first sits behind an `Arc`.
876        control.install_swap_promotion_observer();
877        let forwarding = control.forwarding();
878        let counters = control.counters();
879        Self {
880            backends: HashMap::new(),
881            control,
882            forwarding: Arc::clone(&forwarding),
883            forward_backend: ForwardBackend::new(forwarding),
884            counters,
885            // ConnectionId::LOCAL is 0; real socket ids start at 1 and never collide.
886            next_connection_id: AtomicU64::new(1),
887            orphan_goodbyes: Arc::default(),
888        }
889    }
890
891    /// Install only the inherited-channel service; the supervisor must not own
892    /// this router or retain it after the listeners have stopped serving.
893    pub(crate) fn install_registration_service(self: &Arc<Self>, auth: crate::server::ServerAuth) {
894        self.control
895            .install_registration_service(Arc::downgrade(self), auth);
896    }
897
898    pub fn with_default_self_handler() -> Self {
899        Self::with_control_handler(Arc::new(ControlHandler::default()))
900    }
901
902    pub fn forwarding(&self) -> Arc<ForwardingTable> {
903        Arc::clone(&self.forwarding)
904    }
905
906    pub fn register_backend(
907        &mut self,
908        channel: u16,
909        backend: impl Into<Backend>,
910    ) -> Result<(), RouterError> {
911        self.register_backend_arc(channel, Arc::new(backend.into()))
912    }
913
914    pub(crate) fn register_backend_arc(
915        &mut self,
916        channel: u16,
917        backend: Arc<Backend>,
918    ) -> Result<(), RouterError> {
919        if channel == 0 {
920            return Err(RouterError::ReservedChannelZero);
921        }
922        if self.backends.contains_key(&channel) {
923            return Err(RouterError::DuplicateChannel { channel });
924        }
925        self.backends.insert(channel, backend.as_ref().clone());
926        Ok(())
927    }
928
929    /// Start a connection-scoped routing context. Dropping the guard releases
930    /// any control-plane registrations owned by the connection.
931    fn record_module_frame_drop(&self, connection_id: ConnectionId) -> Result<(), RouterError> {
932        let module_id = self
933            .forwarding
934            .module_id_for_connection(connection_id)
935            .map_err(RouterError::Forwarding)?;
936        self.counters
937            .increment_module_frames_dropped_no_route(module_id.as_deref());
938        Ok(())
939    }
940
941    /// A module sent a non-request frame on a (channel, epoch) the daemon holds
942    /// no route for: most often a route the daemon released whose GOODBYE the
943    /// module never received, so the module still believes it is open and keeps
944    /// sending. Count the drop, and tell the module to let go of exactly that
945    /// (channel, epoch) with a route GOODBYE, at most once per
946    /// [`ORPHAN_ROUTE_GOODBYE_INTERVAL`] per connection and channel.
947    ///
948    /// The answer uses `try_send`: it is a best-effort nudge, and if the queue
949    /// refuses it, the module's next frame on that route after the interval
950    /// asks again. A GOODBYE from the module is not answered, since the module
951    /// is already letting go of the route.
952    fn handle_orphan_module_frame(&self, ctx: &RouteCtx, frame: &Frame) -> Result<(), RouterError> {
953        let channel = frame.header.channel;
954        let epoch = frame.header.epoch;
955        let module_id = self
956            .forwarding
957            .module_id_for_connection(ctx.connection_id)
958            .map_err(RouterError::Forwarding)?;
959        self.counters
960            .increment_module_frames_dropped_no_route(module_id.as_deref());
961        if self
962            .forwarding
963            .module_route_epoch_was_allocated(ctx.connection_id, channel, epoch)
964            .map_err(RouterError::Forwarding)?
965        {
966            self.counters
967                .increment_module_frames_dropped_released_route(module_id.as_deref());
968        }
969        if frame.header.ty == FrameType::Goodbye
970            || !self.orphan_goodbyes.claim(ctx.connection_id, channel)
971        {
972            return Ok(());
973        }
974        let goodbye = Frame::build_with_version(
975            frame.header.ver,
976            FrameType::Goodbye,
977            Flags::new(false, Priority::Passive, false),
978            channel,
979            epoch,
980            0,
981            Vec::new(),
982        )
983        .map_err(RouterError::FrameBuild)?;
984        match ctx.egress.try_send(goodbye) {
985            Ok(()) => {
986                self.counters.increment_module_orphan_route_goodbyes_sent();
987                debug!(
988                    connection_id = ctx.connection_id.get(),
989                    module_id = module_id.as_deref().unwrap_or("unknown"),
990                    channel,
991                    epoch,
992                    "answered module frame on a route the daemon does not hold with a route GOODBYE"
993                );
994            }
995            Err(err) => {
996                self.forwarding.warn_frame_drop(
997                    ctx.connection_id,
998                    channel,
999                    epoch,
1000                    "daemon_to_module",
1001                    if ctx.egress.is_closed() {
1002                        "endpoint_gone"
1003                    } else {
1004                        "egress_budget"
1005                    },
1006                );
1007                debug!(
1008                connection_id = ctx.connection_id.get(),
1009                module_id = module_id.as_deref().unwrap_or("unknown"),
1010                channel,
1011                epoch,
1012                error = %err,
1013                "could not enqueue route GOODBYE for module frame on a route the daemon does not hold; the next such frame retries"
1014                );
1015            }
1016        }
1017        Ok(())
1018    }
1019
1020    pub fn begin_connection(&self) -> RouterConnection {
1021        let raw = self.next_connection_id.fetch_add(1, Ordering::Relaxed);
1022        let id = ConnectionId::new(raw);
1023        let close_receiver = self.forwarding.register_connection_close(id);
1024        RouterConnection {
1025            id,
1026            control_handler: Arc::clone(&self.control),
1027            forwarding: Arc::clone(&self.forwarding),
1028            close_receiver: Some(close_receiver),
1029            orphan_goodbyes: Arc::clone(&self.orphan_goodbyes),
1030        }
1031    }
1032
1033    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1034        self.control.route_open_target(frame)
1035    }
1036
1037    pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
1038        self.control.record_local_connection(connection);
1039    }
1040
1041    pub(crate) fn record_captured_peer(
1042        &self,
1043        connection: ConnectionId,
1044        peer: subc_os::local_peer::CapturedPeer,
1045    ) {
1046        self.control.record_captured_peer(connection, peer);
1047    }
1048
1049    pub(crate) fn record_caller_endpoints(
1050        &self,
1051        connection: ConnectionId,
1052        endpoints: Option<(std::net::SocketAddr, std::net::SocketAddr)>,
1053    ) {
1054        self.control.record_caller_endpoints(connection, endpoints);
1055    }
1056
1057    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1058        self.control.defers_privileged_control(ctx, frame)
1059    }
1060
1061    pub(crate) fn defers_config_approval(&self, frame: &Frame) -> bool {
1062        self.control.defers_config_approval(frame)
1063    }
1064
1065    pub(crate) fn reserve_config_approval(
1066        &self,
1067        ctx: &RouteCtx,
1068        frame: &Frame,
1069    ) -> Result<
1070        Option<crate::config_gate::runtime::ApprovalRequestGuard>,
1071        crate::config_gate::GateError,
1072    > {
1073        self.control.reserve_config_approval(ctx, frame)
1074    }
1075
1076    pub(crate) fn route_open_capacity_refusal(
1077        &self,
1078        ctx: &RouteCtx,
1079        frame: &Frame,
1080        target_module_id: &str,
1081        in_flight: usize,
1082        limit: usize,
1083    ) -> Result<Frame, RouterError> {
1084        self.control
1085            .route_open_capacity_refusal(ctx, frame, target_module_id, in_flight, limit)
1086    }
1087
1088    pub async fn route_for_connection(
1089        &self,
1090        ctx: &RouteCtx,
1091        frame: Frame,
1092    ) -> Result<(), RouterError> {
1093        self.route_for_connection_started(ctx, frame, None).await
1094    }
1095
1096    pub(crate) async fn observe_connection_wait<F: std::future::Future>(
1097        &self,
1098        ctx: &RouteCtx,
1099        kind: &'static str,
1100        future: F,
1101    ) -> F::Output {
1102        self.counters
1103            .observe_reader_wait(
1104                || {
1105                    let module = self
1106                        .forwarding
1107                        .module_id_for_connection(ctx.connection_id)
1108                        .ok()
1109                        .flatten();
1110                    let principal = if module.is_some() { "module" } else { "direct" };
1111                    (
1112                        module.unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
1113                        principal.into(),
1114                    )
1115                },
1116                kind,
1117                future,
1118            )
1119            .await
1120    }
1121
1122    pub(crate) async fn route_for_connection_started(
1123        &self,
1124        ctx: &RouteCtx,
1125        frame: Frame,
1126        dispatch_started_at: Option<Instant>,
1127    ) -> Result<(), RouterError> {
1128        // Embedded callers have no socket reader; mark at their ingress instead.
1129        let first = ctx.egress.first_ingress_frame();
1130        self.route_for_connection_ingress(ctx, frame, dispatch_started_at, first)
1131            .await
1132    }
1133
1134    pub(crate) async fn route_for_connection_ingress(
1135        &self,
1136        ctx: &RouteCtx,
1137        frame: Frame,
1138        dispatch_started_at: Option<Instant>,
1139        first: bool,
1140    ) -> Result<(), RouterError> {
1141        if ctx.egress.ingress_terminal() {
1142            return Ok(());
1143        }
1144        let channel = frame.header.channel;
1145        let epoch = frame.header.epoch;
1146        let corr = frame.header.corr;
1147        if channel == 0 {
1148            debug!(
1149                connection_id = ctx.connection_id.get(),
1150                corr,
1151                frame_type = ?frame.header.ty,
1152                "routing control frame"
1153            );
1154            // The connection loop dispatches every control operation directly
1155            // except `route.open`, activity writes, held config approvals and
1156            // privileged requests. Each
1157            // task passes a timestamp captured before
1158            // spawn, so slow-dispatch timing includes scheduler delay but no
1159            // wait in an application-owned queue.
1160            let dispatch_started_at = (frame.header.ty == FrameType::Request)
1161                .then(|| dispatch_started_at.unwrap_or_else(Instant::now));
1162            let responses = self
1163                .control
1164                .handle_control_frame_ingress(ctx, frame, dispatch_started_at, first)
1165                .await?;
1166            for response in responses {
1167                if let Some(view) = self
1168                    .control
1169                    .config_desktop_review_response(ctx.connection_id, &response)
1170                {
1171                    self.observe_connection_wait(
1172                        ctx,
1173                        "reply_egress",
1174                        self.control
1175                            .deliver_config_desktop_review(ctx, response, view),
1176                    )
1177                    .await?;
1178                    continue;
1179                }
1180                self.observe_connection_wait(ctx, "reply_egress", ctx.egress.send(response))
1181                    .await?;
1182            }
1183            return Ok(());
1184        }
1185
1186        let data_route = self
1187            .forwarding
1188            .lookup_data_route(ctx.connection_id, channel, epoch)
1189            .map_err(RouterError::Forwarding)?;
1190
1191        match data_route {
1192            DataRoute::Module(DataRouteState::EpochMismatch) => {
1193                self.forwarding.warn_frame_drop(
1194                    ctx.connection_id,
1195                    channel,
1196                    epoch,
1197                    "module_to_client",
1198                    "stale_epoch",
1199                );
1200                if frame.header.ty == FrameType::Request {
1201                    self.counters
1202                        .increment_module_requests_dropped_stale_route();
1203                    let err = RouterError::StaleRouteEpoch {
1204                        channel,
1205                        epoch,
1206                        corr,
1207                    };
1208                    if let Some(error_frame) = err.to_error_frame() {
1209                        self.observe_connection_wait(
1210                            ctx,
1211                            "reply_egress",
1212                            ctx.egress.send(error_frame),
1213                        )
1214                        .await?;
1215                    }
1216                } else {
1217                    self.handle_orphan_module_frame(ctx, &frame)?;
1218                }
1219                debug!(
1220                    connection_id = ctx.connection_id.get(),
1221                    channel, epoch, corr, "dropping module frame for stale route epoch"
1222                );
1223                return Ok(());
1224            }
1225            DataRoute::Module(DataRouteState::Reserved) => {
1226                self.forwarding.warn_frame_drop(
1227                    ctx.connection_id,
1228                    channel,
1229                    epoch,
1230                    "module_to_client",
1231                    "missing_binding",
1232                );
1233                if frame.header.ty == FrameType::Request {
1234                    self.counters
1235                        .increment_module_requests_dropped_stale_route();
1236                    let err = RouterError::UnknownChannel {
1237                        channel,
1238                        epoch,
1239                        corr,
1240                    };
1241                    if let Some(error_frame) = err.to_error_frame() {
1242                        self.observe_connection_wait(
1243                            ctx,
1244                            "reply_egress",
1245                            ctx.egress.send(error_frame),
1246                        )
1247                        .await?;
1248                    }
1249                } else {
1250                    self.record_module_frame_drop(ctx.connection_id)?;
1251                }
1252                debug!(
1253                    connection_id = ctx.connection_id.get(),
1254                    channel, epoch, corr, "dropping module frame for reserved route handle"
1255                );
1256                return Ok(());
1257            }
1258            DataRoute::Module(DataRouteState::Absent) => {
1259                self.forwarding.warn_frame_drop(
1260                    ctx.connection_id,
1261                    channel,
1262                    epoch,
1263                    "module_to_client",
1264                    "unknown_channel",
1265                );
1266                if frame.header.ty == FrameType::Request {
1267                    self.counters
1268                        .increment_module_requests_dropped_stale_route();
1269                    let err = RouterError::UnknownChannel {
1270                        channel,
1271                        epoch,
1272                        corr,
1273                    };
1274                    if let Some(error_frame) = err.to_error_frame() {
1275                        self.observe_connection_wait(
1276                            ctx,
1277                            "reply_egress",
1278                            ctx.egress.send(error_frame),
1279                        )
1280                        .await?;
1281                    }
1282                } else {
1283                    self.handle_orphan_module_frame(ctx, &frame)?;
1284                }
1285                debug!(
1286                    connection_id = ctx.connection_id.get(),
1287                    channel, epoch, corr, "dropping module frame for absent route handle"
1288                );
1289                return Ok(());
1290            }
1291            DataRoute::Module(DataRouteState::Bound(route)) => {
1292                if frame.header.ty == FrameType::Goodbye {
1293                    if let RouteRelease::Removed(target) = self
1294                        .forwarding
1295                        .release_module_route(ctx.connection_id, channel, epoch)
1296                        .map_err(RouterError::Forwarding)?
1297                    {
1298                        let mut goodbye = frame;
1299                        goodbye.header.channel = target.channel;
1300                        goodbye.header.epoch = target.epoch;
1301                        // A module must never be able to make a frame look
1302                        // daemon-authored, so the origin bit is cleared on
1303                        // every module frame relayed to a client.
1304                        goodbye.header.flags = goodbye.header.flags.without_daemon_origin();
1305                        if let Err(err) = self
1306                            .observe_connection_wait(
1307                                ctx,
1308                                "module_to_client",
1309                                target.sink.send_to_client(goodbye),
1310                            )
1311                            .await
1312                        {
1313                            if !target.close_on_delivery_failure() {
1314                                self.forwarding.warn_frame_drop(
1315                                    target.connection_id,
1316                                    target.channel,
1317                                    target.epoch,
1318                                    "module_to_client",
1319                                    err.diagnosis.cause,
1320                                );
1321                            }
1322                            if target.close_on_delivery_failure()
1323                                && self
1324                                    .forwarding
1325                                    .escalate_client_delivery_failure_diagnosed(
1326                                        target.connection_id,
1327                                        target.channel,
1328                                        target.epoch,
1329                                        CloseReason::new(
1330                                            "route_goodbye_delivery_failed",
1331                                            format!(
1332                                                "failed to enqueue route GOODBYE for client channel {}: {err}",
1333                                                target.channel
1334                                            ),
1335                                        ),
1336                                        UndeliveredFrame {
1337                                            module_id: Some(&route.module_id),
1338                                            sink: &target.sink,
1339                                        },
1340                                        Some(&err.diagnosis),
1341                                    )
1342                                    .map_err(RouterError::Forwarding)?
1343                            {
1344                                self.counters.increment_goodbye_relay_client_failed();
1345                            }
1346                        }
1347                    } else {
1348                        self.forwarding.warn_frame_drop(
1349                            ctx.connection_id,
1350                            channel,
1351                            epoch,
1352                            "module_to_client",
1353                            "missing_binding",
1354                        );
1355                    }
1356                    return Ok(());
1357                }
1358
1359                // Accounting allows an oversized frame to occupy an empty queue
1360                // on other send paths. Refuse it at this splice before admission,
1361                // so the next ordinary reply cannot close a multiplexed client.
1362                let bytes = queued_frame_bytes(&frame);
1363                let limit = route.client_sink.accounting.byte_budget;
1364                if bytes > limit {
1365                    let ty = frame.header.ty;
1366                    let flags = frame.header.flags;
1367                    drop(frame);
1368                    return self
1369                        .refuse_oversized_module_frame(ctx, &route, (ty, flags), corr, bytes)
1370                        .await;
1371                }
1372
1373                // A terminal frame ends the request at the module whether or not
1374                // the client can still take it, so its credit is released before
1375                // delivery is attempted. Releasing only after a successful
1376                // enqueue would leave a drain counting a finished request until
1377                // the client connection's cleanup removes the route.
1378                let releases_credit = is_terminal_frame(frame.header.ty);
1379                if releases_credit {
1380                    route.flow.release_corr(corr);
1381                }
1382                let mut frame = frame;
1383                frame.header.channel = route.client_channel;
1384                frame.header.epoch = route.client_epoch;
1385                // Responses, errors, stream data and ends, and reverse
1386                // requests all pass through here. A module must never be able
1387                // to make a frame look daemon-authored, so the origin bit a
1388                // module may have set is cleared; every other flag bit and
1389                // the body travel unchanged.
1390                frame.header.flags = frame.header.flags.without_daemon_origin();
1391                if let Err(err) = self
1392                    .counters
1393                    .observe_reader_wait(
1394                        || (route.module_id.clone(), principal_label(&route.principal)),
1395                        "module_to_client",
1396                        route.client_sink.send_to_client(frame),
1397                    )
1398                    .await
1399                {
1400                    if self
1401                        .forwarding
1402                        .escalate_client_delivery_failure_diagnosed(
1403                            route.client_connection_id,
1404                            route.client_channel,
1405                            route.client_epoch,
1406                            CloseReason::new(
1407                                "module_to_client_delivery_failed",
1408                                format!(
1409                                    "failed to enqueue module frame for client channel {} corr {corr}: {err}",
1410                                    route.client_channel
1411                                ),
1412                            ),
1413                            UndeliveredFrame {
1414                                module_id: Some(&route.module_id),
1415                                sink: &route.client_sink,
1416                            },
1417                            Some(&err.diagnosis),
1418                        )
1419                        .map_err(RouterError::Forwarding)?
1420                    {
1421                        self.counters
1422                            .increment_client_egress_close_delivery_failed();
1423                    }
1424                    return Ok(());
1425                }
1426                return Ok(());
1427            }
1428            DataRoute::Client(DataRouteState::EpochMismatch) => {
1429                self.forwarding.warn_frame_drop(
1430                    ctx.connection_id,
1431                    channel,
1432                    epoch,
1433                    "client_to_module",
1434                    "stale_epoch",
1435                );
1436                if frame.header.ty == FrameType::Request {
1437                    self.counters.increment_client_frames_dropped_stale_route();
1438                    // Dropped before forwarding; a re-bind retry cannot double-execute this request.
1439                    let err = RouterError::StaleRouteEpoch {
1440                        channel,
1441                        epoch,
1442                        corr,
1443                    };
1444                    if let Some(error_frame) = err.to_error_frame() {
1445                        self.observe_connection_wait(
1446                            ctx,
1447                            "reply_egress",
1448                            ctx.egress.send(error_frame),
1449                        )
1450                        .await?;
1451                    }
1452                }
1453                debug!(
1454                    connection_id = ctx.connection_id.get(),
1455                    channel, epoch, corr, "dropping client frame for stale route epoch"
1456                );
1457                return Ok(());
1458            }
1459            DataRoute::Client(DataRouteState::Reserved) => {
1460                self.forwarding.warn_frame_drop(
1461                    ctx.connection_id,
1462                    channel,
1463                    epoch,
1464                    "client_to_module",
1465                    "missing_binding",
1466                );
1467                if frame.header.ty == FrameType::Request {
1468                    let err = RouterError::UnknownChannel {
1469                        channel,
1470                        epoch,
1471                        corr,
1472                    };
1473                    if let Some(error_frame) = err.to_error_frame() {
1474                        self.observe_connection_wait(
1475                            ctx,
1476                            "reply_egress",
1477                            ctx.egress.send(error_frame),
1478                        )
1479                        .await?;
1480                    }
1481                }
1482                return Ok(());
1483            }
1484            DataRoute::Client(DataRouteState::Bound(route)) => {
1485                if frame.header.ty == FrameType::Goodbye {
1486                    let _ = self
1487                        .control
1488                        .handle_route_goodbye(ctx.connection_id, channel, epoch)?;
1489                    return Ok(());
1490                }
1491                return self.forward_backend.handle_bound(frame, route).await;
1492            }
1493            DataRoute::Client(DataRouteState::Absent) => {}
1494        }
1495
1496        if let Some(backend) = self.backends.get(&channel) {
1497            return backend.handle(ctx.clone(), frame).await.inspect_err(|_| {
1498                self.forwarding.warn_frame_drop(
1499                    ctx.connection_id,
1500                    channel,
1501                    epoch,
1502                    "client_to_backend",
1503                    "endpoint_gone",
1504                );
1505            });
1506        }
1507        self.forwarding.warn_frame_drop(
1508            ctx.connection_id,
1509            channel,
1510            epoch,
1511            "client_to_module",
1512            "unknown_channel",
1513        );
1514        if frame.header.ty == FrameType::Request {
1515            let err = RouterError::UnknownChannel {
1516                channel,
1517                epoch,
1518                corr,
1519            };
1520            if let Some(error_frame) = err.to_error_frame() {
1521                self.observe_connection_wait(ctx, "reply_egress", ctx.egress.send(error_frame))
1522                    .await?;
1523            }
1524        }
1525        Ok(())
1526    }
1527
1528    async fn refuse_oversized_module_frame(
1529        &self,
1530        ctx: &RouteCtx,
1531        route: &RouteBinding,
1532        header: (FrameType, Flags),
1533        corr: u64,
1534        bytes: usize,
1535    ) -> Result<(), RouterError> {
1536        let (ty, flags) = header;
1537        let limit = route.client_sink.accounting.byte_budget;
1538        if ty == FrameType::Push {
1539            // Deployed Rust clients cannot decode a new route.closed reason.
1540            // Until that vocabulary can be extended compatibly, an oversized
1541            // unsolicited push still closes the connection, with its cause named.
1542            let diagnosis = ClientDeliveryDiagnosis {
1543                frame_type: ty,
1544                flags,
1545                cause: "frame_too_large",
1546            };
1547            if self
1548                .forwarding
1549                .escalate_client_delivery_failure_diagnosed(
1550                    route.client_connection_id,
1551                    route.client_channel,
1552                    route.client_epoch,
1553                    CloseReason::new(
1554                        error_codes::FRAME_TOO_LARGE,
1555                        format!(
1556                            "oversized push: size={bytes} limit={limit} (header plus body bytes)"
1557                        ),
1558                    ),
1559                    UndeliveredFrame {
1560                        module_id: Some(&route.module_id),
1561                        sink: &route.client_sink,
1562                    },
1563                    Some(&diagnosis),
1564                )
1565                .map_err(RouterError::Forwarding)?
1566            {
1567                self.counters
1568                    .increment_client_egress_close_delivery_failed();
1569            }
1570            return Ok(());
1571        }
1572
1573        warn!(
1574            connection_id = route.client_connection_id.get(),
1575            module_id = %route.module_id,
1576            client_principal = %principal_label(&route.principal),
1577            client_channel = route.client_channel,
1578            frame_type = ?ty,
1579            frame_bytes = bytes,
1580            byte_limit = limit,
1581            "refusing oversized module-to-client frame with ERROR frame_too_large"
1582        );
1583        if ty == FrameType::Request {
1584            // A reverse request belongs to the module, not to a waiting client.
1585            // Answer its sender without forwarding any part of it to the client.
1586            let error = frame_too_large_error(
1587                route.module_channel,
1588                route.module_epoch,
1589                corr,
1590                bytes,
1591                limit,
1592            )?;
1593            self.observe_connection_wait(ctx, "reply_egress", ctx.egress.send(error))
1594                .await?;
1595        } else {
1596            // A refused stream chunk also ends the waiting request. Releasing
1597            // its credit prevents a finished request from keeping a drain alive.
1598            route.flow.release_corr(corr);
1599            let error = frame_too_large_error(
1600                route.client_channel,
1601                route.client_epoch,
1602                corr,
1603                bytes,
1604                limit,
1605            )?;
1606            // The replacement shares the reply's bounded burst grace, so a
1607            // stopped client cannot indefinitely stall the module frame reader.
1608            if let Err(err) = self
1609                .counters
1610                .observe_reader_wait(
1611                    || (route.module_id.clone(), principal_label(&route.principal)),
1612                    "module_to_client",
1613                    route.client_sink.send_to_client(error),
1614                )
1615                .await
1616            {
1617                if self
1618                    .forwarding
1619                    .escalate_client_delivery_failure_diagnosed(
1620                        route.client_connection_id,
1621                        route.client_channel,
1622                        route.client_epoch,
1623                        CloseReason::new(
1624                            "module_to_client_delivery_failed",
1625                            format!(
1626                                "failed to enqueue frame_too_large for client channel {} corr {corr}: {err}",
1627                                route.client_channel
1628                            ),
1629                        ),
1630                        UndeliveredFrame {
1631                            module_id: Some(&route.module_id),
1632                            sink: &route.client_sink,
1633                        },
1634                        Some(&err.diagnosis),
1635                    )
1636                    .map_err(RouterError::Forwarding)?
1637                {
1638                    self.counters
1639                        .increment_client_egress_close_delivery_failed();
1640                }
1641            }
1642        }
1643        Ok(())
1644    }
1645}
1646
1647impl Default for Router {
1648    fn default() -> Self {
1649        Self::with_default_self_handler()
1650    }
1651}
1652
1653/// Connection-scoped cleanup guard returned by [`Router::begin_connection`].
1654#[must_use]
1655pub struct RouterConnection {
1656    id: ConnectionId,
1657    control_handler: Arc<ControlHandler>,
1658    forwarding: Arc<ForwardingTable>,
1659    close_receiver: Option<ConnectionCloseReceiver>,
1660    orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
1661}
1662
1663impl RouterConnection {
1664    pub fn id(&self) -> ConnectionId {
1665        self.id
1666    }
1667
1668    pub(crate) fn take_close_receiver(&mut self) -> ConnectionCloseReceiver {
1669        self.close_receiver
1670            .take()
1671            .expect("connection close receiver can only be taken once")
1672    }
1673}
1674
1675impl Drop for RouterConnection {
1676    fn drop(&mut self) {
1677        self.orphan_goodbyes.forget_connection(self.id);
1678        // GOODBYE (explicit) and connection-drop cleanup both call the same
1679        // idempotent deregistration path.
1680        let _ = self.control_handler.cleanup_connection(self.id);
1681        // The read loop and its tasks have ended, but the writer can still
1682        // drain. Keep the connection-to-launch association until pending relays
1683        // and routes are gone.
1684        self.forwarding.unregister_connection_close(self.id);
1685    }
1686}
1687
1688/// Minimal in-memory backend used by tests and early wiring: it emits a
1689/// `RESPONSE` on the same channel/correlation id with the exact same body bytes.
1690#[derive(Debug, Default, Clone, Copy)]
1691pub struct EchoBackend;
1692
1693impl EchoBackend {
1694    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1695        let response = Frame::build_with_version(
1696            frame.header.ver,
1697            FrameType::Response,
1698            frame.header.flags,
1699            frame.header.channel,
1700            frame.header.epoch,
1701            frame.header.corr,
1702            frame.body,
1703        )
1704        .map_err(RouterError::FrameBuild)?;
1705        ctx.egress.send(response).await
1706    }
1707}
1708
1709/// Data-plane backend that splices client frames to the module connection bound at attach time.
1710#[derive(Debug, Clone)]
1711pub struct ForwardBackend {
1712    forwarding: Arc<ForwardingTable>,
1713}
1714
1715impl ForwardBackend {
1716    pub fn new(forwarding: Arc<ForwardingTable>) -> Self {
1717        Self { forwarding }
1718    }
1719
1720    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1721        let channel = frame.header.channel;
1722        let corr = frame.header.corr;
1723        let route = match self
1724            .forwarding
1725            .lookup_data_route(ctx.connection_id, channel, frame.header.epoch)
1726            .map_err(RouterError::Forwarding)?
1727        {
1728            DataRoute::Client(DataRouteState::Bound(route)) => route,
1729            DataRoute::Client(_) | DataRoute::Module(_) => {
1730                self.forwarding.warn_frame_drop(
1731                    ctx.connection_id,
1732                    channel,
1733                    frame.header.epoch,
1734                    "client_to_module",
1735                    "missing_binding",
1736                );
1737                return Err(RouterError::UnknownChannel {
1738                    channel,
1739                    epoch: frame.header.epoch,
1740                    corr,
1741                });
1742            }
1743        };
1744        self.handle_bound(frame, route).await
1745    }
1746
1747    pub(crate) async fn handle_bound(
1748        &self,
1749        frame: Frame,
1750        route: Arc<RouteBinding>,
1751    ) -> Result<(), RouterError> {
1752        let channel = frame.header.channel;
1753        let corr = frame.header.corr;
1754        let frame_type = frame.header.ty;
1755
1756        // CANCEL and other non-REQUEST frames bypass the request-credit window;
1757        // the original request's credit is freed only by the module's terminal frame.
1758        let acquired_credit = frame_type == FrameType::Request;
1759        if acquired_credit && route.flow.busy_refusals() {
1760            use crate::forwarding::RequestAdmission;
1761            match self
1762                .forwarding
1763                .try_admit_request(
1764                    &route,
1765                    frame.header.epoch,
1766                    corr,
1767                    frame.header.flags.is_subscription(),
1768                )
1769                .map_err(RouterError::Forwarding)?
1770            {
1771                RequestAdmission::Admitted => {}
1772                RequestAdmission::Busy(detail) => {
1773                    let principal = principal_label(&route.principal);
1774                    let counters = self.forwarding.counters_ref();
1775                    counters.increment_route_busy(&route.module_id, &principal);
1776                    if route.flow.should_warn_busy() {
1777                        warn!(module_id = %route.module_id, client_principal = %principal, client_channel = channel,
1778                            route_epoch = route.client_epoch, ?detail, "request refused: route_busy");
1779                    }
1780                    let error = RouterError::RouteBusy {
1781                        channel,
1782                        epoch: frame.header.epoch,
1783                        corr,
1784                        detail: Box::new(detail),
1785                    };
1786                    let refusal = error
1787                        .to_error_frame()
1788                        .expect("route_busy is a serializable wire error");
1789                    counters
1790                        .observe_reader_wait(
1791                            || (route.module_id.clone(), principal),
1792                            "route_busy_egress",
1793                            route.client_sink.send(refusal),
1794                        )
1795                        .await?;
1796                    return Ok(());
1797                }
1798                RequestAdmission::ModuleReloading => {
1799                    return Err(RouterError::route_error_with_epoch(
1800                        channel,
1801                        frame.header.epoch,
1802                        corr,
1803                        error_codes::MODULE_RELOADING,
1804                        format!("module endpoint for route channel {channel} is reloading"),
1805                    ));
1806                }
1807                RequestAdmission::Stale => {
1808                    return Err(RouterError::StaleRouteEpoch {
1809                        channel,
1810                        epoch: frame.header.epoch,
1811                        corr,
1812                    })
1813                }
1814            }
1815        } else if acquired_credit {
1816            let counters = self.forwarding.counters_ref();
1817            if let Err(err) = counters
1818                .observe_reader_wait(
1819                    || (route.module_id.clone(), principal_label(&route.principal)),
1820                    "legacy_credit",
1821                    route
1822                        .flow
1823                        .acquire_tagged(corr, frame.header.flags.is_subscription()),
1824                )
1825                .await
1826            {
1827                self.forwarding.warn_frame_drop(
1828                    route.client_connection_id,
1829                    channel,
1830                    frame.header.epoch,
1831                    "client_to_module",
1832                    "endpoint_gone",
1833                );
1834                // `module_reloading` here is answered BEFORE the frame is
1835                // forwarded, so the module never sees the request and callers
1836                // may re-dispatch it after reopening the route. That is a wire
1837                // guarantee documented on `error_codes::MODULE_RELOADING`; do
1838                // not emit this code for a request that already reached
1839                // `module_sink.send` below.
1840                if self
1841                    .forwarding
1842                    .endpoint_is_draining(route.module_endpoint)
1843                    .map_err(RouterError::Forwarding)?
1844                {
1845                    return Err(RouterError::route_error_with_epoch(
1846                        channel,
1847                        frame.header.epoch,
1848                        corr,
1849                        "module_reloading",
1850                        format!("module endpoint for route channel {channel} is reloading"),
1851                    ));
1852                }
1853                return Err(RouterError::backend_with_epoch(
1854                    channel,
1855                    frame.header.epoch,
1856                    corr,
1857                    format!("{err} for route channel {channel}"),
1858                ));
1859            }
1860        }
1861
1862        let mut frame = frame;
1863        frame.header.channel = route.module_channel;
1864        frame.header.epoch = route.module_epoch;
1865        // The origin bit is the daemon's own signal to clients. A client
1866        // frame carrying it must not reach a module appearing daemon-authored,
1867        // so it is cleared here; every other flag bit travels unchanged.
1868        frame.header.flags = frame.header.flags.without_daemon_origin();
1869        let counters = self.forwarding.counters_ref();
1870        let result = counters
1871            .observe_reader_wait(
1872                || (route.module_id.clone(), principal_label(&route.principal)),
1873                "module_queue",
1874                route.module_sink.send(frame),
1875            )
1876            .await
1877            .map_err(|err| {
1878                self.forwarding.warn_frame_drop(
1879                    route.client_connection_id,
1880                    channel,
1881                    route.client_epoch,
1882                    "client_to_module",
1883                    "endpoint_gone",
1884                );
1885                RouterError::backend_with_epoch(channel, route.client_epoch, corr, err.to_string())
1886            });
1887        if acquired_credit && result.is_err() {
1888            route.flow.release_corr(corr);
1889        }
1890        result
1891    }
1892}
1893
1894fn is_terminal_frame(frame_type: FrameType) -> bool {
1895    matches!(
1896        frame_type,
1897        FrameType::Response | FrameType::Error | FrameType::StreamEnd
1898    )
1899}
1900
1901/// Typed router errors. Routable failures can be translated to canonical JSON
1902/// `ERROR` frames with [`RouterError::to_error_frame`].
1903#[derive(Debug, Clone, PartialEq, Eq)]
1904pub enum RouterError {
1905    ReservedChannelZero,
1906    DuplicateChannel {
1907        channel: u16,
1908    },
1909    UnknownChannel {
1910        channel: u16,
1911        epoch: u32,
1912        corr: u64,
1913    },
1914    StaleRouteEpoch {
1915        channel: u16,
1916        epoch: u32,
1917        corr: u64,
1918    },
1919    RouteBusy {
1920        channel: u16,
1921        epoch: u32,
1922        corr: u64,
1923        detail: Box<subc_protocol::RouteBusyDetail>,
1924    },
1925    Backend {
1926        channel: u16,
1927        epoch: u32,
1928        corr: u64,
1929        message: String,
1930    },
1931    RouteError {
1932        channel: u16,
1933        epoch: u32,
1934        corr: u64,
1935        code: String,
1936        message: String,
1937    },
1938    FrameBuild(FrameBuildError),
1939    Forwarding(ForwardingError),
1940}
1941
1942impl RouterError {
1943    pub fn backend(channel: u16, corr: u64, message: impl Into<String>) -> Self {
1944        Self::backend_with_epoch(channel, 0, corr, message)
1945    }
1946
1947    pub fn backend_with_epoch(
1948        channel: u16,
1949        epoch: u32,
1950        corr: u64,
1951        message: impl Into<String>,
1952    ) -> Self {
1953        Self::Backend {
1954            channel,
1955            epoch,
1956            corr,
1957            message: message.into(),
1958        }
1959    }
1960
1961    pub fn route_error(
1962        channel: u16,
1963        corr: u64,
1964        code: impl Into<String>,
1965        message: impl Into<String>,
1966    ) -> Self {
1967        Self::route_error_with_epoch(channel, 0, corr, code, message)
1968    }
1969
1970    pub fn route_error_with_epoch(
1971        channel: u16,
1972        epoch: u32,
1973        corr: u64,
1974        code: impl Into<String>,
1975        message: impl Into<String>,
1976    ) -> Self {
1977        Self::RouteError {
1978            channel,
1979            epoch,
1980            corr,
1981            code: code.into(),
1982            message: message.into(),
1983        }
1984    }
1985
1986    /// Translate route failures that belong on the wire into an `ERROR` frame.
1987    pub fn to_error_frame(&self) -> Option<Frame> {
1988        match self {
1989            Self::RouteBusy {
1990                channel,
1991                epoch,
1992                corr,
1993                detail,
1994            } => {
1995                let body = serde_json::to_vec(&ErrorBody {
1996                    code: error_codes::ROUTE_BUSY.into(),
1997                    message: "route has no free request or watch credit; retry on the same route"
1998                        .into(),
1999                    detail: Some(serde_json::to_value(detail).ok()?),
2000                })
2001                .ok()?;
2002                Frame::build(
2003                    FrameType::Error,
2004                    daemon_error_flags(),
2005                    *channel,
2006                    *epoch,
2007                    *corr,
2008                    body,
2009                )
2010                .ok()
2011            }
2012            Self::UnknownChannel {
2013                channel,
2014                epoch,
2015                corr,
2016            } => error_frame(
2017                *channel,
2018                *epoch,
2019                *corr,
2020                error_codes::UNKNOWN_CHANNEL,
2021                format!("unknown channel {channel}"),
2022            ),
2023            Self::StaleRouteEpoch {
2024                channel,
2025                epoch,
2026                corr,
2027            } => error_frame(
2028                *channel,
2029                *epoch,
2030                *corr,
2031                error_codes::STALE_ROUTE_EPOCH,
2032                format!("stale route epoch for channel {channel}"),
2033            ),
2034            Self::Backend {
2035                channel,
2036                epoch,
2037                corr,
2038                message,
2039            } => error_frame(*channel, *epoch, *corr, "backend_error", message.clone()),
2040            Self::RouteError {
2041                channel,
2042                epoch,
2043                corr,
2044                code,
2045                message,
2046            } => error_frame(*channel, *epoch, *corr, code, message.clone()),
2047            Self::ReservedChannelZero
2048            | Self::DuplicateChannel { .. }
2049            | Self::FrameBuild(_)
2050            | Self::Forwarding(_) => None,
2051        }
2052    }
2053}
2054
2055/// Flags for an ERROR frame the daemon authors itself.
2056///
2057/// Bit 6 (`FLAG_DAEMON_ORIGIN`) tells the client the daemon wrote this
2058/// error, never a module. What a code then proves is per code: on a bound
2059/// route the daemon writes `module_reloading`, `unknown_channel` and
2060/// `stale_route_epoch` only before forwarding, so with the bit they mean the
2061/// request was not delivered, while `frame_too_large` replaces a reply the
2062/// module already produced. Every daemon-authored ERROR goes through here;
2063/// frames a module sent keep their own flags with the bit cleared at the
2064/// splice.
2065/// Nothing else about the flags differs: passive priority, not binary, not last.
2066pub(crate) fn daemon_error_flags() -> Flags {
2067    Flags::new(false, Priority::Passive, false).with_daemon_origin()
2068}
2069
2070fn error_frame(channel: u16, epoch: u32, corr: u64, code: &str, message: String) -> Option<Frame> {
2071    let body = serde_json::to_vec(&ErrorBody {
2072        code: code.to_string(),
2073        message,
2074        detail: None,
2075    })
2076    .ok()?;
2077
2078    Frame::build(
2079        FrameType::Error,
2080        daemon_error_flags(),
2081        channel,
2082        epoch,
2083        corr,
2084        body,
2085    )
2086    .ok()
2087}
2088
2089fn frame_too_large_error(
2090    channel: u16,
2091    epoch: u32,
2092    corr: u64,
2093    bytes: usize,
2094    limit: usize,
2095) -> Result<Frame, RouterError> {
2096    let body = serde_json::to_vec(&ErrorBody {
2097        code: error_codes::FRAME_TOO_LARGE.to_string(),
2098        message: format!(
2099            "module frame of {bytes} bytes exceeds client egress limit of {limit} bytes"
2100        ),
2101        detail: Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit })),
2102    })
2103    .expect("serializing a frame size error");
2104    Frame::build(
2105        FrameType::Error,
2106        daemon_error_flags(),
2107        channel,
2108        epoch,
2109        corr,
2110        body,
2111    )
2112    .map_err(RouterError::FrameBuild)
2113}
2114
2115impl fmt::Display for RouterError {
2116    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2117        match self {
2118            Self::ReservedChannelZero => write!(f, "channel 0 is reserved for subc"),
2119            Self::DuplicateChannel { channel } => {
2120                write!(f, "backend already registered for channel {channel}")
2121            }
2122            Self::UnknownChannel { channel, corr, .. } => {
2123                write!(f, "unknown channel {channel} for corr {corr}")
2124            }
2125            Self::StaleRouteEpoch { channel, corr, .. } => {
2126                write!(f, "stale route epoch for channel {channel} corr {corr}")
2127            }
2128            Self::RouteBusy { channel, corr, .. } => {
2129                write!(f, "route busy on channel {channel} corr {corr}")
2130            }
2131            Self::Backend {
2132                channel,
2133                corr,
2134                message,
2135                ..
2136            } => write!(
2137                f,
2138                "backend error on channel {channel} corr {corr}: {message}"
2139            ),
2140            Self::RouteError {
2141                channel,
2142                corr,
2143                code,
2144                message,
2145                ..
2146            } => write!(
2147                f,
2148                "route error {code} on channel {channel} corr {corr}: {message}"
2149            ),
2150            Self::FrameBuild(err) => write!(f, "failed to build routed frame: {err}"),
2151            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
2152        }
2153    }
2154}
2155
2156impl Error for RouterError {
2157    fn source(&self) -> Option<&(dyn Error + 'static)> {
2158        match self {
2159            Self::FrameBuild(err) => Some(err),
2160            Self::Forwarding(err) => Some(err),
2161            Self::ReservedChannelZero
2162            | Self::DuplicateChannel { .. }
2163            | Self::UnknownChannel { .. }
2164            | Self::StaleRouteEpoch { .. }
2165            | Self::RouteBusy { .. }
2166            | Self::Backend { .. }
2167            | Self::RouteError { .. } => None,
2168        }
2169    }
2170}
2171
2172#[cfg(test)]
2173mod tests {
2174    include!("route_busy_router_tests.rs");
2175    use super::*;
2176    use crate::{
2177        forwarding::RouteBindRelayOutcome,
2178        supervise::{ModuleSpec, RestartPolicy, Supervisor, SupervisorHandle},
2179        ControlHandler, Registry,
2180    };
2181    use std::{
2182        sync::{mpsc as std_mpsc, Arc},
2183        time::Duration,
2184    };
2185    use subc_control::ModuleProtocol;
2186    use subc_protocol::{manifest::Concurrency, ErrorBody, Flags, FrameType, Priority};
2187    use tokio::sync::mpsc;
2188
2189    pub(crate) use crate::router::test_log::{captured_logs, log_capture};
2190
2191    fn logged_millis(logs: &str, field: &str) -> u64 {
2192        logs.split_whitespace()
2193            .find_map(|part| part.strip_prefix(field))
2194            .and_then(|value| value.parse().ok())
2195            .unwrap_or_else(|| panic!("missing numeric {field} in logs: {logs}"))
2196    }
2197
2198    fn request(channel: u16, corr: u64, body: &[u8]) -> Frame {
2199        Frame::build(
2200            FrameType::Request,
2201            Flags::new(true, Priority::Interactive, false),
2202            channel,
2203            0,
2204            corr,
2205            body.to_vec(),
2206        )
2207        .unwrap()
2208    }
2209
2210    fn ping(corr: u64) -> Frame {
2211        Frame::build(
2212            FrameType::Ping,
2213            Flags::new(false, Priority::Passive, false),
2214            0,
2215            0,
2216            corr,
2217            Vec::new(),
2218        )
2219        .unwrap()
2220    }
2221
2222    fn route_ctx() -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
2223        let (tx, rx) = mpsc::channel(8);
2224        (
2225            RouteCtx {
2226                connection_id: ConnectionId::LOCAL,
2227                egress: FrameSink::new(tx),
2228            },
2229            rx,
2230        )
2231    }
2232
2233    #[tokio::test]
2234    async fn retirement_egress_all_send_paths_drop_after_fence() {
2235        let (tx, mut rx) = mpsc::channel(8);
2236        let sink = FrameSink::new(tx);
2237        let reserved = sink.reserve_owned().await.unwrap();
2238        let immediate = sink.try_reserve_owned().unwrap();
2239        sink.terminate_egress();
2240        assert!(reserved.send(ping(1)));
2241        assert!(immediate.send(ping(2)));
2242        sink.send(ping(3)).await.unwrap();
2243        sink.try_send(ping(4)).unwrap();
2244        sink.send_to_client(ping(5)).await.unwrap();
2245        let queued = AtomicBool::new(false);
2246        assert!(sink
2247            .send_flushed_notifying_enqueue(ping(6), || queued.store(true, Ordering::Relaxed))
2248            .await
2249            .is_err());
2250        assert!(!queued.load(Ordering::Relaxed));
2251        assert!(sink.send_flushed(ping(7)).await.is_err());
2252        assert!(sink.reserve_owned().await.is_err());
2253        assert!(sink.try_reserve_owned().is_err());
2254        assert!(
2255            rx.try_recv().is_err(),
2256            "retired frames must not enter any writer queue"
2257        );
2258        assert_eq!(sink.backlog().queued_frames, 0);
2259        assert_eq!(sink.backlog().queued_bytes, 0);
2260    }
2261
2262    #[tokio::test]
2263    async fn retirement_egress_reservation_rechecks_after_slot_acquisition() {
2264        let (tx, mut rx) = mpsc::channel(1);
2265        let sink = FrameSink::new(tx);
2266        // Choose retirement after reserve returns its slot, not while waiting
2267        // on egress_stopped, so the final atomic recheck is exercised.
2268        sink.accounting
2269            .retire_after_reserve
2270            .store(true, Ordering::Relaxed);
2271        assert!(
2272            sink.reserve_owned().await.is_err(),
2273            "a slot acquired before retirement must not escape as a live reservation"
2274        );
2275        assert_eq!(
2276            sink.tx.capacity(),
2277            1,
2278            "refusal must release the reserved slot"
2279        );
2280        assert!(rx.try_recv().is_err());
2281    }
2282
2283    #[tokio::test]
2284    async fn retirement_egress_pending_budget_and_slot_sends_stop() {
2285        for budget in [1, usize::MAX] {
2286            let (tx, mut rx) = mpsc::channel(1);
2287            let sink = FrameSink::with_byte_budget(tx, budget);
2288            sink.try_send(ping(1)).unwrap();
2289            let sending = sink.send(ping(2));
2290            tokio::pin!(sending);
2291            assert!(poll_once(sending.as_mut()).await.is_pending());
2292            sink.terminate_egress();
2293            tokio::time::timeout(Duration::from_secs(2), sending)
2294                .await
2295                .unwrap()
2296                .unwrap();
2297            drop(rx.try_recv().unwrap());
2298            assert!(
2299                rx.try_recv().is_err(),
2300                "a send parked before retirement must not enqueue afterward"
2301            );
2302            assert_eq!(sink.backlog().queued_bytes, 0);
2303        }
2304    }
2305
2306    #[tokio::test]
2307    async fn retirement_egress_pending_flush_and_reservation_stop() {
2308        let (tx, mut rx) = mpsc::channel(1);
2309        let sink = FrameSink::new(tx);
2310        sink.try_send(ping(1)).unwrap();
2311        let queued = AtomicBool::new(false);
2312        let sending =
2313            sink.send_flushed_notifying_enqueue(ping(2), || queued.store(true, Ordering::Relaxed));
2314        let reserving = sink.reserve_owned();
2315        tokio::pin!(sending, reserving);
2316        assert!(poll_once(sending.as_mut()).await.is_pending());
2317        assert!(poll_once(reserving.as_mut()).await.is_pending());
2318        sink.terminate_egress();
2319        assert!(tokio::time::timeout(Duration::from_secs(2), sending)
2320            .await
2321            .unwrap()
2322            .is_err());
2323        assert!(tokio::time::timeout(Duration::from_secs(2), reserving)
2324            .await
2325            .unwrap()
2326            .is_err());
2327        assert!(!queued.load(Ordering::Relaxed));
2328        drop(rx.try_recv().unwrap());
2329        assert!(rx.try_recv().is_err());
2330        assert_eq!(sink.backlog().queued_bytes, 0);
2331    }
2332
2333    #[tokio::test]
2334    async fn echo_backend_returns_response_with_byte_identical_body() {
2335        let mut router = Router::with_default_self_handler();
2336        router.register_backend(7, EchoBackend).unwrap();
2337        let (ctx, mut rx) = route_ctx();
2338        let body = b"{not parsed}\0\xff";
2339
2340        router
2341            .route_for_connection(&ctx, request(7, 123, body))
2342            .await
2343            .unwrap();
2344        let response = rx.recv().await.unwrap();
2345
2346        assert_eq!(response.header.ty, FrameType::Response);
2347        assert_eq!(response.header.channel, 7);
2348        assert_eq!(response.header.corr, 123);
2349        assert_eq!(response.body, body);
2350        assert!(rx.try_recv().is_err());
2351    }
2352
2353    #[tokio::test]
2354    async fn unknown_channel_emits_canonical_error_frame() {
2355        let router = Router::with_default_self_handler();
2356        let (ctx, mut rx) = route_ctx();
2357
2358        router
2359            .route_for_connection(&ctx, request(99, 5, b"payload"))
2360            .await
2361            .unwrap();
2362        let error_frame = rx.recv().await.unwrap();
2363
2364        assert_eq!(error_frame.header.ty, FrameType::Error);
2365        assert_eq!(error_frame.header.channel, 99);
2366        assert_eq!(error_frame.header.corr, 5);
2367        let body: ErrorBody = serde_json::from_slice(&error_frame.body).unwrap();
2368        assert_eq!(body.code, "unknown_channel");
2369        assert_eq!(body.message, "unknown channel 99");
2370    }
2371
2372    /// Every ERROR the daemon authors carries the daemon-origin bit, so a
2373    /// client can tell a daemon refusal from a module's reply. Both the
2374    /// unknown-channel and the stale-epoch errors below are received on the
2375    /// client's own egress, not merely built.
2376    #[tokio::test]
2377    async fn daemon_authored_data_plane_errors_carry_daemon_origin() {
2378        let router = Router::with_default_self_handler();
2379        let (ctx, mut rx) = route_ctx();
2380        router
2381            .route_for_connection(&ctx, request(99, 5, b"payload"))
2382            .await
2383            .unwrap();
2384        let unknown = rx.recv().await.unwrap();
2385        assert_eq!(unknown.header.ty, FrameType::Error);
2386        let body: ErrorBody = serde_json::from_slice(&unknown.body).unwrap();
2387        assert_eq!(body.code, "unknown_channel");
2388        assert!(
2389            unknown.header.flags.is_daemon_origin(),
2390            "unknown_channel must carry daemon origin: {:?}",
2391            unknown.header.flags
2392        );
2393
2394        let (
2395            router,
2396            _forwarding,
2397            client_ctx,
2398            mut client_rx,
2399            _module_ctx,
2400            _module_egress_rx,
2401            _module_rx,
2402            pending,
2403        ) = dynamic_route_fixture(true);
2404        let _ = client_rx.recv().await;
2405        router
2406            .route_for_connection(
2407                &client_ctx,
2408                route_frame(
2409                    FrameType::Request,
2410                    pending.client_channel,
2411                    pending.client_epoch + 1,
2412                    702,
2413                ),
2414            )
2415            .await
2416            .unwrap();
2417        let stale = client_rx.recv().await.unwrap();
2418        assert_eq!(stale.header.ty, FrameType::Error);
2419        let body: ErrorBody = serde_json::from_slice(&stale.body).unwrap();
2420        assert_eq!(body.code, "stale_route_epoch");
2421        assert!(
2422            stale.header.flags.is_daemon_origin(),
2423            "stale_route_epoch must carry daemon origin: {:?}",
2424            stale.header.flags
2425        );
2426    }
2427
2428    /// A module ERROR that spoofs a daemon code with the daemon-origin bit
2429    /// set must reach the client with the bit cleared and the body
2430    /// byte-identical: the bit means the daemon wrote the frame, and a module
2431    /// must never be able to claim that.
2432    #[tokio::test]
2433    async fn module_error_with_daemon_bit_is_stripped_before_client_delivery() {
2434        let (
2435            router,
2436            _forwarding,
2437            client_ctx,
2438            mut client_rx,
2439            module_ctx,
2440            _module_egress_rx,
2441            mut module_rx,
2442            pending,
2443        ) = dynamic_route_fixture(true);
2444        let _ = client_rx.recv().await;
2445        // Open flow credit the way a real request would, so the terminal
2446        // error exercises the credit-release path too.
2447        router
2448            .route_for_connection(
2449                &client_ctx,
2450                route_frame(
2451                    FrameType::Request,
2452                    pending.client_channel,
2453                    pending.client_epoch,
2454                    701,
2455                ),
2456            )
2457            .await
2458            .unwrap();
2459        let _ = module_rx.recv().await;
2460
2461        let body = serde_json::to_vec(&ErrorBody::new("module_reloading", "try again")).unwrap();
2462        let spoof = Frame::build(
2463            FrameType::Error,
2464            Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2465            pending.module_channel,
2466            pending.module_epoch,
2467            701,
2468            body.clone(),
2469        )
2470        .unwrap();
2471        router
2472            .route_for_connection(&module_ctx, spoof)
2473            .await
2474            .unwrap();
2475        let delivered = client_rx.recv().await.unwrap();
2476        assert_eq!(delivered.header.ty, FrameType::Error);
2477        assert_eq!(delivered.header.channel, pending.client_channel);
2478        assert_eq!(delivered.header.epoch, pending.client_epoch);
2479        assert_eq!(delivered.header.corr, 701);
2480        assert!(
2481            !delivered.header.flags.is_daemon_origin(),
2482            "module-set daemon origin must be cleared: {:?}",
2483            delivered.header.flags
2484        );
2485        assert_eq!(delivered.body, body, "the body travels byte-identical");
2486    }
2487
2488    /// Clearing the origin bit strips exactly that bit: priority, LAST and
2489    /// subscription survive the splice unchanged.
2490    #[tokio::test]
2491    async fn module_response_with_daemon_bit_is_stripped_preserving_other_flags() {
2492        use subc_protocol::FLAG_SUBSCRIPTION;
2493        let (
2494            router,
2495            _forwarding,
2496            client_ctx,
2497            mut client_rx,
2498            module_ctx,
2499            _module_egress_rx,
2500            mut module_rx,
2501            pending,
2502        ) = dynamic_route_fixture(true);
2503        let _ = client_rx.recv().await;
2504        router
2505            .route_for_connection(
2506                &client_ctx,
2507                route_frame(
2508                    FrameType::Request,
2509                    pending.client_channel,
2510                    pending.client_epoch,
2511                    701,
2512                ),
2513            )
2514            .await
2515            .unwrap();
2516        let _ = module_rx.recv().await;
2517
2518        let sent_flags = Flags(
2519            Flags::new(true, Priority::Background, true).0
2520                | FLAG_SUBSCRIPTION
2521                | subc_protocol::FLAG_DAEMON_ORIGIN,
2522        );
2523        assert!(sent_flags.is_daemon_origin());
2524        let response = Frame::build(
2525            FrameType::Response,
2526            sent_flags,
2527            pending.module_channel,
2528            pending.module_epoch,
2529            701,
2530            b"route-body".to_vec(),
2531        )
2532        .unwrap();
2533        router
2534            .route_for_connection(&module_ctx, response)
2535            .await
2536            .unwrap();
2537        let delivered = client_rx.recv().await.unwrap();
2538        assert_eq!(delivered.header.ty, FrameType::Response);
2539        assert!(
2540            !delivered.header.flags.is_daemon_origin(),
2541            "module-set daemon origin must be cleared: {:?}",
2542            delivered.header.flags
2543        );
2544        assert!(
2545            delivered.header.flags.is_binary(),
2546            "binary flag must survive: {:?}",
2547            delivered.header.flags
2548        );
2549        assert!(
2550            delivered.header.flags.is_last(),
2551            "LAST flag must survive: {:?}",
2552            delivered.header.flags
2553        );
2554        assert!(
2555            delivered.header.flags.is_subscription(),
2556            "subscription flag must survive: {:?}",
2557            delivered.header.flags
2558        );
2559        assert_eq!(
2560            delivered.header.flags.priority(),
2561            Some(Priority::Background),
2562            "priority must survive"
2563        );
2564    }
2565
2566    /// A client frame carrying the origin bit must not reach the module
2567    /// looking daemon-authored either: the splice clears it symmetrically.
2568    #[tokio::test]
2569    async fn client_frame_with_daemon_bit_is_stripped_before_module_delivery() {
2570        let (
2571            router,
2572            _forwarding,
2573            client_ctx,
2574            mut client_rx,
2575            _module_ctx,
2576            _module_egress_rx,
2577            mut module_rx,
2578            pending,
2579        ) = dynamic_route_fixture(true);
2580        let _ = client_rx.recv().await;
2581        let spoof = Frame::build(
2582            FrameType::Request,
2583            Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2584            pending.client_channel,
2585            pending.client_epoch,
2586            701,
2587            b"route-body".to_vec(),
2588        )
2589        .unwrap();
2590        router
2591            .route_for_connection(&client_ctx, spoof)
2592            .await
2593            .unwrap();
2594        let delivered = module_rx.recv().await.unwrap();
2595        assert_eq!(delivered.header.ty, FrameType::Request);
2596        assert!(
2597            !delivered.header.flags.is_daemon_origin(),
2598            "client-set daemon origin must be cleared: {:?}",
2599            delivered.header.flags
2600        );
2601    }
2602
2603    #[tokio::test]
2604    async fn channel_zero_uses_control_handler_not_backend_registry() {
2605        let mut router = Router::with_default_self_handler();
2606        router.register_backend(1, EchoBackend).unwrap();
2607        let (ctx, mut rx) = route_ctx();
2608
2609        router.route_for_connection(&ctx, ping(77)).await.unwrap();
2610        let response = rx.recv().await.unwrap();
2611
2612        assert_eq!(response.header.ty, FrameType::Pong);
2613        assert_eq!(response.header.channel, 0);
2614        assert_eq!(response.header.corr, 77);
2615        assert!(response.body.is_empty());
2616    }
2617
2618    #[tokio::test]
2619    async fn slow_control_dispatch_logs_decoded_op_and_elapsed_time() {
2620        let control = Arc::new(
2621            ControlHandler::new(Arc::new(Registry::default()))
2622                .with_control_dispatch_delay(Duration::from_millis(1050)),
2623        );
2624        let router = Router::with_control_handler(control);
2625        let (ctx, mut rx) = route_ctx();
2626        let (output, guard) = log_capture(tracing::Level::WARN);
2627
2628        router
2629            .route_for_connection(&ctx, request(0, 41, br#"{"op":"server.describe"}"#))
2630            .await
2631            .expect("slow request routes");
2632        assert!(rx.recv().await.is_some(), "request receives a response");
2633        drop(guard);
2634
2635        let logs = captured_logs(&output);
2636        assert!(logs.contains("slow control dispatch"));
2637        assert!(logs.contains("op=server.describe"));
2638        assert!(logs.contains("connection_id=0"));
2639        assert!(logs.contains("corr=41"));
2640        assert!(
2641            logged_millis(&logs, "elapsed_ms=") >= 1050,
2642            "elapsed must include the injected handler delay: {logs}"
2643        );
2644    }
2645
2646    #[tokio::test]
2647    async fn fast_control_dispatch_emits_arrival_without_slow_warning() {
2648        let router = Router::with_default_self_handler();
2649        let (ctx, mut rx) = route_ctx();
2650        let (output, guard) = log_capture(tracing::Level::DEBUG);
2651
2652        router
2653            .route_for_connection(&ctx, request(0, 42, br#"{"op":"server.describe"}"#))
2654            .await
2655            .expect("fast request routes");
2656        assert!(rx.recv().await.is_some(), "request receives a response");
2657        drop(guard);
2658
2659        let logs = captured_logs(&output);
2660        assert!(logs.contains("control dispatch op=server.describe connection_id=0 corr=42"));
2661        assert!(!logs.contains("slow control dispatch"));
2662    }
2663
2664    #[tokio::test]
2665    async fn control_dispatch_arrival_is_hidden_at_info() {
2666        let router = Router::with_default_self_handler();
2667        let (ctx, mut rx) = route_ctx();
2668        let (output, guard) = log_capture(tracing::Level::INFO);
2669
2670        router
2671            .route_for_connection(&ctx, request(0, 43, br#"{"op":"server.describe"}"#))
2672            .await
2673            .expect("fast request routes");
2674        assert!(rx.recv().await.is_some(), "request receives a response");
2675        drop(guard);
2676
2677        assert!(
2678            !captured_logs(&output).contains("control dispatch"),
2679            "arrival logging must stay hidden at INFO"
2680        );
2681    }
2682
2683    #[tokio::test]
2684    async fn supervisor_list_logs_contended_snapshot_lock_only() {
2685        let registry = Arc::new(Registry::default());
2686        let handle = SupervisorHandle::new();
2687        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
2688            .with_handle(handle.clone());
2689        let module = supervisor
2690            .supervise_configured(
2691                ModuleSpec {
2692                    script: None,
2693                    carry_containment: false,
2694                    module_id: "held-module".to_string(),
2695                    program: "test-module".into(),
2696                    args: Vec::new(),
2697                    env: Vec::new(),
2698                    reserved: false,
2699                    reserved_prefixes: Vec::new(),
2700                    protocol: ModuleProtocol::Subc,
2701                    overlap: Default::default(),
2702                },
2703                false,
2704            )
2705            .expect("disabled test module is supervised");
2706        let router = Router::with_control_handler(Arc::new(
2707            ControlHandler::new(Arc::clone(&registry)).with_supervisor(handle),
2708        ));
2709        let (ctx, mut rx) = route_ctx();
2710        let (acquired, ready) = std_mpsc::channel();
2711        // Park the disabled supervisor actor before holding its snapshot. The
2712        // detached list reader, not actor startup, must be the lock contender.
2713        tokio::task::yield_now().await;
2714        let holder = module.hold_snapshot_for_test(acquired, Duration::from_millis(400));
2715        ready.recv().expect("holder acquired snapshot lock");
2716        let (output, guard) = log_capture(tracing::Level::WARN);
2717
2718        router
2719            .route_for_connection(&ctx, request(0, 44, br#"{"op":"supervisor.list"}"#))
2720            .await
2721            .expect("list request routes after the lock releases");
2722        assert!(
2723            rx.recv().await.is_some(),
2724            "list request receives a response"
2725        );
2726        holder.join().expect("snapshot holder exits cleanly");
2727        drop(guard);
2728
2729        let logs = captured_logs(&output);
2730        assert!(logs.contains("slow snapshot lock"));
2731        assert!(logs.contains("module_id=held-module"));
2732        assert!(logs.contains("caller=list"));
2733        assert!(
2734            logged_millis(&logs, "waited_ms=") >= 250,
2735            "wait must exceed the slow-lock threshold: {logs}"
2736        );
2737
2738        let (output, guard) = log_capture(tracing::Level::WARN);
2739        router
2740            .route_for_connection(&ctx, request(0, 45, br#"{"op":"supervisor.list"}"#))
2741            .await
2742            .expect("uncontended list request routes");
2743        assert!(
2744            rx.recv().await.is_some(),
2745            "uncontended list receives a response"
2746        );
2747        drop(guard);
2748        assert!(
2749            !captured_logs(&output).contains("slow snapshot lock"),
2750            "uncontended list acquisition must not warn"
2751        );
2752    }
2753
2754    #[tokio::test]
2755    async fn full_module_to_client_sink_requests_client_close_without_erroring_module() {
2756        let forwarding = Arc::new(ForwardingTable::default());
2757        let control = Arc::new(ControlHandler::with_forwarding(
2758            Arc::new(crate::Registry::default()),
2759            Arc::clone(&forwarding),
2760        ));
2761        let router = Router::with_control_handler(control);
2762        let module_connection = ConnectionId::new(10);
2763        let client_connection = ConnectionId::new(20);
2764        let mut close_receiver = forwarding.register_connection_close(client_connection);
2765        let (module_tx, _module_rx) = mpsc::channel(1);
2766        forwarding
2767            .register_module_connection(
2768                module_connection,
2769                "full-sink-provider".to_string(),
2770                1,
2771                Concurrency::ModuleManaged,
2772                FrameSink::new(module_tx),
2773            )
2774            .unwrap();
2775        let (client_tx, mut client_rx) = mpsc::channel(1);
2776        let pending = forwarding
2777            .begin_route_bind_relay_for_test(
2778                client_connection,
2779                FrameSink::new(client_tx),
2780                700,
2781                "full-sink-provider",
2782            )
2783            .unwrap();
2784        forwarding
2785            .complete_pending_relay(
2786                module_connection,
2787                pending.corr,
2788                RouteBindRelayOutcome::Accepted,
2789            )
2790            .unwrap();
2791
2792        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2793        let module_ctx = RouteCtx {
2794            connection_id: module_connection,
2795            egress: FrameSink::new(module_egress_tx),
2796        };
2797        let terminal = Frame::build(
2798            FrameType::Response,
2799            Flags::new(false, Priority::Interactive, true),
2800            pending.module_channel,
2801            pending.module_epoch,
2802            701,
2803            b"terminal".to_vec(),
2804        )
2805        .unwrap();
2806
2807        router
2808            .route_for_connection(&module_ctx, terminal)
2809            .await
2810            .unwrap();
2811        let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2812            .await
2813            .expect("close request should be sent for the full client sink")
2814            .expect("close sender should include a reason");
2815        assert!(
2816            reason
2817                .to_string()
2818                .contains("module_to_client_delivery_failed"),
2819            "unexpected close reason: {reason}"
2820        );
2821        assert_eq!(client_rx.try_recv().unwrap().header.corr, 700);
2822        assert!(client_rx.try_recv().is_err());
2823        assert_eq!(
2824            router.counters.snapshot()["client_egress_close_delivery_failed"],
2825            1
2826        );
2827    }
2828
2829    /// A terminal frame ends the request at the module even when the client
2830    /// cannot take it, so the drain must stop counting it at once rather than
2831    /// when the client connection's cleanup later removes the route.
2832    #[tokio::test]
2833    async fn terminal_frame_releases_its_credit_even_when_client_delivery_fails() {
2834        let forwarding = Arc::new(ForwardingTable::default());
2835        let control = Arc::new(ControlHandler::with_forwarding(
2836            Arc::new(crate::Registry::default()),
2837            Arc::clone(&forwarding),
2838        ));
2839        let router = Router::with_control_handler(control);
2840        let module_connection = ConnectionId::new(11);
2841        let client_connection = ConnectionId::new(21);
2842        let _close_receiver = forwarding.register_connection_close(client_connection);
2843        let (module_tx, _module_rx) = mpsc::channel(1);
2844        forwarding
2845            .register_module_connection(
2846                module_connection,
2847                "credit-provider".to_string(),
2848                1,
2849                Concurrency::ModuleManaged,
2850                FrameSink::new(module_tx),
2851            )
2852            .unwrap();
2853        // Capacity one, filled by the route.open response, so the terminal
2854        // frame below cannot be enqueued for the client.
2855        let (client_tx, _client_rx) = mpsc::channel(1);
2856        let pending = forwarding
2857            .begin_route_bind_relay_for_test(
2858                client_connection,
2859                FrameSink::new(client_tx),
2860                800,
2861                "credit-provider",
2862            )
2863            .unwrap();
2864        forwarding
2865            .complete_pending_relay(
2866                module_connection,
2867                pending.corr,
2868                RouteBindRelayOutcome::Accepted,
2869            )
2870            .unwrap();
2871        let DataRoute::Client(DataRouteState::Bound(route)) = forwarding
2872            .lookup_data_route(
2873                client_connection,
2874                pending.client_channel,
2875                pending.client_epoch,
2876            )
2877            .unwrap()
2878        else {
2879            panic!("expected a bound client route");
2880        };
2881        route.flow.acquire_tagged(801, false).await.unwrap();
2882        assert_eq!(route.flow.drain_in_flight(), 1);
2883
2884        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2885        let module_ctx = RouteCtx {
2886            connection_id: module_connection,
2887            egress: FrameSink::new(module_egress_tx),
2888        };
2889        let terminal = Frame::build(
2890            FrameType::Response,
2891            Flags::new(false, Priority::Interactive, true),
2892            pending.module_channel,
2893            pending.module_epoch,
2894            801,
2895            b"terminal".to_vec(),
2896        )
2897        .unwrap();
2898        router
2899            .route_for_connection(&module_ctx, terminal)
2900            .await
2901            .unwrap();
2902
2903        assert_eq!(
2904            router.counters.snapshot()["client_egress_close_delivery_failed"],
2905            1,
2906            "the client delivery must have failed for this test to mean anything"
2907        );
2908        assert_eq!(
2909            route.flow.drain_in_flight(),
2910            0,
2911            "the module's terminal frame must release its credit even though the client could not take it"
2912        );
2913    }
2914
2915    #[tokio::test]
2916    async fn full_route_goodbye_sink_requests_target_close_without_erroring_module() {
2917        let forwarding = Arc::new(ForwardingTable::default());
2918        let control = Arc::new(ControlHandler::with_forwarding(
2919            Arc::new(crate::Registry::default()),
2920            Arc::clone(&forwarding),
2921        ));
2922        let router = Router::with_control_handler(control);
2923        let module_connection = ConnectionId::new(30);
2924        let client_connection = ConnectionId::new(40);
2925        let mut close_receiver = forwarding.register_connection_close(client_connection);
2926        let (module_tx, _module_rx) = mpsc::channel(1);
2927        forwarding
2928            .register_module_connection(
2929                module_connection,
2930                "goodbye-full-provider".to_string(),
2931                1,
2932                Concurrency::ModuleManaged,
2933                FrameSink::new(module_tx),
2934            )
2935            .unwrap();
2936        let (client_tx, mut client_rx) = mpsc::channel(1);
2937        let pending = forwarding
2938            .begin_route_bind_relay_for_test(
2939                client_connection,
2940                FrameSink::new(client_tx),
2941                800,
2942                "goodbye-full-provider",
2943            )
2944            .unwrap();
2945        forwarding
2946            .complete_pending_relay(
2947                module_connection,
2948                pending.corr,
2949                RouteBindRelayOutcome::Accepted,
2950            )
2951            .unwrap();
2952
2953        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2954        let module_ctx = RouteCtx {
2955            connection_id: module_connection,
2956            egress: FrameSink::new(module_egress_tx),
2957        };
2958        let goodbye = Frame::build(
2959            FrameType::Goodbye,
2960            Flags::new(false, Priority::Passive, true),
2961            pending.module_channel,
2962            pending.module_epoch,
2963            801,
2964            Vec::new(),
2965        )
2966        .unwrap();
2967
2968        router
2969            .route_for_connection(&module_ctx, goodbye)
2970            .await
2971            .unwrap();
2972        let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2973            .await
2974            .expect("close request should be sent for the full GOODBYE sink")
2975            .expect("close sender should include a reason");
2976        assert!(
2977            reason.to_string().contains("route_goodbye_delivery_failed"),
2978            "unexpected close reason: {reason}"
2979        );
2980        assert_eq!(client_rx.try_recv().unwrap().header.corr, 800);
2981        assert!(client_rx.try_recv().is_err());
2982        assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 1);
2983        assert_eq!(router.counters.snapshot()["route_released_epoch_fenced"], 1);
2984    }
2985
2986    /// One module and one client connection with `routes` routes bound between
2987    /// them. The client uses a real connection egress queue (the same byte
2988    /// budget and frame-count backstop as a live connection), and its route.open
2989    /// responses are drained so the queue starts empty.
2990    async fn multi_route_client(
2991        module_id: &str,
2992        module_connection: ConnectionId,
2993        client_connection: ConnectionId,
2994        routes: usize,
2995    ) -> (
2996        Router,
2997        FrameSink,
2998        mpsc::Receiver<OutboundFrame>,
2999        RouteCtx,
3000        Vec<crate::forwarding::PendingRouteBindRelay>,
3001        ConnectionCloseReceiver,
3002    ) {
3003        let forwarding = Arc::new(ForwardingTable::default());
3004        let control = Arc::new(ControlHandler::with_forwarding(
3005            Arc::new(crate::Registry::default()),
3006            Arc::clone(&forwarding),
3007        ));
3008        let router = Router::with_control_handler(control);
3009        let close_receiver = forwarding.register_connection_close(client_connection);
3010        let (module_tx, _module_rx) = mpsc::channel(8);
3011        forwarding
3012            .register_module_connection(
3013                module_connection,
3014                module_id.to_string(),
3015                1,
3016                Concurrency::ModuleManaged,
3017                FrameSink::new(module_tx),
3018            )
3019            .unwrap();
3020        let (client_sink, mut client_rx) = crate::server::connection_egress();
3021        let mut bound = Vec::with_capacity(routes);
3022        for index in 0..routes {
3023            let pending = forwarding
3024                .begin_route_bind_relay_for_test(
3025                    client_connection,
3026                    client_sink.clone(),
3027                    900 + index as u64,
3028                    module_id,
3029                )
3030                .unwrap();
3031            forwarding
3032                .complete_pending_relay(
3033                    module_connection,
3034                    pending.corr,
3035                    RouteBindRelayOutcome::Accepted,
3036                )
3037                .unwrap();
3038            assert_eq!(
3039                client_rx.recv().await.unwrap().header.corr,
3040                900 + index as u64
3041            );
3042            bound.push(pending);
3043        }
3044        let (module_egress_tx, _module_egress_rx) = mpsc::channel(8);
3045        let module_ctx = RouteCtx {
3046            connection_id: module_connection,
3047            egress: FrameSink::new(module_egress_tx),
3048        };
3049        (
3050            router,
3051            client_sink,
3052            client_rx,
3053            module_ctx,
3054            bound,
3055            close_receiver,
3056        )
3057    }
3058
3059    /// Per-frame cost of the egress sink's admission and release accounting:
3060    /// one million 200-byte frames enqueued with `try_send` and taken off the
3061    /// queue the way the connection writer does, in batches of 1,000 so the
3062    /// queue stays well inside its budget. Prints nanoseconds per frame; it is
3063    /// a measurement, not a gate. Run with
3064    /// `cargo test --release -p subc-daemon --lib egress_sink_per_frame_cost -- --ignored --nocapture`.
3065    #[test]
3066    #[ignore = "timing measurement, run on demand"]
3067    fn egress_sink_per_frame_cost() {
3068        const FRAMES: usize = 1_000_000;
3069        const BATCH: usize = 1_000;
3070        let (sink, mut rx) = crate::server::connection_egress();
3071        let template = stream_frame(9, 1, 0, vec![b't'; 200]);
3072        let started = Instant::now();
3073        for _ in 0..FRAMES / BATCH {
3074            for _ in 0..BATCH {
3075                sink.try_send(template.clone()).unwrap();
3076            }
3077            for _ in 0..BATCH {
3078                let outbound = rx.try_recv().unwrap();
3079                if let Some(charge) = &outbound.charge {
3080                    charge.taken_by_writer();
3081                }
3082                drop(std::hint::black_box(outbound));
3083            }
3084        }
3085        let elapsed = started.elapsed();
3086        assert_eq!(sink.backlog().queued_bytes, 0);
3087        println!(
3088            "egress sink: {FRAMES} frames in {elapsed:?}, {:.1} ns/frame",
3089            elapsed.as_nanos() as f64 / FRAMES as f64
3090        );
3091    }
3092
3093    fn stream_frame(channel: u16, epoch: u32, corr: u64, body: Vec<u8>) -> Frame {
3094        Frame::build(
3095            FrameType::StreamData,
3096            Flags::new(false, Priority::Interactive, false),
3097            channel,
3098            epoch,
3099            corr,
3100            body,
3101        )
3102        .unwrap()
3103    }
3104
3105    fn module_route_for_test(
3106        router: &Router,
3107        ctx: &RouteCtx,
3108        route: &crate::forwarding::PendingRouteBindRelay,
3109    ) -> Arc<RouteBinding> {
3110        match router
3111            .forwarding
3112            .lookup_data_route(ctx.connection_id, route.module_channel, route.module_epoch)
3113            .unwrap()
3114        {
3115            DataRoute::Module(DataRouteState::Bound(binding)) => binding,
3116            other => panic!("expected a bound module route, got {other:?}"),
3117        }
3118    }
3119
3120    fn assert_frame_size_error(frame: &Frame, channel: u16, epoch: u32, corr: u64, bytes: usize) {
3121        assert_eq!(frame.header.ty, FrameType::Error);
3122        assert_eq!(frame.header.channel, channel);
3123        assert_eq!(frame.header.epoch, epoch);
3124        assert_eq!(frame.header.corr, corr);
3125        assert!(frame.header.flags.is_daemon_origin());
3126        let body: ErrorBody = serde_json::from_slice(&frame.body).unwrap();
3127        assert_eq!(body.code, error_codes::FRAME_TOO_LARGE);
3128        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3129        assert!(body.message.contains(&bytes.to_string()));
3130        assert!(body.message.contains(&limit.to_string()));
3131        assert_eq!(
3132            body.detail,
3133            Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit }))
3134        );
3135    }
3136
3137    #[tokio::test]
3138    async fn module_egress_response_oversized_reply_keeps_connection_and_other_route() {
3139        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3140        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3141            "oversize-provider",
3142            ConnectionId::new(80),
3143            ConnectionId::new(81),
3144            2,
3145        )
3146        .await;
3147        let binding = module_route_for_test(&router, &ctx, &routes[0]);
3148        binding.flow.acquire_tagged(101, false).await.unwrap();
3149        module_route_for_test(&router, &ctx, &routes[1])
3150            .flow
3151            .acquire_tagged(102, false)
3152            .await
3153            .unwrap();
3154        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3155        let oversized = Frame::build(
3156            FrameType::Response,
3157            Flags::new(false, Priority::Interactive, true),
3158            routes[0].module_channel,
3159            routes[0].module_epoch,
3160            101,
3161            vec![b'x'; limit - subc_protocol::HEADER_LEN + 1],
3162        )
3163        .unwrap();
3164        assert_eq!(sink.backlog().queued_bytes, 0);
3165        router.route_for_connection(&ctx, oversized).await.unwrap();
3166
3167        // Reproduce a large reply followed by ordinary traffic on another route
3168        // before the client reads anything. The large reply must not monopolize
3169        // the queue and make that second delivery close the shared connection.
3170        router
3171            .route_for_connection(
3172                &ctx,
3173                route_frame(
3174                    FrameType::Response,
3175                    routes[1].module_channel,
3176                    routes[1].module_epoch,
3177                    102,
3178                ),
3179            )
3180            .await
3181            .unwrap();
3182        assert!(
3183            close.try_recv().is_err(),
3184            "an oversized reply must not close the multiplexed client"
3185        );
3186        let error = rx.try_recv().expect("the waiting client gets an ERROR");
3187        assert_frame_size_error(
3188            &error,
3189            routes[0].client_channel,
3190            routes[0].client_epoch,
3191            101,
3192            limit + 1,
3193        );
3194        let second = rx.try_recv().expect("the other route's reply arrives");
3195        assert_eq!(second.header.ty, FrameType::Response);
3196        assert_eq!(second.header.channel, routes[1].client_channel);
3197        assert_eq!(second.header.epoch, routes[1].client_epoch);
3198        assert_eq!(second.header.corr, 102);
3199        assert_eq!(second.body, b"route-body");
3200        assert_eq!(binding.flow.drain_in_flight(), 0);
3201        for route in &routes {
3202            module_route_for_test(&router, &ctx, route);
3203        }
3204        assert!(rx.try_recv().is_err());
3205        drop((error, second));
3206        assert_eq!(sink.backlog().queued_bytes, 0);
3207        assert_eq!(
3208            router.counters.snapshot()["client_egress_close_delivery_failed"],
3209            0
3210        );
3211
3212        let captured = test_log::captured_logs(&logs);
3213        let warnings: Vec<_> = captured
3214            .lines()
3215            .filter(|line| line.contains("WARN"))
3216            .collect();
3217        assert_eq!(warnings.len(), 1, "{captured}");
3218        let warning = warnings[0];
3219        for expected in [
3220            "module_id=oversize-provider".to_string(),
3221            "client_principal=direct".to_string(),
3222            format!("client_channel={}", routes[0].client_channel),
3223            format!("frame_bytes={}", limit + 1),
3224            format!("byte_limit={limit}"),
3225        ] {
3226            assert!(warning.contains(&expected), "missing {expected}: {warning}");
3227        }
3228    }
3229
3230    async fn poll_once<F: std::future::Future>(
3231        mut future: std::pin::Pin<&mut F>,
3232    ) -> std::task::Poll<F::Output> {
3233        std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
3234    }
3235
3236    fn fill_client_budget(sink: &FrameSink, route: &crate::forwarding::PendingRouteBindRelay) {
3237        sink.try_send(
3238            Frame::build(
3239                FrameType::Response,
3240                Flags::new(false, Priority::Interactive, true),
3241                route.client_channel,
3242                route.client_epoch,
3243                100,
3244                vec![
3245                    b'x';
3246                    crate::server::CONNECTION_EGRESS_BYTE_BUDGET - subc_protocol::HEADER_LEN
3247                ],
3248            )
3249            .unwrap(),
3250        )
3251        .unwrap();
3252        assert_eq!(
3253            sink.backlog().queued_bytes,
3254            crate::server::CONNECTION_EGRESS_BYTE_BUDGET
3255        );
3256    }
3257
3258    async fn assert_full_client_burst_survives(ty: FrameType, oversized: bool) {
3259        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3260            "draining-burst",
3261            ConnectionId::new(92),
3262            ConnectionId::new(93),
3263            2,
3264        )
3265        .await;
3266        fill_client_budget(&sink, &routes[0]);
3267        let frame = if oversized {
3268            Frame::build(
3269                ty,
3270                Flags::new(false, Priority::Interactive, false),
3271                routes[1].module_channel,
3272                routes[1].module_epoch,
3273                101,
3274                vec![b's'; crate::server::CONNECTION_EGRESS_BYTE_BUDGET],
3275            )
3276            .unwrap()
3277        } else {
3278            route_frame(ty, routes[1].module_channel, routes[1].module_epoch, 101)
3279        };
3280        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3281        assert!(poll_once(delivery.as_mut()).await.is_pending());
3282        assert!(
3283            close.try_recv().is_err(),
3284            "fresh bursts must not close: {ty:?}"
3285        );
3286        tokio::time::advance(Duration::from_millis(50)).await;
3287        let queued = rx.try_recv().unwrap();
3288        queued.charge.as_ref().unwrap().taken_by_writer();
3289        drop(queued);
3290        assert!(matches!(
3291            poll_once(delivery.as_mut()).await,
3292            std::task::Poll::Ready(Ok(()))
3293        ));
3294        let delivered = rx
3295            .try_recv()
3296            .expect("the burst frame must be delivered after draining");
3297        assert_eq!(delivered.header.channel, routes[1].client_channel);
3298        assert_eq!(delivered.header.epoch, routes[1].client_epoch);
3299        assert_eq!(delivered.header.corr, 101);
3300        if oversized {
3301            assert_frame_size_error(
3302                &delivered,
3303                routes[1].client_channel,
3304                routes[1].client_epoch,
3305                101,
3306                crate::server::CONNECTION_EGRESS_BYTE_BUDGET + subc_protocol::HEADER_LEN,
3307            );
3308        } else {
3309            assert_eq!(delivered.header.ty, ty);
3310        }
3311        assert!(rx.try_recv().is_err());
3312        assert!(close.try_recv().is_err());
3313        assert_eq!(
3314            router.counters.snapshot()["client_egress_close_delivery_failed"],
3315            0
3316        );
3317        assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 0);
3318    }
3319
3320    #[tokio::test(start_paused = true)]
3321    async fn module_egress_fresh_full_queue_burst_survives() {
3322        for ty in [
3323            FrameType::Response,
3324            FrameType::StreamData,
3325            FrameType::Request,
3326            FrameType::Push,
3327        ] {
3328            assert_full_client_burst_survives(ty, false).await;
3329        }
3330    }
3331
3332    #[tokio::test(start_paused = true)]
3333    async fn module_goodbye_fresh_full_queue_burst_survives() {
3334        assert_full_client_burst_survives(FrameType::Goodbye, false).await;
3335    }
3336
3337    #[tokio::test(start_paused = true)]
3338    async fn frame_count_backstop_also_allows_a_draining_burst() {
3339        let (tx, mut rx) = mpsc::channel(1);
3340        let sink = FrameSink::with_byte_budget(tx, 2_000);
3341        sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
3342            .unwrap();
3343        let mut delivery = Box::pin(sink.send_to_client(stream_frame(1, 1, 2, vec![b'y'; 100])));
3344        assert!(poll_once(delivery.as_mut()).await.is_pending());
3345        assert!(
3346            sink.backlog().queued_bytes < 2_000,
3347            "only frame capacity is exhausted"
3348        );
3349        tokio::time::advance(Duration::from_millis(50)).await;
3350        drop(rx.try_recv().unwrap());
3351        assert!(matches!(
3352            poll_once(delivery.as_mut()).await,
3353            std::task::Poll::Ready(Ok(()))
3354        ));
3355        assert_eq!(rx.try_recv().unwrap().header.corr, 2);
3356    }
3357
3358    #[tokio::test(start_paused = true)]
3359    async fn oversized_reply_replacement_fresh_full_queue_burst_survives() {
3360        for ty in [FrameType::Response, FrameType::StreamData] {
3361            assert_full_client_burst_survives(ty, true).await;
3362        }
3363    }
3364
3365    #[tokio::test(start_paused = true)]
3366    async fn module_egress_stopped_reader_closes_without_waiting() {
3367        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3368        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3369            "oldest-stopped-reader",
3370            ConnectionId::new(94),
3371            ConnectionId::new(95),
3372            2,
3373        )
3374        .await;
3375        fill_client_budget(&sink, &routes[0]);
3376        // The writer has taken the head but cannot finish writing it. Its
3377        // bytes stay charged, and the atomic age still diagnoses that stall.
3378        let writing = rx.try_recv().unwrap();
3379        writing.charge.as_ref().unwrap().taken_by_writer();
3380        tokio::time::advance(Duration::from_millis(251)).await;
3381        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(251)));
3382        assert_eq!(sink.backlog().queued_bytes, queued_frame_bytes(&writing));
3383        let started = tokio::time::Instant::now();
3384        let frame = stream_frame(
3385            routes[0].module_channel,
3386            routes[0].module_epoch,
3387            101,
3388            b"secret-body-never-log".to_vec(),
3389        );
3390        let flags = frame.header.flags.0;
3391        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3392        assert!(
3393            matches!(
3394                poll_once(delivery.as_mut()).await,
3395                std::task::Poll::Ready(Ok(()))
3396            ),
3397            "an oldest frame older than 250 ms must close immediately, without a new wait"
3398        );
3399        assert_eq!(started.elapsed(), Duration::ZERO);
3400        assert!(close
3401            .try_recv()
3402            .unwrap()
3403            .to_string()
3404            .contains("module_to_client_delivery_failed"));
3405        let captured = test_log::captured_logs(&logs);
3406        for expected in [
3407            "frame_type=Some(StreamData)".to_string(),
3408            format!("frame_flags=Some({flags})"),
3409            "oldest_queued_ms=251".to_string(),
3410            "close_cause=\"oldest_frame_older_than_250_ms\"".to_string(),
3411        ] {
3412            assert!(
3413                captured.contains(&expected),
3414                "missing {expected}: {captured}"
3415            );
3416        }
3417        assert!(!captured.contains("secret-body-never-log"));
3418    }
3419
3420    #[tokio::test(start_paused = true)]
3421    async fn client_gone_mid_wait_resumes_module_reader_immediately() {
3422        for writer_gone in [false, true] {
3423            let (router, sink, mut rx, ctx, routes, _close) = multi_route_client(
3424                "gone-during-burst",
3425                ConnectionId::new(96),
3426                ConnectionId::new(97),
3427                1,
3428            )
3429            .await;
3430            fill_client_budget(&sink, &routes[0]);
3431            let frame = route_frame(
3432                FrameType::Response,
3433                routes[0].module_channel,
3434                routes[0].module_epoch,
3435                101,
3436            );
3437            let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3438            assert!(poll_once(delivery.as_mut()).await.is_pending());
3439            tokio::time::advance(Duration::from_millis(50)).await;
3440            let started = tokio::time::Instant::now();
3441            if writer_gone {
3442                rx.close();
3443            } else {
3444                sink.close_client_delivery();
3445            }
3446            assert!(
3447                matches!(
3448                    poll_once(delivery.as_mut()).await,
3449                    std::task::Poll::Ready(Ok(()))
3450                ),
3451                "client teardown must wake the module reader before the writer drains"
3452            );
3453            assert_eq!(started.elapsed(), Duration::ZERO);
3454            assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
3455            assert_eq!(rx.try_recv().unwrap().header.corr, 100);
3456            assert!(rx.try_recv().is_err());
3457        }
3458    }
3459
3460    #[tokio::test(start_paused = true)]
3461    async fn cancelled_module_delivery_releases_waiter_without_closing_client() {
3462        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3463            "cancelled-burst",
3464            ConnectionId::new(98),
3465            ConnectionId::new(99),
3466            1,
3467        )
3468        .await;
3469        fill_client_budget(&sink, &routes[0]);
3470        let frame = route_frame(
3471            FrameType::Response,
3472            routes[0].module_channel,
3473            routes[0].module_epoch,
3474            101,
3475        );
3476        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3477        assert!(poll_once(delivery.as_mut()).await.is_pending());
3478        assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 1);
3479        drop(delivery);
3480        assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
3481        tokio::time::advance(Duration::from_secs(1)).await;
3482        assert!(close.try_recv().is_err());
3483        drop(rx.try_recv().unwrap());
3484        assert_eq!(sink.backlog().queued_bytes, 0);
3485        assert!(sink.backlog().oldest_age.is_none());
3486    }
3487
3488    #[tokio::test(start_paused = true)]
3489    async fn egress_atomic_age_follows_writer_and_clears_when_drained() {
3490        let (sink, mut rx) = crate::server::connection_egress();
3491        sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
3492            .unwrap();
3493        tokio::time::advance(Duration::from_millis(200)).await;
3494        sink.try_send(stream_frame(2, 1, 2, vec![b'y'; 100]))
3495            .unwrap();
3496        let writing = rx.try_recv().unwrap();
3497        writing.charge.as_ref().unwrap().taken_by_writer();
3498        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3499        drop(writing);
3500        // Between frames the lock-free marker can still name the previous
3501        // head. The next writer take advances it without scanning the queue.
3502        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3503        tokio::time::advance(Duration::from_millis(50)).await;
3504        let next = rx.try_recv().unwrap();
3505        next.charge.as_ref().unwrap().taken_by_writer();
3506        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(50)));
3507        assert!(
3508            sink.backlog().queued_bytes > 0,
3509            "in-flight bytes remain charged until written"
3510        );
3511        drop(next);
3512        assert_eq!(sink.backlog().queued_bytes, 0);
3513        assert_eq!(sink.backlog().oldest_age, None);
3514    }
3515
3516    #[tokio::test(start_paused = true)]
3517    async fn module_egress_response_full_client_waits_only_remaining_grace() {
3518        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3519            "backpressure-provider",
3520            ConnectionId::new(82),
3521            ConnectionId::new(83),
3522            2,
3523        )
3524        .await;
3525        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3526        // Exactly the limit, including its header, must be admitted to an empty
3527        // queue. Leave it unread to deterministically exhaust only byte capacity.
3528        router
3529            .route_for_connection(
3530                &ctx,
3531                Frame::build(
3532                    FrameType::Response,
3533                    Flags::new(false, Priority::Interactive, true),
3534                    routes[0].module_channel,
3535                    routes[0].module_epoch,
3536                    201,
3537                    vec![b'x'; limit - subc_protocol::HEADER_LEN],
3538                )
3539                .unwrap(),
3540            )
3541            .await
3542            .unwrap();
3543        assert_eq!(sink.backlog().queued_bytes, limit);
3544        assert_eq!(sink.backlog().queued_frames, 1);
3545        let ordinary = route_frame(
3546            FrameType::Response,
3547            routes[1].module_channel,
3548            routes[1].module_epoch,
3549            202,
3550        );
3551        assert!(queued_frame_bytes(&ordinary) < limit);
3552        // A fitting reply gets only the unused portion of the oldest frame's
3553        // 250 ms grace, not an immediate close or a fresh 250 ms wait.
3554        tokio::time::advance(Duration::from_millis(100)).await;
3555        let mut delivery = Box::pin(router.route_for_connection(&ctx, ordinary));
3556        assert!(poll_once(delivery.as_mut()).await.is_pending());
3557        assert!(close.try_recv().is_err());
3558        tokio::time::advance(Duration::from_millis(149)).await;
3559        assert!(poll_once(delivery.as_mut()).await.is_pending());
3560        assert!(close.try_recv().is_err());
3561        tokio::time::advance(Duration::from_millis(1)).await;
3562        assert!(matches!(
3563            poll_once(delivery.as_mut()).await,
3564            std::task::Poll::Ready(Ok(()))
3565        ));
3566        drop(delivery);
3567        let reason = close.try_recv().expect("the full client must close");
3568        assert!(reason
3569            .to_string()
3570            .contains("module_to_client_delivery_failed"));
3571        assert!(!reason.to_string().contains("frame_too_large"));
3572        assert_eq!(
3573            router.counters.snapshot()["client_egress_close_delivery_failed"],
3574            1
3575        );
3576        let admitted = rx.try_recv().expect("the budget-sized frame was admitted");
3577        assert_eq!(admitted.header.ty, FrameType::Response);
3578        assert_eq!(admitted.header.corr, 201);
3579        assert!(
3580            rx.try_recv().is_err(),
3581            "no oversized-frame ERROR is substituted for backpressure"
3582        );
3583    }
3584
3585    #[tokio::test(start_paused = true)]
3586    async fn oversized_reply_to_a_full_client_waits_only_remaining_grace() {
3587        let (router, sink, _rx, ctx, routes, mut close) = multi_route_client(
3588            "full-oversize-provider",
3589            ConnectionId::new(90),
3590            ConnectionId::new(91),
3591            2,
3592        )
3593        .await;
3594        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3595        // Fill the client's queue to its budget and leave it unread.
3596        router
3597            .route_for_connection(
3598                &ctx,
3599                Frame::build(
3600                    FrameType::Response,
3601                    Flags::new(false, Priority::Interactive, true),
3602                    routes[0].module_channel,
3603                    routes[0].module_epoch,
3604                    501,
3605                    vec![b'x'; limit - subc_protocol::HEADER_LEN],
3606                )
3607                .unwrap(),
3608            )
3609            .await
3610            .unwrap();
3611        assert_eq!(sink.backlog().queued_bytes, limit);
3612        let oversized = Frame::build(
3613            FrameType::Response,
3614            Flags::new(false, Priority::Interactive, true),
3615            routes[1].module_channel,
3616            routes[1].module_epoch,
3617            502,
3618            vec![b'y'; limit - subc_protocol::HEADER_LEN + 1],
3619        )
3620        .unwrap();
3621        // The replacement ERROR gets the same bounded burst grace as the
3622        // ordinary reply; oversized payloads themselves are never queued.
3623        tokio::time::advance(Duration::from_millis(80)).await;
3624        let mut delivery = Box::pin(router.route_for_connection(&ctx, oversized));
3625        assert!(poll_once(delivery.as_mut()).await.is_pending());
3626        assert!(close.try_recv().is_err());
3627        tokio::time::advance(Duration::from_millis(169)).await;
3628        assert!(poll_once(delivery.as_mut()).await.is_pending());
3629        assert!(close.try_recv().is_err());
3630        tokio::time::advance(Duration::from_millis(1)).await;
3631        assert!(matches!(
3632            poll_once(delivery.as_mut()).await,
3633            std::task::Poll::Ready(Ok(()))
3634        ));
3635        drop(delivery);
3636        let reason = close.try_recv().expect("the full client must close");
3637        assert!(reason
3638            .to_string()
3639            .contains("module_to_client_delivery_failed"));
3640    }
3641
3642    #[tokio::test]
3643    async fn oversized_reverse_request_errors_module_without_reaching_client() {
3644        let (router, _sink, mut rx, mut ctx, routes, mut close) = multi_route_client(
3645            "reverse-oversize-provider",
3646            ConnectionId::new(84),
3647            ConnectionId::new(85),
3648            2,
3649        )
3650        .await;
3651        let (module_tx, mut module_rx) = mpsc::channel(8);
3652        ctx.egress = FrameSink::new(module_tx);
3653        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3654        router
3655            .route_for_connection(
3656                &ctx,
3657                Frame::build(
3658                    FrameType::Request,
3659                    Flags::new(false, Priority::Interactive, false),
3660                    routes[0].module_channel,
3661                    routes[0].module_epoch,
3662                    301,
3663                    vec![b'r'; limit - subc_protocol::HEADER_LEN + 1],
3664                )
3665                .unwrap(),
3666            )
3667            .await
3668            .unwrap();
3669        assert!(
3670            rx.try_recv().is_err(),
3671            "the reverse request never reaches the client"
3672        );
3673        let error = module_rx.try_recv().expect("the module gets the refusal");
3674        assert_frame_size_error(
3675            &error,
3676            routes[0].module_channel,
3677            routes[0].module_epoch,
3678            301,
3679            limit + 1,
3680        );
3681        assert!(module_rx.try_recv().is_err());
3682        for (index, route) in routes.iter().enumerate() {
3683            module_route_for_test(&router, &ctx, route);
3684            router
3685                .route_for_connection(
3686                    &ctx,
3687                    route_frame(
3688                        FrameType::Response,
3689                        route.module_channel,
3690                        route.module_epoch,
3691                        302 + index as u64,
3692                    ),
3693                )
3694                .await
3695                .unwrap();
3696            let reply = rx.try_recv().expect("both routes still deliver");
3697            assert_eq!(reply.header.ty, FrameType::Response);
3698            assert_eq!(reply.header.channel, route.client_channel);
3699        }
3700        assert!(close.try_recv().is_err());
3701        assert_eq!(
3702            router.counters.snapshot()["client_egress_close_delivery_failed"],
3703            0
3704        );
3705    }
3706
3707    #[tokio::test]
3708    async fn oversized_push_closes_client_with_precise_warn() {
3709        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3710        let (router, _sink, mut rx, ctx, routes, mut close) = multi_route_client(
3711            "push-oversize-provider",
3712            ConnectionId::new(86),
3713            ConnectionId::new(87),
3714            2,
3715        )
3716        .await;
3717        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3718        router
3719            .route_for_connection(
3720                &ctx,
3721                Frame::build(
3722                    FrameType::Push,
3723                    Flags::new(false, Priority::Interactive, false),
3724                    routes[0].module_channel,
3725                    routes[0].module_epoch,
3726                    0,
3727                    vec![b'p'; limit - subc_protocol::HEADER_LEN + 1],
3728                )
3729                .unwrap(),
3730            )
3731            .await
3732            .unwrap();
3733        let reason = close
3734            .try_recv()
3735            .expect("unsolicited oversize still closes the client");
3736        assert!(reason
3737            .to_string()
3738            .contains("frame_too_large: oversized push"));
3739        assert!(
3740            rx.try_recv().is_err(),
3741            "no part of the oversized push is forwarded"
3742        );
3743        let captured = test_log::captured_logs(&logs);
3744        let warnings: Vec<_> = captured
3745            .lines()
3746            .filter(|line| line.contains("WARN"))
3747            .collect();
3748        assert_eq!(warnings.len(), 1, "{captured}");
3749        let warning = warnings[0];
3750        for expected in [
3751            "oversized push".to_string(),
3752            "module_id=\"push-oversize-provider\"".to_string(),
3753            "principals=direct".to_string(),
3754            format!("client_channel={}", routes[0].client_channel),
3755            format!("size={}", limit + 1),
3756            format!("limit={limit}"),
3757        ] {
3758            assert!(warning.contains(&expected), "missing {expected}: {warning}");
3759        }
3760        assert_eq!(
3761            router.counters.snapshot()["client_egress_close_delivery_failed"],
3762            1
3763        );
3764    }
3765
3766    #[tokio::test]
3767    async fn oversized_stream_and_error_frames_end_request_even_with_queued_traffic() {
3768        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3769            "stream-oversize-provider",
3770            ConnectionId::new(88),
3771            ConnectionId::new(89),
3772            2,
3773        )
3774        .await;
3775        let binding = module_route_for_test(&router, &ctx, &routes[0]);
3776        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3777        for ty in [
3778            FrameType::StreamData,
3779            FrameType::StreamEnd,
3780            FrameType::Error,
3781        ] {
3782            binding.flow.acquire_tagged(401, false).await.unwrap();
3783            sink.try_send(route_frame(
3784                FrameType::Response,
3785                routes[1].client_channel,
3786                routes[1].client_epoch,
3787                402,
3788            ))
3789            .unwrap();
3790            router
3791                .route_for_connection(
3792                    &ctx,
3793                    Frame::build(
3794                        ty,
3795                        Flags::new(false, Priority::Interactive, false),
3796                        routes[0].module_channel,
3797                        routes[0].module_epoch,
3798                        401,
3799                        vec![b's'; limit - subc_protocol::HEADER_LEN + 1],
3800                    )
3801                    .unwrap(),
3802                )
3803                .await
3804                .unwrap();
3805            assert!(close.try_recv().is_err());
3806            assert_eq!(rx.try_recv().unwrap().header.corr, 402);
3807            let error = rx
3808                .try_recv()
3809                .expect("every correlated oversize frame becomes ERROR");
3810            assert_frame_size_error(
3811                &error,
3812                routes[0].client_channel,
3813                routes[0].client_epoch,
3814                401,
3815                limit + 1,
3816            );
3817            assert_eq!(binding.flow.drain_in_flight(), 0, "{ty:?} ends the request");
3818            assert!(rx.try_recv().is_err());
3819        }
3820    }
3821
3822    /// An awaited send (a control or error reply) is never refused by the byte
3823    /// budget: it parks until the writer frees bytes. If a release could miss a
3824    /// parked sender, that reply would hang for the connection's lifetime, so
3825    /// this pins the wakeup: the send stays parked while the queue is full and
3826    /// completes as soon as one frame leaves it.
3827    #[tokio::test]
3828    async fn awaited_send_parked_behind_a_full_byte_budget_wakes_when_bytes_free() {
3829        let (tx, mut rx) = mpsc::channel(1024);
3830        let sink = FrameSink::with_byte_budget(tx, 2_000);
3831        let mut queued = 0u64;
3832        while sink
3833            .try_send(stream_frame(7, 1, queued, vec![b'x'; 200]))
3834            .is_ok()
3835        {
3836            queued += 1;
3837        }
3838        assert!(queued > 0, "the budget admitted nothing");
3839
3840        let parked = tokio::spawn({
3841            let sink = sink.clone();
3842            async move { sink.send(stream_frame(0, 0, 999, vec![b'r'; 200])).await }
3843        });
3844        tokio::time::sleep(Duration::from_millis(100)).await;
3845        assert!(
3846            !parked.is_finished(),
3847            "the awaited send must wait while the byte budget is full"
3848        );
3849
3850        // The writer takes one frame and drops it, releasing its bytes.
3851        drop(rx.recv().await.expect("a queued frame"));
3852        tokio::time::timeout(Duration::from_secs(2), parked)
3853            .await
3854            .expect("the parked send was never woken after bytes were freed")
3855            .unwrap()
3856            .unwrap();
3857    }
3858
3859    /// A client multiplexing several token streams pauses its reader while the
3860    /// module keeps producing small frames. Far more frames than the old
3861    /// 64-frame queue allowed, but far fewer bytes than the budget, must all be
3862    /// held without closing the connection and delivered in order afterwards.
3863    #[tokio::test]
3864    async fn paused_client_reader_keeps_connection_through_small_frame_burst() {
3865        const ROUTES: usize = 4;
3866        const FRAMES_PER_ROUTE: usize = 250;
3867        let (router, client_sink, mut client_rx, module_ctx, routes, mut close_receiver) =
3868            multi_route_client(
3869                "burst-provider",
3870                ConnectionId::new(60),
3871                ConnectionId::new(61),
3872                ROUTES,
3873            )
3874            .await;
3875
3876        // The reader is paused: nothing is received until every frame is sent.
3877        for seq in 0..FRAMES_PER_ROUTE as u64 {
3878            for (index, route) in routes.iter().enumerate() {
3879                let body = format!("route-{index}-token-{seq:05}-{}", "t".repeat(170));
3880                router
3881                    .route_for_connection(
3882                        &module_ctx,
3883                        stream_frame(
3884                            route.module_channel,
3885                            route.module_epoch,
3886                            seq,
3887                            body.into_bytes(),
3888                        ),
3889                    )
3890                    .await
3891                    .unwrap();
3892            }
3893        }
3894
3895        let backlog = client_sink.backlog();
3896        assert_eq!(backlog.queued_frames, ROUTES * FRAMES_PER_ROUTE);
3897        assert!(backlog.queued_bytes < crate::server::CONNECTION_EGRESS_BYTE_BUDGET);
3898        assert!(
3899            close_receiver.try_recv().is_err(),
3900            "a paused reader under the byte budget must not be closed"
3901        );
3902        assert_eq!(
3903            router.counters.snapshot()["client_egress_close_delivery_failed"],
3904            0
3905        );
3906
3907        // The reader resumes: every frame arrives, in order within each route.
3908        let mut next_seq = vec![0u64; ROUTES];
3909        for _ in 0..ROUTES * FRAMES_PER_ROUTE {
3910            let frame = client_rx.try_recv().expect("every queued frame arrives");
3911            let index = routes
3912                .iter()
3913                .position(|route| route.client_channel == frame.header.channel)
3914                .expect("frame arrives on one of the bound client channels");
3915            assert_eq!(frame.header.corr, next_seq[index], "route {index} order");
3916            let expected_prefix = format!("route-{index}-token-{:05}-", next_seq[index]);
3917            assert!(frame.body.starts_with(expected_prefix.as_bytes()));
3918            next_seq[index] += 1;
3919        }
3920        assert!(client_rx.try_recv().is_err());
3921        assert_eq!(next_seq, vec![FRAMES_PER_ROUTE as u64; ROUTES]);
3922        assert_eq!(client_sink.backlog().queued_bytes, 0);
3923    }
3924
3925    /// A client that never reads exhausts its byte budget and the oldest
3926    /// frame's 250 ms grace, then closes with one header-only diagnosis.
3927    #[tokio::test(start_paused = true)]
3928    async fn never_reading_client_is_closed_at_byte_budget_with_warn_diagnosis() {
3929        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3930        const BODY: usize = 16 * 1024;
3931        let (router, client_sink, _client_rx, module_ctx, routes, mut close_receiver) =
3932            multi_route_client(
3933                "stuck-reader-provider",
3934                ConnectionId::new(70),
3935                ConnectionId::new(71),
3936                2,
3937            )
3938            .await;
3939
3940        let mut admitted = 0usize;
3941        let mut sent = 0u64;
3942        let started = tokio::time::Instant::now();
3943        while router.counters.snapshot()["client_egress_close_delivery_failed"] == 0 {
3944            assert!(sent < 1_000, "the byte budget never refused a frame");
3945            let route = &routes[(sent % 2) as usize];
3946            router
3947                .route_for_connection(
3948                    &module_ctx,
3949                    stream_frame(
3950                        route.module_channel,
3951                        route.module_epoch,
3952                        sent,
3953                        vec![b'z'; BODY],
3954                    ),
3955                )
3956                .await
3957                .unwrap();
3958            sent += 1;
3959            admitted = client_sink.backlog().queued_frames;
3960        }
3961        assert_eq!(
3962            started.elapsed(),
3963            crate::server::CONNECTION_EGRESS_STALL_GRACE
3964        );
3965        // The budget, not the frame-count backstop, did the refusing.
3966        let frame_bytes = subc_protocol::HEADER_LEN + BODY;
3967        assert_eq!(
3968            admitted,
3969            crate::server::CONNECTION_EGRESS_BYTE_BUDGET / frame_bytes
3970        );
3971        let reason = close_receiver
3972            .try_recv()
3973            .expect("the client connection must be asked to close");
3974        assert!(reason
3975            .to_string()
3976            .contains("module_to_client_delivery_failed"));
3977
3978        // A second refused frame for the same connection adds no second WARN.
3979        router
3980            .route_for_connection(
3981                &module_ctx,
3982                stream_frame(
3983                    routes[0].module_channel,
3984                    routes[0].module_epoch,
3985                    sent,
3986                    vec![b'z'; BODY],
3987                ),
3988            )
3989            .await
3990            .unwrap();
3991
3992        let captured = test_log::captured_logs(&logs);
3993        let warn_lines = captured
3994            .lines()
3995            .filter(|line| {
3996                line.contains("closing client connection: its egress queue could not take a frame")
3997            })
3998            .collect::<Vec<_>>();
3999        assert_eq!(warn_lines.len(), 1, "exactly one WARN, got: {captured}");
4000        let line = warn_lines[0];
4001        assert!(line.contains("WARN"), "{line}");
4002        assert!(line.contains("connection_id=71"), "{line}");
4003        assert!(
4004            line.contains("module_id=\"stuck-reader-provider\""),
4005            "{line}"
4006        );
4007        assert!(line.contains("client_channel="), "{line}");
4008        assert!(line.contains("principals=direct"), "{line}");
4009        let queued_bytes: usize = line
4010            .split("queued_bytes=")
4011            .nth(1)
4012            .and_then(|rest| rest.split_whitespace().next())
4013            .and_then(|value| value.parse().ok())
4014            .expect("queued_bytes is logged");
4015        assert_eq!(queued_bytes, admitted * frame_bytes);
4016        assert!(
4017            line.contains(&format!("queued_frames={admitted}")),
4018            "{line}"
4019        );
4020        assert!(line.contains("oldest_queued_ms="), "{line}");
4021        assert!(line.contains("oldest_queued_ms=250"), "{line}");
4022        assert!(line.contains("frame_type=Some(StreamData)"), "{line}");
4023        assert!(
4024            line.contains(&format!(
4025                "frame_flags=Some({})",
4026                Flags::new(false, Priority::Interactive, false).0
4027            )),
4028            "{line}"
4029        );
4030        assert!(
4031            line.contains("close_cause=\"waited_250_ms_without_room\""),
4032            "{line}"
4033        );
4034        assert!(
4035            !line.contains(&"z".repeat(20)),
4036            "body bytes must never be logged: {line}"
4037        );
4038    }
4039
4040    fn route_frame(ty: FrameType, channel: u16, epoch: u32, corr: u64) -> Frame {
4041        Frame::build(
4042            ty,
4043            Flags::new(false, Priority::Interactive, false),
4044            channel,
4045            epoch,
4046            corr,
4047            if ty == FrameType::Request || ty == FrameType::Response {
4048                b"route-body".to_vec()
4049            } else {
4050                Vec::new()
4051            },
4052        )
4053        .unwrap()
4054    }
4055
4056    type DynamicRouteFixture = (
4057        Router,
4058        Arc<ForwardingTable>,
4059        RouteCtx,
4060        mpsc::Receiver<crate::router::OutboundFrame>,
4061        RouteCtx,
4062        mpsc::Receiver<crate::router::OutboundFrame>,
4063        mpsc::Receiver<crate::router::OutboundFrame>,
4064        crate::forwarding::PendingRouteBindRelay,
4065    );
4066
4067    fn dynamic_route_fixture(commit: bool) -> DynamicRouteFixture {
4068        dynamic_route_fixture_with_busy(commit, false)
4069    }
4070
4071    fn dynamic_route_fixture_with_busy(commit: bool, busy_refusals: bool) -> DynamicRouteFixture {
4072        let forwarding = Arc::new(ForwardingTable::default());
4073        let control = Arc::new(crate::ControlHandler::with_forwarding(
4074            Arc::new(crate::Registry::default()),
4075            Arc::clone(&forwarding),
4076        ));
4077        let router = Router::with_control_handler(control);
4078        let module_connection = ConnectionId::new(500);
4079        let client_connection = ConnectionId::new(501);
4080        let (module_tx, module_rx) = mpsc::channel(if busy_refusals { 64 } else { 8 });
4081        forwarding
4082            .register_module_connection(
4083                module_connection,
4084                "epoch-router".into(),
4085                2,
4086                Concurrency::ModuleManaged,
4087                FrameSink::new(module_tx),
4088            )
4089            .unwrap();
4090        let (client_tx, client_rx) = mpsc::channel(if busy_refusals { 64 } else { 8 });
4091        let client_sink = FrameSink::new(client_tx);
4092        let pending = forwarding
4093            .begin_route_bind_relay_with_busy_for_test(
4094                client_connection,
4095                client_sink.clone(),
4096                700,
4097                "epoch-router",
4098                busy_refusals,
4099            )
4100            .unwrap();
4101        if commit {
4102            forwarding
4103                .complete_pending_relay(
4104                    module_connection,
4105                    pending.corr,
4106                    RouteBindRelayOutcome::Accepted,
4107                )
4108                .unwrap();
4109        }
4110        let (module_egress_tx, module_egress_rx) = mpsc::channel(8);
4111        (
4112            router,
4113            forwarding,
4114            RouteCtx {
4115                connection_id: client_connection,
4116                egress: client_sink,
4117            },
4118            client_rx,
4119            RouteCtx {
4120                connection_id: module_connection,
4121                egress: FrameSink::new(module_egress_tx),
4122            },
4123            module_egress_rx,
4124            module_rx,
4125            pending,
4126        )
4127    }
4128
4129    #[tokio::test]
4130    async fn route_epochs_validate_both_directions_and_rewrite_to_peer_handle() {
4131        let (
4132            router,
4133            _forwarding,
4134            client_ctx,
4135            mut client_rx,
4136            module_ctx,
4137            _module_egress_rx,
4138            mut module_rx,
4139            pending,
4140        ) = dynamic_route_fixture(true);
4141        let route_open = client_rx.recv().await.unwrap();
4142        assert_eq!(route_open.header.corr, 700);
4143
4144        router
4145            .route_for_connection(
4146                &client_ctx,
4147                route_frame(
4148                    FrameType::Request,
4149                    pending.client_channel,
4150                    pending.client_epoch,
4151                    701,
4152                ),
4153            )
4154            .await
4155            .unwrap();
4156        let forwarded = module_rx.recv().await.unwrap();
4157        assert_eq!(forwarded.header.channel, pending.module_channel);
4158        assert_eq!(forwarded.header.epoch, pending.module_epoch);
4159
4160        router
4161            .route_for_connection(
4162                &module_ctx,
4163                route_frame(
4164                    FrameType::Response,
4165                    pending.module_channel,
4166                    pending.module_epoch,
4167                    701,
4168                ),
4169            )
4170            .await
4171            .unwrap();
4172        let delivered = client_rx.recv().await.unwrap();
4173        assert_eq!(delivered.header.channel, pending.client_channel);
4174        assert_eq!(delivered.header.epoch, pending.client_epoch);
4175
4176        router
4177            .route_for_connection(
4178                &client_ctx,
4179                route_frame(
4180                    FrameType::Request,
4181                    pending.client_channel,
4182                    pending.client_epoch + 1,
4183                    702,
4184                ),
4185            )
4186            .await
4187            .unwrap();
4188        router
4189            .route_for_connection(
4190                &module_ctx,
4191                route_frame(
4192                    FrameType::Response,
4193                    pending.module_channel,
4194                    pending.module_epoch + 1,
4195                    703,
4196                ),
4197            )
4198            .await
4199            .unwrap();
4200        let stale_error = client_rx.recv().await.unwrap();
4201        assert_eq!(stale_error.header.ty, FrameType::Error);
4202        assert_eq!(stale_error.header.channel, pending.client_channel);
4203        assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
4204        assert_eq!(stale_error.header.corr, 702);
4205        let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
4206        assert_eq!(body.code, "stale_route_epoch");
4207        assert!(module_rx.try_recv().is_err());
4208        assert!(client_rx.try_recv().is_err());
4209        let counters = router.counters.snapshot();
4210        assert_eq!(counters["client_frames_dropped_stale_route"], 1);
4211        assert_eq!(counters["module_frames_dropped_no_route"], 1);
4212    }
4213
4214    #[tokio::test]
4215    async fn accepted_route_publishes_route_open_before_immediate_reverse_request() {
4216        let (
4217            router,
4218            _,
4219            _client_ctx,
4220            mut client_rx,
4221            module_ctx,
4222            _module_egress_rx,
4223            _module_rx,
4224            pending,
4225        ) = dynamic_route_fixture(true);
4226        router
4227            .route_for_connection(
4228                &module_ctx,
4229                route_frame(
4230                    FrameType::Request,
4231                    pending.module_channel,
4232                    pending.module_epoch,
4233                    800,
4234                ),
4235            )
4236            .await
4237            .unwrap();
4238
4239        let first = client_rx.recv().await.unwrap();
4240        let second = client_rx.recv().await.unwrap();
4241        assert_eq!(first.header.channel, 0);
4242        assert_eq!(first.header.corr, 700);
4243        assert_eq!(second.header.channel, pending.client_channel);
4244        assert_eq!(second.header.epoch, pending.client_epoch);
4245        assert_eq!(second.header.corr, 800);
4246    }
4247
4248    #[tokio::test]
4249    async fn reserved_slot_ingress_errors_only_matching_client_requests() {
4250        let (
4251            router,
4252            _forwarding,
4253            client_ctx,
4254            mut client_rx,
4255            _module_ctx,
4256            _module_egress_rx,
4257            mut module_rx,
4258            pending,
4259        ) = dynamic_route_fixture(false);
4260        router
4261            .route_for_connection(
4262                &client_ctx,
4263                route_frame(
4264                    FrameType::Request,
4265                    pending.client_channel,
4266                    pending.client_epoch,
4267                    900,
4268                ),
4269            )
4270            .await
4271            .unwrap();
4272        let error = client_rx.recv().await.unwrap();
4273        assert_eq!(error.header.ty, FrameType::Error);
4274        assert_eq!(error.header.channel, pending.client_channel);
4275        assert_eq!(error.header.epoch, pending.client_epoch);
4276        assert_eq!(error.header.corr, 900);
4277
4278        router
4279            .route_for_connection(
4280                &client_ctx,
4281                route_frame(
4282                    FrameType::Response,
4283                    pending.client_channel,
4284                    pending.client_epoch,
4285                    901,
4286                ),
4287            )
4288            .await
4289            .unwrap();
4290        router
4291            .route_for_connection(
4292                &client_ctx,
4293                route_frame(
4294                    FrameType::Request,
4295                    pending.client_channel,
4296                    pending.client_epoch + 1,
4297                    902,
4298                ),
4299            )
4300            .await
4301            .unwrap();
4302        let stale_error = client_rx.recv().await.unwrap();
4303        assert_eq!(stale_error.header.ty, FrameType::Error);
4304        assert_eq!(stale_error.header.channel, pending.client_channel);
4305        assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
4306        assert_eq!(stale_error.header.corr, 902);
4307        let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
4308        assert_eq!(body.code, "stale_route_epoch");
4309        assert!(module_rx.try_recv().is_err());
4310        let counters = router.counters.snapshot();
4311        assert_eq!(counters["client_frames_dropped_stale_route"], 1);
4312        assert_eq!(counters["module_frames_dropped_no_route"], 0);
4313    }
4314
4315    #[tokio::test]
4316    async fn dropped_module_route_goodbye_increments_counter() {
4317        let (
4318            router,
4319            _forwarding,
4320            client_ctx,
4321            mut client_rx,
4322            _module_ctx,
4323            _module_egress_rx,
4324            mut module_rx,
4325            pending,
4326        ) = dynamic_route_fixture(true);
4327        let _ = client_rx.recv().await;
4328        module_rx.close();
4329
4330        router
4331            .route_for_connection(
4332                &client_ctx,
4333                route_frame(
4334                    FrameType::Goodbye,
4335                    pending.client_channel,
4336                    pending.client_epoch,
4337                    999,
4338                ),
4339            )
4340            .await
4341            .unwrap();
4342
4343        let counters = router.counters.snapshot();
4344        assert_eq!(counters["goodbye_relay_module_dropped"], 1);
4345        assert_eq!(
4346            counters["goodbye_relay_module_dropped_by_module"],
4347            serde_json::json!({ "epoch-router": 1 })
4348        );
4349        assert_eq!(counters["route_released_epoch_fenced"], 1);
4350    }
4351
4352    #[tokio::test]
4353    async fn module_request_on_stale_epoch_receives_stale_route_epoch() {
4354        let (
4355            router,
4356            _forwarding,
4357            _client_ctx,
4358            _client_rx,
4359            module_ctx,
4360            mut module_egress_rx,
4361            mut module_rx,
4362            pending,
4363        ) = dynamic_route_fixture(true);
4364
4365        router
4366            .route_for_connection(
4367                &module_ctx,
4368                route_frame(
4369                    FrameType::Request,
4370                    pending.module_channel,
4371                    pending.module_epoch + 1,
4372                    1_000,
4373                ),
4374            )
4375            .await
4376            .unwrap();
4377
4378        let error = module_egress_rx.try_recv().unwrap();
4379        assert_eq!(error.header.ty, FrameType::Error);
4380        assert_eq!(error.header.channel, pending.module_channel);
4381        assert_eq!(error.header.epoch, pending.module_epoch + 1);
4382        assert_eq!(error.header.corr, 1_000);
4383        let body: ErrorBody = serde_json::from_slice(&error.body).unwrap();
4384        assert_eq!(body.code, "stale_route_epoch");
4385        assert!(module_rx.try_recv().is_err());
4386        let counters = router.counters.snapshot();
4387        assert_eq!(counters["module_requests_dropped_stale_route"], 1);
4388        assert_eq!(counters["module_frames_dropped_no_route"], 0);
4389    }
4390
4391    #[tokio::test(start_paused = true)]
4392    async fn stale_epoch_drop_warns_once_per_minute_with_suppressed_count() {
4393        let (output, _guard) = test_log::log_capture(tracing::Level::WARN);
4394        let (
4395            router,
4396            _forwarding,
4397            client_ctx,
4398            mut client_rx,
4399            _module_ctx,
4400            _module_egress_rx,
4401            mut module_rx,
4402            pending,
4403        ) = dynamic_route_fixture(true);
4404        let _ = client_rx.recv().await;
4405        for corr in 1..=3 {
4406            router
4407                .route_for_connection(
4408                    &client_ctx,
4409                    route_frame(
4410                        FrameType::Request,
4411                        pending.client_channel,
4412                        pending.client_epoch + 1,
4413                        corr,
4414                    ),
4415                )
4416                .await
4417                .unwrap();
4418            assert_eq!(client_rx.recv().await.unwrap().header.ty, FrameType::Error);
4419        }
4420        assert!(module_rx.try_recv().is_err());
4421        let logs = test_log::captured_logs(&output);
4422        assert_eq!(
4423            logs.matches("dropping data-plane frame").count(),
4424            1,
4425            "{logs}"
4426        );
4427        assert!(logs.contains("WARN"), "{logs}");
4428        assert!(
4429            logs.contains(&format!("connection_id={}", client_ctx.connection_id.get())),
4430            "{logs}"
4431        );
4432        assert!(
4433            logs.contains(&format!("channel={}", pending.client_channel)),
4434            "{logs}"
4435        );
4436        assert!(
4437            logs.contains(&format!("epoch={}", pending.client_epoch + 1)),
4438            "{logs}"
4439        );
4440        assert!(
4441            logs.contains("client_to_module") && logs.contains("stale_epoch"),
4442            "{logs}"
4443        );
4444        assert!(logs.contains("suppressed=0"), "{logs}");
4445
4446        tokio::time::advance(Duration::from_secs(60)).await;
4447        router
4448            .route_for_connection(
4449                &client_ctx,
4450                route_frame(
4451                    FrameType::Request,
4452                    pending.client_channel,
4453                    pending.client_epoch + 2,
4454                    4,
4455                ),
4456            )
4457            .await
4458            .unwrap();
4459        let logs = test_log::captured_logs(&output);
4460        assert_eq!(
4461            logs.matches("dropping data-plane frame").count(),
4462            2,
4463            "{logs}"
4464        );
4465        assert!(logs.contains("suppressed=2"), "{logs}");
4466    }
4467
4468    #[tokio::test]
4469    async fn module_request_on_reserved_or_absent_route_receives_unknown_channel() {
4470        let (
4471            reserved_router,
4472            _forwarding,
4473            _client_ctx,
4474            _client_rx,
4475            reserved_module_ctx,
4476            mut reserved_module_egress_rx,
4477            _module_rx,
4478            reserved,
4479        ) = dynamic_route_fixture(false);
4480        reserved_router
4481            .route_for_connection(
4482                &reserved_module_ctx,
4483                route_frame(
4484                    FrameType::Request,
4485                    reserved.module_channel,
4486                    reserved.module_epoch,
4487                    1_001,
4488                ),
4489            )
4490            .await
4491            .unwrap();
4492        let reserved_error = reserved_module_egress_rx.try_recv().unwrap();
4493        let reserved_body: ErrorBody = serde_json::from_slice(&reserved_error.body).unwrap();
4494        assert_eq!(reserved_error.header.ty, FrameType::Error);
4495        assert_eq!(reserved_error.header.channel, reserved.module_channel);
4496        assert_eq!(reserved_error.header.epoch, reserved.module_epoch);
4497        assert_eq!(reserved_error.header.corr, 1_001);
4498        assert_eq!(reserved_body.code, "unknown_channel");
4499        assert_eq!(
4500            reserved_router.counters.snapshot()["module_requests_dropped_stale_route"],
4501            1
4502        );
4503
4504        let (
4505            absent_router,
4506            _forwarding,
4507            _client_ctx,
4508            _client_rx,
4509            absent_module_ctx,
4510            mut absent_module_egress_rx,
4511            _module_rx,
4512            absent,
4513        ) = dynamic_route_fixture(false);
4514        absent_router
4515            .route_for_connection(
4516                &absent_module_ctx,
4517                route_frame(
4518                    FrameType::Request,
4519                    absent.module_channel + 1,
4520                    absent.module_epoch,
4521                    1_002,
4522                ),
4523            )
4524            .await
4525            .unwrap();
4526        let absent_error = absent_module_egress_rx.try_recv().unwrap();
4527        let absent_body: ErrorBody = serde_json::from_slice(&absent_error.body).unwrap();
4528        assert_eq!(absent_error.header.ty, FrameType::Error);
4529        assert_eq!(absent_error.header.channel, absent.module_channel + 1);
4530        assert_eq!(absent_error.header.epoch, absent.module_epoch);
4531        assert_eq!(absent_error.header.corr, 1_002);
4532        assert_eq!(absent_body.code, "unknown_channel");
4533        assert_eq!(
4534            absent_router.counters.snapshot()["module_requests_dropped_stale_route"],
4535            1
4536        );
4537    }
4538
4539    #[tokio::test]
4540    async fn non_request_module_frame_on_dead_route_is_counted_without_error() {
4541        let (
4542            router,
4543            forwarding,
4544            client_ctx,
4545            mut client_rx,
4546            module_ctx,
4547            mut module_egress_rx,
4548            mut module_rx,
4549            pending,
4550        ) = dynamic_route_fixture(true);
4551        let (other_module_tx, _other_module_rx) = mpsc::channel(8);
4552        forwarding
4553            .register_module_connection(
4554                ConnectionId::new(502),
4555                "other-module".into(),
4556                2,
4557                Concurrency::ModuleManaged,
4558                FrameSink::new(other_module_tx),
4559            )
4560            .unwrap();
4561        let _ = client_rx.recv().await.unwrap();
4562
4563        router
4564            .route_for_connection(
4565                &client_ctx,
4566                route_frame(
4567                    FrameType::Goodbye,
4568                    pending.client_channel,
4569                    pending.client_epoch,
4570                    1_003,
4571                ),
4572            )
4573            .await
4574            .unwrap();
4575        let _ = module_rx.recv().await.unwrap();
4576
4577        router
4578            .route_for_connection(
4579                &module_ctx,
4580                route_frame(
4581                    FrameType::StreamData,
4582                    pending.module_channel,
4583                    pending.module_epoch,
4584                    1_004,
4585                ),
4586            )
4587            .await
4588            .unwrap();
4589
4590        // No ERROR goes back for a non-request frame. The one reply is the
4591        // route GOODBYE telling the module to let go of the released route.
4592        let reply = module_egress_rx.try_recv().unwrap();
4593        assert_eq!(reply.header.ty, FrameType::Goodbye);
4594        assert!(module_egress_rx.try_recv().is_err());
4595        let counters = router.counters.snapshot();
4596        assert_eq!(counters["module_frames_dropped_no_route"], 1);
4597        assert_eq!(
4598            counters["module_frames_dropped_no_route_by_module"],
4599            serde_json::json!({ "epoch-router": 1 })
4600        );
4601        assert_eq!(counters["module_requests_dropped_stale_route"], 0);
4602    }
4603
4604    /// Drain whatever the module connection's queue holds right now, the way a
4605    /// module's reader would before it stalls.
4606    fn drain_now(rx: &mut mpsc::Receiver<OutboundFrame>) {
4607        while rx.try_recv().is_ok() {}
4608    }
4609
4610    /// A module that stops reading for a moment when a client closes one of
4611    /// its routes still learns the route is gone: the GOODBYE its full egress
4612    /// queue refused is delivered as soon as it reads again, rather than
4613    /// dropped, which would leave the module holding the route for the rest of
4614    /// its connection.
4615    #[tokio::test]
4616    async fn route_goodbye_refused_by_stalled_module_is_delivered_when_it_resumes_reading() {
4617        const BUDGET: usize = 4_096;
4618        let forwarding = Arc::new(ForwardingTable::default());
4619        let control = Arc::new(ControlHandler::with_forwarding(
4620            Arc::new(Registry::default()),
4621            Arc::clone(&forwarding),
4622        ));
4623        let router = Router::with_control_handler(control);
4624        let module_connection = ConnectionId::new(80);
4625        let client_connection = ConnectionId::new(81);
4626        let (module_tx, mut module_rx) = mpsc::channel(64);
4627        let module_sink = FrameSink::with_byte_budget(module_tx, BUDGET);
4628        forwarding
4629            .register_module_connection(
4630                module_connection,
4631                "stalled-provider".into(),
4632                2,
4633                Concurrency::ModuleManaged,
4634                module_sink.clone(),
4635            )
4636            .unwrap();
4637        let (client_tx, mut client_rx) = mpsc::channel(8);
4638        let client_sink = FrameSink::new(client_tx);
4639        let pending = forwarding
4640            .begin_route_bind_relay_for_test(
4641                client_connection,
4642                client_sink.clone(),
4643                1_100,
4644                "stalled-provider",
4645            )
4646            .unwrap();
4647        forwarding
4648            .complete_pending_relay(
4649                module_connection,
4650                pending.corr,
4651                RouteBindRelayOutcome::Accepted,
4652            )
4653            .unwrap();
4654        let _ = client_rx.recv().await.unwrap();
4655        drain_now(&mut module_rx);
4656
4657        // The module stalls: its queue holds more than the byte budget, so the
4658        // queue refuses anything further.
4659        module_sink
4660            .try_send(stream_frame(9, 1, 0, vec![b'f'; BUDGET]))
4661            .unwrap();
4662        assert!(module_sink
4663            .try_send(stream_frame(9, 1, 1, Vec::new()))
4664            .is_err());
4665
4666        let client_ctx = RouteCtx {
4667            connection_id: client_connection,
4668            egress: client_sink,
4669        };
4670        router
4671            .route_for_connection(
4672                &client_ctx,
4673                route_frame(
4674                    FrameType::Goodbye,
4675                    pending.client_channel,
4676                    pending.client_epoch,
4677                    1_101,
4678                ),
4679            )
4680            .await
4681            .unwrap();
4682        tokio::task::yield_now().await;
4683        assert_eq!(
4684            router.counters.snapshot()["goodbye_relay_module_dropped"],
4685            0,
4686            "a GOODBYE refused by a momentarily full module queue must not be dropped"
4687        );
4688
4689        // The module reads again: the filler comes off, then the GOODBYE.
4690        let filler = module_rx.recv().await.unwrap();
4691        assert_eq!(filler.header.ty, FrameType::StreamData);
4692        drop(filler);
4693        let goodbye = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
4694            .await
4695            .expect("the refused GOODBYE must be delivered once the module frees room")
4696            .unwrap();
4697        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
4698        assert_eq!(goodbye.header.channel, pending.module_channel);
4699        assert_eq!(goodbye.header.epoch, pending.module_epoch);
4700        assert_eq!(
4701            router.counters.snapshot()["goodbye_relay_module_dropped"],
4702            0
4703        );
4704    }
4705
4706    /// A module still sending on a route the daemon released is told, with a
4707    /// route GOODBYE for exactly the (channel, epoch) it sent on. The same
4708    /// happens for a stale epoch on a channel whose route moved on and for a
4709    /// channel that never had a route; only the first is counted as traffic on
4710    /// a released route.
4711    #[tokio::test]
4712    async fn module_frame_on_route_the_daemon_does_not_hold_is_answered_with_goodbye() {
4713        let (
4714            router,
4715            _forwarding,
4716            client_ctx,
4717            mut client_rx,
4718            module_ctx,
4719            mut module_egress_rx,
4720            mut module_rx,
4721            pending,
4722        ) = dynamic_route_fixture(true);
4723        let _ = client_rx.recv().await.unwrap();
4724        router
4725            .route_for_connection(
4726                &client_ctx,
4727                route_frame(
4728                    FrameType::Goodbye,
4729                    pending.client_channel,
4730                    pending.client_epoch,
4731                    1_200,
4732                ),
4733            )
4734            .await
4735            .unwrap();
4736        drain_now(&mut module_rx);
4737
4738        let cases = [
4739            // Released route: the daemon allocated this (channel, epoch).
4740            (pending.module_channel, pending.module_epoch),
4741            // A channel the daemon never allocated on this connection.
4742            (pending.module_channel + 1, 1),
4743        ];
4744        for (channel, epoch) in cases {
4745            router
4746                .route_for_connection(
4747                    &module_ctx,
4748                    route_frame(FrameType::StreamData, channel, epoch, 1_201),
4749                )
4750                .await
4751                .unwrap();
4752            let reply = module_egress_rx
4753                .try_recv()
4754                .expect("a frame on a route the daemon does not hold is answered");
4755            assert_eq!(reply.header.ty, FrameType::Goodbye);
4756            assert_eq!(reply.header.channel, channel);
4757            assert_eq!(reply.header.epoch, epoch);
4758            assert_eq!(reply.header.corr, 0);
4759            assert!(module_egress_rx.try_recv().is_err());
4760        }
4761
4762        // A GOODBYE from the module for a route the daemon already released
4763        // is not answered: the module is letting go already.
4764        router
4765            .route_for_connection(
4766                &module_ctx,
4767                route_frame(FrameType::Goodbye, pending.module_channel + 2, 1, 0),
4768            )
4769            .await
4770            .unwrap();
4771        assert!(module_egress_rx.try_recv().is_err());
4772
4773        let counters = router.counters.snapshot();
4774        assert_eq!(counters["module_frames_dropped_no_route"], 3);
4775        assert_eq!(counters["module_frames_dropped_released_route"], 1);
4776        assert_eq!(
4777            counters["module_frames_dropped_released_route_by_module"],
4778            serde_json::json!({ "epoch-router": 1 })
4779        );
4780        assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4781    }
4782
4783    /// A chatty orphan (a streaming module still producing on a route it
4784    /// missed the GOODBYE for) is answered once per interval, not once per
4785    /// frame, and answered again once the interval has passed.
4786    #[tokio::test(start_paused = true)]
4787    async fn burst_of_orphan_module_frames_is_answered_once_per_interval() {
4788        let (
4789            router,
4790            _forwarding,
4791            client_ctx,
4792            mut client_rx,
4793            module_ctx,
4794            mut module_egress_rx,
4795            mut module_rx,
4796            pending,
4797        ) = dynamic_route_fixture(true);
4798        let _ = client_rx.recv().await.unwrap();
4799        router
4800            .route_for_connection(
4801                &client_ctx,
4802                route_frame(
4803                    FrameType::Goodbye,
4804                    pending.client_channel,
4805                    pending.client_epoch,
4806                    1_300,
4807                ),
4808            )
4809            .await
4810            .unwrap();
4811        drain_now(&mut module_rx);
4812
4813        let send_burst = |corr: u64| {
4814            route_frame(
4815                FrameType::StreamData,
4816                pending.module_channel,
4817                pending.module_epoch,
4818                corr,
4819            )
4820        };
4821        for corr in 0..20 {
4822            router
4823                .route_for_connection(&module_ctx, send_burst(corr))
4824                .await
4825                .unwrap();
4826        }
4827        let mut replies = 0;
4828        while let Ok(reply) = module_egress_rx.try_recv() {
4829            assert_eq!(reply.header.ty, FrameType::Goodbye);
4830            replies += 1;
4831        }
4832        assert_eq!(replies, 1, "a burst within the interval gets one GOODBYE");
4833
4834        tokio::time::advance(ORPHAN_ROUTE_GOODBYE_INTERVAL).await;
4835        router
4836            .route_for_connection(&module_ctx, send_burst(20))
4837            .await
4838            .unwrap();
4839        let retry = module_egress_rx
4840            .try_recv()
4841            .expect("the first orphan frame after the interval is answered again");
4842        assert_eq!(retry.header.channel, pending.module_channel);
4843        assert_eq!(retry.header.epoch, pending.module_epoch);
4844
4845        let counters = router.counters.snapshot();
4846        assert_eq!(counters["module_frames_dropped_no_route"], 21);
4847        assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4848    }
4849
4850    /// The rate-limit memory is per connection and goes away with it.
4851    #[test]
4852    fn orphan_goodbye_rate_limit_state_is_released_with_the_connection() {
4853        let router = Router::with_default_self_handler();
4854        let connection = router.begin_connection();
4855        let id = connection.id();
4856        assert!(router.orphan_goodbyes.claim(id, 7));
4857        assert!(!router.orphan_goodbyes.claim(id, 7));
4858        assert!(router.orphan_goodbyes.claim(id, 8));
4859        drop(connection);
4860        assert!(router
4861            .orphan_goodbyes
4862            .last_sent
4863            .lock()
4864            .unwrap()
4865            .get(&id)
4866            .is_none());
4867    }
4868
4869    #[test]
4870    fn channel_zero_cannot_be_registered_as_backend() {
4871        let mut router = Router::with_default_self_handler();
4872
4873        let err = router.register_backend(0, EchoBackend).unwrap_err();
4874
4875        assert_eq!(err, RouterError::ReservedChannelZero);
4876    }
4877}