Skip to main content

subc_daemon/
router.rs

1use std::{
2    collections::HashMap,
3    error::Error,
4    fmt,
5    sync::{
6        atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
7        Arc, Mutex,
8    },
9    time::{Duration, Instant},
10};
11
12use subc_protocol::{error_codes, ErrorBody, Flags, FrameType, Priority};
13use tokio::sync::{mpsc, Notify};
14use tracing::{debug, warn};
15
16use crate::{
17    control::ControlHandler,
18    forwarding::{
19        principal_label, CloseReason, ConnectionCloseReceiver, DataRoute, DataRouteState,
20        ForwardingError, ForwardingTable, RouteBinding, RouteRelease, UndeliveredFrame,
21    },
22    registry::ConnectionId,
23    DaemonCounters, Frame, FrameBuildError,
24};
25
26/// One queued outbound frame plus the instant it entered the writer queue.
27///
28/// The stamp exists for the reply-path half of slow-control diagnosis: a
29/// handler can finish in microseconds while the reply sits in this queue
30/// waiting for the writer task to be scheduled, and without a per-item stamp
31/// that wait is invisible to every other timing point (the client's round
32/// trip is the only witness, and it cannot say which side ate the time).
33/// Constructed exclusively inside [`FrameSink`] so no caller can forget it.
34#[derive(Debug)]
35pub struct OutboundFrame {
36    pub frame: Frame,
37    pub enqueued_at: std::time::Instant,
38    pub(crate) flushed: Option<tokio::sync::oneshot::Sender<()>>,
39    /// This frame's share of the connection's queued-byte count. Dropping the
40    /// frame (after the writer has written it, or when the queue itself is
41    /// dropped with the frame still in it) gives the bytes back. `None` only for
42    /// frames built outside a [`FrameSink`], which exist in tests alone.
43    pub(crate) charge: Option<EgressCharge>,
44}
45
46impl OutboundFrame {
47    fn charged(frame: Frame, charge: EgressCharge) -> Self {
48        Self {
49            frame,
50            enqueued_at: charge.enqueued_at,
51            flushed: None,
52            charge: Some(charge),
53        }
54    }
55}
56
57/// Bytes a frame occupies in the connection's egress queue for budget
58/// purposes: the fixed envelope header plus the body.
59fn queued_frame_bytes(frame: &Frame) -> usize {
60    subc_protocol::HEADER_LEN + frame.body.len()
61}
62
63/// A point-in-time view of one connection's egress queue, for diagnosing why a
64/// frame did not fit.
65#[derive(Debug, Clone, Copy, PartialEq, Eq)]
66pub(crate) struct EgressBacklog {
67    pub queued_bytes: usize,
68    pub queued_frames: usize,
69    /// Now minus the enqueue time of the frame the writer most recently took
70    /// off the queue (the frame it is writing, or has just written), or, if
71    /// the writer has taken nothing since the queue was last empty, of the
72    /// frame that made the queue non-empty. While the writer is blocked on a
73    /// frame this is exactly the oldest frame's age; between frames it can be
74    /// one frame older than the true head. `None` when nothing is queued.
75    pub oldest_age: Option<Duration>,
76}
77
78/// Queued-byte accounting shared by every clone of one connection's
79/// [`FrameSink`] and by every frame that sink has admitted. Its per-frame
80/// accounting uses atomics: this is on the path of every frame a module sends
81/// to a client, so it takes no lock per frame.
82#[derive(Debug)]
83struct EgressAccounting {
84    // Shared by the connection's sink clones so even an invalid data frame
85    // spends the one opportunity to present a claim after authentication.
86    ingress_seen: AtomicBool,
87    ingress_terminal: AtomicBool,
88    egress_terminal: AtomicBool,
89    egress_stopped: Notify,
90    byte_budget: usize,
91    queued_bytes: AtomicUsize,
92    queued_frames: AtomicUsize,
93    /// Origin for the nanosecond timestamps in `oldest_enqueued_nanos`.
94    time_base: Instant,
95    /// Enqueue time, as nanoseconds after `time_base` plus one, of the frame
96    /// described by [`EgressBacklog::oldest_age`]; 0 means none.
97    oldest_enqueued_nanos: AtomicU64,
98    /// Awaited senders currently parked on `freed`. A release only pays for a
99    /// notification when this is non-zero.
100    waiters: AtomicUsize,
101    /// Woken when a charged frame releases its bytes while a sender waits, so
102    /// the awaited send can re-check for room.
103    freed: Notify,
104    client_delivery_closed: AtomicBool,
105    client_delivery_stopped: Notify,
106    #[cfg(test)]
107    retire_after_reserve: AtomicBool,
108}
109
110impl EgressAccounting {
111    fn new(byte_budget: usize) -> Self {
112        Self {
113            ingress_seen: AtomicBool::new(false),
114            ingress_terminal: AtomicBool::new(false),
115            egress_terminal: AtomicBool::new(false),
116            egress_stopped: Notify::new(),
117            byte_budget,
118            queued_bytes: AtomicUsize::new(0),
119            queued_frames: AtomicUsize::new(0),
120            time_base: tokio::time::Instant::now().into_std(),
121            oldest_enqueued_nanos: AtomicU64::new(0),
122            waiters: AtomicUsize::new(0),
123            freed: Notify::new(),
124            client_delivery_closed: AtomicBool::new(false),
125            client_delivery_stopped: Notify::new(),
126            #[cfg(test)]
127            retire_after_reserve: AtomicBool::new(false),
128        }
129    }
130
131    fn stamp(&self, at: Instant) -> u64 {
132        (at.saturating_duration_since(self.time_base).as_nanos() as u64).saturating_add(1)
133    }
134
135    /// Charge `bytes` if they fit in the budget. A frame larger than the whole
136    /// budget (bodies may be up to 64 MiB) is still admitted into an EMPTY
137    /// queue, since it could otherwise never be sent at all; it simply has the
138    /// queue to itself until it is written.
139    fn try_charge(self: &Arc<Self>, bytes: usize) -> Option<EgressCharge> {
140        // SeqCst pairs with `release`: either this load sees bytes a release
141        // just freed, or that release sees this sender counted in `waiters`.
142        let mut current = self.queued_bytes.load(Ordering::SeqCst);
143        loop {
144            if current != 0 && current.saturating_add(bytes) > self.byte_budget {
145                return None;
146            }
147            match self.queued_bytes.compare_exchange_weak(
148                current,
149                current + bytes,
150                Ordering::SeqCst,
151                Ordering::SeqCst,
152            ) {
153                Ok(_) => return Some(self.record(bytes)),
154                Err(actual) => current = actual,
155            }
156        }
157    }
158
159    /// Charge `bytes` regardless of the budget. Used only for a frame whose
160    /// queue slot was reserved in advance, which must be sendable.
161    fn charge_unconditionally(self: &Arc<Self>, bytes: usize) -> EgressCharge {
162        self.queued_bytes.fetch_add(bytes, Ordering::SeqCst);
163        self.record(bytes)
164    }
165
166    fn record(self: &Arc<Self>, bytes: usize) -> EgressCharge {
167        let enqueued_at = tokio::time::Instant::now().into_std();
168        let stamp = self.stamp(enqueued_at);
169        if self.queued_frames.fetch_add(1, Ordering::AcqRel) == 0 {
170            // This frame made the queue non-empty, so it is the head until the
171            // writer takes something.
172            self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
173        }
174        EgressCharge {
175            accounting: Arc::clone(self),
176            bytes,
177            stamp,
178            enqueued_at,
179        }
180    }
181
182    /// The writer has taken the frame stamped `stamp` off the queue.
183    fn taken_by_writer(&self, stamp: u64) {
184        self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
185    }
186
187    fn release(&self, bytes: usize, stamp: u64) {
188        self.queued_bytes.fetch_sub(bytes, Ordering::SeqCst);
189        if self.queued_frames.fetch_sub(1, Ordering::AcqRel) == 1 {
190            // The queue drained. Clear the marker only if it still names this
191            // frame, so a frame admitted in the meantime keeps its stamp.
192            let _ = self.oldest_enqueued_nanos.compare_exchange(
193                stamp,
194                0,
195                Ordering::AcqRel,
196                Ordering::Acquire,
197            );
198        }
199        if self.waiters.load(Ordering::SeqCst) != 0 {
200            self.freed.notify_waiters();
201        }
202    }
203
204    fn backlog(&self) -> EgressBacklog {
205        let queued_frames = self.queued_frames.load(Ordering::Acquire);
206        let stamp = self.oldest_enqueued_nanos.load(Ordering::Acquire);
207        let oldest_age = (queued_frames != 0 && stamp != 0).then(|| {
208            let enqueued = self.time_base + Duration::from_nanos(stamp - 1);
209            tokio::time::Instant::from_std(enqueued).elapsed()
210        });
211        EgressBacklog {
212            queued_bytes: self.queued_bytes.load(Ordering::Acquire),
213            queued_frames,
214            oldest_age,
215        }
216    }
217}
218
219/// One admitted frame's claim on its connection's egress byte budget,
220/// returned when the frame is dropped.
221#[derive(Debug)]
222pub(crate) struct EgressCharge {
223    accounting: Arc<EgressAccounting>,
224    bytes: usize,
225    stamp: u64,
226    enqueued_at: Instant,
227}
228
229impl EgressCharge {
230    /// Called by the connection writer when it takes this frame off the queue,
231    /// so the backlog's oldest-age figure follows the writer.
232    pub(crate) fn taken_by_writer(&self) {
233        self.accounting.taken_by_writer(self.stamp);
234    }
235}
236
237impl Drop for EgressCharge {
238    fn drop(&mut self) {
239        self.accounting.release(self.bytes, self.stamp);
240    }
241}
242
243/// A queue slot reserved ahead of time (a pending `route.open` holds one until
244/// its module answers). Sending through it never waits and never fails for
245/// lack of room: the slot is already held, and its frame is charged to the
246/// byte count outside the budget check. At most
247/// `MAX_PENDING_ROUTE_OPENS_PER_CONNECTION` such frames exist per connection.
248#[derive(Debug)]
249pub(crate) struct EgressPermit {
250    permit: mpsc::OwnedPermit<OutboundFrame>,
251    accounting: Arc<EgressAccounting>,
252}
253
254impl EgressPermit {
255    /// Enqueue `frame` in the reserved slot. Returns true when the connection's
256    /// writer had already gone away or egress was retired, so the frame will
257    /// never be written.
258    pub(crate) fn send(self, frame: Frame) -> bool {
259        if self.accounting.egress_terminal.load(Ordering::Acquire) {
260            return true;
261        }
262        let charge = self
263            .accounting
264            .charge_unconditionally(queued_frame_bytes(&frame));
265        let sender = self.permit.send(OutboundFrame::charged(frame, charge));
266        sender.is_closed()
267    }
268}
269
270/// An `OutboundFrame` is a stamped `Frame`; deref keeps every existing frame
271/// read (headers, bodies, assertions) working on queued items unchanged.
272impl std::ops::Deref for OutboundFrame {
273    type Target = Frame;
274
275    fn deref(&self) -> &Frame {
276        &self.frame
277    }
278}
279
280/// Shared tracing-capture helpers for timing-observability tests across
281/// modules (router dispatch, server reply path). Test-only.
282#[cfg(test)]
283pub(crate) mod test_log {
284    use std::{
285        io::Write,
286        sync::{Arc, Mutex, OnceLock},
287    };
288
289    #[derive(Clone)]
290    struct TestLogWriter(Arc<Mutex<Vec<u8>>>);
291
292    impl Write for TestLogWriter {
293        fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
294            self.0
295                .lock()
296                .expect("test log capture is not poisoned")
297                .extend(buffer);
298            Ok(buffer.len())
299        }
300
301        fn flush(&mut self) -> std::io::Result<()> {
302            Ok(())
303        }
304    }
305
306    pub(crate) fn log_capture(
307        level: tracing::Level,
308    ) -> (Arc<Mutex<Vec<u8>>>, tracing::dispatcher::DefaultGuard) {
309        // tracing-core's single-dispatch fast path registers callsites using
310        // the emitting thread's default subscriber. A parallel test with no
311        // subscriber can therefore cache `never` for a callsite this capture
312        // needs. Keep a second dispatch alive so registration considers all
313        // live subscribers instead. This registry is never installed as a
314        // default: each test still records only into its own thread's writer.
315        static INTEREST_ANCHOR: OnceLock<tracing::Dispatch> = OnceLock::new();
316        INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
317        let output = Arc::new(Mutex::new(Vec::new()));
318        let writer = Arc::clone(&output);
319        let subscriber = tracing_subscriber::fmt()
320            .with_max_level(level)
321            .with_ansi(false)
322            .without_time()
323            .with_target(false)
324            .with_writer(move || TestLogWriter(Arc::clone(&writer)))
325            .finish();
326        let guard = tracing::subscriber::set_default(subscriber);
327        (output, guard)
328    }
329
330    pub(crate) fn captured_logs(output: &Arc<Mutex<Vec<u8>>>) -> String {
331        String::from_utf8(
332            output
333                .lock()
334                .expect("test log capture is not poisoned")
335                .clone(),
336        )
337        .expect("tracing output is UTF-8")
338    }
339
340    #[cfg(test)]
341    mod tests {
342        use super::*;
343
344        fn emit_capture_probe(source: &str) {
345            tracing::error!(source, "parallel capture probe");
346        }
347
348        #[test]
349        fn parallel_callsite_registration_keeps_the_test_capture_enabled() {
350            let (output, _guard) = log_capture(tracing::Level::ERROR);
351            // First register the very same callsite on a thread without a
352            // default subscriber, while the test's capture remains live.
353            std::thread::spawn(|| emit_capture_probe("other-thread"))
354                .join()
355                .expect("unsubscribed thread exits");
356            emit_capture_probe("captured-thread");
357
358            let logs = captured_logs(&output);
359            assert!(
360                logs.contains("captured-thread"),
361                "capture was disabled: {logs}"
362            );
363            assert!(!logs.contains("other-thread"), "capture leaked: {logs}");
364        }
365    }
366}
367
368/// Cheaply cloneable handle to one connection's bounded outbound frame queue.
369///
370/// Backends emit responses, streaming frames, and future PUSH frames through this
371/// single path. The queue is bounded twice: by queued BYTES (the budget that
372/// matters for memory and for how long a slow reader may pause), and by the
373/// `mpsc` channel's frame count (a backstop). The socket layer owns the sole
374/// receiver/writer.
375#[derive(Debug, Clone)]
376pub struct FrameSink {
377    tx: mpsc::Sender<OutboundFrame>,
378    accounting: Arc<EgressAccounting>,
379}
380
381impl FrameSink {
382    pub(crate) fn terminate_ingress(&self) {
383        self.accounting
384            .ingress_terminal
385            .store(true, Ordering::Release);
386    }
387
388    pub(crate) fn ingress_terminal(&self) -> bool {
389        self.accounting.ingress_terminal.load(Ordering::Acquire)
390    }
391
392    // Retirement fences every sink clone and reserved permit synchronously,
393    // including senders already waiting for bytes or a queue slot.
394    pub(crate) fn terminate_egress(&self) {
395        self.accounting
396            .egress_terminal
397            .store(true, Ordering::Release);
398        self.accounting.egress_stopped.notify_waiters();
399        self.accounting.freed.notify_waiters();
400    }
401
402    pub(crate) fn egress_terminal(&self) -> bool {
403        self.accounting.egress_terminal.load(Ordering::Acquire)
404    }
405
406    pub(crate) fn first_ingress_frame(&self) -> bool {
407        !self.accounting.ingress_seen.swap(true, Ordering::Relaxed)
408    }
409    /// A sink with the standard per-connection byte budget
410    /// ([`crate::server::CONNECTION_EGRESS_BYTE_BUDGET`]); the frame-count bound
411    /// is whatever capacity `tx`'s channel was created with.
412    pub fn new(tx: mpsc::Sender<OutboundFrame>) -> Self {
413        Self::with_byte_budget(tx, crate::server::CONNECTION_EGRESS_BYTE_BUDGET)
414    }
415
416    pub(crate) fn with_byte_budget(tx: mpsc::Sender<OutboundFrame>, byte_budget: usize) -> Self {
417        Self {
418            tx,
419            accounting: Arc::new(EgressAccounting::new(byte_budget)),
420        }
421    }
422
423    /// Wait until the frame's bytes fit the budget, then charge them. Awaited
424    /// senders (control replies, error replies, shutdown notices) are never
425    /// refused by the byte budget: they wait for the writer to free bytes, just
426    /// as they wait for a free slot, which is the same backpressure they had
427    /// when the queue was bounded by frame count alone. Returns `None` if the
428    /// writer goes away or egress is retired while waiting.
429    async fn charge_waiting(&self, bytes: usize) -> Option<EgressCharge> {
430        if self.egress_terminal() {
431            return None;
432        }
433        if let Some(charge) = self.accounting.try_charge(bytes) {
434            return Some(charge);
435        }
436        // Counted as a waiter for as long as this future is parked, including
437        // when it is cancelled, so releases notify only while someone waits.
438        struct Waiting<'a>(&'a AtomicUsize);
439        impl Drop for Waiting<'_> {
440            fn drop(&mut self) {
441                self.0.fetch_sub(1, Ordering::SeqCst);
442            }
443        }
444        self.accounting.waiters.fetch_add(1, Ordering::SeqCst);
445        let _waiting = Waiting(&self.accounting.waiters);
446        loop {
447            let freed = self.accounting.freed.notified();
448            tokio::pin!(freed);
449            // Register interest before checking, so a release that happens
450            // between the check and the await still wakes this sender.
451            freed.as_mut().enable();
452            if self.egress_terminal() {
453                return None;
454            }
455            if let Some(charge) = self.accounting.try_charge(bytes) {
456                return Some(charge);
457            }
458            tokio::select! {
459                _ = &mut freed => {}
460                _ = self.tx.closed() => return None,
461            }
462        }
463    }
464
465    // Queue admission must recheck retirement after waiting for a frame slot,
466    // including when the bytes were charged before retirement began. Ordinary
467    // sends silently discard retired traffic; flush callers must not report it
468    // as queued or written.
469    async fn send_outbound(&self, outbound: OutboundFrame) -> Result<bool, RouterError> {
470        let stopped = self.accounting.egress_stopped.notified();
471        tokio::pin!(stopped);
472        stopped.as_mut().enable();
473        if self.egress_terminal() {
474            return Ok(false);
475        }
476        let header = &outbound.frame.header;
477        let closed = || {
478            RouterError::backend_with_epoch(
479                header.channel,
480                header.epoch,
481                header.corr,
482                "connection writer closed",
483            )
484        };
485        let permit = tokio::select! {
486            biased;
487            _ = &mut stopped => return Ok(false),
488            permit = self.tx.reserve() => permit.map_err(|_| closed())?,
489        };
490        if self.egress_terminal() {
491            return Ok(false);
492        }
493        permit.send(outbound);
494        Ok(true)
495    }
496
497    pub async fn send(&self, frame: Frame) -> Result<(), RouterError> {
498        if self.egress_terminal() {
499            return Ok(());
500        }
501        let channel = frame.header.channel;
502        let epoch = frame.header.epoch;
503        let corr = frame.header.corr;
504        let closed =
505            || RouterError::backend_with_epoch(channel, epoch, corr, "connection writer closed");
506        let Some(charge) = self.charge_waiting(queued_frame_bytes(&frame)).await else {
507            return if self.egress_terminal() {
508                Ok(())
509            } else {
510                Err(closed())
511            };
512        };
513        self.send_outbound(OutboundFrame::charged(frame, charge))
514            .await
515            .map(|_| ())
516    }
517
518    /// Shutdown notices must leave the socket writer before an idle daemon exits.
519    /// Queue admission alone does not prove this; the writer acknowledges flush.
520    ///
521    /// The local endpoint uses it the same way for its clients-only refusal: the
522    /// connection closes right after the refusal is sent, so the writer must
523    /// flush it first.
524    pub async fn send_flushed(&self, frame: Frame) -> Result<(), RouterError> {
525        self.send_flushed_notifying_enqueue(frame, || {}).await
526    }
527
528    /// Notify the caller after queue admission, before waiting for the writer's
529    /// flush, so cancellation withdraws only requests that could have arrived.
530    pub(crate) async fn send_flushed_notifying_enqueue(
531        &self,
532        frame: Frame,
533        queued: impl FnOnce(),
534    ) -> Result<(), RouterError> {
535        let (tx, rx) = tokio::sync::oneshot::channel();
536        let charge = self
537            .charge_waiting(queued_frame_bytes(&frame))
538            .await
539            .ok_or_else(|| RouterError::backend(0, 0, "connection writer closed"))?;
540        let mut outbound = OutboundFrame::charged(frame, charge);
541        outbound.flushed = Some(tx);
542        if !self.send_outbound(outbound).await? {
543            return Err(RouterError::backend(0, 0, "connection egress retired"));
544        }
545        queued();
546        rx.await
547            .map_err(|_| RouterError::backend(0, 0, "connection flush failed"))
548    }
549
550    pub(crate) async fn reserve_owned(&self) -> Result<EgressPermit, RouterError> {
551        let stopped = self.accounting.egress_stopped.notified();
552        tokio::pin!(stopped);
553        stopped.as_mut().enable();
554        if self.egress_terminal() {
555            return Err(RouterError::backend(0, 0, "connection egress retired"));
556        }
557        let permit = tokio::select! {
558            biased;
559            _ = &mut stopped => return Err(RouterError::backend(0, 0, "connection egress retired")),
560            permit = self.tx.clone().reserve_owned() => permit
561                .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?,
562        };
563        #[cfg(test)]
564        if self
565            .accounting
566            .retire_after_reserve
567            .swap(false, Ordering::Relaxed)
568        {
569            self.terminate_egress();
570        }
571        if self.egress_terminal() {
572            return Err(RouterError::backend(0, 0, "connection egress retired"));
573        }
574        Ok(EgressPermit {
575            permit,
576            accounting: Arc::clone(&self.accounting),
577        })
578    }
579
580    #[cfg(test)]
581    pub(crate) fn try_reserve_owned(&self) -> Result<EgressPermit, RouterError> {
582        if self.egress_terminal() {
583            return Err(RouterError::backend(0, 0, "connection egress retired"));
584        }
585        let permit = self
586            .tx
587            .clone()
588            .try_reserve_owned()
589            .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
590        Ok(EgressPermit {
591            permit,
592            accounting: Arc::clone(&self.accounting),
593        })
594    }
595
596    pub(crate) fn is_closed(&self) -> bool {
597        self.tx.is_closed()
598    }
599
600    /// What the connection's egress queue holds right now.
601    pub(crate) fn backlog(&self) -> EgressBacklog {
602        self.accounting.backlog()
603    }
604
605    /// Stop module deliveries as soon as the client reader exits, without
606    /// waiting for the socket writer's separate teardown drain grace.
607    pub(crate) fn close_client_delivery(&self) {
608        self.accounting
609            .client_delivery_closed
610            .store(true, Ordering::Release);
611        self.accounting.client_delivery_stopped.notify_waiters();
612    }
613
614    /// A full queue can be a short fanout burst rather than a stopped reader.
615    /// Spend only the grace the atomic backlog age has not already used.
616    async fn send_to_client(&self, frame: Frame) -> Result<(), ClientDeliveryFailure> {
617        if self.egress_terminal() {
618            return Ok(());
619        }
620        let frame_type = frame.header.ty;
621        let flags = frame.header.flags;
622        let stopped = self.accounting.client_delivery_stopped.notified();
623        tokio::pin!(stopped);
624        stopped.as_mut().enable();
625        let failure = |cause, error| ClientDeliveryFailure {
626            diagnosis: ClientDeliveryDiagnosis {
627                frame_type,
628                flags,
629                cause,
630            },
631            error,
632        };
633        let closed = || {
634            RouterError::backend_with_epoch(
635                frame.header.channel,
636                frame.header.epoch,
637                frame.header.corr,
638                "client connection closed",
639            )
640        };
641        if self
642            .accounting
643            .client_delivery_closed
644            .load(Ordering::Acquire)
645        {
646            return Err(failure("client_closed", closed()));
647        }
648        let (frame, error) = match self.try_send_owned(frame) {
649            Ok(()) => return Ok(()),
650            Err(refused) => refused,
651        };
652        if self.is_closed() {
653            return Err(failure("client_closed", error));
654        }
655        let grace = crate::server::CONNECTION_EGRESS_STALL_GRACE;
656        let age = self.backlog().oldest_age.unwrap_or_default();
657        if age > grace {
658            return Err(failure("oldest_frame_older_than_250_ms", error));
659        }
660        let result = tokio::select! {
661            biased;
662            _ = &mut stopped => return Err(failure("client_closed", error)),
663            result = tokio::time::timeout(grace.saturating_sub(age), self.send(frame)) => result,
664        };
665        match result {
666            Ok(Ok(())) => Ok(()),
667            Ok(Err(error)) => Err(failure("client_closed", error)),
668            Err(_) => Err(failure("waited_250_ms_without_room", error)),
669        }
670    }
671
672    /// Enqueue without waiting. Fails when the frame's bytes would take the
673    /// queue past its byte budget, when the channel's frame-count backstop is
674    /// full, or when the writer is gone.
675    pub(crate) fn try_send(&self, frame: Frame) -> Result<(), RouterError> {
676        self.try_send_owned(frame).map_err(|(_, error)| error)
677    }
678
679    fn try_send_owned(&self, frame: Frame) -> Result<(), (Frame, RouterError)> {
680        if self.egress_terminal() {
681            return Ok(());
682        }
683        let channel = frame.header.channel;
684        let epoch = frame.header.epoch;
685        let corr = frame.header.corr;
686        let unavailable = |why: String| {
687            RouterError::backend_with_epoch(
688                channel,
689                epoch,
690                corr,
691                format!("connection writer unavailable: {why}"),
692            )
693        };
694        let bytes = queued_frame_bytes(&frame);
695        let Some(charge) = self.accounting.try_charge(bytes) else {
696            if self.tx.is_closed() {
697                return Err((frame, unavailable("channel closed".to_string())));
698            }
699            return Err((
700                frame,
701                unavailable(format!(
702                    "egress byte budget exhausted ({} queued bytes, frame of {bytes} bytes, budget {})",
703                    self.accounting.queued_bytes.load(Ordering::Acquire),
704                    self.accounting.byte_budget
705                )),
706            ));
707        };
708        // Return the refused frame for a bounded retry; dropping the rest of
709        // the outbound wrapper returns its charge.
710        if self.egress_terminal() {
711            return Ok(());
712        }
713        self.tx
714            .try_send(OutboundFrame::charged(frame, charge))
715            .map_err(|err| {
716                let error = unavailable(err.to_string());
717                (err.into_inner().frame, error)
718            })
719    }
720}
721
722#[derive(Debug)]
723struct ClientDeliveryFailure {
724    diagnosis: ClientDeliveryDiagnosis,
725    error: RouterError,
726}
727
728/// Header facts only: payloads can contain secrets and must never enter logs.
729#[derive(Debug)]
730pub(crate) struct ClientDeliveryDiagnosis {
731    pub frame_type: FrameType,
732    pub flags: Flags,
733    pub cause: &'static str,
734}
735
736impl fmt::Display for ClientDeliveryFailure {
737    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
738        self.error.fmt(f)
739    }
740}
741
742/// Minimum time between two route GOODBYEs the daemon sends one module
743/// connection for one channel in answer to the module's frames on a (channel,
744/// epoch) the daemon holds no route for.
745///
746/// A module that missed a GOODBYE keeps sending on that route (a streaming
747/// module, many frames a second), and every one of those frames lands here. One
748/// answer is enough when it arrives, so answering each frame would only flood
749/// the module's egress queue at the moment it is already behind. Five seconds
750/// is long enough for an answer queued behind a deep backlog to reach the module
751/// and take effect before a second is sent, and short enough that an answer
752/// the queue refused is retried by the next orphan frame well within a minute.
753const ORPHAN_ROUTE_GOODBYE_INTERVAL: Duration = Duration::from_secs(5);
754
755/// Once one connection has this many remembered channels, entries older than
756/// [`ORPHAN_ROUTE_GOODBYE_INTERVAL`] are pruned before another is added. An
757/// expired entry has no effect on rate limiting, so pruning never changes a
758/// decision; it keeps a module that orphaned many channels long ago from
759/// pinning memory for all of them. The map is bounded in any case by the
760/// 16-bit channel space and is dropped whole when the connection ends.
761const ORPHAN_ROUTE_GOODBYE_PRUNE_AT: usize = 256;
762
763/// When each module connection was last sent an orphan-route GOODBYE, per
764/// channel. Shared by the [`Router`] (which consults it) and every
765/// [`RouterConnection`] (which removes its own entry when the connection ends).
766#[derive(Debug, Default)]
767struct OrphanGoodbyeLimiter {
768    last_sent: Mutex<HashMap<ConnectionId, HashMap<u16, tokio::time::Instant>>>,
769}
770
771impl OrphanGoodbyeLimiter {
772    /// True, and the send recorded, when `channel` on `connection_id` has not
773    /// been answered within the interval.
774    fn claim(&self, connection_id: ConnectionId, channel: u16) -> bool {
775        let now = tokio::time::Instant::now();
776        let mut last_sent = self
777            .last_sent
778            .lock()
779            .unwrap_or_else(std::sync::PoisonError::into_inner);
780        let channels = last_sent.entry(connection_id).or_default();
781        if channels.get(&channel).is_some_and(|sent| {
782            now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
783        }) {
784            return false;
785        }
786        if channels.len() >= ORPHAN_ROUTE_GOODBYE_PRUNE_AT {
787            channels.retain(|_, sent| {
788                now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
789            });
790        }
791        channels.insert(channel, now);
792        true
793    }
794
795    fn forget_connection(&self, connection_id: ConnectionId) {
796        self.last_sent
797            .lock()
798            .unwrap_or_else(std::sync::PoisonError::into_inner)
799            .remove(&connection_id);
800    }
801}
802
803/// Per-route context shared with backends besides the frame itself.
804#[derive(Debug, Clone)]
805pub struct RouteCtx {
806    pub connection_id: ConnectionId,
807    pub egress: FrameSink,
808}
809
810/// Closed set of data-plane backends for non-zero channels.
811///
812/// Channel 0 is structurally special and remains `Router::control`, not an
813/// enum variant. Static/test backends live in `Router::backends`; the forwarding
814/// backend is selected dynamically only when a per-connection forwarding binding exists.
815#[derive(Debug, Clone)]
816pub enum Backend {
817    Echo(EchoBackend),
818    Forward(ForwardBackend),
819}
820
821impl From<EchoBackend> for Backend {
822    fn from(backend: EchoBackend) -> Self {
823        Self::Echo(backend)
824    }
825}
826
827impl From<ForwardBackend> for Backend {
828    fn from(backend: ForwardBackend) -> Self {
829        Self::Forward(backend)
830    }
831}
832
833impl Backend {
834    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
835        match self {
836            Self::Echo(backend) => backend.handle(ctx, frame).await,
837            Self::Forward(backend) => backend.handle(ctx, frame).await,
838        }
839    }
840}
841
842/// I/O-agnostic splice router keyed by envelope `channel`.
843///
844/// Channel 0 is reserved for subc itself and is always dispatched to the
845/// dedicated control handler. Other channels must be explicitly registered.
846/// Unknown client-originated non-zero channels are translated to canonical JSON `ERROR` frames on
847/// the connection sink so the peer can continue using the same socket. Module-originated frames for
848/// released route channels are logged and dropped as the channel-gone race backstop.
849///
850/// DATA-PLANE BODIES ARE NEVER DECODED HERE, and the consequence is worth stating
851/// because it looks like a guarantee and is not. Additive fields in a request or
852/// response body reach the far side untouched — not because anything permits them,
853/// but because routing reads only the 21-byte header and treats the body as opaque
854/// bytes. That is a performance property, so **nothing prevents it from changing**:
855/// a future reason to inspect a body would convert a wire-transparent path into a
856/// filtering one, and nobody would think of it as a contract change.
857///
858/// So when a peer asks whether subc sees an additive field, the answer is per-path
859/// and this path's zero means WE NEVER LOOK rather than WE LOOK AT EVERYTHING. The
860/// control plane (typed enums at the frame boundary) and the MCP gateway (envelope
861/// unwrap plus a named struct) both narrow; only this one does not.
862pub struct Router {
863    backends: HashMap<u16, Backend>,
864    control: Arc<ControlHandler>,
865    forwarding: Arc<ForwardingTable>,
866    forward_backend: ForwardBackend,
867    counters: DaemonCounters,
868    next_connection_id: AtomicU64,
869    orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
870}
871
872impl Router {
873    pub fn with_control_handler(control: Arc<ControlHandler>) -> Self {
874        // The handler the router serves is the one a swap must tell when it
875        // promotes a candidate; this is where it first sits behind an `Arc`.
876        control.install_swap_promotion_observer();
877        let forwarding = control.forwarding();
878        let counters = control.counters();
879        Self {
880            backends: HashMap::new(),
881            control,
882            forwarding: Arc::clone(&forwarding),
883            forward_backend: ForwardBackend::new(forwarding),
884            counters,
885            // ConnectionId::LOCAL is 0; real socket ids start at 1 and never collide.
886            next_connection_id: AtomicU64::new(1),
887            orphan_goodbyes: Arc::default(),
888        }
889    }
890
891    /// Install only the inherited-channel service; the supervisor must not own
892    /// this router or retain it after the listeners have stopped serving.
893    pub(crate) fn install_registration_service(self: &Arc<Self>, auth: crate::server::ServerAuth) {
894        self.control
895            .install_registration_service(Arc::downgrade(self), auth);
896    }
897
898    pub fn with_default_self_handler() -> Self {
899        Self::with_control_handler(Arc::new(ControlHandler::default()))
900    }
901
902    pub fn forwarding(&self) -> Arc<ForwardingTable> {
903        Arc::clone(&self.forwarding)
904    }
905
906    pub fn register_backend(
907        &mut self,
908        channel: u16,
909        backend: impl Into<Backend>,
910    ) -> Result<(), RouterError> {
911        self.register_backend_arc(channel, Arc::new(backend.into()))
912    }
913
914    pub(crate) fn register_backend_arc(
915        &mut self,
916        channel: u16,
917        backend: Arc<Backend>,
918    ) -> Result<(), RouterError> {
919        if channel == 0 {
920            return Err(RouterError::ReservedChannelZero);
921        }
922        if self.backends.contains_key(&channel) {
923            return Err(RouterError::DuplicateChannel { channel });
924        }
925        self.backends.insert(channel, backend.as_ref().clone());
926        Ok(())
927    }
928
929    /// Start a connection-scoped routing context. Dropping the guard releases
930    /// any control-plane registrations owned by the connection.
931    fn record_module_frame_drop(&self, connection_id: ConnectionId) -> Result<(), RouterError> {
932        let module_id = self
933            .forwarding
934            .module_id_for_connection(connection_id)
935            .map_err(RouterError::Forwarding)?;
936        self.counters
937            .increment_module_frames_dropped_no_route(module_id.as_deref());
938        Ok(())
939    }
940
941    /// A module sent a non-request frame on a (channel, epoch) the daemon holds
942    /// no route for: most often a route the daemon released whose GOODBYE the
943    /// module never received, so the module still believes it is open and keeps
944    /// sending. Count the drop, and tell the module to let go of exactly that
945    /// (channel, epoch) with a route GOODBYE, at most once per
946    /// [`ORPHAN_ROUTE_GOODBYE_INTERVAL`] per connection and channel.
947    ///
948    /// The answer uses `try_send`: it is a best-effort nudge, and if the queue
949    /// refuses it, the module's next frame on that route after the interval
950    /// asks again. A GOODBYE from the module is not answered, since the module
951    /// is already letting go of the route.
952    fn handle_orphan_module_frame(&self, ctx: &RouteCtx, frame: &Frame) -> Result<(), RouterError> {
953        let channel = frame.header.channel;
954        let epoch = frame.header.epoch;
955        let module_id = self
956            .forwarding
957            .module_id_for_connection(ctx.connection_id)
958            .map_err(RouterError::Forwarding)?;
959        self.counters
960            .increment_module_frames_dropped_no_route(module_id.as_deref());
961        if self
962            .forwarding
963            .module_route_epoch_was_allocated(ctx.connection_id, channel, epoch)
964            .map_err(RouterError::Forwarding)?
965        {
966            self.counters
967                .increment_module_frames_dropped_released_route(module_id.as_deref());
968        }
969        if frame.header.ty == FrameType::Goodbye
970            || !self.orphan_goodbyes.claim(ctx.connection_id, channel)
971        {
972            return Ok(());
973        }
974        let goodbye = Frame::build_with_version(
975            frame.header.ver,
976            FrameType::Goodbye,
977            Flags::new(false, Priority::Passive, false),
978            channel,
979            epoch,
980            0,
981            Vec::new(),
982        )
983        .map_err(RouterError::FrameBuild)?;
984        match ctx.egress.try_send(goodbye) {
985            Ok(()) => {
986                self.counters.increment_module_orphan_route_goodbyes_sent();
987                debug!(
988                    connection_id = ctx.connection_id.get(),
989                    module_id = module_id.as_deref().unwrap_or("unknown"),
990                    channel,
991                    epoch,
992                    "answered module frame on a route the daemon does not hold with a route GOODBYE"
993                );
994            }
995            Err(err) => {
996                self.forwarding.warn_frame_drop(
997                    ctx.connection_id,
998                    channel,
999                    epoch,
1000                    "daemon_to_module",
1001                    if ctx.egress.is_closed() {
1002                        "endpoint_gone"
1003                    } else {
1004                        "egress_budget"
1005                    },
1006                );
1007                debug!(
1008                connection_id = ctx.connection_id.get(),
1009                module_id = module_id.as_deref().unwrap_or("unknown"),
1010                channel,
1011                epoch,
1012                error = %err,
1013                "could not enqueue route GOODBYE for module frame on a route the daemon does not hold; the next such frame retries"
1014                );
1015            }
1016        }
1017        Ok(())
1018    }
1019
1020    pub fn begin_connection(&self) -> RouterConnection {
1021        let raw = self.next_connection_id.fetch_add(1, Ordering::Relaxed);
1022        let id = ConnectionId::new(raw);
1023        let close_receiver = self.forwarding.register_connection_close(id);
1024        RouterConnection {
1025            id,
1026            control_handler: Arc::clone(&self.control),
1027            forwarding: Arc::clone(&self.forwarding),
1028            close_receiver: Some(close_receiver),
1029            orphan_goodbyes: Arc::clone(&self.orphan_goodbyes),
1030        }
1031    }
1032
1033    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1034        self.control.route_open_target(frame)
1035    }
1036
1037    pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
1038        self.control.record_local_connection(connection);
1039    }
1040
1041    pub(crate) fn record_captured_peer(
1042        &self,
1043        connection: ConnectionId,
1044        peer: subc_os::local_peer::CapturedPeer,
1045    ) {
1046        self.control.record_captured_peer(connection, peer);
1047    }
1048
1049    pub(crate) fn record_caller_endpoints(
1050        &self,
1051        connection: ConnectionId,
1052        endpoints: Option<(std::net::SocketAddr, std::net::SocketAddr)>,
1053    ) {
1054        self.control.record_caller_endpoints(connection, endpoints);
1055    }
1056
1057    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1058        self.control.defers_privileged_control(ctx, frame)
1059    }
1060
1061    pub(crate) fn defers_config_approval(&self, frame: &Frame) -> bool {
1062        self.control.defers_config_approval(frame)
1063    }
1064
1065    pub(crate) fn reserve_config_approval(
1066        &self,
1067        ctx: &RouteCtx,
1068        frame: &Frame,
1069    ) -> Result<
1070        Option<crate::config_gate::runtime::ApprovalRequestGuard>,
1071        crate::config_gate::GateError,
1072    > {
1073        self.control.reserve_config_approval(ctx, frame)
1074    }
1075
1076    pub(crate) fn route_open_capacity_refusal(
1077        &self,
1078        ctx: &RouteCtx,
1079        frame: &Frame,
1080        target_module_id: &str,
1081        in_flight: usize,
1082        limit: usize,
1083    ) -> Result<Frame, RouterError> {
1084        self.control
1085            .route_open_capacity_refusal(ctx, frame, target_module_id, in_flight, limit)
1086    }
1087
1088    pub async fn route_for_connection(
1089        &self,
1090        ctx: &RouteCtx,
1091        frame: Frame,
1092    ) -> Result<(), RouterError> {
1093        self.route_for_connection_started(ctx, frame, None).await
1094    }
1095
1096    pub(crate) async fn observe_connection_wait<F: std::future::Future>(
1097        &self,
1098        ctx: &RouteCtx,
1099        kind: &'static str,
1100        future: F,
1101    ) -> F::Output {
1102        self.counters
1103            .observe_reader_wait(
1104                || {
1105                    let module = self
1106                        .forwarding
1107                        .module_id_for_connection(ctx.connection_id)
1108                        .ok()
1109                        .flatten();
1110                    let principal = if module.is_some() { "module" } else { "direct" };
1111                    (
1112                        module.unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
1113                        principal.into(),
1114                    )
1115                },
1116                kind,
1117                future,
1118            )
1119            .await
1120    }
1121
1122    pub(crate) async fn route_for_connection_started(
1123        &self,
1124        ctx: &RouteCtx,
1125        frame: Frame,
1126        dispatch_started_at: Option<Instant>,
1127    ) -> Result<(), RouterError> {
1128        // Embedded callers have no socket reader; mark at their ingress instead.
1129        let first = ctx.egress.first_ingress_frame();
1130        self.route_for_connection_ingress(ctx, frame, dispatch_started_at, first)
1131            .await
1132    }
1133
1134    pub(crate) async fn route_for_connection_ingress(
1135        &self,
1136        ctx: &RouteCtx,
1137        frame: Frame,
1138        dispatch_started_at: Option<Instant>,
1139        first: bool,
1140    ) -> Result<(), RouterError> {
1141        if ctx.egress.ingress_terminal() {
1142            return Ok(());
1143        }
1144        let channel = frame.header.channel;
1145        let epoch = frame.header.epoch;
1146        let corr = frame.header.corr;
1147        if channel == 0 {
1148            debug!(
1149                connection_id = ctx.connection_id.get(),
1150                corr,
1151                frame_type = ?frame.header.ty,
1152                "routing control frame"
1153            );
1154            // The connection loop dispatches every control operation directly
1155            // except `route.open`, activity writes, held config approvals and
1156            // privileged requests. Each
1157            // task passes a timestamp captured before
1158            // spawn, so slow-dispatch timing includes scheduler delay but no
1159            // wait in an application-owned queue.
1160            let dispatch_started_at = (frame.header.ty == FrameType::Request)
1161                .then(|| dispatch_started_at.unwrap_or_else(Instant::now));
1162            // Only review-producing requests need the post-flush receipt hook.
1163            // Probing every response would scan large catalog bodies again.
1164            let review_request = self.control.config_desktop_review_request(&frame);
1165            let responses = self
1166                .control
1167                .handle_control_frame_ingress(ctx, frame, dispatch_started_at, first)
1168                .await?;
1169            for response in responses {
1170                if let Some(view) = review_request
1171                    .then(|| {
1172                        self.control
1173                            .config_desktop_review_response(ctx.connection_id, &response)
1174                    })
1175                    .flatten()
1176                {
1177                    self.observe_connection_wait(
1178                        ctx,
1179                        "reply_egress",
1180                        self.control
1181                            .deliver_config_desktop_review(ctx, response, view),
1182                    )
1183                    .await?;
1184                    continue;
1185                }
1186                self.observe_connection_wait(ctx, "reply_egress", ctx.egress.send(response))
1187                    .await?;
1188            }
1189            return Ok(());
1190        }
1191
1192        let data_route = self
1193            .forwarding
1194            .lookup_data_route(ctx.connection_id, channel, epoch)
1195            .map_err(RouterError::Forwarding)?;
1196
1197        match data_route {
1198            DataRoute::Module(DataRouteState::EpochMismatch) => {
1199                self.forwarding.warn_frame_drop(
1200                    ctx.connection_id,
1201                    channel,
1202                    epoch,
1203                    "module_to_client",
1204                    "stale_epoch",
1205                );
1206                if frame.header.ty == FrameType::Request {
1207                    self.counters
1208                        .increment_module_requests_dropped_stale_route();
1209                    let err = RouterError::StaleRouteEpoch {
1210                        channel,
1211                        epoch,
1212                        corr,
1213                    };
1214                    if let Some(error_frame) = err.to_error_frame() {
1215                        self.observe_connection_wait(
1216                            ctx,
1217                            "reply_egress",
1218                            ctx.egress.send(error_frame),
1219                        )
1220                        .await?;
1221                    }
1222                } else {
1223                    self.handle_orphan_module_frame(ctx, &frame)?;
1224                }
1225                debug!(
1226                    connection_id = ctx.connection_id.get(),
1227                    channel, epoch, corr, "dropping module frame for stale route epoch"
1228                );
1229                return Ok(());
1230            }
1231            DataRoute::Module(DataRouteState::Reserved) => {
1232                self.forwarding.warn_frame_drop(
1233                    ctx.connection_id,
1234                    channel,
1235                    epoch,
1236                    "module_to_client",
1237                    "missing_binding",
1238                );
1239                if frame.header.ty == FrameType::Request {
1240                    self.counters
1241                        .increment_module_requests_dropped_stale_route();
1242                    let err = RouterError::UnknownChannel {
1243                        channel,
1244                        epoch,
1245                        corr,
1246                    };
1247                    if let Some(error_frame) = err.to_error_frame() {
1248                        self.observe_connection_wait(
1249                            ctx,
1250                            "reply_egress",
1251                            ctx.egress.send(error_frame),
1252                        )
1253                        .await?;
1254                    }
1255                } else {
1256                    self.record_module_frame_drop(ctx.connection_id)?;
1257                }
1258                debug!(
1259                    connection_id = ctx.connection_id.get(),
1260                    channel, epoch, corr, "dropping module frame for reserved route handle"
1261                );
1262                return Ok(());
1263            }
1264            DataRoute::Module(DataRouteState::Absent) => {
1265                self.forwarding.warn_frame_drop(
1266                    ctx.connection_id,
1267                    channel,
1268                    epoch,
1269                    "module_to_client",
1270                    "unknown_channel",
1271                );
1272                if frame.header.ty == FrameType::Request {
1273                    self.counters
1274                        .increment_module_requests_dropped_stale_route();
1275                    let err = RouterError::UnknownChannel {
1276                        channel,
1277                        epoch,
1278                        corr,
1279                    };
1280                    if let Some(error_frame) = err.to_error_frame() {
1281                        self.observe_connection_wait(
1282                            ctx,
1283                            "reply_egress",
1284                            ctx.egress.send(error_frame),
1285                        )
1286                        .await?;
1287                    }
1288                } else {
1289                    self.handle_orphan_module_frame(ctx, &frame)?;
1290                }
1291                debug!(
1292                    connection_id = ctx.connection_id.get(),
1293                    channel, epoch, corr, "dropping module frame for absent route handle"
1294                );
1295                return Ok(());
1296            }
1297            DataRoute::Module(DataRouteState::Bound(route)) => {
1298                if frame.header.ty == FrameType::Goodbye {
1299                    if let RouteRelease::Removed(target) = self
1300                        .forwarding
1301                        .release_module_route(ctx.connection_id, channel, epoch)
1302                        .map_err(RouterError::Forwarding)?
1303                    {
1304                        let mut goodbye = frame;
1305                        goodbye.header.channel = target.channel;
1306                        goodbye.header.epoch = target.epoch;
1307                        // A module must never be able to make a frame look
1308                        // daemon-authored, so the origin bit is cleared on
1309                        // every module frame relayed to a client.
1310                        goodbye.header.flags = goodbye.header.flags.without_daemon_origin();
1311                        if let Err(err) = self
1312                            .observe_connection_wait(
1313                                ctx,
1314                                "module_to_client",
1315                                target.sink.send_to_client(goodbye),
1316                            )
1317                            .await
1318                        {
1319                            if !target.close_on_delivery_failure() {
1320                                self.forwarding.warn_frame_drop(
1321                                    target.connection_id,
1322                                    target.channel,
1323                                    target.epoch,
1324                                    "module_to_client",
1325                                    err.diagnosis.cause,
1326                                );
1327                            }
1328                            if target.close_on_delivery_failure()
1329                                && self
1330                                    .forwarding
1331                                    .escalate_client_delivery_failure_diagnosed(
1332                                        target.connection_id,
1333                                        target.channel,
1334                                        target.epoch,
1335                                        CloseReason::new(
1336                                            "route_goodbye_delivery_failed",
1337                                            format!(
1338                                                "failed to enqueue route GOODBYE for client channel {}: {err}",
1339                                                target.channel
1340                                            ),
1341                                        ),
1342                                        UndeliveredFrame {
1343                                            module_id: Some(&route.module_id),
1344                                            sink: &target.sink,
1345                                        },
1346                                        Some(&err.diagnosis),
1347                                    )
1348                                    .map_err(RouterError::Forwarding)?
1349                            {
1350                                self.counters.increment_goodbye_relay_client_failed();
1351                            }
1352                        }
1353                    } else {
1354                        self.forwarding.warn_frame_drop(
1355                            ctx.connection_id,
1356                            channel,
1357                            epoch,
1358                            "module_to_client",
1359                            "missing_binding",
1360                        );
1361                    }
1362                    return Ok(());
1363                }
1364
1365                // Accounting allows an oversized frame to occupy an empty queue
1366                // on other send paths. Refuse it at this splice before admission,
1367                // so the next ordinary reply cannot close a multiplexed client.
1368                let bytes = queued_frame_bytes(&frame);
1369                let limit = route.client_sink.accounting.byte_budget;
1370                if bytes > limit {
1371                    let ty = frame.header.ty;
1372                    let flags = frame.header.flags;
1373                    drop(frame);
1374                    return self
1375                        .refuse_oversized_module_frame(ctx, &route, (ty, flags), corr, bytes)
1376                        .await;
1377                }
1378
1379                // A terminal frame ends the request at the module whether or not
1380                // the client can still take it, so its credit is released before
1381                // delivery is attempted. Releasing only after a successful
1382                // enqueue would leave a drain counting a finished request until
1383                // the client connection's cleanup removes the route.
1384                let releases_credit = is_terminal_frame(frame.header.ty);
1385                if releases_credit {
1386                    route.flow.release_corr(corr);
1387                }
1388                let mut frame = frame;
1389                frame.header.channel = route.client_channel;
1390                frame.header.epoch = route.client_epoch;
1391                // Responses, errors, stream data and ends, and reverse
1392                // requests all pass through here. A module must never be able
1393                // to make a frame look daemon-authored, so the origin bit a
1394                // module may have set is cleared; every other flag bit and
1395                // the body travel unchanged.
1396                frame.header.flags = frame.header.flags.without_daemon_origin();
1397                if let Err(err) = self
1398                    .counters
1399                    .observe_reader_wait(
1400                        || (route.module_id.clone(), principal_label(&route.principal)),
1401                        "module_to_client",
1402                        route.client_sink.send_to_client(frame),
1403                    )
1404                    .await
1405                {
1406                    if self
1407                        .forwarding
1408                        .escalate_client_delivery_failure_diagnosed(
1409                            route.client_connection_id,
1410                            route.client_channel,
1411                            route.client_epoch,
1412                            CloseReason::new(
1413                                "module_to_client_delivery_failed",
1414                                format!(
1415                                    "failed to enqueue module frame for client channel {} corr {corr}: {err}",
1416                                    route.client_channel
1417                                ),
1418                            ),
1419                            UndeliveredFrame {
1420                                module_id: Some(&route.module_id),
1421                                sink: &route.client_sink,
1422                            },
1423                            Some(&err.diagnosis),
1424                        )
1425                        .map_err(RouterError::Forwarding)?
1426                    {
1427                        self.counters
1428                            .increment_client_egress_close_delivery_failed();
1429                    }
1430                    return Ok(());
1431                }
1432                return Ok(());
1433            }
1434            DataRoute::Client(DataRouteState::EpochMismatch) => {
1435                self.forwarding.warn_frame_drop(
1436                    ctx.connection_id,
1437                    channel,
1438                    epoch,
1439                    "client_to_module",
1440                    "stale_epoch",
1441                );
1442                if frame.header.ty == FrameType::Request {
1443                    self.counters.increment_client_frames_dropped_stale_route();
1444                    // Dropped before forwarding; a re-bind retry cannot double-execute this request.
1445                    let err = RouterError::StaleRouteEpoch {
1446                        channel,
1447                        epoch,
1448                        corr,
1449                    };
1450                    if let Some(error_frame) = err.to_error_frame() {
1451                        self.observe_connection_wait(
1452                            ctx,
1453                            "reply_egress",
1454                            ctx.egress.send(error_frame),
1455                        )
1456                        .await?;
1457                    }
1458                }
1459                debug!(
1460                    connection_id = ctx.connection_id.get(),
1461                    channel, epoch, corr, "dropping client frame for stale route epoch"
1462                );
1463                return Ok(());
1464            }
1465            DataRoute::Client(DataRouteState::Reserved) => {
1466                self.forwarding.warn_frame_drop(
1467                    ctx.connection_id,
1468                    channel,
1469                    epoch,
1470                    "client_to_module",
1471                    "missing_binding",
1472                );
1473                if frame.header.ty == FrameType::Request {
1474                    let err = RouterError::UnknownChannel {
1475                        channel,
1476                        epoch,
1477                        corr,
1478                    };
1479                    if let Some(error_frame) = err.to_error_frame() {
1480                        self.observe_connection_wait(
1481                            ctx,
1482                            "reply_egress",
1483                            ctx.egress.send(error_frame),
1484                        )
1485                        .await?;
1486                    }
1487                }
1488                return Ok(());
1489            }
1490            DataRoute::Client(DataRouteState::Bound(route)) => {
1491                if frame.header.ty == FrameType::Goodbye {
1492                    let _ = self
1493                        .control
1494                        .handle_route_goodbye(ctx.connection_id, channel, epoch)?;
1495                    return Ok(());
1496                }
1497                return self.forward_backend.handle_bound(frame, route).await;
1498            }
1499            DataRoute::Client(DataRouteState::Absent) => {}
1500        }
1501
1502        if let Some(backend) = self.backends.get(&channel) {
1503            return backend.handle(ctx.clone(), frame).await.inspect_err(|_| {
1504                self.forwarding.warn_frame_drop(
1505                    ctx.connection_id,
1506                    channel,
1507                    epoch,
1508                    "client_to_backend",
1509                    "endpoint_gone",
1510                );
1511            });
1512        }
1513        self.forwarding.warn_frame_drop(
1514            ctx.connection_id,
1515            channel,
1516            epoch,
1517            "client_to_module",
1518            "unknown_channel",
1519        );
1520        if frame.header.ty == FrameType::Request {
1521            let err = RouterError::UnknownChannel {
1522                channel,
1523                epoch,
1524                corr,
1525            };
1526            if let Some(error_frame) = err.to_error_frame() {
1527                self.observe_connection_wait(ctx, "reply_egress", ctx.egress.send(error_frame))
1528                    .await?;
1529            }
1530        }
1531        Ok(())
1532    }
1533
1534    async fn refuse_oversized_module_frame(
1535        &self,
1536        ctx: &RouteCtx,
1537        route: &RouteBinding,
1538        header: (FrameType, Flags),
1539        corr: u64,
1540        bytes: usize,
1541    ) -> Result<(), RouterError> {
1542        let (ty, flags) = header;
1543        let limit = route.client_sink.accounting.byte_budget;
1544        if ty == FrameType::Push {
1545            // Deployed Rust clients cannot decode a new route.closed reason.
1546            // Until that vocabulary can be extended compatibly, an oversized
1547            // unsolicited push still closes the connection, with its cause named.
1548            let diagnosis = ClientDeliveryDiagnosis {
1549                frame_type: ty,
1550                flags,
1551                cause: "frame_too_large",
1552            };
1553            if self
1554                .forwarding
1555                .escalate_client_delivery_failure_diagnosed(
1556                    route.client_connection_id,
1557                    route.client_channel,
1558                    route.client_epoch,
1559                    CloseReason::new(
1560                        error_codes::FRAME_TOO_LARGE,
1561                        format!(
1562                            "oversized push: size={bytes} limit={limit} (header plus body bytes)"
1563                        ),
1564                    ),
1565                    UndeliveredFrame {
1566                        module_id: Some(&route.module_id),
1567                        sink: &route.client_sink,
1568                    },
1569                    Some(&diagnosis),
1570                )
1571                .map_err(RouterError::Forwarding)?
1572            {
1573                self.counters
1574                    .increment_client_egress_close_delivery_failed();
1575            }
1576            return Ok(());
1577        }
1578
1579        warn!(
1580            connection_id = route.client_connection_id.get(),
1581            module_id = %route.module_id,
1582            client_principal = %principal_label(&route.principal),
1583            client_channel = route.client_channel,
1584            frame_type = ?ty,
1585            frame_bytes = bytes,
1586            byte_limit = limit,
1587            "refusing oversized module-to-client frame with ERROR frame_too_large"
1588        );
1589        if ty == FrameType::Request {
1590            // A reverse request belongs to the module, not to a waiting client.
1591            // Answer its sender without forwarding any part of it to the client.
1592            let error = frame_too_large_error(
1593                route.module_channel,
1594                route.module_epoch,
1595                corr,
1596                bytes,
1597                limit,
1598            )?;
1599            self.observe_connection_wait(ctx, "reply_egress", ctx.egress.send(error))
1600                .await?;
1601        } else {
1602            // A refused stream chunk also ends the waiting request. Releasing
1603            // its credit prevents a finished request from keeping a drain alive.
1604            route.flow.release_corr(corr);
1605            let error = frame_too_large_error(
1606                route.client_channel,
1607                route.client_epoch,
1608                corr,
1609                bytes,
1610                limit,
1611            )?;
1612            // The replacement shares the reply's bounded burst grace, so a
1613            // stopped client cannot indefinitely stall the module frame reader.
1614            if let Err(err) = self
1615                .counters
1616                .observe_reader_wait(
1617                    || (route.module_id.clone(), principal_label(&route.principal)),
1618                    "module_to_client",
1619                    route.client_sink.send_to_client(error),
1620                )
1621                .await
1622            {
1623                if self
1624                    .forwarding
1625                    .escalate_client_delivery_failure_diagnosed(
1626                        route.client_connection_id,
1627                        route.client_channel,
1628                        route.client_epoch,
1629                        CloseReason::new(
1630                            "module_to_client_delivery_failed",
1631                            format!(
1632                                "failed to enqueue frame_too_large for client channel {} corr {corr}: {err}",
1633                                route.client_channel
1634                            ),
1635                        ),
1636                        UndeliveredFrame {
1637                            module_id: Some(&route.module_id),
1638                            sink: &route.client_sink,
1639                        },
1640                        Some(&err.diagnosis),
1641                    )
1642                    .map_err(RouterError::Forwarding)?
1643                {
1644                    self.counters
1645                        .increment_client_egress_close_delivery_failed();
1646                }
1647            }
1648        }
1649        Ok(())
1650    }
1651}
1652
1653impl Default for Router {
1654    fn default() -> Self {
1655        Self::with_default_self_handler()
1656    }
1657}
1658
1659/// Connection-scoped cleanup guard returned by [`Router::begin_connection`].
1660#[must_use]
1661pub struct RouterConnection {
1662    id: ConnectionId,
1663    control_handler: Arc<ControlHandler>,
1664    forwarding: Arc<ForwardingTable>,
1665    close_receiver: Option<ConnectionCloseReceiver>,
1666    orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
1667}
1668
1669impl RouterConnection {
1670    pub fn id(&self) -> ConnectionId {
1671        self.id
1672    }
1673
1674    pub(crate) fn take_close_receiver(&mut self) -> ConnectionCloseReceiver {
1675        self.close_receiver
1676            .take()
1677            .expect("connection close receiver can only be taken once")
1678    }
1679}
1680
1681impl Drop for RouterConnection {
1682    fn drop(&mut self) {
1683        self.orphan_goodbyes.forget_connection(self.id);
1684        // GOODBYE (explicit) and connection-drop cleanup both call the same
1685        // idempotent deregistration path.
1686        let _ = self.control_handler.cleanup_connection(self.id);
1687        // The read loop and its tasks have ended, but the writer can still
1688        // drain. Keep the connection-to-launch association until pending relays
1689        // and routes are gone.
1690        self.forwarding.unregister_connection_close(self.id);
1691    }
1692}
1693
1694/// Minimal in-memory backend used by tests and early wiring: it emits a
1695/// `RESPONSE` on the same channel/correlation id with the exact same body bytes.
1696#[derive(Debug, Default, Clone, Copy)]
1697pub struct EchoBackend;
1698
1699impl EchoBackend {
1700    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1701        let response = Frame::build_with_version(
1702            frame.header.ver,
1703            FrameType::Response,
1704            frame.header.flags,
1705            frame.header.channel,
1706            frame.header.epoch,
1707            frame.header.corr,
1708            frame.body,
1709        )
1710        .map_err(RouterError::FrameBuild)?;
1711        ctx.egress.send(response).await
1712    }
1713}
1714
1715/// Data-plane backend that splices client frames to the module connection bound at attach time.
1716#[derive(Debug, Clone)]
1717pub struct ForwardBackend {
1718    forwarding: Arc<ForwardingTable>,
1719}
1720
1721impl ForwardBackend {
1722    pub fn new(forwarding: Arc<ForwardingTable>) -> Self {
1723        Self { forwarding }
1724    }
1725
1726    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1727        let channel = frame.header.channel;
1728        let corr = frame.header.corr;
1729        let route = match self
1730            .forwarding
1731            .lookup_data_route(ctx.connection_id, channel, frame.header.epoch)
1732            .map_err(RouterError::Forwarding)?
1733        {
1734            DataRoute::Client(DataRouteState::Bound(route)) => route,
1735            DataRoute::Client(_) | DataRoute::Module(_) => {
1736                self.forwarding.warn_frame_drop(
1737                    ctx.connection_id,
1738                    channel,
1739                    frame.header.epoch,
1740                    "client_to_module",
1741                    "missing_binding",
1742                );
1743                return Err(RouterError::UnknownChannel {
1744                    channel,
1745                    epoch: frame.header.epoch,
1746                    corr,
1747                });
1748            }
1749        };
1750        self.handle_bound(frame, route).await
1751    }
1752
1753    pub(crate) async fn handle_bound(
1754        &self,
1755        frame: Frame,
1756        route: Arc<RouteBinding>,
1757    ) -> Result<(), RouterError> {
1758        let channel = frame.header.channel;
1759        let corr = frame.header.corr;
1760        let frame_type = frame.header.ty;
1761
1762        // CANCEL and other non-REQUEST frames bypass the request-credit window;
1763        // the original request's credit is freed only by the module's terminal frame.
1764        let acquired_credit = frame_type == FrameType::Request;
1765        if acquired_credit && route.flow.busy_refusals() {
1766            use crate::forwarding::RequestAdmission;
1767            match self
1768                .forwarding
1769                .try_admit_request(
1770                    &route,
1771                    frame.header.epoch,
1772                    corr,
1773                    frame.header.flags.is_subscription(),
1774                )
1775                .map_err(RouterError::Forwarding)?
1776            {
1777                RequestAdmission::Admitted => {}
1778                RequestAdmission::Busy(detail) => {
1779                    let principal = principal_label(&route.principal);
1780                    let counters = self.forwarding.counters_ref();
1781                    counters.increment_route_busy(&route.module_id, &principal);
1782                    if route.flow.should_warn_busy() {
1783                        warn!(module_id = %route.module_id, client_principal = %principal, client_channel = channel,
1784                            route_epoch = route.client_epoch, ?detail, "request refused: route_busy");
1785                    }
1786                    let error = RouterError::RouteBusy {
1787                        channel,
1788                        epoch: frame.header.epoch,
1789                        corr,
1790                        detail: Box::new(detail),
1791                    };
1792                    let refusal = error
1793                        .to_error_frame()
1794                        .expect("route_busy is a serializable wire error");
1795                    counters
1796                        .observe_reader_wait(
1797                            || (route.module_id.clone(), principal),
1798                            "route_busy_egress",
1799                            route.client_sink.send(refusal),
1800                        )
1801                        .await?;
1802                    return Ok(());
1803                }
1804                RequestAdmission::ModuleReloading => {
1805                    return Err(RouterError::route_error_with_epoch(
1806                        channel,
1807                        frame.header.epoch,
1808                        corr,
1809                        error_codes::MODULE_RELOADING,
1810                        format!("module endpoint for route channel {channel} is reloading"),
1811                    ));
1812                }
1813                RequestAdmission::Stale => {
1814                    return Err(RouterError::StaleRouteEpoch {
1815                        channel,
1816                        epoch: frame.header.epoch,
1817                        corr,
1818                    })
1819                }
1820            }
1821        } else if acquired_credit {
1822            let counters = self.forwarding.counters_ref();
1823            if let Err(err) = counters
1824                .observe_reader_wait(
1825                    || (route.module_id.clone(), principal_label(&route.principal)),
1826                    "legacy_credit",
1827                    route
1828                        .flow
1829                        .acquire_tagged(corr, frame.header.flags.is_subscription()),
1830                )
1831                .await
1832            {
1833                self.forwarding.warn_frame_drop(
1834                    route.client_connection_id,
1835                    channel,
1836                    frame.header.epoch,
1837                    "client_to_module",
1838                    "endpoint_gone",
1839                );
1840                // `module_reloading` here is answered BEFORE the frame is
1841                // forwarded, so the module never sees the request and callers
1842                // may re-dispatch it after reopening the route. That is a wire
1843                // guarantee documented on `error_codes::MODULE_RELOADING`; do
1844                // not emit this code for a request that already reached
1845                // `module_sink.send` below.
1846                if self
1847                    .forwarding
1848                    .endpoint_is_draining(route.module_endpoint)
1849                    .map_err(RouterError::Forwarding)?
1850                {
1851                    return Err(RouterError::route_error_with_epoch(
1852                        channel,
1853                        frame.header.epoch,
1854                        corr,
1855                        "module_reloading",
1856                        format!("module endpoint for route channel {channel} is reloading"),
1857                    ));
1858                }
1859                return Err(RouterError::backend_with_epoch(
1860                    channel,
1861                    frame.header.epoch,
1862                    corr,
1863                    format!("{err} for route channel {channel}"),
1864                ));
1865            }
1866        }
1867
1868        let mut frame = frame;
1869        frame.header.channel = route.module_channel;
1870        frame.header.epoch = route.module_epoch;
1871        // The origin bit is the daemon's own signal to clients. A client
1872        // frame carrying it must not reach a module appearing daemon-authored,
1873        // so it is cleared here; every other flag bit travels unchanged.
1874        frame.header.flags = frame.header.flags.without_daemon_origin();
1875        let counters = self.forwarding.counters_ref();
1876        let result = counters
1877            .observe_reader_wait(
1878                || (route.module_id.clone(), principal_label(&route.principal)),
1879                "module_queue",
1880                route.module_sink.send(frame),
1881            )
1882            .await
1883            .map_err(|err| {
1884                self.forwarding.warn_frame_drop(
1885                    route.client_connection_id,
1886                    channel,
1887                    route.client_epoch,
1888                    "client_to_module",
1889                    "endpoint_gone",
1890                );
1891                RouterError::backend_with_epoch(channel, route.client_epoch, corr, err.to_string())
1892            });
1893        if acquired_credit && result.is_err() {
1894            route.flow.release_corr(corr);
1895        }
1896        result
1897    }
1898}
1899
1900fn is_terminal_frame(frame_type: FrameType) -> bool {
1901    matches!(
1902        frame_type,
1903        FrameType::Response | FrameType::Error | FrameType::StreamEnd
1904    )
1905}
1906
1907/// Typed router errors. Routable failures can be translated to canonical JSON
1908/// `ERROR` frames with [`RouterError::to_error_frame`].
1909#[derive(Debug, Clone, PartialEq, Eq)]
1910pub enum RouterError {
1911    ReservedChannelZero,
1912    DuplicateChannel {
1913        channel: u16,
1914    },
1915    UnknownChannel {
1916        channel: u16,
1917        epoch: u32,
1918        corr: u64,
1919    },
1920    StaleRouteEpoch {
1921        channel: u16,
1922        epoch: u32,
1923        corr: u64,
1924    },
1925    RouteBusy {
1926        channel: u16,
1927        epoch: u32,
1928        corr: u64,
1929        detail: Box<subc_protocol::RouteBusyDetail>,
1930    },
1931    Backend {
1932        channel: u16,
1933        epoch: u32,
1934        corr: u64,
1935        message: String,
1936    },
1937    RouteError {
1938        channel: u16,
1939        epoch: u32,
1940        corr: u64,
1941        code: String,
1942        message: String,
1943    },
1944    FrameBuild(FrameBuildError),
1945    Forwarding(ForwardingError),
1946}
1947
1948impl RouterError {
1949    pub fn backend(channel: u16, corr: u64, message: impl Into<String>) -> Self {
1950        Self::backend_with_epoch(channel, 0, corr, message)
1951    }
1952
1953    pub fn backend_with_epoch(
1954        channel: u16,
1955        epoch: u32,
1956        corr: u64,
1957        message: impl Into<String>,
1958    ) -> Self {
1959        Self::Backend {
1960            channel,
1961            epoch,
1962            corr,
1963            message: message.into(),
1964        }
1965    }
1966
1967    pub fn route_error(
1968        channel: u16,
1969        corr: u64,
1970        code: impl Into<String>,
1971        message: impl Into<String>,
1972    ) -> Self {
1973        Self::route_error_with_epoch(channel, 0, corr, code, message)
1974    }
1975
1976    pub fn route_error_with_epoch(
1977        channel: u16,
1978        epoch: u32,
1979        corr: u64,
1980        code: impl Into<String>,
1981        message: impl Into<String>,
1982    ) -> Self {
1983        Self::RouteError {
1984            channel,
1985            epoch,
1986            corr,
1987            code: code.into(),
1988            message: message.into(),
1989        }
1990    }
1991
1992    /// Translate route failures that belong on the wire into an `ERROR` frame.
1993    pub fn to_error_frame(&self) -> Option<Frame> {
1994        match self {
1995            Self::RouteBusy {
1996                channel,
1997                epoch,
1998                corr,
1999                detail,
2000            } => {
2001                let body = serde_json::to_vec(&ErrorBody {
2002                    code: error_codes::ROUTE_BUSY.into(),
2003                    message: "route has no free request or watch credit; retry on the same route"
2004                        .into(),
2005                    detail: Some(serde_json::to_value(detail).ok()?),
2006                })
2007                .ok()?;
2008                Frame::build(
2009                    FrameType::Error,
2010                    daemon_error_flags(),
2011                    *channel,
2012                    *epoch,
2013                    *corr,
2014                    body,
2015                )
2016                .ok()
2017            }
2018            Self::UnknownChannel {
2019                channel,
2020                epoch,
2021                corr,
2022            } => error_frame(
2023                *channel,
2024                *epoch,
2025                *corr,
2026                error_codes::UNKNOWN_CHANNEL,
2027                format!("unknown channel {channel}"),
2028            ),
2029            Self::StaleRouteEpoch {
2030                channel,
2031                epoch,
2032                corr,
2033            } => error_frame(
2034                *channel,
2035                *epoch,
2036                *corr,
2037                error_codes::STALE_ROUTE_EPOCH,
2038                format!("stale route epoch for channel {channel}"),
2039            ),
2040            Self::Backend {
2041                channel,
2042                epoch,
2043                corr,
2044                message,
2045            } => error_frame(*channel, *epoch, *corr, "backend_error", message.clone()),
2046            Self::RouteError {
2047                channel,
2048                epoch,
2049                corr,
2050                code,
2051                message,
2052            } => error_frame(*channel, *epoch, *corr, code, message.clone()),
2053            Self::ReservedChannelZero
2054            | Self::DuplicateChannel { .. }
2055            | Self::FrameBuild(_)
2056            | Self::Forwarding(_) => None,
2057        }
2058    }
2059}
2060
2061/// Flags for an ERROR frame the daemon authors itself.
2062///
2063/// Bit 6 (`FLAG_DAEMON_ORIGIN`) tells the client the daemon wrote this
2064/// error, never a module. What a code then proves is per code: on a bound
2065/// route the daemon writes `module_reloading`, `unknown_channel` and
2066/// `stale_route_epoch` only before forwarding, so with the bit they mean the
2067/// request was not delivered, while `frame_too_large` replaces a reply the
2068/// module already produced. Every daemon-authored ERROR goes through here;
2069/// frames a module sent keep their own flags with the bit cleared at the
2070/// splice.
2071/// Nothing else about the flags differs: passive priority, not binary, not last.
2072pub(crate) fn daemon_error_flags() -> Flags {
2073    Flags::new(false, Priority::Passive, false).with_daemon_origin()
2074}
2075
2076fn error_frame(channel: u16, epoch: u32, corr: u64, code: &str, message: String) -> Option<Frame> {
2077    let body = serde_json::to_vec(&ErrorBody {
2078        code: code.to_string(),
2079        message,
2080        detail: None,
2081    })
2082    .ok()?;
2083
2084    Frame::build(
2085        FrameType::Error,
2086        daemon_error_flags(),
2087        channel,
2088        epoch,
2089        corr,
2090        body,
2091    )
2092    .ok()
2093}
2094
2095fn frame_too_large_error(
2096    channel: u16,
2097    epoch: u32,
2098    corr: u64,
2099    bytes: usize,
2100    limit: usize,
2101) -> Result<Frame, RouterError> {
2102    let body = serde_json::to_vec(&ErrorBody {
2103        code: error_codes::FRAME_TOO_LARGE.to_string(),
2104        message: format!(
2105            "module frame of {bytes} bytes exceeds client egress limit of {limit} bytes"
2106        ),
2107        detail: Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit })),
2108    })
2109    .expect("serializing a frame size error");
2110    Frame::build(
2111        FrameType::Error,
2112        daemon_error_flags(),
2113        channel,
2114        epoch,
2115        corr,
2116        body,
2117    )
2118    .map_err(RouterError::FrameBuild)
2119}
2120
2121impl fmt::Display for RouterError {
2122    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2123        match self {
2124            Self::ReservedChannelZero => write!(f, "channel 0 is reserved for subc"),
2125            Self::DuplicateChannel { channel } => {
2126                write!(f, "backend already registered for channel {channel}")
2127            }
2128            Self::UnknownChannel { channel, corr, .. } => {
2129                write!(f, "unknown channel {channel} for corr {corr}")
2130            }
2131            Self::StaleRouteEpoch { channel, corr, .. } => {
2132                write!(f, "stale route epoch for channel {channel} corr {corr}")
2133            }
2134            Self::RouteBusy { channel, corr, .. } => {
2135                write!(f, "route busy on channel {channel} corr {corr}")
2136            }
2137            Self::Backend {
2138                channel,
2139                corr,
2140                message,
2141                ..
2142            } => write!(
2143                f,
2144                "backend error on channel {channel} corr {corr}: {message}"
2145            ),
2146            Self::RouteError {
2147                channel,
2148                corr,
2149                code,
2150                message,
2151                ..
2152            } => write!(
2153                f,
2154                "route error {code} on channel {channel} corr {corr}: {message}"
2155            ),
2156            Self::FrameBuild(err) => write!(f, "failed to build routed frame: {err}"),
2157            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
2158        }
2159    }
2160}
2161
2162impl Error for RouterError {
2163    fn source(&self) -> Option<&(dyn Error + 'static)> {
2164        match self {
2165            Self::FrameBuild(err) => Some(err),
2166            Self::Forwarding(err) => Some(err),
2167            Self::ReservedChannelZero
2168            | Self::DuplicateChannel { .. }
2169            | Self::UnknownChannel { .. }
2170            | Self::StaleRouteEpoch { .. }
2171            | Self::RouteBusy { .. }
2172            | Self::Backend { .. }
2173            | Self::RouteError { .. } => None,
2174        }
2175    }
2176}
2177
2178#[cfg(test)]
2179mod tests {
2180    include!("route_busy_router_tests.rs");
2181    use super::*;
2182    use crate::{
2183        forwarding::RouteBindRelayOutcome,
2184        supervise::{ModuleSpec, RestartPolicy, Supervisor, SupervisorHandle},
2185        ControlHandler, Registry,
2186    };
2187    use std::{
2188        sync::{mpsc as std_mpsc, Arc},
2189        time::Duration,
2190    };
2191    use subc_control::ModuleProtocol;
2192    use subc_protocol::{manifest::Concurrency, ErrorBody, Flags, FrameType, Priority};
2193    use tokio::sync::mpsc;
2194
2195    pub(crate) use crate::router::test_log::{captured_logs, log_capture};
2196
2197    fn logged_millis(logs: &str, field: &str) -> u64 {
2198        logs.split_whitespace()
2199            .find_map(|part| part.strip_prefix(field))
2200            .and_then(|value| value.parse().ok())
2201            .unwrap_or_else(|| panic!("missing numeric {field} in logs: {logs}"))
2202    }
2203
2204    fn request(channel: u16, corr: u64, body: &[u8]) -> Frame {
2205        Frame::build(
2206            FrameType::Request,
2207            Flags::new(true, Priority::Interactive, false),
2208            channel,
2209            0,
2210            corr,
2211            body.to_vec(),
2212        )
2213        .unwrap()
2214    }
2215
2216    fn ping(corr: u64) -> Frame {
2217        Frame::build(
2218            FrameType::Ping,
2219            Flags::new(false, Priority::Passive, false),
2220            0,
2221            0,
2222            corr,
2223            Vec::new(),
2224        )
2225        .unwrap()
2226    }
2227
2228    fn route_ctx() -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
2229        let (tx, rx) = mpsc::channel(8);
2230        (
2231            RouteCtx {
2232                connection_id: ConnectionId::LOCAL,
2233                egress: FrameSink::new(tx),
2234            },
2235            rx,
2236        )
2237    }
2238
2239    #[tokio::test]
2240    async fn retirement_egress_all_send_paths_drop_after_fence() {
2241        let (tx, mut rx) = mpsc::channel(8);
2242        let sink = FrameSink::new(tx);
2243        let reserved = sink.reserve_owned().await.unwrap();
2244        let immediate = sink.try_reserve_owned().unwrap();
2245        sink.terminate_egress();
2246        assert!(reserved.send(ping(1)));
2247        assert!(immediate.send(ping(2)));
2248        sink.send(ping(3)).await.unwrap();
2249        sink.try_send(ping(4)).unwrap();
2250        sink.send_to_client(ping(5)).await.unwrap();
2251        let queued = AtomicBool::new(false);
2252        assert!(sink
2253            .send_flushed_notifying_enqueue(ping(6), || queued.store(true, Ordering::Relaxed))
2254            .await
2255            .is_err());
2256        assert!(!queued.load(Ordering::Relaxed));
2257        assert!(sink.send_flushed(ping(7)).await.is_err());
2258        assert!(sink.reserve_owned().await.is_err());
2259        assert!(sink.try_reserve_owned().is_err());
2260        assert!(
2261            rx.try_recv().is_err(),
2262            "retired frames must not enter any writer queue"
2263        );
2264        assert_eq!(sink.backlog().queued_frames, 0);
2265        assert_eq!(sink.backlog().queued_bytes, 0);
2266    }
2267
2268    #[tokio::test]
2269    async fn retirement_egress_reservation_rechecks_after_slot_acquisition() {
2270        let (tx, mut rx) = mpsc::channel(1);
2271        let sink = FrameSink::new(tx);
2272        // Choose retirement after reserve returns its slot, not while waiting
2273        // on egress_stopped, so the final atomic recheck is exercised.
2274        sink.accounting
2275            .retire_after_reserve
2276            .store(true, Ordering::Relaxed);
2277        assert!(
2278            sink.reserve_owned().await.is_err(),
2279            "a slot acquired before retirement must not escape as a live reservation"
2280        );
2281        assert_eq!(
2282            sink.tx.capacity(),
2283            1,
2284            "refusal must release the reserved slot"
2285        );
2286        assert!(rx.try_recv().is_err());
2287    }
2288
2289    #[tokio::test]
2290    async fn retirement_egress_pending_budget_and_slot_sends_stop() {
2291        for budget in [1, usize::MAX] {
2292            let (tx, mut rx) = mpsc::channel(1);
2293            let sink = FrameSink::with_byte_budget(tx, budget);
2294            sink.try_send(ping(1)).unwrap();
2295            let sending = sink.send(ping(2));
2296            tokio::pin!(sending);
2297            assert!(poll_once(sending.as_mut()).await.is_pending());
2298            sink.terminate_egress();
2299            tokio::time::timeout(Duration::from_secs(2), sending)
2300                .await
2301                .unwrap()
2302                .unwrap();
2303            drop(rx.try_recv().unwrap());
2304            assert!(
2305                rx.try_recv().is_err(),
2306                "a send parked before retirement must not enqueue afterward"
2307            );
2308            assert_eq!(sink.backlog().queued_bytes, 0);
2309        }
2310    }
2311
2312    #[tokio::test]
2313    async fn retirement_egress_pending_flush_and_reservation_stop() {
2314        let (tx, mut rx) = mpsc::channel(1);
2315        let sink = FrameSink::new(tx);
2316        sink.try_send(ping(1)).unwrap();
2317        let queued = AtomicBool::new(false);
2318        let sending =
2319            sink.send_flushed_notifying_enqueue(ping(2), || queued.store(true, Ordering::Relaxed));
2320        let reserving = sink.reserve_owned();
2321        tokio::pin!(sending, reserving);
2322        assert!(poll_once(sending.as_mut()).await.is_pending());
2323        assert!(poll_once(reserving.as_mut()).await.is_pending());
2324        sink.terminate_egress();
2325        assert!(tokio::time::timeout(Duration::from_secs(2), sending)
2326            .await
2327            .unwrap()
2328            .is_err());
2329        assert!(tokio::time::timeout(Duration::from_secs(2), reserving)
2330            .await
2331            .unwrap()
2332            .is_err());
2333        assert!(!queued.load(Ordering::Relaxed));
2334        drop(rx.try_recv().unwrap());
2335        assert!(rx.try_recv().is_err());
2336        assert_eq!(sink.backlog().queued_bytes, 0);
2337    }
2338
2339    #[tokio::test]
2340    async fn echo_backend_returns_response_with_byte_identical_body() {
2341        let mut router = Router::with_default_self_handler();
2342        router.register_backend(7, EchoBackend).unwrap();
2343        let (ctx, mut rx) = route_ctx();
2344        let body = b"{not parsed}\0\xff";
2345
2346        router
2347            .route_for_connection(&ctx, request(7, 123, body))
2348            .await
2349            .unwrap();
2350        let response = rx.recv().await.unwrap();
2351
2352        assert_eq!(response.header.ty, FrameType::Response);
2353        assert_eq!(response.header.channel, 7);
2354        assert_eq!(response.header.corr, 123);
2355        assert_eq!(response.body, body);
2356        assert!(rx.try_recv().is_err());
2357    }
2358
2359    #[tokio::test]
2360    async fn unknown_channel_emits_canonical_error_frame() {
2361        let router = Router::with_default_self_handler();
2362        let (ctx, mut rx) = route_ctx();
2363
2364        router
2365            .route_for_connection(&ctx, request(99, 5, b"payload"))
2366            .await
2367            .unwrap();
2368        let error_frame = rx.recv().await.unwrap();
2369
2370        assert_eq!(error_frame.header.ty, FrameType::Error);
2371        assert_eq!(error_frame.header.channel, 99);
2372        assert_eq!(error_frame.header.corr, 5);
2373        let body: ErrorBody = serde_json::from_slice(&error_frame.body).unwrap();
2374        assert_eq!(body.code, "unknown_channel");
2375        assert_eq!(body.message, "unknown channel 99");
2376    }
2377
2378    /// Every ERROR the daemon authors carries the daemon-origin bit, so a
2379    /// client can tell a daemon refusal from a module's reply. Both the
2380    /// unknown-channel and the stale-epoch errors below are received on the
2381    /// client's own egress, not merely built.
2382    #[tokio::test]
2383    async fn daemon_authored_data_plane_errors_carry_daemon_origin() {
2384        let router = Router::with_default_self_handler();
2385        let (ctx, mut rx) = route_ctx();
2386        router
2387            .route_for_connection(&ctx, request(99, 5, b"payload"))
2388            .await
2389            .unwrap();
2390        let unknown = rx.recv().await.unwrap();
2391        assert_eq!(unknown.header.ty, FrameType::Error);
2392        let body: ErrorBody = serde_json::from_slice(&unknown.body).unwrap();
2393        assert_eq!(body.code, "unknown_channel");
2394        assert!(
2395            unknown.header.flags.is_daemon_origin(),
2396            "unknown_channel must carry daemon origin: {:?}",
2397            unknown.header.flags
2398        );
2399
2400        let (
2401            router,
2402            _forwarding,
2403            client_ctx,
2404            mut client_rx,
2405            _module_ctx,
2406            _module_egress_rx,
2407            _module_rx,
2408            pending,
2409        ) = dynamic_route_fixture(true);
2410        let _ = client_rx.recv().await;
2411        router
2412            .route_for_connection(
2413                &client_ctx,
2414                route_frame(
2415                    FrameType::Request,
2416                    pending.client_channel,
2417                    pending.client_epoch + 1,
2418                    702,
2419                ),
2420            )
2421            .await
2422            .unwrap();
2423        let stale = client_rx.recv().await.unwrap();
2424        assert_eq!(stale.header.ty, FrameType::Error);
2425        let body: ErrorBody = serde_json::from_slice(&stale.body).unwrap();
2426        assert_eq!(body.code, "stale_route_epoch");
2427        assert!(
2428            stale.header.flags.is_daemon_origin(),
2429            "stale_route_epoch must carry daemon origin: {:?}",
2430            stale.header.flags
2431        );
2432    }
2433
2434    /// A module ERROR that spoofs a daemon code with the daemon-origin bit
2435    /// set must reach the client with the bit cleared and the body
2436    /// byte-identical: the bit means the daemon wrote the frame, and a module
2437    /// must never be able to claim that.
2438    #[tokio::test]
2439    async fn module_error_with_daemon_bit_is_stripped_before_client_delivery() {
2440        let (
2441            router,
2442            _forwarding,
2443            client_ctx,
2444            mut client_rx,
2445            module_ctx,
2446            _module_egress_rx,
2447            mut module_rx,
2448            pending,
2449        ) = dynamic_route_fixture(true);
2450        let _ = client_rx.recv().await;
2451        // Open flow credit the way a real request would, so the terminal
2452        // error exercises the credit-release path too.
2453        router
2454            .route_for_connection(
2455                &client_ctx,
2456                route_frame(
2457                    FrameType::Request,
2458                    pending.client_channel,
2459                    pending.client_epoch,
2460                    701,
2461                ),
2462            )
2463            .await
2464            .unwrap();
2465        let _ = module_rx.recv().await;
2466
2467        let body = serde_json::to_vec(&ErrorBody::new("module_reloading", "try again")).unwrap();
2468        let spoof = Frame::build(
2469            FrameType::Error,
2470            Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2471            pending.module_channel,
2472            pending.module_epoch,
2473            701,
2474            body.clone(),
2475        )
2476        .unwrap();
2477        router
2478            .route_for_connection(&module_ctx, spoof)
2479            .await
2480            .unwrap();
2481        let delivered = client_rx.recv().await.unwrap();
2482        assert_eq!(delivered.header.ty, FrameType::Error);
2483        assert_eq!(delivered.header.channel, pending.client_channel);
2484        assert_eq!(delivered.header.epoch, pending.client_epoch);
2485        assert_eq!(delivered.header.corr, 701);
2486        assert!(
2487            !delivered.header.flags.is_daemon_origin(),
2488            "module-set daemon origin must be cleared: {:?}",
2489            delivered.header.flags
2490        );
2491        assert_eq!(delivered.body, body, "the body travels byte-identical");
2492    }
2493
2494    /// Clearing the origin bit strips exactly that bit: priority, LAST and
2495    /// subscription survive the splice unchanged.
2496    #[tokio::test]
2497    async fn module_response_with_daemon_bit_is_stripped_preserving_other_flags() {
2498        use subc_protocol::FLAG_SUBSCRIPTION;
2499        let (
2500            router,
2501            _forwarding,
2502            client_ctx,
2503            mut client_rx,
2504            module_ctx,
2505            _module_egress_rx,
2506            mut module_rx,
2507            pending,
2508        ) = dynamic_route_fixture(true);
2509        let _ = client_rx.recv().await;
2510        router
2511            .route_for_connection(
2512                &client_ctx,
2513                route_frame(
2514                    FrameType::Request,
2515                    pending.client_channel,
2516                    pending.client_epoch,
2517                    701,
2518                ),
2519            )
2520            .await
2521            .unwrap();
2522        let _ = module_rx.recv().await;
2523
2524        let sent_flags = Flags(
2525            Flags::new(true, Priority::Background, true).0
2526                | FLAG_SUBSCRIPTION
2527                | subc_protocol::FLAG_DAEMON_ORIGIN,
2528        );
2529        assert!(sent_flags.is_daemon_origin());
2530        let response = Frame::build(
2531            FrameType::Response,
2532            sent_flags,
2533            pending.module_channel,
2534            pending.module_epoch,
2535            701,
2536            b"route-body".to_vec(),
2537        )
2538        .unwrap();
2539        router
2540            .route_for_connection(&module_ctx, response)
2541            .await
2542            .unwrap();
2543        let delivered = client_rx.recv().await.unwrap();
2544        assert_eq!(delivered.header.ty, FrameType::Response);
2545        assert!(
2546            !delivered.header.flags.is_daemon_origin(),
2547            "module-set daemon origin must be cleared: {:?}",
2548            delivered.header.flags
2549        );
2550        assert!(
2551            delivered.header.flags.is_binary(),
2552            "binary flag must survive: {:?}",
2553            delivered.header.flags
2554        );
2555        assert!(
2556            delivered.header.flags.is_last(),
2557            "LAST flag must survive: {:?}",
2558            delivered.header.flags
2559        );
2560        assert!(
2561            delivered.header.flags.is_subscription(),
2562            "subscription flag must survive: {:?}",
2563            delivered.header.flags
2564        );
2565        assert_eq!(
2566            delivered.header.flags.priority(),
2567            Some(Priority::Background),
2568            "priority must survive"
2569        );
2570    }
2571
2572    /// A client frame carrying the origin bit must not reach the module
2573    /// looking daemon-authored either: the splice clears it symmetrically.
2574    #[tokio::test]
2575    async fn client_frame_with_daemon_bit_is_stripped_before_module_delivery() {
2576        let (
2577            router,
2578            _forwarding,
2579            client_ctx,
2580            mut client_rx,
2581            _module_ctx,
2582            _module_egress_rx,
2583            mut module_rx,
2584            pending,
2585        ) = dynamic_route_fixture(true);
2586        let _ = client_rx.recv().await;
2587        let spoof = Frame::build(
2588            FrameType::Request,
2589            Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2590            pending.client_channel,
2591            pending.client_epoch,
2592            701,
2593            b"route-body".to_vec(),
2594        )
2595        .unwrap();
2596        router
2597            .route_for_connection(&client_ctx, spoof)
2598            .await
2599            .unwrap();
2600        let delivered = module_rx.recv().await.unwrap();
2601        assert_eq!(delivered.header.ty, FrameType::Request);
2602        assert!(
2603            !delivered.header.flags.is_daemon_origin(),
2604            "client-set daemon origin must be cleared: {:?}",
2605            delivered.header.flags
2606        );
2607    }
2608
2609    #[tokio::test]
2610    async fn channel_zero_uses_control_handler_not_backend_registry() {
2611        let mut router = Router::with_default_self_handler();
2612        router.register_backend(1, EchoBackend).unwrap();
2613        let (ctx, mut rx) = route_ctx();
2614
2615        router.route_for_connection(&ctx, ping(77)).await.unwrap();
2616        let response = rx.recv().await.unwrap();
2617
2618        assert_eq!(response.header.ty, FrameType::Pong);
2619        assert_eq!(response.header.channel, 0);
2620        assert_eq!(response.header.corr, 77);
2621        assert!(response.body.is_empty());
2622    }
2623
2624    #[tokio::test]
2625    async fn slow_control_dispatch_logs_decoded_op_and_elapsed_time() {
2626        let control = Arc::new(
2627            ControlHandler::new(Arc::new(Registry::default()))
2628                .with_control_dispatch_delay(Duration::from_millis(1050)),
2629        );
2630        let router = Router::with_control_handler(control);
2631        let (ctx, mut rx) = route_ctx();
2632        let (output, guard) = log_capture(tracing::Level::WARN);
2633
2634        router
2635            .route_for_connection(&ctx, request(0, 41, br#"{"op":"server.describe"}"#))
2636            .await
2637            .expect("slow request routes");
2638        assert!(rx.recv().await.is_some(), "request receives a response");
2639        drop(guard);
2640
2641        let logs = captured_logs(&output);
2642        assert!(logs.contains("slow control dispatch"));
2643        assert!(logs.contains("op=server.describe"));
2644        assert!(logs.contains("connection_id=0"));
2645        assert!(logs.contains("corr=41"));
2646        assert!(
2647            logged_millis(&logs, "elapsed_ms=") >= 1050,
2648            "elapsed must include the injected handler delay: {logs}"
2649        );
2650    }
2651
2652    #[tokio::test]
2653    async fn fast_control_dispatch_emits_arrival_without_slow_warning() {
2654        let router = Router::with_default_self_handler();
2655        let (ctx, mut rx) = route_ctx();
2656        let (output, guard) = log_capture(tracing::Level::DEBUG);
2657
2658        router
2659            .route_for_connection(&ctx, request(0, 42, br#"{"op":"server.describe"}"#))
2660            .await
2661            .expect("fast request routes");
2662        assert!(rx.recv().await.is_some(), "request receives a response");
2663        drop(guard);
2664
2665        let logs = captured_logs(&output);
2666        assert!(logs.contains("control dispatch op=server.describe connection_id=0 corr=42"));
2667        assert!(!logs.contains("slow control dispatch"));
2668    }
2669
2670    #[tokio::test]
2671    async fn control_dispatch_arrival_is_hidden_at_info() {
2672        let router = Router::with_default_self_handler();
2673        let (ctx, mut rx) = route_ctx();
2674        let (output, guard) = log_capture(tracing::Level::INFO);
2675
2676        router
2677            .route_for_connection(&ctx, request(0, 43, br#"{"op":"server.describe"}"#))
2678            .await
2679            .expect("fast request routes");
2680        assert!(rx.recv().await.is_some(), "request receives a response");
2681        drop(guard);
2682
2683        assert!(
2684            !captured_logs(&output).contains("control dispatch"),
2685            "arrival logging must stay hidden at INFO"
2686        );
2687    }
2688
2689    #[tokio::test]
2690    async fn supervisor_list_logs_contended_snapshot_lock_only() {
2691        let registry = Arc::new(Registry::default());
2692        let handle = SupervisorHandle::new();
2693        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
2694            .with_handle(handle.clone());
2695        let module = supervisor
2696            .supervise_configured(
2697                ModuleSpec {
2698                    script: None,
2699                    carry_containment: false,
2700                    module_id: "held-module".to_string(),
2701                    program: "test-module".into(),
2702                    args: Vec::new(),
2703                    env: Vec::new(),
2704                    reserved: false,
2705                    reserved_prefixes: Vec::new(),
2706                    protocol: ModuleProtocol::Subc,
2707                    overlap: Default::default(),
2708                },
2709                false,
2710            )
2711            .expect("disabled test module is supervised");
2712        let router = Router::with_control_handler(Arc::new(
2713            ControlHandler::new(Arc::clone(&registry)).with_supervisor(handle),
2714        ));
2715        let (ctx, mut rx) = route_ctx();
2716        let (acquired, ready) = std_mpsc::channel();
2717        // Park the disabled supervisor actor before holding its snapshot. The
2718        // detached list reader, not actor startup, must be the lock contender.
2719        tokio::task::yield_now().await;
2720        let holder = module.hold_snapshot_for_test(acquired, Duration::from_millis(400));
2721        ready.recv().expect("holder acquired snapshot lock");
2722        let (output, guard) = log_capture(tracing::Level::WARN);
2723
2724        router
2725            .route_for_connection(&ctx, request(0, 44, br#"{"op":"supervisor.list"}"#))
2726            .await
2727            .expect("list request routes after the lock releases");
2728        assert!(
2729            rx.recv().await.is_some(),
2730            "list request receives a response"
2731        );
2732        holder.join().expect("snapshot holder exits cleanly");
2733        drop(guard);
2734
2735        let logs = captured_logs(&output);
2736        assert!(logs.contains("slow snapshot lock"));
2737        assert!(logs.contains("module_id=held-module"));
2738        assert!(logs.contains("caller=list"));
2739        assert!(
2740            logged_millis(&logs, "waited_ms=") >= 250,
2741            "wait must exceed the slow-lock threshold: {logs}"
2742        );
2743
2744        let (output, guard) = log_capture(tracing::Level::WARN);
2745        router
2746            .route_for_connection(&ctx, request(0, 45, br#"{"op":"supervisor.list"}"#))
2747            .await
2748            .expect("uncontended list request routes");
2749        assert!(
2750            rx.recv().await.is_some(),
2751            "uncontended list receives a response"
2752        );
2753        drop(guard);
2754        assert!(
2755            !captured_logs(&output).contains("slow snapshot lock"),
2756            "uncontended list acquisition must not warn"
2757        );
2758    }
2759
2760    #[tokio::test]
2761    async fn full_module_to_client_sink_requests_client_close_without_erroring_module() {
2762        let forwarding = Arc::new(ForwardingTable::default());
2763        let control = Arc::new(ControlHandler::with_forwarding(
2764            Arc::new(crate::Registry::default()),
2765            Arc::clone(&forwarding),
2766        ));
2767        let router = Router::with_control_handler(control);
2768        let module_connection = ConnectionId::new(10);
2769        let client_connection = ConnectionId::new(20);
2770        let mut close_receiver = forwarding.register_connection_close(client_connection);
2771        let (module_tx, _module_rx) = mpsc::channel(1);
2772        forwarding
2773            .register_module_connection(
2774                module_connection,
2775                "full-sink-provider".to_string(),
2776                1,
2777                Concurrency::ModuleManaged,
2778                FrameSink::new(module_tx),
2779            )
2780            .unwrap();
2781        let (client_tx, mut client_rx) = mpsc::channel(1);
2782        let pending = forwarding
2783            .begin_route_bind_relay_for_test(
2784                client_connection,
2785                FrameSink::new(client_tx),
2786                700,
2787                "full-sink-provider",
2788            )
2789            .unwrap();
2790        forwarding
2791            .complete_pending_relay(
2792                module_connection,
2793                pending.corr,
2794                RouteBindRelayOutcome::Accepted,
2795            )
2796            .unwrap();
2797
2798        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2799        let module_ctx = RouteCtx {
2800            connection_id: module_connection,
2801            egress: FrameSink::new(module_egress_tx),
2802        };
2803        let terminal = Frame::build(
2804            FrameType::Response,
2805            Flags::new(false, Priority::Interactive, true),
2806            pending.module_channel,
2807            pending.module_epoch,
2808            701,
2809            b"terminal".to_vec(),
2810        )
2811        .unwrap();
2812
2813        router
2814            .route_for_connection(&module_ctx, terminal)
2815            .await
2816            .unwrap();
2817        let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2818            .await
2819            .expect("close request should be sent for the full client sink")
2820            .expect("close sender should include a reason");
2821        assert!(
2822            reason
2823                .to_string()
2824                .contains("module_to_client_delivery_failed"),
2825            "unexpected close reason: {reason}"
2826        );
2827        assert_eq!(client_rx.try_recv().unwrap().header.corr, 700);
2828        assert!(client_rx.try_recv().is_err());
2829        assert_eq!(
2830            router.counters.snapshot()["client_egress_close_delivery_failed"],
2831            1
2832        );
2833    }
2834
2835    /// A terminal frame ends the request at the module even when the client
2836    /// cannot take it, so the drain must stop counting it at once rather than
2837    /// when the client connection's cleanup later removes the route.
2838    #[tokio::test]
2839    async fn terminal_frame_releases_its_credit_even_when_client_delivery_fails() {
2840        let forwarding = Arc::new(ForwardingTable::default());
2841        let control = Arc::new(ControlHandler::with_forwarding(
2842            Arc::new(crate::Registry::default()),
2843            Arc::clone(&forwarding),
2844        ));
2845        let router = Router::with_control_handler(control);
2846        let module_connection = ConnectionId::new(11);
2847        let client_connection = ConnectionId::new(21);
2848        let _close_receiver = forwarding.register_connection_close(client_connection);
2849        let (module_tx, _module_rx) = mpsc::channel(1);
2850        forwarding
2851            .register_module_connection(
2852                module_connection,
2853                "credit-provider".to_string(),
2854                1,
2855                Concurrency::ModuleManaged,
2856                FrameSink::new(module_tx),
2857            )
2858            .unwrap();
2859        // Capacity one, filled by the route.open response, so the terminal
2860        // frame below cannot be enqueued for the client.
2861        let (client_tx, _client_rx) = mpsc::channel(1);
2862        let pending = forwarding
2863            .begin_route_bind_relay_for_test(
2864                client_connection,
2865                FrameSink::new(client_tx),
2866                800,
2867                "credit-provider",
2868            )
2869            .unwrap();
2870        forwarding
2871            .complete_pending_relay(
2872                module_connection,
2873                pending.corr,
2874                RouteBindRelayOutcome::Accepted,
2875            )
2876            .unwrap();
2877        let DataRoute::Client(DataRouteState::Bound(route)) = forwarding
2878            .lookup_data_route(
2879                client_connection,
2880                pending.client_channel,
2881                pending.client_epoch,
2882            )
2883            .unwrap()
2884        else {
2885            panic!("expected a bound client route");
2886        };
2887        route.flow.acquire_tagged(801, false).await.unwrap();
2888        assert_eq!(route.flow.drain_in_flight(), 1);
2889
2890        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2891        let module_ctx = RouteCtx {
2892            connection_id: module_connection,
2893            egress: FrameSink::new(module_egress_tx),
2894        };
2895        let terminal = Frame::build(
2896            FrameType::Response,
2897            Flags::new(false, Priority::Interactive, true),
2898            pending.module_channel,
2899            pending.module_epoch,
2900            801,
2901            b"terminal".to_vec(),
2902        )
2903        .unwrap();
2904        router
2905            .route_for_connection(&module_ctx, terminal)
2906            .await
2907            .unwrap();
2908
2909        assert_eq!(
2910            router.counters.snapshot()["client_egress_close_delivery_failed"],
2911            1,
2912            "the client delivery must have failed for this test to mean anything"
2913        );
2914        assert_eq!(
2915            route.flow.drain_in_flight(),
2916            0,
2917            "the module's terminal frame must release its credit even though the client could not take it"
2918        );
2919    }
2920
2921    #[tokio::test]
2922    async fn full_route_goodbye_sink_requests_target_close_without_erroring_module() {
2923        let forwarding = Arc::new(ForwardingTable::default());
2924        let control = Arc::new(ControlHandler::with_forwarding(
2925            Arc::new(crate::Registry::default()),
2926            Arc::clone(&forwarding),
2927        ));
2928        let router = Router::with_control_handler(control);
2929        let module_connection = ConnectionId::new(30);
2930        let client_connection = ConnectionId::new(40);
2931        let mut close_receiver = forwarding.register_connection_close(client_connection);
2932        let (module_tx, _module_rx) = mpsc::channel(1);
2933        forwarding
2934            .register_module_connection(
2935                module_connection,
2936                "goodbye-full-provider".to_string(),
2937                1,
2938                Concurrency::ModuleManaged,
2939                FrameSink::new(module_tx),
2940            )
2941            .unwrap();
2942        let (client_tx, mut client_rx) = mpsc::channel(1);
2943        let pending = forwarding
2944            .begin_route_bind_relay_for_test(
2945                client_connection,
2946                FrameSink::new(client_tx),
2947                800,
2948                "goodbye-full-provider",
2949            )
2950            .unwrap();
2951        forwarding
2952            .complete_pending_relay(
2953                module_connection,
2954                pending.corr,
2955                RouteBindRelayOutcome::Accepted,
2956            )
2957            .unwrap();
2958
2959        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2960        let module_ctx = RouteCtx {
2961            connection_id: module_connection,
2962            egress: FrameSink::new(module_egress_tx),
2963        };
2964        let goodbye = Frame::build(
2965            FrameType::Goodbye,
2966            Flags::new(false, Priority::Passive, true),
2967            pending.module_channel,
2968            pending.module_epoch,
2969            801,
2970            Vec::new(),
2971        )
2972        .unwrap();
2973
2974        router
2975            .route_for_connection(&module_ctx, goodbye)
2976            .await
2977            .unwrap();
2978        let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2979            .await
2980            .expect("close request should be sent for the full GOODBYE sink")
2981            .expect("close sender should include a reason");
2982        assert!(
2983            reason.to_string().contains("route_goodbye_delivery_failed"),
2984            "unexpected close reason: {reason}"
2985        );
2986        assert_eq!(client_rx.try_recv().unwrap().header.corr, 800);
2987        assert!(client_rx.try_recv().is_err());
2988        assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 1);
2989        assert_eq!(router.counters.snapshot()["route_released_epoch_fenced"], 1);
2990    }
2991
2992    /// One module and one client connection with `routes` routes bound between
2993    /// them. The client uses a real connection egress queue (the same byte
2994    /// budget and frame-count backstop as a live connection), and its route.open
2995    /// responses are drained so the queue starts empty.
2996    async fn multi_route_client(
2997        module_id: &str,
2998        module_connection: ConnectionId,
2999        client_connection: ConnectionId,
3000        routes: usize,
3001    ) -> (
3002        Router,
3003        FrameSink,
3004        mpsc::Receiver<OutboundFrame>,
3005        RouteCtx,
3006        Vec<crate::forwarding::PendingRouteBindRelay>,
3007        ConnectionCloseReceiver,
3008    ) {
3009        let forwarding = Arc::new(ForwardingTable::default());
3010        let control = Arc::new(ControlHandler::with_forwarding(
3011            Arc::new(crate::Registry::default()),
3012            Arc::clone(&forwarding),
3013        ));
3014        let router = Router::with_control_handler(control);
3015        let close_receiver = forwarding.register_connection_close(client_connection);
3016        let (module_tx, _module_rx) = mpsc::channel(8);
3017        forwarding
3018            .register_module_connection(
3019                module_connection,
3020                module_id.to_string(),
3021                1,
3022                Concurrency::ModuleManaged,
3023                FrameSink::new(module_tx),
3024            )
3025            .unwrap();
3026        let (client_sink, mut client_rx) = crate::server::connection_egress();
3027        let mut bound = Vec::with_capacity(routes);
3028        for index in 0..routes {
3029            let pending = forwarding
3030                .begin_route_bind_relay_for_test(
3031                    client_connection,
3032                    client_sink.clone(),
3033                    900 + index as u64,
3034                    module_id,
3035                )
3036                .unwrap();
3037            forwarding
3038                .complete_pending_relay(
3039                    module_connection,
3040                    pending.corr,
3041                    RouteBindRelayOutcome::Accepted,
3042                )
3043                .unwrap();
3044            assert_eq!(
3045                client_rx.recv().await.unwrap().header.corr,
3046                900 + index as u64
3047            );
3048            bound.push(pending);
3049        }
3050        let (module_egress_tx, _module_egress_rx) = mpsc::channel(8);
3051        let module_ctx = RouteCtx {
3052            connection_id: module_connection,
3053            egress: FrameSink::new(module_egress_tx),
3054        };
3055        (
3056            router,
3057            client_sink,
3058            client_rx,
3059            module_ctx,
3060            bound,
3061            close_receiver,
3062        )
3063    }
3064
3065    /// Per-frame cost of the egress sink's admission and release accounting:
3066    /// one million 200-byte frames enqueued with `try_send` and taken off the
3067    /// queue the way the connection writer does, in batches of 1,000 so the
3068    /// queue stays well inside its budget. Prints nanoseconds per frame; it is
3069    /// a measurement, not a gate. Run with
3070    /// `cargo test --release -p subc-daemon --lib egress_sink_per_frame_cost -- --ignored --nocapture`.
3071    #[test]
3072    #[ignore = "timing measurement, run on demand"]
3073    fn egress_sink_per_frame_cost() {
3074        const FRAMES: usize = 1_000_000;
3075        const BATCH: usize = 1_000;
3076        let (sink, mut rx) = crate::server::connection_egress();
3077        let template = stream_frame(9, 1, 0, vec![b't'; 200]);
3078        let started = Instant::now();
3079        for _ in 0..FRAMES / BATCH {
3080            for _ in 0..BATCH {
3081                sink.try_send(template.clone()).unwrap();
3082            }
3083            for _ in 0..BATCH {
3084                let outbound = rx.try_recv().unwrap();
3085                if let Some(charge) = &outbound.charge {
3086                    charge.taken_by_writer();
3087                }
3088                drop(std::hint::black_box(outbound));
3089            }
3090        }
3091        let elapsed = started.elapsed();
3092        assert_eq!(sink.backlog().queued_bytes, 0);
3093        println!(
3094            "egress sink: {FRAMES} frames in {elapsed:?}, {:.1} ns/frame",
3095            elapsed.as_nanos() as f64 / FRAMES as f64
3096        );
3097    }
3098
3099    fn stream_frame(channel: u16, epoch: u32, corr: u64, body: Vec<u8>) -> Frame {
3100        Frame::build(
3101            FrameType::StreamData,
3102            Flags::new(false, Priority::Interactive, false),
3103            channel,
3104            epoch,
3105            corr,
3106            body,
3107        )
3108        .unwrap()
3109    }
3110
3111    fn module_route_for_test(
3112        router: &Router,
3113        ctx: &RouteCtx,
3114        route: &crate::forwarding::PendingRouteBindRelay,
3115    ) -> Arc<RouteBinding> {
3116        match router
3117            .forwarding
3118            .lookup_data_route(ctx.connection_id, route.module_channel, route.module_epoch)
3119            .unwrap()
3120        {
3121            DataRoute::Module(DataRouteState::Bound(binding)) => binding,
3122            other => panic!("expected a bound module route, got {other:?}"),
3123        }
3124    }
3125
3126    fn assert_frame_size_error(frame: &Frame, channel: u16, epoch: u32, corr: u64, bytes: usize) {
3127        assert_eq!(frame.header.ty, FrameType::Error);
3128        assert_eq!(frame.header.channel, channel);
3129        assert_eq!(frame.header.epoch, epoch);
3130        assert_eq!(frame.header.corr, corr);
3131        assert!(frame.header.flags.is_daemon_origin());
3132        let body: ErrorBody = serde_json::from_slice(&frame.body).unwrap();
3133        assert_eq!(body.code, error_codes::FRAME_TOO_LARGE);
3134        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3135        assert!(body.message.contains(&bytes.to_string()));
3136        assert!(body.message.contains(&limit.to_string()));
3137        assert_eq!(
3138            body.detail,
3139            Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit }))
3140        );
3141    }
3142
3143    #[tokio::test]
3144    async fn module_egress_response_oversized_reply_keeps_connection_and_other_route() {
3145        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3146        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3147            "oversize-provider",
3148            ConnectionId::new(80),
3149            ConnectionId::new(81),
3150            2,
3151        )
3152        .await;
3153        let binding = module_route_for_test(&router, &ctx, &routes[0]);
3154        binding.flow.acquire_tagged(101, false).await.unwrap();
3155        module_route_for_test(&router, &ctx, &routes[1])
3156            .flow
3157            .acquire_tagged(102, false)
3158            .await
3159            .unwrap();
3160        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3161        let oversized = Frame::build(
3162            FrameType::Response,
3163            Flags::new(false, Priority::Interactive, true),
3164            routes[0].module_channel,
3165            routes[0].module_epoch,
3166            101,
3167            vec![b'x'; limit - subc_protocol::HEADER_LEN + 1],
3168        )
3169        .unwrap();
3170        assert_eq!(sink.backlog().queued_bytes, 0);
3171        router.route_for_connection(&ctx, oversized).await.unwrap();
3172
3173        // Reproduce a large reply followed by ordinary traffic on another route
3174        // before the client reads anything. The large reply must not monopolize
3175        // the queue and make that second delivery close the shared connection.
3176        router
3177            .route_for_connection(
3178                &ctx,
3179                route_frame(
3180                    FrameType::Response,
3181                    routes[1].module_channel,
3182                    routes[1].module_epoch,
3183                    102,
3184                ),
3185            )
3186            .await
3187            .unwrap();
3188        assert!(
3189            close.try_recv().is_err(),
3190            "an oversized reply must not close the multiplexed client"
3191        );
3192        let error = rx.try_recv().expect("the waiting client gets an ERROR");
3193        assert_frame_size_error(
3194            &error,
3195            routes[0].client_channel,
3196            routes[0].client_epoch,
3197            101,
3198            limit + 1,
3199        );
3200        let second = rx.try_recv().expect("the other route's reply arrives");
3201        assert_eq!(second.header.ty, FrameType::Response);
3202        assert_eq!(second.header.channel, routes[1].client_channel);
3203        assert_eq!(second.header.epoch, routes[1].client_epoch);
3204        assert_eq!(second.header.corr, 102);
3205        assert_eq!(second.body, b"route-body");
3206        assert_eq!(binding.flow.drain_in_flight(), 0);
3207        for route in &routes {
3208            module_route_for_test(&router, &ctx, route);
3209        }
3210        assert!(rx.try_recv().is_err());
3211        drop((error, second));
3212        assert_eq!(sink.backlog().queued_bytes, 0);
3213        assert_eq!(
3214            router.counters.snapshot()["client_egress_close_delivery_failed"],
3215            0
3216        );
3217
3218        let captured = test_log::captured_logs(&logs);
3219        let warnings: Vec<_> = captured
3220            .lines()
3221            .filter(|line| line.contains("WARN"))
3222            .collect();
3223        assert_eq!(warnings.len(), 1, "{captured}");
3224        let warning = warnings[0];
3225        for expected in [
3226            "module_id=oversize-provider".to_string(),
3227            "client_principal=direct".to_string(),
3228            format!("client_channel={}", routes[0].client_channel),
3229            format!("frame_bytes={}", limit + 1),
3230            format!("byte_limit={limit}"),
3231        ] {
3232            assert!(warning.contains(&expected), "missing {expected}: {warning}");
3233        }
3234    }
3235
3236    async fn poll_once<F: std::future::Future>(
3237        mut future: std::pin::Pin<&mut F>,
3238    ) -> std::task::Poll<F::Output> {
3239        std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
3240    }
3241
3242    fn fill_client_budget(sink: &FrameSink, route: &crate::forwarding::PendingRouteBindRelay) {
3243        sink.try_send(
3244            Frame::build(
3245                FrameType::Response,
3246                Flags::new(false, Priority::Interactive, true),
3247                route.client_channel,
3248                route.client_epoch,
3249                100,
3250                vec![
3251                    b'x';
3252                    crate::server::CONNECTION_EGRESS_BYTE_BUDGET - subc_protocol::HEADER_LEN
3253                ],
3254            )
3255            .unwrap(),
3256        )
3257        .unwrap();
3258        assert_eq!(
3259            sink.backlog().queued_bytes,
3260            crate::server::CONNECTION_EGRESS_BYTE_BUDGET
3261        );
3262    }
3263
3264    async fn assert_full_client_burst_survives(ty: FrameType, oversized: bool) {
3265        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3266            "draining-burst",
3267            ConnectionId::new(92),
3268            ConnectionId::new(93),
3269            2,
3270        )
3271        .await;
3272        fill_client_budget(&sink, &routes[0]);
3273        let frame = if oversized {
3274            Frame::build(
3275                ty,
3276                Flags::new(false, Priority::Interactive, false),
3277                routes[1].module_channel,
3278                routes[1].module_epoch,
3279                101,
3280                vec![b's'; crate::server::CONNECTION_EGRESS_BYTE_BUDGET],
3281            )
3282            .unwrap()
3283        } else {
3284            route_frame(ty, routes[1].module_channel, routes[1].module_epoch, 101)
3285        };
3286        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3287        assert!(poll_once(delivery.as_mut()).await.is_pending());
3288        assert!(
3289            close.try_recv().is_err(),
3290            "fresh bursts must not close: {ty:?}"
3291        );
3292        tokio::time::advance(Duration::from_millis(50)).await;
3293        let queued = rx.try_recv().unwrap();
3294        queued.charge.as_ref().unwrap().taken_by_writer();
3295        drop(queued);
3296        assert!(matches!(
3297            poll_once(delivery.as_mut()).await,
3298            std::task::Poll::Ready(Ok(()))
3299        ));
3300        let delivered = rx
3301            .try_recv()
3302            .expect("the burst frame must be delivered after draining");
3303        assert_eq!(delivered.header.channel, routes[1].client_channel);
3304        assert_eq!(delivered.header.epoch, routes[1].client_epoch);
3305        assert_eq!(delivered.header.corr, 101);
3306        if oversized {
3307            assert_frame_size_error(
3308                &delivered,
3309                routes[1].client_channel,
3310                routes[1].client_epoch,
3311                101,
3312                crate::server::CONNECTION_EGRESS_BYTE_BUDGET + subc_protocol::HEADER_LEN,
3313            );
3314        } else {
3315            assert_eq!(delivered.header.ty, ty);
3316        }
3317        assert!(rx.try_recv().is_err());
3318        assert!(close.try_recv().is_err());
3319        assert_eq!(
3320            router.counters.snapshot()["client_egress_close_delivery_failed"],
3321            0
3322        );
3323        assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 0);
3324    }
3325
3326    #[tokio::test(start_paused = true)]
3327    async fn module_egress_fresh_full_queue_burst_survives() {
3328        for ty in [
3329            FrameType::Response,
3330            FrameType::StreamData,
3331            FrameType::Request,
3332            FrameType::Push,
3333        ] {
3334            assert_full_client_burst_survives(ty, false).await;
3335        }
3336    }
3337
3338    #[tokio::test(start_paused = true)]
3339    async fn module_goodbye_fresh_full_queue_burst_survives() {
3340        assert_full_client_burst_survives(FrameType::Goodbye, false).await;
3341    }
3342
3343    #[tokio::test(start_paused = true)]
3344    async fn frame_count_backstop_also_allows_a_draining_burst() {
3345        let (tx, mut rx) = mpsc::channel(1);
3346        let sink = FrameSink::with_byte_budget(tx, 2_000);
3347        sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
3348            .unwrap();
3349        let mut delivery = Box::pin(sink.send_to_client(stream_frame(1, 1, 2, vec![b'y'; 100])));
3350        assert!(poll_once(delivery.as_mut()).await.is_pending());
3351        assert!(
3352            sink.backlog().queued_bytes < 2_000,
3353            "only frame capacity is exhausted"
3354        );
3355        tokio::time::advance(Duration::from_millis(50)).await;
3356        drop(rx.try_recv().unwrap());
3357        assert!(matches!(
3358            poll_once(delivery.as_mut()).await,
3359            std::task::Poll::Ready(Ok(()))
3360        ));
3361        assert_eq!(rx.try_recv().unwrap().header.corr, 2);
3362    }
3363
3364    #[tokio::test(start_paused = true)]
3365    async fn oversized_reply_replacement_fresh_full_queue_burst_survives() {
3366        for ty in [FrameType::Response, FrameType::StreamData] {
3367            assert_full_client_burst_survives(ty, true).await;
3368        }
3369    }
3370
3371    #[tokio::test(start_paused = true)]
3372    async fn module_egress_stopped_reader_closes_without_waiting() {
3373        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3374        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3375            "oldest-stopped-reader",
3376            ConnectionId::new(94),
3377            ConnectionId::new(95),
3378            2,
3379        )
3380        .await;
3381        fill_client_budget(&sink, &routes[0]);
3382        // The writer has taken the head but cannot finish writing it. Its
3383        // bytes stay charged, and the atomic age still diagnoses that stall.
3384        let writing = rx.try_recv().unwrap();
3385        writing.charge.as_ref().unwrap().taken_by_writer();
3386        tokio::time::advance(Duration::from_millis(251)).await;
3387        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(251)));
3388        assert_eq!(sink.backlog().queued_bytes, queued_frame_bytes(&writing));
3389        let started = tokio::time::Instant::now();
3390        let frame = stream_frame(
3391            routes[0].module_channel,
3392            routes[0].module_epoch,
3393            101,
3394            b"secret-body-never-log".to_vec(),
3395        );
3396        let flags = frame.header.flags.0;
3397        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3398        assert!(
3399            matches!(
3400                poll_once(delivery.as_mut()).await,
3401                std::task::Poll::Ready(Ok(()))
3402            ),
3403            "an oldest frame older than 250 ms must close immediately, without a new wait"
3404        );
3405        assert_eq!(started.elapsed(), Duration::ZERO);
3406        assert!(close
3407            .try_recv()
3408            .unwrap()
3409            .to_string()
3410            .contains("module_to_client_delivery_failed"));
3411        let captured = test_log::captured_logs(&logs);
3412        for expected in [
3413            "frame_type=Some(StreamData)".to_string(),
3414            format!("frame_flags=Some({flags})"),
3415            "oldest_queued_ms=251".to_string(),
3416            "close_cause=\"oldest_frame_older_than_250_ms\"".to_string(),
3417        ] {
3418            assert!(
3419                captured.contains(&expected),
3420                "missing {expected}: {captured}"
3421            );
3422        }
3423        assert!(!captured.contains("secret-body-never-log"));
3424    }
3425
3426    #[tokio::test(start_paused = true)]
3427    async fn client_gone_mid_wait_resumes_module_reader_immediately() {
3428        for writer_gone in [false, true] {
3429            let (router, sink, mut rx, ctx, routes, _close) = multi_route_client(
3430                "gone-during-burst",
3431                ConnectionId::new(96),
3432                ConnectionId::new(97),
3433                1,
3434            )
3435            .await;
3436            fill_client_budget(&sink, &routes[0]);
3437            let frame = route_frame(
3438                FrameType::Response,
3439                routes[0].module_channel,
3440                routes[0].module_epoch,
3441                101,
3442            );
3443            let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3444            assert!(poll_once(delivery.as_mut()).await.is_pending());
3445            tokio::time::advance(Duration::from_millis(50)).await;
3446            let started = tokio::time::Instant::now();
3447            if writer_gone {
3448                rx.close();
3449            } else {
3450                sink.close_client_delivery();
3451            }
3452            assert!(
3453                matches!(
3454                    poll_once(delivery.as_mut()).await,
3455                    std::task::Poll::Ready(Ok(()))
3456                ),
3457                "client teardown must wake the module reader before the writer drains"
3458            );
3459            assert_eq!(started.elapsed(), Duration::ZERO);
3460            assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
3461            assert_eq!(rx.try_recv().unwrap().header.corr, 100);
3462            assert!(rx.try_recv().is_err());
3463        }
3464    }
3465
3466    #[tokio::test(start_paused = true)]
3467    async fn cancelled_module_delivery_releases_waiter_without_closing_client() {
3468        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3469            "cancelled-burst",
3470            ConnectionId::new(98),
3471            ConnectionId::new(99),
3472            1,
3473        )
3474        .await;
3475        fill_client_budget(&sink, &routes[0]);
3476        let frame = route_frame(
3477            FrameType::Response,
3478            routes[0].module_channel,
3479            routes[0].module_epoch,
3480            101,
3481        );
3482        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3483        assert!(poll_once(delivery.as_mut()).await.is_pending());
3484        assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 1);
3485        drop(delivery);
3486        assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
3487        tokio::time::advance(Duration::from_secs(1)).await;
3488        assert!(close.try_recv().is_err());
3489        drop(rx.try_recv().unwrap());
3490        assert_eq!(sink.backlog().queued_bytes, 0);
3491        assert!(sink.backlog().oldest_age.is_none());
3492    }
3493
3494    #[tokio::test(start_paused = true)]
3495    async fn egress_atomic_age_follows_writer_and_clears_when_drained() {
3496        let (sink, mut rx) = crate::server::connection_egress();
3497        sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
3498            .unwrap();
3499        tokio::time::advance(Duration::from_millis(200)).await;
3500        sink.try_send(stream_frame(2, 1, 2, vec![b'y'; 100]))
3501            .unwrap();
3502        let writing = rx.try_recv().unwrap();
3503        writing.charge.as_ref().unwrap().taken_by_writer();
3504        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3505        drop(writing);
3506        // Between frames the lock-free marker can still name the previous
3507        // head. The next writer take advances it without scanning the queue.
3508        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3509        tokio::time::advance(Duration::from_millis(50)).await;
3510        let next = rx.try_recv().unwrap();
3511        next.charge.as_ref().unwrap().taken_by_writer();
3512        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(50)));
3513        assert!(
3514            sink.backlog().queued_bytes > 0,
3515            "in-flight bytes remain charged until written"
3516        );
3517        drop(next);
3518        assert_eq!(sink.backlog().queued_bytes, 0);
3519        assert_eq!(sink.backlog().oldest_age, None);
3520    }
3521
3522    #[tokio::test(start_paused = true)]
3523    async fn module_egress_response_full_client_waits_only_remaining_grace() {
3524        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3525            "backpressure-provider",
3526            ConnectionId::new(82),
3527            ConnectionId::new(83),
3528            2,
3529        )
3530        .await;
3531        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3532        // Exactly the limit, including its header, must be admitted to an empty
3533        // queue. Leave it unread to deterministically exhaust only byte capacity.
3534        router
3535            .route_for_connection(
3536                &ctx,
3537                Frame::build(
3538                    FrameType::Response,
3539                    Flags::new(false, Priority::Interactive, true),
3540                    routes[0].module_channel,
3541                    routes[0].module_epoch,
3542                    201,
3543                    vec![b'x'; limit - subc_protocol::HEADER_LEN],
3544                )
3545                .unwrap(),
3546            )
3547            .await
3548            .unwrap();
3549        assert_eq!(sink.backlog().queued_bytes, limit);
3550        assert_eq!(sink.backlog().queued_frames, 1);
3551        let ordinary = route_frame(
3552            FrameType::Response,
3553            routes[1].module_channel,
3554            routes[1].module_epoch,
3555            202,
3556        );
3557        assert!(queued_frame_bytes(&ordinary) < limit);
3558        // A fitting reply gets only the unused portion of the oldest frame's
3559        // 250 ms grace, not an immediate close or a fresh 250 ms wait.
3560        tokio::time::advance(Duration::from_millis(100)).await;
3561        let mut delivery = Box::pin(router.route_for_connection(&ctx, ordinary));
3562        assert!(poll_once(delivery.as_mut()).await.is_pending());
3563        assert!(close.try_recv().is_err());
3564        tokio::time::advance(Duration::from_millis(149)).await;
3565        assert!(poll_once(delivery.as_mut()).await.is_pending());
3566        assert!(close.try_recv().is_err());
3567        tokio::time::advance(Duration::from_millis(1)).await;
3568        assert!(matches!(
3569            poll_once(delivery.as_mut()).await,
3570            std::task::Poll::Ready(Ok(()))
3571        ));
3572        drop(delivery);
3573        let reason = close.try_recv().expect("the full client must close");
3574        assert!(reason
3575            .to_string()
3576            .contains("module_to_client_delivery_failed"));
3577        assert!(!reason.to_string().contains("frame_too_large"));
3578        assert_eq!(
3579            router.counters.snapshot()["client_egress_close_delivery_failed"],
3580            1
3581        );
3582        let admitted = rx.try_recv().expect("the budget-sized frame was admitted");
3583        assert_eq!(admitted.header.ty, FrameType::Response);
3584        assert_eq!(admitted.header.corr, 201);
3585        assert!(
3586            rx.try_recv().is_err(),
3587            "no oversized-frame ERROR is substituted for backpressure"
3588        );
3589    }
3590
3591    #[tokio::test(start_paused = true)]
3592    async fn oversized_reply_to_a_full_client_waits_only_remaining_grace() {
3593        let (router, sink, _rx, ctx, routes, mut close) = multi_route_client(
3594            "full-oversize-provider",
3595            ConnectionId::new(90),
3596            ConnectionId::new(91),
3597            2,
3598        )
3599        .await;
3600        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3601        // Fill the client's queue to its budget and leave it unread.
3602        router
3603            .route_for_connection(
3604                &ctx,
3605                Frame::build(
3606                    FrameType::Response,
3607                    Flags::new(false, Priority::Interactive, true),
3608                    routes[0].module_channel,
3609                    routes[0].module_epoch,
3610                    501,
3611                    vec![b'x'; limit - subc_protocol::HEADER_LEN],
3612                )
3613                .unwrap(),
3614            )
3615            .await
3616            .unwrap();
3617        assert_eq!(sink.backlog().queued_bytes, limit);
3618        let oversized = Frame::build(
3619            FrameType::Response,
3620            Flags::new(false, Priority::Interactive, true),
3621            routes[1].module_channel,
3622            routes[1].module_epoch,
3623            502,
3624            vec![b'y'; limit - subc_protocol::HEADER_LEN + 1],
3625        )
3626        .unwrap();
3627        // The replacement ERROR gets the same bounded burst grace as the
3628        // ordinary reply; oversized payloads themselves are never queued.
3629        tokio::time::advance(Duration::from_millis(80)).await;
3630        let mut delivery = Box::pin(router.route_for_connection(&ctx, oversized));
3631        assert!(poll_once(delivery.as_mut()).await.is_pending());
3632        assert!(close.try_recv().is_err());
3633        tokio::time::advance(Duration::from_millis(169)).await;
3634        assert!(poll_once(delivery.as_mut()).await.is_pending());
3635        assert!(close.try_recv().is_err());
3636        tokio::time::advance(Duration::from_millis(1)).await;
3637        assert!(matches!(
3638            poll_once(delivery.as_mut()).await,
3639            std::task::Poll::Ready(Ok(()))
3640        ));
3641        drop(delivery);
3642        let reason = close.try_recv().expect("the full client must close");
3643        assert!(reason
3644            .to_string()
3645            .contains("module_to_client_delivery_failed"));
3646    }
3647
3648    #[tokio::test]
3649    async fn oversized_reverse_request_errors_module_without_reaching_client() {
3650        let (router, _sink, mut rx, mut ctx, routes, mut close) = multi_route_client(
3651            "reverse-oversize-provider",
3652            ConnectionId::new(84),
3653            ConnectionId::new(85),
3654            2,
3655        )
3656        .await;
3657        let (module_tx, mut module_rx) = mpsc::channel(8);
3658        ctx.egress = FrameSink::new(module_tx);
3659        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3660        router
3661            .route_for_connection(
3662                &ctx,
3663                Frame::build(
3664                    FrameType::Request,
3665                    Flags::new(false, Priority::Interactive, false),
3666                    routes[0].module_channel,
3667                    routes[0].module_epoch,
3668                    301,
3669                    vec![b'r'; limit - subc_protocol::HEADER_LEN + 1],
3670                )
3671                .unwrap(),
3672            )
3673            .await
3674            .unwrap();
3675        assert!(
3676            rx.try_recv().is_err(),
3677            "the reverse request never reaches the client"
3678        );
3679        let error = module_rx.try_recv().expect("the module gets the refusal");
3680        assert_frame_size_error(
3681            &error,
3682            routes[0].module_channel,
3683            routes[0].module_epoch,
3684            301,
3685            limit + 1,
3686        );
3687        assert!(module_rx.try_recv().is_err());
3688        for (index, route) in routes.iter().enumerate() {
3689            module_route_for_test(&router, &ctx, route);
3690            router
3691                .route_for_connection(
3692                    &ctx,
3693                    route_frame(
3694                        FrameType::Response,
3695                        route.module_channel,
3696                        route.module_epoch,
3697                        302 + index as u64,
3698                    ),
3699                )
3700                .await
3701                .unwrap();
3702            let reply = rx.try_recv().expect("both routes still deliver");
3703            assert_eq!(reply.header.ty, FrameType::Response);
3704            assert_eq!(reply.header.channel, route.client_channel);
3705        }
3706        assert!(close.try_recv().is_err());
3707        assert_eq!(
3708            router.counters.snapshot()["client_egress_close_delivery_failed"],
3709            0
3710        );
3711    }
3712
3713    #[tokio::test]
3714    async fn oversized_push_closes_client_with_precise_warn() {
3715        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3716        let (router, _sink, mut rx, ctx, routes, mut close) = multi_route_client(
3717            "push-oversize-provider",
3718            ConnectionId::new(86),
3719            ConnectionId::new(87),
3720            2,
3721        )
3722        .await;
3723        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3724        router
3725            .route_for_connection(
3726                &ctx,
3727                Frame::build(
3728                    FrameType::Push,
3729                    Flags::new(false, Priority::Interactive, false),
3730                    routes[0].module_channel,
3731                    routes[0].module_epoch,
3732                    0,
3733                    vec![b'p'; limit - subc_protocol::HEADER_LEN + 1],
3734                )
3735                .unwrap(),
3736            )
3737            .await
3738            .unwrap();
3739        let reason = close
3740            .try_recv()
3741            .expect("unsolicited oversize still closes the client");
3742        assert!(reason
3743            .to_string()
3744            .contains("frame_too_large: oversized push"));
3745        assert!(
3746            rx.try_recv().is_err(),
3747            "no part of the oversized push is forwarded"
3748        );
3749        let captured = test_log::captured_logs(&logs);
3750        let warnings: Vec<_> = captured
3751            .lines()
3752            .filter(|line| line.contains("WARN"))
3753            .collect();
3754        assert_eq!(warnings.len(), 1, "{captured}");
3755        let warning = warnings[0];
3756        for expected in [
3757            "oversized push".to_string(),
3758            "module_id=\"push-oversize-provider\"".to_string(),
3759            "principals=direct".to_string(),
3760            format!("client_channel={}", routes[0].client_channel),
3761            format!("size={}", limit + 1),
3762            format!("limit={limit}"),
3763        ] {
3764            assert!(warning.contains(&expected), "missing {expected}: {warning}");
3765        }
3766        assert_eq!(
3767            router.counters.snapshot()["client_egress_close_delivery_failed"],
3768            1
3769        );
3770    }
3771
3772    #[tokio::test]
3773    async fn oversized_stream_and_error_frames_end_request_even_with_queued_traffic() {
3774        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3775            "stream-oversize-provider",
3776            ConnectionId::new(88),
3777            ConnectionId::new(89),
3778            2,
3779        )
3780        .await;
3781        let binding = module_route_for_test(&router, &ctx, &routes[0]);
3782        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3783        for ty in [
3784            FrameType::StreamData,
3785            FrameType::StreamEnd,
3786            FrameType::Error,
3787        ] {
3788            binding.flow.acquire_tagged(401, false).await.unwrap();
3789            sink.try_send(route_frame(
3790                FrameType::Response,
3791                routes[1].client_channel,
3792                routes[1].client_epoch,
3793                402,
3794            ))
3795            .unwrap();
3796            router
3797                .route_for_connection(
3798                    &ctx,
3799                    Frame::build(
3800                        ty,
3801                        Flags::new(false, Priority::Interactive, false),
3802                        routes[0].module_channel,
3803                        routes[0].module_epoch,
3804                        401,
3805                        vec![b's'; limit - subc_protocol::HEADER_LEN + 1],
3806                    )
3807                    .unwrap(),
3808                )
3809                .await
3810                .unwrap();
3811            assert!(close.try_recv().is_err());
3812            assert_eq!(rx.try_recv().unwrap().header.corr, 402);
3813            let error = rx
3814                .try_recv()
3815                .expect("every correlated oversize frame becomes ERROR");
3816            assert_frame_size_error(
3817                &error,
3818                routes[0].client_channel,
3819                routes[0].client_epoch,
3820                401,
3821                limit + 1,
3822            );
3823            assert_eq!(binding.flow.drain_in_flight(), 0, "{ty:?} ends the request");
3824            assert!(rx.try_recv().is_err());
3825        }
3826    }
3827
3828    /// An awaited send (a control or error reply) is never refused by the byte
3829    /// budget: it parks until the writer frees bytes. If a release could miss a
3830    /// parked sender, that reply would hang for the connection's lifetime, so
3831    /// this pins the wakeup: the send stays parked while the queue is full and
3832    /// completes as soon as one frame leaves it.
3833    #[tokio::test]
3834    async fn awaited_send_parked_behind_a_full_byte_budget_wakes_when_bytes_free() {
3835        let (tx, mut rx) = mpsc::channel(1024);
3836        let sink = FrameSink::with_byte_budget(tx, 2_000);
3837        let mut queued = 0u64;
3838        while sink
3839            .try_send(stream_frame(7, 1, queued, vec![b'x'; 200]))
3840            .is_ok()
3841        {
3842            queued += 1;
3843        }
3844        assert!(queued > 0, "the budget admitted nothing");
3845
3846        let parked = tokio::spawn({
3847            let sink = sink.clone();
3848            async move { sink.send(stream_frame(0, 0, 999, vec![b'r'; 200])).await }
3849        });
3850        tokio::time::sleep(Duration::from_millis(100)).await;
3851        assert!(
3852            !parked.is_finished(),
3853            "the awaited send must wait while the byte budget is full"
3854        );
3855
3856        // The writer takes one frame and drops it, releasing its bytes.
3857        drop(rx.recv().await.expect("a queued frame"));
3858        tokio::time::timeout(Duration::from_secs(2), parked)
3859            .await
3860            .expect("the parked send was never woken after bytes were freed")
3861            .unwrap()
3862            .unwrap();
3863    }
3864
3865    /// A client multiplexing several token streams pauses its reader while the
3866    /// module keeps producing small frames. Far more frames than the old
3867    /// 64-frame queue allowed, but far fewer bytes than the budget, must all be
3868    /// held without closing the connection and delivered in order afterwards.
3869    #[tokio::test]
3870    async fn paused_client_reader_keeps_connection_through_small_frame_burst() {
3871        const ROUTES: usize = 4;
3872        const FRAMES_PER_ROUTE: usize = 250;
3873        let (router, client_sink, mut client_rx, module_ctx, routes, mut close_receiver) =
3874            multi_route_client(
3875                "burst-provider",
3876                ConnectionId::new(60),
3877                ConnectionId::new(61),
3878                ROUTES,
3879            )
3880            .await;
3881
3882        // The reader is paused: nothing is received until every frame is sent.
3883        for seq in 0..FRAMES_PER_ROUTE as u64 {
3884            for (index, route) in routes.iter().enumerate() {
3885                let body = format!("route-{index}-token-{seq:05}-{}", "t".repeat(170));
3886                router
3887                    .route_for_connection(
3888                        &module_ctx,
3889                        stream_frame(
3890                            route.module_channel,
3891                            route.module_epoch,
3892                            seq,
3893                            body.into_bytes(),
3894                        ),
3895                    )
3896                    .await
3897                    .unwrap();
3898            }
3899        }
3900
3901        let backlog = client_sink.backlog();
3902        assert_eq!(backlog.queued_frames, ROUTES * FRAMES_PER_ROUTE);
3903        assert!(backlog.queued_bytes < crate::server::CONNECTION_EGRESS_BYTE_BUDGET);
3904        assert!(
3905            close_receiver.try_recv().is_err(),
3906            "a paused reader under the byte budget must not be closed"
3907        );
3908        assert_eq!(
3909            router.counters.snapshot()["client_egress_close_delivery_failed"],
3910            0
3911        );
3912
3913        // The reader resumes: every frame arrives, in order within each route.
3914        let mut next_seq = vec![0u64; ROUTES];
3915        for _ in 0..ROUTES * FRAMES_PER_ROUTE {
3916            let frame = client_rx.try_recv().expect("every queued frame arrives");
3917            let index = routes
3918                .iter()
3919                .position(|route| route.client_channel == frame.header.channel)
3920                .expect("frame arrives on one of the bound client channels");
3921            assert_eq!(frame.header.corr, next_seq[index], "route {index} order");
3922            let expected_prefix = format!("route-{index}-token-{:05}-", next_seq[index]);
3923            assert!(frame.body.starts_with(expected_prefix.as_bytes()));
3924            next_seq[index] += 1;
3925        }
3926        assert!(client_rx.try_recv().is_err());
3927        assert_eq!(next_seq, vec![FRAMES_PER_ROUTE as u64; ROUTES]);
3928        assert_eq!(client_sink.backlog().queued_bytes, 0);
3929    }
3930
3931    /// A client that never reads exhausts its byte budget and the oldest
3932    /// frame's 250 ms grace, then closes with one header-only diagnosis.
3933    #[tokio::test(start_paused = true)]
3934    async fn never_reading_client_is_closed_at_byte_budget_with_warn_diagnosis() {
3935        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3936        const BODY: usize = 16 * 1024;
3937        let (router, client_sink, _client_rx, module_ctx, routes, mut close_receiver) =
3938            multi_route_client(
3939                "stuck-reader-provider",
3940                ConnectionId::new(70),
3941                ConnectionId::new(71),
3942                2,
3943            )
3944            .await;
3945
3946        let mut admitted = 0usize;
3947        let mut sent = 0u64;
3948        let started = tokio::time::Instant::now();
3949        while router.counters.snapshot()["client_egress_close_delivery_failed"] == 0 {
3950            assert!(sent < 1_000, "the byte budget never refused a frame");
3951            let route = &routes[(sent % 2) as usize];
3952            router
3953                .route_for_connection(
3954                    &module_ctx,
3955                    stream_frame(
3956                        route.module_channel,
3957                        route.module_epoch,
3958                        sent,
3959                        vec![b'z'; BODY],
3960                    ),
3961                )
3962                .await
3963                .unwrap();
3964            sent += 1;
3965            admitted = client_sink.backlog().queued_frames;
3966        }
3967        assert_eq!(
3968            started.elapsed(),
3969            crate::server::CONNECTION_EGRESS_STALL_GRACE
3970        );
3971        // The budget, not the frame-count backstop, did the refusing.
3972        let frame_bytes = subc_protocol::HEADER_LEN + BODY;
3973        assert_eq!(
3974            admitted,
3975            crate::server::CONNECTION_EGRESS_BYTE_BUDGET / frame_bytes
3976        );
3977        let reason = close_receiver
3978            .try_recv()
3979            .expect("the client connection must be asked to close");
3980        assert!(reason
3981            .to_string()
3982            .contains("module_to_client_delivery_failed"));
3983
3984        // A second refused frame for the same connection adds no second WARN.
3985        router
3986            .route_for_connection(
3987                &module_ctx,
3988                stream_frame(
3989                    routes[0].module_channel,
3990                    routes[0].module_epoch,
3991                    sent,
3992                    vec![b'z'; BODY],
3993                ),
3994            )
3995            .await
3996            .unwrap();
3997
3998        let captured = test_log::captured_logs(&logs);
3999        let warn_lines = captured
4000            .lines()
4001            .filter(|line| {
4002                line.contains("closing client connection: its egress queue could not take a frame")
4003            })
4004            .collect::<Vec<_>>();
4005        assert_eq!(warn_lines.len(), 1, "exactly one WARN, got: {captured}");
4006        let line = warn_lines[0];
4007        assert!(line.contains("WARN"), "{line}");
4008        assert!(line.contains("connection_id=71"), "{line}");
4009        assert!(
4010            line.contains("module_id=\"stuck-reader-provider\""),
4011            "{line}"
4012        );
4013        assert!(line.contains("client_channel="), "{line}");
4014        assert!(line.contains("principals=direct"), "{line}");
4015        let queued_bytes: usize = line
4016            .split("queued_bytes=")
4017            .nth(1)
4018            .and_then(|rest| rest.split_whitespace().next())
4019            .and_then(|value| value.parse().ok())
4020            .expect("queued_bytes is logged");
4021        assert_eq!(queued_bytes, admitted * frame_bytes);
4022        assert!(
4023            line.contains(&format!("queued_frames={admitted}")),
4024            "{line}"
4025        );
4026        assert!(line.contains("oldest_queued_ms="), "{line}");
4027        assert!(line.contains("oldest_queued_ms=250"), "{line}");
4028        assert!(line.contains("frame_type=Some(StreamData)"), "{line}");
4029        assert!(
4030            line.contains(&format!(
4031                "frame_flags=Some({})",
4032                Flags::new(false, Priority::Interactive, false).0
4033            )),
4034            "{line}"
4035        );
4036        assert!(
4037            line.contains("close_cause=\"waited_250_ms_without_room\""),
4038            "{line}"
4039        );
4040        assert!(
4041            !line.contains(&"z".repeat(20)),
4042            "body bytes must never be logged: {line}"
4043        );
4044    }
4045
4046    fn route_frame(ty: FrameType, channel: u16, epoch: u32, corr: u64) -> Frame {
4047        Frame::build(
4048            ty,
4049            Flags::new(false, Priority::Interactive, false),
4050            channel,
4051            epoch,
4052            corr,
4053            if ty == FrameType::Request || ty == FrameType::Response {
4054                b"route-body".to_vec()
4055            } else {
4056                Vec::new()
4057            },
4058        )
4059        .unwrap()
4060    }
4061
4062    type DynamicRouteFixture = (
4063        Router,
4064        Arc<ForwardingTable>,
4065        RouteCtx,
4066        mpsc::Receiver<crate::router::OutboundFrame>,
4067        RouteCtx,
4068        mpsc::Receiver<crate::router::OutboundFrame>,
4069        mpsc::Receiver<crate::router::OutboundFrame>,
4070        crate::forwarding::PendingRouteBindRelay,
4071    );
4072
4073    fn dynamic_route_fixture(commit: bool) -> DynamicRouteFixture {
4074        dynamic_route_fixture_with_busy(commit, false)
4075    }
4076
4077    fn dynamic_route_fixture_with_busy(commit: bool, busy_refusals: bool) -> DynamicRouteFixture {
4078        let forwarding = Arc::new(ForwardingTable::default());
4079        let control = Arc::new(crate::ControlHandler::with_forwarding(
4080            Arc::new(crate::Registry::default()),
4081            Arc::clone(&forwarding),
4082        ));
4083        let router = Router::with_control_handler(control);
4084        let module_connection = ConnectionId::new(500);
4085        let client_connection = ConnectionId::new(501);
4086        let (module_tx, module_rx) = mpsc::channel(if busy_refusals { 64 } else { 8 });
4087        forwarding
4088            .register_module_connection(
4089                module_connection,
4090                "epoch-router".into(),
4091                2,
4092                Concurrency::ModuleManaged,
4093                FrameSink::new(module_tx),
4094            )
4095            .unwrap();
4096        let (client_tx, client_rx) = mpsc::channel(if busy_refusals { 64 } else { 8 });
4097        let client_sink = FrameSink::new(client_tx);
4098        let pending = forwarding
4099            .begin_route_bind_relay_with_busy_for_test(
4100                client_connection,
4101                client_sink.clone(),
4102                700,
4103                "epoch-router",
4104                busy_refusals,
4105            )
4106            .unwrap();
4107        if commit {
4108            forwarding
4109                .complete_pending_relay(
4110                    module_connection,
4111                    pending.corr,
4112                    RouteBindRelayOutcome::Accepted,
4113                )
4114                .unwrap();
4115        }
4116        let (module_egress_tx, module_egress_rx) = mpsc::channel(8);
4117        (
4118            router,
4119            forwarding,
4120            RouteCtx {
4121                connection_id: client_connection,
4122                egress: client_sink,
4123            },
4124            client_rx,
4125            RouteCtx {
4126                connection_id: module_connection,
4127                egress: FrameSink::new(module_egress_tx),
4128            },
4129            module_egress_rx,
4130            module_rx,
4131            pending,
4132        )
4133    }
4134
4135    #[tokio::test]
4136    async fn route_epochs_validate_both_directions_and_rewrite_to_peer_handle() {
4137        let (
4138            router,
4139            _forwarding,
4140            client_ctx,
4141            mut client_rx,
4142            module_ctx,
4143            _module_egress_rx,
4144            mut module_rx,
4145            pending,
4146        ) = dynamic_route_fixture(true);
4147        let route_open = client_rx.recv().await.unwrap();
4148        assert_eq!(route_open.header.corr, 700);
4149
4150        router
4151            .route_for_connection(
4152                &client_ctx,
4153                route_frame(
4154                    FrameType::Request,
4155                    pending.client_channel,
4156                    pending.client_epoch,
4157                    701,
4158                ),
4159            )
4160            .await
4161            .unwrap();
4162        let forwarded = module_rx.recv().await.unwrap();
4163        assert_eq!(forwarded.header.channel, pending.module_channel);
4164        assert_eq!(forwarded.header.epoch, pending.module_epoch);
4165
4166        router
4167            .route_for_connection(
4168                &module_ctx,
4169                route_frame(
4170                    FrameType::Response,
4171                    pending.module_channel,
4172                    pending.module_epoch,
4173                    701,
4174                ),
4175            )
4176            .await
4177            .unwrap();
4178        let delivered = client_rx.recv().await.unwrap();
4179        assert_eq!(delivered.header.channel, pending.client_channel);
4180        assert_eq!(delivered.header.epoch, pending.client_epoch);
4181
4182        router
4183            .route_for_connection(
4184                &client_ctx,
4185                route_frame(
4186                    FrameType::Request,
4187                    pending.client_channel,
4188                    pending.client_epoch + 1,
4189                    702,
4190                ),
4191            )
4192            .await
4193            .unwrap();
4194        router
4195            .route_for_connection(
4196                &module_ctx,
4197                route_frame(
4198                    FrameType::Response,
4199                    pending.module_channel,
4200                    pending.module_epoch + 1,
4201                    703,
4202                ),
4203            )
4204            .await
4205            .unwrap();
4206        let stale_error = client_rx.recv().await.unwrap();
4207        assert_eq!(stale_error.header.ty, FrameType::Error);
4208        assert_eq!(stale_error.header.channel, pending.client_channel);
4209        assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
4210        assert_eq!(stale_error.header.corr, 702);
4211        let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
4212        assert_eq!(body.code, "stale_route_epoch");
4213        assert!(module_rx.try_recv().is_err());
4214        assert!(client_rx.try_recv().is_err());
4215        let counters = router.counters.snapshot();
4216        assert_eq!(counters["client_frames_dropped_stale_route"], 1);
4217        assert_eq!(counters["module_frames_dropped_no_route"], 1);
4218    }
4219
4220    #[tokio::test]
4221    async fn accepted_route_publishes_route_open_before_immediate_reverse_request() {
4222        let (
4223            router,
4224            _,
4225            _client_ctx,
4226            mut client_rx,
4227            module_ctx,
4228            _module_egress_rx,
4229            _module_rx,
4230            pending,
4231        ) = dynamic_route_fixture(true);
4232        router
4233            .route_for_connection(
4234                &module_ctx,
4235                route_frame(
4236                    FrameType::Request,
4237                    pending.module_channel,
4238                    pending.module_epoch,
4239                    800,
4240                ),
4241            )
4242            .await
4243            .unwrap();
4244
4245        let first = client_rx.recv().await.unwrap();
4246        let second = client_rx.recv().await.unwrap();
4247        assert_eq!(first.header.channel, 0);
4248        assert_eq!(first.header.corr, 700);
4249        assert_eq!(second.header.channel, pending.client_channel);
4250        assert_eq!(second.header.epoch, pending.client_epoch);
4251        assert_eq!(second.header.corr, 800);
4252    }
4253
4254    #[tokio::test]
4255    async fn reserved_slot_ingress_errors_only_matching_client_requests() {
4256        let (
4257            router,
4258            _forwarding,
4259            client_ctx,
4260            mut client_rx,
4261            _module_ctx,
4262            _module_egress_rx,
4263            mut module_rx,
4264            pending,
4265        ) = dynamic_route_fixture(false);
4266        router
4267            .route_for_connection(
4268                &client_ctx,
4269                route_frame(
4270                    FrameType::Request,
4271                    pending.client_channel,
4272                    pending.client_epoch,
4273                    900,
4274                ),
4275            )
4276            .await
4277            .unwrap();
4278        let error = client_rx.recv().await.unwrap();
4279        assert_eq!(error.header.ty, FrameType::Error);
4280        assert_eq!(error.header.channel, pending.client_channel);
4281        assert_eq!(error.header.epoch, pending.client_epoch);
4282        assert_eq!(error.header.corr, 900);
4283
4284        router
4285            .route_for_connection(
4286                &client_ctx,
4287                route_frame(
4288                    FrameType::Response,
4289                    pending.client_channel,
4290                    pending.client_epoch,
4291                    901,
4292                ),
4293            )
4294            .await
4295            .unwrap();
4296        router
4297            .route_for_connection(
4298                &client_ctx,
4299                route_frame(
4300                    FrameType::Request,
4301                    pending.client_channel,
4302                    pending.client_epoch + 1,
4303                    902,
4304                ),
4305            )
4306            .await
4307            .unwrap();
4308        let stale_error = client_rx.recv().await.unwrap();
4309        assert_eq!(stale_error.header.ty, FrameType::Error);
4310        assert_eq!(stale_error.header.channel, pending.client_channel);
4311        assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
4312        assert_eq!(stale_error.header.corr, 902);
4313        let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
4314        assert_eq!(body.code, "stale_route_epoch");
4315        assert!(module_rx.try_recv().is_err());
4316        let counters = router.counters.snapshot();
4317        assert_eq!(counters["client_frames_dropped_stale_route"], 1);
4318        assert_eq!(counters["module_frames_dropped_no_route"], 0);
4319    }
4320
4321    #[tokio::test]
4322    async fn dropped_module_route_goodbye_increments_counter() {
4323        let (
4324            router,
4325            _forwarding,
4326            client_ctx,
4327            mut client_rx,
4328            _module_ctx,
4329            _module_egress_rx,
4330            mut module_rx,
4331            pending,
4332        ) = dynamic_route_fixture(true);
4333        let _ = client_rx.recv().await;
4334        module_rx.close();
4335
4336        router
4337            .route_for_connection(
4338                &client_ctx,
4339                route_frame(
4340                    FrameType::Goodbye,
4341                    pending.client_channel,
4342                    pending.client_epoch,
4343                    999,
4344                ),
4345            )
4346            .await
4347            .unwrap();
4348
4349        let counters = router.counters.snapshot();
4350        assert_eq!(counters["goodbye_relay_module_dropped"], 1);
4351        assert_eq!(
4352            counters["goodbye_relay_module_dropped_by_module"],
4353            serde_json::json!({ "epoch-router": 1 })
4354        );
4355        assert_eq!(counters["route_released_epoch_fenced"], 1);
4356    }
4357
4358    #[tokio::test]
4359    async fn module_request_on_stale_epoch_receives_stale_route_epoch() {
4360        let (
4361            router,
4362            _forwarding,
4363            _client_ctx,
4364            _client_rx,
4365            module_ctx,
4366            mut module_egress_rx,
4367            mut module_rx,
4368            pending,
4369        ) = dynamic_route_fixture(true);
4370
4371        router
4372            .route_for_connection(
4373                &module_ctx,
4374                route_frame(
4375                    FrameType::Request,
4376                    pending.module_channel,
4377                    pending.module_epoch + 1,
4378                    1_000,
4379                ),
4380            )
4381            .await
4382            .unwrap();
4383
4384        let error = module_egress_rx.try_recv().unwrap();
4385        assert_eq!(error.header.ty, FrameType::Error);
4386        assert_eq!(error.header.channel, pending.module_channel);
4387        assert_eq!(error.header.epoch, pending.module_epoch + 1);
4388        assert_eq!(error.header.corr, 1_000);
4389        let body: ErrorBody = serde_json::from_slice(&error.body).unwrap();
4390        assert_eq!(body.code, "stale_route_epoch");
4391        assert!(module_rx.try_recv().is_err());
4392        let counters = router.counters.snapshot();
4393        assert_eq!(counters["module_requests_dropped_stale_route"], 1);
4394        assert_eq!(counters["module_frames_dropped_no_route"], 0);
4395    }
4396
4397    #[tokio::test(start_paused = true)]
4398    async fn stale_epoch_drop_warns_once_per_minute_with_suppressed_count() {
4399        let (output, _guard) = test_log::log_capture(tracing::Level::WARN);
4400        let (
4401            router,
4402            _forwarding,
4403            client_ctx,
4404            mut client_rx,
4405            _module_ctx,
4406            _module_egress_rx,
4407            mut module_rx,
4408            pending,
4409        ) = dynamic_route_fixture(true);
4410        let _ = client_rx.recv().await;
4411        for corr in 1..=3 {
4412            router
4413                .route_for_connection(
4414                    &client_ctx,
4415                    route_frame(
4416                        FrameType::Request,
4417                        pending.client_channel,
4418                        pending.client_epoch + 1,
4419                        corr,
4420                    ),
4421                )
4422                .await
4423                .unwrap();
4424            assert_eq!(client_rx.recv().await.unwrap().header.ty, FrameType::Error);
4425        }
4426        assert!(module_rx.try_recv().is_err());
4427        let logs = test_log::captured_logs(&output);
4428        assert_eq!(
4429            logs.matches("dropping data-plane frame").count(),
4430            1,
4431            "{logs}"
4432        );
4433        assert!(logs.contains("WARN"), "{logs}");
4434        assert!(
4435            logs.contains(&format!("connection_id={}", client_ctx.connection_id.get())),
4436            "{logs}"
4437        );
4438        assert!(
4439            logs.contains(&format!("channel={}", pending.client_channel)),
4440            "{logs}"
4441        );
4442        assert!(
4443            logs.contains(&format!("epoch={}", pending.client_epoch + 1)),
4444            "{logs}"
4445        );
4446        assert!(
4447            logs.contains("client_to_module") && logs.contains("stale_epoch"),
4448            "{logs}"
4449        );
4450        assert!(logs.contains("suppressed=0"), "{logs}");
4451
4452        tokio::time::advance(Duration::from_secs(60)).await;
4453        router
4454            .route_for_connection(
4455                &client_ctx,
4456                route_frame(
4457                    FrameType::Request,
4458                    pending.client_channel,
4459                    pending.client_epoch + 2,
4460                    4,
4461                ),
4462            )
4463            .await
4464            .unwrap();
4465        let logs = test_log::captured_logs(&output);
4466        assert_eq!(
4467            logs.matches("dropping data-plane frame").count(),
4468            2,
4469            "{logs}"
4470        );
4471        assert!(logs.contains("suppressed=2"), "{logs}");
4472    }
4473
4474    #[tokio::test]
4475    async fn module_request_on_reserved_or_absent_route_receives_unknown_channel() {
4476        let (
4477            reserved_router,
4478            _forwarding,
4479            _client_ctx,
4480            _client_rx,
4481            reserved_module_ctx,
4482            mut reserved_module_egress_rx,
4483            _module_rx,
4484            reserved,
4485        ) = dynamic_route_fixture(false);
4486        reserved_router
4487            .route_for_connection(
4488                &reserved_module_ctx,
4489                route_frame(
4490                    FrameType::Request,
4491                    reserved.module_channel,
4492                    reserved.module_epoch,
4493                    1_001,
4494                ),
4495            )
4496            .await
4497            .unwrap();
4498        let reserved_error = reserved_module_egress_rx.try_recv().unwrap();
4499        let reserved_body: ErrorBody = serde_json::from_slice(&reserved_error.body).unwrap();
4500        assert_eq!(reserved_error.header.ty, FrameType::Error);
4501        assert_eq!(reserved_error.header.channel, reserved.module_channel);
4502        assert_eq!(reserved_error.header.epoch, reserved.module_epoch);
4503        assert_eq!(reserved_error.header.corr, 1_001);
4504        assert_eq!(reserved_body.code, "unknown_channel");
4505        assert_eq!(
4506            reserved_router.counters.snapshot()["module_requests_dropped_stale_route"],
4507            1
4508        );
4509
4510        let (
4511            absent_router,
4512            _forwarding,
4513            _client_ctx,
4514            _client_rx,
4515            absent_module_ctx,
4516            mut absent_module_egress_rx,
4517            _module_rx,
4518            absent,
4519        ) = dynamic_route_fixture(false);
4520        absent_router
4521            .route_for_connection(
4522                &absent_module_ctx,
4523                route_frame(
4524                    FrameType::Request,
4525                    absent.module_channel + 1,
4526                    absent.module_epoch,
4527                    1_002,
4528                ),
4529            )
4530            .await
4531            .unwrap();
4532        let absent_error = absent_module_egress_rx.try_recv().unwrap();
4533        let absent_body: ErrorBody = serde_json::from_slice(&absent_error.body).unwrap();
4534        assert_eq!(absent_error.header.ty, FrameType::Error);
4535        assert_eq!(absent_error.header.channel, absent.module_channel + 1);
4536        assert_eq!(absent_error.header.epoch, absent.module_epoch);
4537        assert_eq!(absent_error.header.corr, 1_002);
4538        assert_eq!(absent_body.code, "unknown_channel");
4539        assert_eq!(
4540            absent_router.counters.snapshot()["module_requests_dropped_stale_route"],
4541            1
4542        );
4543    }
4544
4545    #[tokio::test]
4546    async fn non_request_module_frame_on_dead_route_is_counted_without_error() {
4547        let (
4548            router,
4549            forwarding,
4550            client_ctx,
4551            mut client_rx,
4552            module_ctx,
4553            mut module_egress_rx,
4554            mut module_rx,
4555            pending,
4556        ) = dynamic_route_fixture(true);
4557        let (other_module_tx, _other_module_rx) = mpsc::channel(8);
4558        forwarding
4559            .register_module_connection(
4560                ConnectionId::new(502),
4561                "other-module".into(),
4562                2,
4563                Concurrency::ModuleManaged,
4564                FrameSink::new(other_module_tx),
4565            )
4566            .unwrap();
4567        let _ = client_rx.recv().await.unwrap();
4568
4569        router
4570            .route_for_connection(
4571                &client_ctx,
4572                route_frame(
4573                    FrameType::Goodbye,
4574                    pending.client_channel,
4575                    pending.client_epoch,
4576                    1_003,
4577                ),
4578            )
4579            .await
4580            .unwrap();
4581        let _ = module_rx.recv().await.unwrap();
4582
4583        router
4584            .route_for_connection(
4585                &module_ctx,
4586                route_frame(
4587                    FrameType::StreamData,
4588                    pending.module_channel,
4589                    pending.module_epoch,
4590                    1_004,
4591                ),
4592            )
4593            .await
4594            .unwrap();
4595
4596        // No ERROR goes back for a non-request frame. The one reply is the
4597        // route GOODBYE telling the module to let go of the released route.
4598        let reply = module_egress_rx.try_recv().unwrap();
4599        assert_eq!(reply.header.ty, FrameType::Goodbye);
4600        assert!(module_egress_rx.try_recv().is_err());
4601        let counters = router.counters.snapshot();
4602        assert_eq!(counters["module_frames_dropped_no_route"], 1);
4603        assert_eq!(
4604            counters["module_frames_dropped_no_route_by_module"],
4605            serde_json::json!({ "epoch-router": 1 })
4606        );
4607        assert_eq!(counters["module_requests_dropped_stale_route"], 0);
4608    }
4609
4610    /// Drain whatever the module connection's queue holds right now, the way a
4611    /// module's reader would before it stalls.
4612    fn drain_now(rx: &mut mpsc::Receiver<OutboundFrame>) {
4613        while rx.try_recv().is_ok() {}
4614    }
4615
4616    /// A module that stops reading for a moment when a client closes one of
4617    /// its routes still learns the route is gone: the GOODBYE its full egress
4618    /// queue refused is delivered as soon as it reads again, rather than
4619    /// dropped, which would leave the module holding the route for the rest of
4620    /// its connection.
4621    #[tokio::test]
4622    async fn route_goodbye_refused_by_stalled_module_is_delivered_when_it_resumes_reading() {
4623        const BUDGET: usize = 4_096;
4624        let forwarding = Arc::new(ForwardingTable::default());
4625        let control = Arc::new(ControlHandler::with_forwarding(
4626            Arc::new(Registry::default()),
4627            Arc::clone(&forwarding),
4628        ));
4629        let router = Router::with_control_handler(control);
4630        let module_connection = ConnectionId::new(80);
4631        let client_connection = ConnectionId::new(81);
4632        let (module_tx, mut module_rx) = mpsc::channel(64);
4633        let module_sink = FrameSink::with_byte_budget(module_tx, BUDGET);
4634        forwarding
4635            .register_module_connection(
4636                module_connection,
4637                "stalled-provider".into(),
4638                2,
4639                Concurrency::ModuleManaged,
4640                module_sink.clone(),
4641            )
4642            .unwrap();
4643        let (client_tx, mut client_rx) = mpsc::channel(8);
4644        let client_sink = FrameSink::new(client_tx);
4645        let pending = forwarding
4646            .begin_route_bind_relay_for_test(
4647                client_connection,
4648                client_sink.clone(),
4649                1_100,
4650                "stalled-provider",
4651            )
4652            .unwrap();
4653        forwarding
4654            .complete_pending_relay(
4655                module_connection,
4656                pending.corr,
4657                RouteBindRelayOutcome::Accepted,
4658            )
4659            .unwrap();
4660        let _ = client_rx.recv().await.unwrap();
4661        drain_now(&mut module_rx);
4662
4663        // The module stalls: its queue holds more than the byte budget, so the
4664        // queue refuses anything further.
4665        module_sink
4666            .try_send(stream_frame(9, 1, 0, vec![b'f'; BUDGET]))
4667            .unwrap();
4668        assert!(module_sink
4669            .try_send(stream_frame(9, 1, 1, Vec::new()))
4670            .is_err());
4671
4672        let client_ctx = RouteCtx {
4673            connection_id: client_connection,
4674            egress: client_sink,
4675        };
4676        router
4677            .route_for_connection(
4678                &client_ctx,
4679                route_frame(
4680                    FrameType::Goodbye,
4681                    pending.client_channel,
4682                    pending.client_epoch,
4683                    1_101,
4684                ),
4685            )
4686            .await
4687            .unwrap();
4688        tokio::task::yield_now().await;
4689        assert_eq!(
4690            router.counters.snapshot()["goodbye_relay_module_dropped"],
4691            0,
4692            "a GOODBYE refused by a momentarily full module queue must not be dropped"
4693        );
4694
4695        // The module reads again: the filler comes off, then the GOODBYE.
4696        let filler = module_rx.recv().await.unwrap();
4697        assert_eq!(filler.header.ty, FrameType::StreamData);
4698        drop(filler);
4699        let goodbye = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
4700            .await
4701            .expect("the refused GOODBYE must be delivered once the module frees room")
4702            .unwrap();
4703        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
4704        assert_eq!(goodbye.header.channel, pending.module_channel);
4705        assert_eq!(goodbye.header.epoch, pending.module_epoch);
4706        assert_eq!(
4707            router.counters.snapshot()["goodbye_relay_module_dropped"],
4708            0
4709        );
4710    }
4711
4712    /// A module still sending on a route the daemon released is told, with a
4713    /// route GOODBYE for exactly the (channel, epoch) it sent on. The same
4714    /// happens for a stale epoch on a channel whose route moved on and for a
4715    /// channel that never had a route; only the first is counted as traffic on
4716    /// a released route.
4717    #[tokio::test]
4718    async fn module_frame_on_route_the_daemon_does_not_hold_is_answered_with_goodbye() {
4719        let (
4720            router,
4721            _forwarding,
4722            client_ctx,
4723            mut client_rx,
4724            module_ctx,
4725            mut module_egress_rx,
4726            mut module_rx,
4727            pending,
4728        ) = dynamic_route_fixture(true);
4729        let _ = client_rx.recv().await.unwrap();
4730        router
4731            .route_for_connection(
4732                &client_ctx,
4733                route_frame(
4734                    FrameType::Goodbye,
4735                    pending.client_channel,
4736                    pending.client_epoch,
4737                    1_200,
4738                ),
4739            )
4740            .await
4741            .unwrap();
4742        drain_now(&mut module_rx);
4743
4744        let cases = [
4745            // Released route: the daemon allocated this (channel, epoch).
4746            (pending.module_channel, pending.module_epoch),
4747            // A channel the daemon never allocated on this connection.
4748            (pending.module_channel + 1, 1),
4749        ];
4750        for (channel, epoch) in cases {
4751            router
4752                .route_for_connection(
4753                    &module_ctx,
4754                    route_frame(FrameType::StreamData, channel, epoch, 1_201),
4755                )
4756                .await
4757                .unwrap();
4758            let reply = module_egress_rx
4759                .try_recv()
4760                .expect("a frame on a route the daemon does not hold is answered");
4761            assert_eq!(reply.header.ty, FrameType::Goodbye);
4762            assert_eq!(reply.header.channel, channel);
4763            assert_eq!(reply.header.epoch, epoch);
4764            assert_eq!(reply.header.corr, 0);
4765            assert!(module_egress_rx.try_recv().is_err());
4766        }
4767
4768        // A GOODBYE from the module for a route the daemon already released
4769        // is not answered: the module is letting go already.
4770        router
4771            .route_for_connection(
4772                &module_ctx,
4773                route_frame(FrameType::Goodbye, pending.module_channel + 2, 1, 0),
4774            )
4775            .await
4776            .unwrap();
4777        assert!(module_egress_rx.try_recv().is_err());
4778
4779        let counters = router.counters.snapshot();
4780        assert_eq!(counters["module_frames_dropped_no_route"], 3);
4781        assert_eq!(counters["module_frames_dropped_released_route"], 1);
4782        assert_eq!(
4783            counters["module_frames_dropped_released_route_by_module"],
4784            serde_json::json!({ "epoch-router": 1 })
4785        );
4786        assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4787    }
4788
4789    /// A chatty orphan (a streaming module still producing on a route it
4790    /// missed the GOODBYE for) is answered once per interval, not once per
4791    /// frame, and answered again once the interval has passed.
4792    #[tokio::test(start_paused = true)]
4793    async fn burst_of_orphan_module_frames_is_answered_once_per_interval() {
4794        let (
4795            router,
4796            _forwarding,
4797            client_ctx,
4798            mut client_rx,
4799            module_ctx,
4800            mut module_egress_rx,
4801            mut module_rx,
4802            pending,
4803        ) = dynamic_route_fixture(true);
4804        let _ = client_rx.recv().await.unwrap();
4805        router
4806            .route_for_connection(
4807                &client_ctx,
4808                route_frame(
4809                    FrameType::Goodbye,
4810                    pending.client_channel,
4811                    pending.client_epoch,
4812                    1_300,
4813                ),
4814            )
4815            .await
4816            .unwrap();
4817        drain_now(&mut module_rx);
4818
4819        let send_burst = |corr: u64| {
4820            route_frame(
4821                FrameType::StreamData,
4822                pending.module_channel,
4823                pending.module_epoch,
4824                corr,
4825            )
4826        };
4827        for corr in 0..20 {
4828            router
4829                .route_for_connection(&module_ctx, send_burst(corr))
4830                .await
4831                .unwrap();
4832        }
4833        let mut replies = 0;
4834        while let Ok(reply) = module_egress_rx.try_recv() {
4835            assert_eq!(reply.header.ty, FrameType::Goodbye);
4836            replies += 1;
4837        }
4838        assert_eq!(replies, 1, "a burst within the interval gets one GOODBYE");
4839
4840        tokio::time::advance(ORPHAN_ROUTE_GOODBYE_INTERVAL).await;
4841        router
4842            .route_for_connection(&module_ctx, send_burst(20))
4843            .await
4844            .unwrap();
4845        let retry = module_egress_rx
4846            .try_recv()
4847            .expect("the first orphan frame after the interval is answered again");
4848        assert_eq!(retry.header.channel, pending.module_channel);
4849        assert_eq!(retry.header.epoch, pending.module_epoch);
4850
4851        let counters = router.counters.snapshot();
4852        assert_eq!(counters["module_frames_dropped_no_route"], 21);
4853        assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4854    }
4855
4856    /// The rate-limit memory is per connection and goes away with it.
4857    #[test]
4858    fn orphan_goodbye_rate_limit_state_is_released_with_the_connection() {
4859        let router = Router::with_default_self_handler();
4860        let connection = router.begin_connection();
4861        let id = connection.id();
4862        assert!(router.orphan_goodbyes.claim(id, 7));
4863        assert!(!router.orphan_goodbyes.claim(id, 7));
4864        assert!(router.orphan_goodbyes.claim(id, 8));
4865        drop(connection);
4866        assert!(router
4867            .orphan_goodbyes
4868            .last_sent
4869            .lock()
4870            .unwrap()
4871            .get(&id)
4872            .is_none());
4873    }
4874
4875    #[test]
4876    fn channel_zero_cannot_be_registered_as_backend() {
4877        let mut router = Router::with_default_self_handler();
4878
4879        let err = router.register_backend(0, EchoBackend).unwrap_err();
4880
4881        assert_eq!(err, RouterError::ReservedChannelZero);
4882    }
4883}