Skip to main content

subc_daemon/
router.rs

1use std::{
2    collections::HashMap,
3    error::Error,
4    fmt,
5    sync::{
6        atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
7        Arc, Mutex,
8    },
9    time::{Duration, Instant},
10};
11
12use subc_protocol::{error_codes, ErrorBody, Flags, FrameType, Priority};
13use tokio::sync::{mpsc, Notify};
14use tracing::{debug, warn};
15
16use crate::{
17    control::ControlHandler,
18    forwarding::{
19        principal_label, CloseReason, ConnectionCloseReceiver, DataRoute, DataRouteState,
20        ForwardingError, ForwardingTable, RouteBinding, RouteRelease, UndeliveredFrame,
21    },
22    registry::ConnectionId,
23    DaemonCounters, Frame, FrameBuildError,
24};
25
26/// One queued outbound frame plus the instant it entered the writer queue.
27///
28/// The stamp exists for the reply-path half of slow-control diagnosis: a
29/// handler can finish in microseconds while the reply sits in this queue
30/// waiting for the writer task to be scheduled, and without a per-item stamp
31/// that wait is invisible to every other timing point (the client's round
32/// trip is the only witness, and it cannot say which side ate the time).
33/// Constructed exclusively inside [`FrameSink`] so no caller can forget it.
34#[derive(Debug)]
35pub struct OutboundFrame {
36    pub frame: Frame,
37    pub enqueued_at: std::time::Instant,
38    pub(crate) flushed: Option<tokio::sync::oneshot::Sender<()>>,
39    /// This frame's share of the connection's queued-byte count. Dropping the
40    /// frame (after the writer has written it, or when the queue itself is
41    /// dropped with the frame still in it) gives the bytes back. `None` only for
42    /// frames built outside a [`FrameSink`], which exist in tests alone.
43    pub(crate) charge: Option<EgressCharge>,
44}
45
46impl OutboundFrame {
47    fn charged(frame: Frame, charge: EgressCharge) -> Self {
48        Self {
49            frame,
50            enqueued_at: charge.enqueued_at,
51            flushed: None,
52            charge: Some(charge),
53        }
54    }
55}
56
57/// Bytes a frame occupies in the connection's egress queue for budget
58/// purposes: the fixed envelope header plus the body.
59fn queued_frame_bytes(frame: &Frame) -> usize {
60    subc_protocol::HEADER_LEN + frame.body.len()
61}
62
63/// A point-in-time view of one connection's egress queue, for diagnosing why a
64/// frame did not fit.
65#[derive(Debug, Clone, Copy, PartialEq, Eq)]
66pub(crate) struct EgressBacklog {
67    pub queued_bytes: usize,
68    pub queued_frames: usize,
69    /// Now minus the enqueue time of the frame the writer most recently took
70    /// off the queue (the frame it is writing, or has just written), or, if
71    /// the writer has taken nothing since the queue was last empty, of the
72    /// frame that made the queue non-empty. While the writer is blocked on a
73    /// frame this is exactly the oldest frame's age; between frames it can be
74    /// one frame older than the true head. `None` when nothing is queued.
75    pub oldest_age: Option<Duration>,
76}
77
78/// Queued-byte accounting shared by every clone of one connection's
79/// [`FrameSink`] and by every frame that sink has admitted. Its per-frame
80/// accounting uses atomics: this is on the path of every frame a module sends
81/// to a client, so it takes no lock per frame.
82#[derive(Debug)]
83struct EgressAccounting {
84    byte_budget: usize,
85    queued_bytes: AtomicUsize,
86    queued_frames: AtomicUsize,
87    /// Origin for the nanosecond timestamps in `oldest_enqueued_nanos`.
88    time_base: Instant,
89    /// Enqueue time, as nanoseconds after `time_base` plus one, of the frame
90    /// described by [`EgressBacklog::oldest_age`]; 0 means none.
91    oldest_enqueued_nanos: AtomicU64,
92    /// Awaited senders currently parked on `freed`. A release only pays for a
93    /// notification when this is non-zero.
94    waiters: AtomicUsize,
95    /// Woken when a charged frame releases its bytes while a sender waits, so
96    /// the awaited send can re-check for room.
97    freed: Notify,
98    client_delivery_closed: AtomicBool,
99    client_delivery_stopped: Notify,
100}
101
102impl EgressAccounting {
103    fn new(byte_budget: usize) -> Self {
104        Self {
105            byte_budget,
106            queued_bytes: AtomicUsize::new(0),
107            queued_frames: AtomicUsize::new(0),
108            time_base: tokio::time::Instant::now().into_std(),
109            oldest_enqueued_nanos: AtomicU64::new(0),
110            waiters: AtomicUsize::new(0),
111            freed: Notify::new(),
112            client_delivery_closed: AtomicBool::new(false),
113            client_delivery_stopped: Notify::new(),
114        }
115    }
116
117    fn stamp(&self, at: Instant) -> u64 {
118        (at.saturating_duration_since(self.time_base).as_nanos() as u64).saturating_add(1)
119    }
120
121    /// Charge `bytes` if they fit in the budget. A frame larger than the whole
122    /// budget (bodies may be up to 64 MiB) is still admitted into an EMPTY
123    /// queue, since it could otherwise never be sent at all; it simply has the
124    /// queue to itself until it is written.
125    fn try_charge(self: &Arc<Self>, bytes: usize) -> Option<EgressCharge> {
126        // SeqCst pairs with `release`: either this load sees bytes a release
127        // just freed, or that release sees this sender counted in `waiters`.
128        let mut current = self.queued_bytes.load(Ordering::SeqCst);
129        loop {
130            if current != 0 && current.saturating_add(bytes) > self.byte_budget {
131                return None;
132            }
133            match self.queued_bytes.compare_exchange_weak(
134                current,
135                current + bytes,
136                Ordering::SeqCst,
137                Ordering::SeqCst,
138            ) {
139                Ok(_) => return Some(self.record(bytes)),
140                Err(actual) => current = actual,
141            }
142        }
143    }
144
145    /// Charge `bytes` regardless of the budget. Used only for a frame whose
146    /// queue slot was reserved in advance, which must be sendable.
147    fn charge_unconditionally(self: &Arc<Self>, bytes: usize) -> EgressCharge {
148        self.queued_bytes.fetch_add(bytes, Ordering::SeqCst);
149        self.record(bytes)
150    }
151
152    fn record(self: &Arc<Self>, bytes: usize) -> EgressCharge {
153        let enqueued_at = tokio::time::Instant::now().into_std();
154        let stamp = self.stamp(enqueued_at);
155        if self.queued_frames.fetch_add(1, Ordering::AcqRel) == 0 {
156            // This frame made the queue non-empty, so it is the head until the
157            // writer takes something.
158            self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
159        }
160        EgressCharge {
161            accounting: Arc::clone(self),
162            bytes,
163            stamp,
164            enqueued_at,
165        }
166    }
167
168    /// The writer has taken the frame stamped `stamp` off the queue.
169    fn taken_by_writer(&self, stamp: u64) {
170        self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
171    }
172
173    fn release(&self, bytes: usize, stamp: u64) {
174        self.queued_bytes.fetch_sub(bytes, Ordering::SeqCst);
175        if self.queued_frames.fetch_sub(1, Ordering::AcqRel) == 1 {
176            // The queue drained. Clear the marker only if it still names this
177            // frame, so a frame admitted in the meantime keeps its stamp.
178            let _ = self.oldest_enqueued_nanos.compare_exchange(
179                stamp,
180                0,
181                Ordering::AcqRel,
182                Ordering::Acquire,
183            );
184        }
185        if self.waiters.load(Ordering::SeqCst) != 0 {
186            self.freed.notify_waiters();
187        }
188    }
189
190    fn backlog(&self) -> EgressBacklog {
191        let queued_frames = self.queued_frames.load(Ordering::Acquire);
192        let stamp = self.oldest_enqueued_nanos.load(Ordering::Acquire);
193        let oldest_age = (queued_frames != 0 && stamp != 0).then(|| {
194            let enqueued = self.time_base + Duration::from_nanos(stamp - 1);
195            tokio::time::Instant::from_std(enqueued).elapsed()
196        });
197        EgressBacklog {
198            queued_bytes: self.queued_bytes.load(Ordering::Acquire),
199            queued_frames,
200            oldest_age,
201        }
202    }
203}
204
205/// One admitted frame's claim on its connection's egress byte budget,
206/// returned when the frame is dropped.
207#[derive(Debug)]
208pub(crate) struct EgressCharge {
209    accounting: Arc<EgressAccounting>,
210    bytes: usize,
211    stamp: u64,
212    enqueued_at: Instant,
213}
214
215impl EgressCharge {
216    /// Called by the connection writer when it takes this frame off the queue,
217    /// so the backlog's oldest-age figure follows the writer.
218    pub(crate) fn taken_by_writer(&self) {
219        self.accounting.taken_by_writer(self.stamp);
220    }
221}
222
223impl Drop for EgressCharge {
224    fn drop(&mut self) {
225        self.accounting.release(self.bytes, self.stamp);
226    }
227}
228
229/// A queue slot reserved ahead of time (a pending `route.open` holds one until
230/// its module answers). Sending through it never waits and never fails for
231/// lack of room: the slot is already held, and its frame is charged to the
232/// byte count outside the budget check. At most
233/// `MAX_PENDING_ROUTE_OPENS_PER_CONNECTION` such frames exist per connection.
234#[derive(Debug)]
235pub(crate) struct EgressPermit {
236    permit: mpsc::OwnedPermit<OutboundFrame>,
237    accounting: Arc<EgressAccounting>,
238}
239
240impl EgressPermit {
241    /// Enqueue `frame` in the reserved slot. Returns true when the connection's
242    /// writer had already gone away, meaning the frame will never be written.
243    pub(crate) fn send(self, frame: Frame) -> bool {
244        let charge = self
245            .accounting
246            .charge_unconditionally(queued_frame_bytes(&frame));
247        let sender = self.permit.send(OutboundFrame::charged(frame, charge));
248        sender.is_closed()
249    }
250}
251
252/// An `OutboundFrame` is a stamped `Frame`; deref keeps every existing frame
253/// read (headers, bodies, assertions) working on queued items unchanged.
254impl std::ops::Deref for OutboundFrame {
255    type Target = Frame;
256
257    fn deref(&self) -> &Frame {
258        &self.frame
259    }
260}
261
262/// Shared tracing-capture helpers for timing-observability tests across
263/// modules (router dispatch, server reply path). Test-only.
264#[cfg(test)]
265pub(crate) mod test_log {
266    use std::{
267        io::Write,
268        sync::{Arc, Mutex, OnceLock},
269    };
270
271    #[derive(Clone)]
272    struct TestLogWriter(Arc<Mutex<Vec<u8>>>);
273
274    impl Write for TestLogWriter {
275        fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
276            self.0
277                .lock()
278                .expect("test log capture is not poisoned")
279                .extend(buffer);
280            Ok(buffer.len())
281        }
282
283        fn flush(&mut self) -> std::io::Result<()> {
284            Ok(())
285        }
286    }
287
288    pub(crate) fn log_capture(
289        level: tracing::Level,
290    ) -> (Arc<Mutex<Vec<u8>>>, tracing::dispatcher::DefaultGuard) {
291        // tracing-core's single-dispatch fast path registers callsites using
292        // the emitting thread's default subscriber. A parallel test with no
293        // subscriber can therefore cache `never` for a callsite this capture
294        // needs. Keep a second dispatch alive so registration considers all
295        // live subscribers instead. This registry is never installed as a
296        // default: each test still records only into its own thread's writer.
297        static INTEREST_ANCHOR: OnceLock<tracing::Dispatch> = OnceLock::new();
298        INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
299        let output = Arc::new(Mutex::new(Vec::new()));
300        let writer = Arc::clone(&output);
301        let subscriber = tracing_subscriber::fmt()
302            .with_max_level(level)
303            .with_ansi(false)
304            .without_time()
305            .with_target(false)
306            .with_writer(move || TestLogWriter(Arc::clone(&writer)))
307            .finish();
308        let guard = tracing::subscriber::set_default(subscriber);
309        (output, guard)
310    }
311
312    pub(crate) fn captured_logs(output: &Arc<Mutex<Vec<u8>>>) -> String {
313        String::from_utf8(
314            output
315                .lock()
316                .expect("test log capture is not poisoned")
317                .clone(),
318        )
319        .expect("tracing output is UTF-8")
320    }
321
322    #[cfg(test)]
323    mod tests {
324        use super::*;
325
326        fn emit_capture_probe(source: &str) {
327            tracing::error!(source, "parallel capture probe");
328        }
329
330        #[test]
331        fn parallel_callsite_registration_keeps_the_test_capture_enabled() {
332            let (output, _guard) = log_capture(tracing::Level::ERROR);
333            // First register the very same callsite on a thread without a
334            // default subscriber, while the test's capture remains live.
335            std::thread::spawn(|| emit_capture_probe("other-thread"))
336                .join()
337                .expect("unsubscribed thread exits");
338            emit_capture_probe("captured-thread");
339
340            let logs = captured_logs(&output);
341            assert!(
342                logs.contains("captured-thread"),
343                "capture was disabled: {logs}"
344            );
345            assert!(!logs.contains("other-thread"), "capture leaked: {logs}");
346        }
347    }
348}
349
350/// Cheaply cloneable handle to one connection's bounded outbound frame queue.
351///
352/// Backends emit responses, streaming frames, and future PUSH frames through this
353/// single path. The queue is bounded twice: by queued BYTES (the budget that
354/// matters for memory and for how long a slow reader may pause), and by the
355/// `mpsc` channel's frame count (a backstop). The socket layer owns the sole
356/// receiver/writer.
357#[derive(Debug, Clone)]
358pub struct FrameSink {
359    tx: mpsc::Sender<OutboundFrame>,
360    accounting: Arc<EgressAccounting>,
361}
362
363impl FrameSink {
364    /// A sink with the standard per-connection byte budget
365    /// ([`crate::server::CONNECTION_EGRESS_BYTE_BUDGET`]); the frame-count bound
366    /// is whatever capacity `tx`'s channel was created with.
367    pub fn new(tx: mpsc::Sender<OutboundFrame>) -> Self {
368        Self::with_byte_budget(tx, crate::server::CONNECTION_EGRESS_BYTE_BUDGET)
369    }
370
371    pub(crate) fn with_byte_budget(tx: mpsc::Sender<OutboundFrame>, byte_budget: usize) -> Self {
372        Self {
373            tx,
374            accounting: Arc::new(EgressAccounting::new(byte_budget)),
375        }
376    }
377
378    /// Wait until the frame's bytes fit the budget, then charge them. Awaited
379    /// senders (control replies, error replies, shutdown notices) are never
380    /// refused by the byte budget: they wait for the writer to free bytes, just
381    /// as they wait for a free slot, which is the same backpressure they had
382    /// when the queue was bounded by frame count alone. Returns `None` if the
383    /// writer goes away while waiting.
384    async fn charge_waiting(&self, bytes: usize) -> Option<EgressCharge> {
385        if let Some(charge) = self.accounting.try_charge(bytes) {
386            return Some(charge);
387        }
388        // Counted as a waiter for as long as this future is parked, including
389        // when it is cancelled, so releases notify only while someone waits.
390        struct Waiting<'a>(&'a AtomicUsize);
391        impl Drop for Waiting<'_> {
392            fn drop(&mut self) {
393                self.0.fetch_sub(1, Ordering::SeqCst);
394            }
395        }
396        self.accounting.waiters.fetch_add(1, Ordering::SeqCst);
397        let _waiting = Waiting(&self.accounting.waiters);
398        loop {
399            let freed = self.accounting.freed.notified();
400            tokio::pin!(freed);
401            // Register interest before checking, so a release that happens
402            // between the check and the await still wakes this sender.
403            freed.as_mut().enable();
404            if let Some(charge) = self.accounting.try_charge(bytes) {
405                return Some(charge);
406            }
407            tokio::select! {
408                _ = &mut freed => {}
409                _ = self.tx.closed() => return None,
410            }
411        }
412    }
413
414    pub async fn send(&self, frame: Frame) -> Result<(), RouterError> {
415        let channel = frame.header.channel;
416        let epoch = frame.header.epoch;
417        let corr = frame.header.corr;
418        let closed =
419            || RouterError::backend_with_epoch(channel, epoch, corr, "connection writer closed");
420        let charge = self
421            .charge_waiting(queued_frame_bytes(&frame))
422            .await
423            .ok_or_else(closed)?;
424        self.tx
425            .send(OutboundFrame::charged(frame, charge))
426            .await
427            .map_err(|_| closed())
428    }
429
430    /// Shutdown notices must leave the socket writer before an idle daemon exits.
431    /// Queue admission alone does not prove this; the writer acknowledges flush.
432    ///
433    /// The local endpoint uses it the same way for its clients-only refusal: the
434    /// connection closes right after the refusal is sent, so the writer must
435    /// flush it first.
436    #[cfg(unix)]
437    pub async fn send_flushed(&self, frame: Frame) -> Result<(), RouterError> {
438        let (tx, rx) = tokio::sync::oneshot::channel();
439        let charge = self
440            .charge_waiting(queued_frame_bytes(&frame))
441            .await
442            .ok_or_else(|| RouterError::backend(0, 0, "connection writer closed"))?;
443        let mut outbound = OutboundFrame::charged(frame, charge);
444        outbound.flushed = Some(tx);
445        self.tx
446            .send(outbound)
447            .await
448            .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
449        rx.await
450            .map_err(|_| RouterError::backend(0, 0, "connection flush failed"))
451    }
452
453    pub(crate) async fn reserve_owned(&self) -> Result<EgressPermit, RouterError> {
454        let permit = self
455            .tx
456            .clone()
457            .reserve_owned()
458            .await
459            .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
460        Ok(EgressPermit {
461            permit,
462            accounting: Arc::clone(&self.accounting),
463        })
464    }
465
466    #[cfg(test)]
467    pub(crate) fn try_reserve_owned(&self) -> Result<EgressPermit, RouterError> {
468        let permit = self
469            .tx
470            .clone()
471            .try_reserve_owned()
472            .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
473        Ok(EgressPermit {
474            permit,
475            accounting: Arc::clone(&self.accounting),
476        })
477    }
478
479    pub(crate) fn is_closed(&self) -> bool {
480        self.tx.is_closed()
481    }
482
483    /// What the connection's egress queue holds right now.
484    pub(crate) fn backlog(&self) -> EgressBacklog {
485        self.accounting.backlog()
486    }
487
488    /// Stop module deliveries as soon as the client reader exits, without
489    /// waiting for the socket writer's separate teardown drain grace.
490    pub(crate) fn close_client_delivery(&self) {
491        self.accounting
492            .client_delivery_closed
493            .store(true, Ordering::Release);
494        self.accounting.client_delivery_stopped.notify_waiters();
495    }
496
497    /// A full queue can be a short fanout burst rather than a stopped reader.
498    /// Spend only the grace the atomic backlog age has not already used.
499    async fn send_to_client(&self, frame: Frame) -> Result<(), ClientDeliveryFailure> {
500        let frame_type = frame.header.ty;
501        let flags = frame.header.flags;
502        let stopped = self.accounting.client_delivery_stopped.notified();
503        tokio::pin!(stopped);
504        stopped.as_mut().enable();
505        let failure = |cause, error| ClientDeliveryFailure {
506            diagnosis: ClientDeliveryDiagnosis {
507                frame_type,
508                flags,
509                cause,
510            },
511            error,
512        };
513        let closed = || {
514            RouterError::backend_with_epoch(
515                frame.header.channel,
516                frame.header.epoch,
517                frame.header.corr,
518                "client connection closed",
519            )
520        };
521        if self
522            .accounting
523            .client_delivery_closed
524            .load(Ordering::Acquire)
525        {
526            return Err(failure("client_closed", closed()));
527        }
528        let (frame, error) = match self.try_send_owned(frame) {
529            Ok(()) => return Ok(()),
530            Err(refused) => refused,
531        };
532        if self.is_closed() {
533            return Err(failure("client_closed", error));
534        }
535        let grace = crate::server::CONNECTION_EGRESS_STALL_GRACE;
536        let age = self.backlog().oldest_age.unwrap_or_default();
537        if age > grace {
538            return Err(failure("oldest_frame_older_than_250_ms", error));
539        }
540        let result = tokio::select! {
541            biased;
542            _ = &mut stopped => return Err(failure("client_closed", error)),
543            result = tokio::time::timeout(grace.saturating_sub(age), self.send(frame)) => result,
544        };
545        match result {
546            Ok(Ok(())) => Ok(()),
547            Ok(Err(error)) => Err(failure("client_closed", error)),
548            Err(_) => Err(failure("waited_250_ms_without_room", error)),
549        }
550    }
551
552    /// Enqueue without waiting. Fails when the frame's bytes would take the
553    /// queue past its byte budget, when the channel's frame-count backstop is
554    /// full, or when the writer is gone.
555    pub(crate) fn try_send(&self, frame: Frame) -> Result<(), RouterError> {
556        self.try_send_owned(frame).map_err(|(_, error)| error)
557    }
558
559    fn try_send_owned(&self, frame: Frame) -> Result<(), (Frame, RouterError)> {
560        let channel = frame.header.channel;
561        let epoch = frame.header.epoch;
562        let corr = frame.header.corr;
563        let unavailable = |why: String| {
564            RouterError::backend_with_epoch(
565                channel,
566                epoch,
567                corr,
568                format!("connection writer unavailable: {why}"),
569            )
570        };
571        let bytes = queued_frame_bytes(&frame);
572        let Some(charge) = self.accounting.try_charge(bytes) else {
573            if self.tx.is_closed() {
574                return Err((frame, unavailable("channel closed".to_string())));
575            }
576            return Err((
577                frame,
578                unavailable(format!(
579                    "egress byte budget exhausted ({} queued bytes, frame of {bytes} bytes, budget {})",
580                    self.accounting.queued_bytes.load(Ordering::Acquire),
581                    self.accounting.byte_budget
582                )),
583            ));
584        };
585        // Return the refused frame for a bounded retry; dropping the rest of
586        // the outbound wrapper returns its charge.
587        self.tx
588            .try_send(OutboundFrame::charged(frame, charge))
589            .map_err(|err| {
590                let error = unavailable(err.to_string());
591                (err.into_inner().frame, error)
592            })
593    }
594}
595
596#[derive(Debug)]
597struct ClientDeliveryFailure {
598    diagnosis: ClientDeliveryDiagnosis,
599    error: RouterError,
600}
601
602/// Header facts only: payloads can contain secrets and must never enter logs.
603#[derive(Debug)]
604pub(crate) struct ClientDeliveryDiagnosis {
605    pub frame_type: FrameType,
606    pub flags: Flags,
607    pub cause: &'static str,
608}
609
610impl fmt::Display for ClientDeliveryFailure {
611    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
612        self.error.fmt(f)
613    }
614}
615
616/// Minimum time between two route GOODBYEs the daemon sends one module
617/// connection for one channel in answer to the module's frames on a (channel,
618/// epoch) the daemon holds no route for.
619///
620/// A module that missed a GOODBYE keeps sending on that route (a streaming
621/// module, many frames a second), and every one of those frames lands here. One
622/// answer is enough when it arrives, so answering each frame would only flood
623/// the module's egress queue at the moment it is already behind. Five seconds
624/// is long enough for an answer queued behind a deep backlog to reach the module
625/// and take effect before a second is sent, and short enough that an answer
626/// the queue refused is retried by the next orphan frame well within a minute.
627const ORPHAN_ROUTE_GOODBYE_INTERVAL: Duration = Duration::from_secs(5);
628
629/// Once one connection has this many remembered channels, entries older than
630/// [`ORPHAN_ROUTE_GOODBYE_INTERVAL`] are pruned before another is added. An
631/// expired entry has no effect on rate limiting, so pruning never changes a
632/// decision; it keeps a module that orphaned many channels long ago from
633/// pinning memory for all of them. The map is bounded in any case by the
634/// 16-bit channel space and is dropped whole when the connection ends.
635const ORPHAN_ROUTE_GOODBYE_PRUNE_AT: usize = 256;
636
637/// When each module connection was last sent an orphan-route GOODBYE, per
638/// channel. Shared by the [`Router`] (which consults it) and every
639/// [`RouterConnection`] (which removes its own entry when the connection ends).
640#[derive(Debug, Default)]
641struct OrphanGoodbyeLimiter {
642    last_sent: Mutex<HashMap<ConnectionId, HashMap<u16, tokio::time::Instant>>>,
643}
644
645impl OrphanGoodbyeLimiter {
646    /// True, and the send recorded, when `channel` on `connection_id` has not
647    /// been answered within the interval.
648    fn claim(&self, connection_id: ConnectionId, channel: u16) -> bool {
649        let now = tokio::time::Instant::now();
650        let mut last_sent = self
651            .last_sent
652            .lock()
653            .unwrap_or_else(std::sync::PoisonError::into_inner);
654        let channels = last_sent.entry(connection_id).or_default();
655        if channels.get(&channel).is_some_and(|sent| {
656            now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
657        }) {
658            return false;
659        }
660        if channels.len() >= ORPHAN_ROUTE_GOODBYE_PRUNE_AT {
661            channels.retain(|_, sent| {
662                now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
663            });
664        }
665        channels.insert(channel, now);
666        true
667    }
668
669    fn forget_connection(&self, connection_id: ConnectionId) {
670        self.last_sent
671            .lock()
672            .unwrap_or_else(std::sync::PoisonError::into_inner)
673            .remove(&connection_id);
674    }
675}
676
677/// Per-route context shared with backends besides the frame itself.
678#[derive(Debug, Clone)]
679pub struct RouteCtx {
680    pub connection_id: ConnectionId,
681    pub egress: FrameSink,
682}
683
684/// Closed set of data-plane backends for non-zero channels.
685///
686/// Channel 0 is structurally special and remains [`Router::control`], not an
687/// enum variant. Static/test backends live in [`Router::backends`]; the forwarding
688/// backend is selected dynamically only when a per-connection forwarding binding exists.
689#[derive(Debug, Clone)]
690pub enum Backend {
691    Echo(EchoBackend),
692    Forward(ForwardBackend),
693}
694
695impl From<EchoBackend> for Backend {
696    fn from(backend: EchoBackend) -> Self {
697        Self::Echo(backend)
698    }
699}
700
701impl From<ForwardBackend> for Backend {
702    fn from(backend: ForwardBackend) -> Self {
703        Self::Forward(backend)
704    }
705}
706
707impl Backend {
708    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
709        match self {
710            Self::Echo(backend) => backend.handle(ctx, frame).await,
711            Self::Forward(backend) => backend.handle(ctx, frame).await,
712        }
713    }
714}
715
716/// I/O-agnostic splice router keyed by envelope `channel`.
717///
718/// Channel 0 is reserved for subc itself and is always dispatched to the
719/// dedicated control handler. Other channels must be explicitly registered.
720/// Unknown client-originated non-zero channels are translated to canonical JSON `ERROR` frames on
721/// the connection sink so the peer can continue using the same socket. Module-originated frames for
722/// released route channels are logged and dropped as the channel-gone race backstop.
723///
724/// DATA-PLANE BODIES ARE NEVER DECODED HERE, and the consequence is worth stating
725/// because it looks like a guarantee and is not. Additive fields in a request or
726/// response body reach the far side untouched — not because anything permits them,
727/// but because routing reads only the 21-byte header and treats the body as opaque
728/// bytes. That is a performance property, so **nothing prevents it from changing**:
729/// a future reason to inspect a body would convert a wire-transparent path into a
730/// filtering one, and nobody would think of it as a contract change.
731///
732/// So when a peer asks whether subc sees an additive field, the answer is per-path
733/// and this path's zero means WE NEVER LOOK rather than WE LOOK AT EVERYTHING. The
734/// control plane (typed enums at the frame boundary) and the MCP gateway (envelope
735/// unwrap plus a named struct) both narrow; only this one does not.
736pub struct Router {
737    backends: HashMap<u16, Backend>,
738    control: Arc<ControlHandler>,
739    forwarding: Arc<ForwardingTable>,
740    forward_backend: ForwardBackend,
741    counters: DaemonCounters,
742    next_connection_id: AtomicU64,
743    orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
744}
745
746impl Router {
747    pub fn with_control_handler(control: Arc<ControlHandler>) -> Self {
748        // The handler the router serves is the one a swap must tell when it
749        // promotes a candidate; this is where it first sits behind an `Arc`.
750        control.install_swap_promotion_observer();
751        let forwarding = control.forwarding();
752        let counters = control.counters();
753        Self {
754            backends: HashMap::new(),
755            control,
756            forwarding: Arc::clone(&forwarding),
757            forward_backend: ForwardBackend::new(forwarding),
758            counters,
759            // ConnectionId::LOCAL is 0; real socket ids start at 1 and never collide.
760            next_connection_id: AtomicU64::new(1),
761            orphan_goodbyes: Arc::default(),
762        }
763    }
764
765    pub fn with_default_self_handler() -> Self {
766        Self::with_control_handler(Arc::new(ControlHandler::default()))
767    }
768
769    pub fn forwarding(&self) -> Arc<ForwardingTable> {
770        Arc::clone(&self.forwarding)
771    }
772
773    pub fn register_backend(
774        &mut self,
775        channel: u16,
776        backend: impl Into<Backend>,
777    ) -> Result<(), RouterError> {
778        self.register_backend_arc(channel, Arc::new(backend.into()))
779    }
780
781    pub(crate) fn register_backend_arc(
782        &mut self,
783        channel: u16,
784        backend: Arc<Backend>,
785    ) -> Result<(), RouterError> {
786        if channel == 0 {
787            return Err(RouterError::ReservedChannelZero);
788        }
789        if self.backends.contains_key(&channel) {
790            return Err(RouterError::DuplicateChannel { channel });
791        }
792        self.backends.insert(channel, backend.as_ref().clone());
793        Ok(())
794    }
795
796    /// Start a connection-scoped routing context. Dropping the guard releases
797    /// any control-plane registrations owned by the connection.
798    fn record_module_frame_drop(&self, connection_id: ConnectionId) -> Result<(), RouterError> {
799        let module_id = self
800            .forwarding
801            .module_id_for_connection(connection_id)
802            .map_err(RouterError::Forwarding)?;
803        self.counters
804            .increment_module_frames_dropped_no_route(module_id.as_deref());
805        Ok(())
806    }
807
808    /// A module sent a non-request frame on a (channel, epoch) the daemon holds
809    /// no route for: most often a route the daemon released whose GOODBYE the
810    /// module never received, so the module still believes it is open and keeps
811    /// sending. Count the drop, and tell the module to let go of exactly that
812    /// (channel, epoch) with a route GOODBYE, at most once per
813    /// [`ORPHAN_ROUTE_GOODBYE_INTERVAL`] per connection and channel.
814    ///
815    /// The answer uses `try_send`: it is a best-effort nudge, and if the queue
816    /// refuses it, the module's next frame on that route after the interval
817    /// asks again. A GOODBYE from the module is not answered, since the module
818    /// is already letting go of the route.
819    fn handle_orphan_module_frame(&self, ctx: &RouteCtx, frame: &Frame) -> Result<(), RouterError> {
820        let channel = frame.header.channel;
821        let epoch = frame.header.epoch;
822        let module_id = self
823            .forwarding
824            .module_id_for_connection(ctx.connection_id)
825            .map_err(RouterError::Forwarding)?;
826        self.counters
827            .increment_module_frames_dropped_no_route(module_id.as_deref());
828        if self
829            .forwarding
830            .module_route_epoch_was_allocated(ctx.connection_id, channel, epoch)
831            .map_err(RouterError::Forwarding)?
832        {
833            self.counters
834                .increment_module_frames_dropped_released_route(module_id.as_deref());
835        }
836        if frame.header.ty == FrameType::Goodbye
837            || !self.orphan_goodbyes.claim(ctx.connection_id, channel)
838        {
839            return Ok(());
840        }
841        let goodbye = Frame::build_with_version(
842            frame.header.ver,
843            FrameType::Goodbye,
844            Flags::new(false, Priority::Passive, false),
845            channel,
846            epoch,
847            0,
848            Vec::new(),
849        )
850        .map_err(RouterError::FrameBuild)?;
851        match ctx.egress.try_send(goodbye) {
852            Ok(()) => {
853                self.counters.increment_module_orphan_route_goodbyes_sent();
854                debug!(
855                    connection_id = ctx.connection_id.get(),
856                    module_id = module_id.as_deref().unwrap_or("unknown"),
857                    channel,
858                    epoch,
859                    "answered module frame on a route the daemon does not hold with a route GOODBYE"
860                );
861            }
862            Err(err) => debug!(
863                connection_id = ctx.connection_id.get(),
864                module_id = module_id.as_deref().unwrap_or("unknown"),
865                channel,
866                epoch,
867                error = %err,
868                "could not enqueue route GOODBYE for module frame on a route the daemon does not hold; the next such frame retries"
869            ),
870        }
871        Ok(())
872    }
873
874    pub fn begin_connection(&self) -> RouterConnection {
875        let raw = self.next_connection_id.fetch_add(1, Ordering::Relaxed);
876        let id = ConnectionId::new(raw);
877        let close_receiver = self.forwarding.register_connection_close(id);
878        RouterConnection {
879            id,
880            control_handler: Arc::clone(&self.control),
881            forwarding: Arc::clone(&self.forwarding),
882            close_receiver: Some(close_receiver),
883            orphan_goodbyes: Arc::clone(&self.orphan_goodbyes),
884        }
885    }
886
887    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
888        self.control.route_open_target(frame)
889    }
890
891    pub(crate) fn record_caller_endpoints(
892        &self,
893        connection: ConnectionId,
894        endpoints: Option<(std::net::SocketAddr, std::net::SocketAddr)>,
895    ) {
896        self.control.record_caller_endpoints(connection, endpoints);
897    }
898
899    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
900        self.control.defers_privileged_control(ctx, frame)
901    }
902
903    pub(crate) fn route_open_capacity_refusal(
904        &self,
905        ctx: &RouteCtx,
906        frame: &Frame,
907        target_module_id: &str,
908        in_flight: usize,
909        limit: usize,
910    ) -> Result<Frame, RouterError> {
911        self.control
912            .route_open_capacity_refusal(ctx, frame, target_module_id, in_flight, limit)
913    }
914
915    pub async fn route_for_connection(
916        &self,
917        ctx: &RouteCtx,
918        frame: Frame,
919    ) -> Result<(), RouterError> {
920        self.route_for_connection_started(ctx, frame, None).await
921    }
922
923    pub(crate) async fn route_for_connection_started(
924        &self,
925        ctx: &RouteCtx,
926        frame: Frame,
927        dispatch_started_at: Option<Instant>,
928    ) -> Result<(), RouterError> {
929        let channel = frame.header.channel;
930        let epoch = frame.header.epoch;
931        let corr = frame.header.corr;
932        if channel == 0 {
933            debug!(
934                connection_id = ctx.connection_id.get(),
935                corr,
936                frame_type = ?frame.header.ty,
937                "routing control frame"
938            );
939            // The connection loop dispatches every control operation directly
940            // except `route.open`, activity writes and privileged requests. Each
941            // task passes a timestamp captured before
942            // spawn, so slow-dispatch timing includes scheduler delay but no
943            // wait in an application-owned queue.
944            let dispatch_started_at = (frame.header.ty == FrameType::Request)
945                .then(|| dispatch_started_at.unwrap_or_else(Instant::now));
946            let responses = self
947                .control
948                .handle_control_frame_timed(ctx, frame, dispatch_started_at)
949                .await?;
950            for response in responses {
951                ctx.egress.send(response).await?;
952            }
953            return Ok(());
954        }
955
956        let data_route = self
957            .forwarding
958            .lookup_data_route(ctx.connection_id, channel, epoch)
959            .map_err(RouterError::Forwarding)?;
960
961        match data_route {
962            DataRoute::Module(DataRouteState::EpochMismatch) => {
963                if frame.header.ty == FrameType::Request {
964                    self.counters
965                        .increment_module_requests_dropped_stale_route();
966                    let err = RouterError::StaleRouteEpoch {
967                        channel,
968                        epoch,
969                        corr,
970                    };
971                    if let Some(error_frame) = err.to_error_frame() {
972                        ctx.egress.send(error_frame).await?;
973                    }
974                } else {
975                    self.handle_orphan_module_frame(ctx, &frame)?;
976                }
977                debug!(
978                    connection_id = ctx.connection_id.get(),
979                    channel, epoch, corr, "dropping module frame for stale route epoch"
980                );
981                return Ok(());
982            }
983            DataRoute::Module(DataRouteState::Reserved) => {
984                if frame.header.ty == FrameType::Request {
985                    self.counters
986                        .increment_module_requests_dropped_stale_route();
987                    let err = RouterError::UnknownChannel {
988                        channel,
989                        epoch,
990                        corr,
991                    };
992                    if let Some(error_frame) = err.to_error_frame() {
993                        ctx.egress.send(error_frame).await?;
994                    }
995                } else {
996                    self.record_module_frame_drop(ctx.connection_id)?;
997                }
998                debug!(
999                    connection_id = ctx.connection_id.get(),
1000                    channel, epoch, corr, "dropping module frame for reserved route handle"
1001                );
1002                return Ok(());
1003            }
1004            DataRoute::Module(DataRouteState::Absent) => {
1005                if frame.header.ty == FrameType::Request {
1006                    self.counters
1007                        .increment_module_requests_dropped_stale_route();
1008                    let err = RouterError::UnknownChannel {
1009                        channel,
1010                        epoch,
1011                        corr,
1012                    };
1013                    if let Some(error_frame) = err.to_error_frame() {
1014                        ctx.egress.send(error_frame).await?;
1015                    }
1016                } else {
1017                    self.handle_orphan_module_frame(ctx, &frame)?;
1018                }
1019                debug!(
1020                    connection_id = ctx.connection_id.get(),
1021                    channel, epoch, corr, "dropping module frame for absent route handle"
1022                );
1023                return Ok(());
1024            }
1025            DataRoute::Module(DataRouteState::Bound(route)) => {
1026                if frame.header.ty == FrameType::Goodbye {
1027                    if let RouteRelease::Removed(target) = self
1028                        .forwarding
1029                        .release_module_route(ctx.connection_id, channel, epoch)
1030                        .map_err(RouterError::Forwarding)?
1031                    {
1032                        let mut goodbye = frame;
1033                        goodbye.header.channel = target.channel;
1034                        goodbye.header.epoch = target.epoch;
1035                        // A module must never be able to make a frame look
1036                        // daemon-authored, so the origin bit is cleared on
1037                        // every module frame relayed to a client.
1038                        goodbye.header.flags = goodbye.header.flags.without_daemon_origin();
1039                        if let Err(err) = target.sink.send_to_client(goodbye).await {
1040                            if target.close_on_delivery_failure()
1041                                && self
1042                                    .forwarding
1043                                    .escalate_client_delivery_failure_diagnosed(
1044                                        target.connection_id,
1045                                        target.channel,
1046                                        target.epoch,
1047                                        CloseReason::new(
1048                                            "route_goodbye_delivery_failed",
1049                                            format!(
1050                                                "failed to enqueue route GOODBYE for client channel {}: {err}",
1051                                                target.channel
1052                                            ),
1053                                        ),
1054                                        UndeliveredFrame {
1055                                            module_id: Some(&route.module_id),
1056                                            sink: &target.sink,
1057                                        },
1058                                        Some(&err.diagnosis),
1059                                    )
1060                                    .map_err(RouterError::Forwarding)?
1061                            {
1062                                self.counters.increment_goodbye_relay_client_failed();
1063                            }
1064                        }
1065                    }
1066                    return Ok(());
1067                }
1068
1069                // Accounting allows an oversized frame to occupy an empty queue
1070                // on other send paths. Refuse it at this splice before admission,
1071                // so the next ordinary reply cannot close a multiplexed client.
1072                let bytes = queued_frame_bytes(&frame);
1073                let limit = route.client_sink.accounting.byte_budget;
1074                if bytes > limit {
1075                    let ty = frame.header.ty;
1076                    let flags = frame.header.flags;
1077                    drop(frame);
1078                    return self
1079                        .refuse_oversized_module_frame(ctx, &route, (ty, flags), corr, bytes)
1080                        .await;
1081                }
1082
1083                // A terminal frame ends the request at the module whether or not
1084                // the client can still take it, so its credit is released before
1085                // delivery is attempted. Releasing only after a successful
1086                // enqueue would leave a drain counting a finished request until
1087                // the client connection's cleanup removes the route.
1088                let releases_credit = is_terminal_frame(frame.header.ty);
1089                if releases_credit {
1090                    route.flow.release_corr(corr);
1091                }
1092                let mut frame = frame;
1093                frame.header.channel = route.client_channel;
1094                frame.header.epoch = route.client_epoch;
1095                // Responses, errors, stream data and ends, and reverse
1096                // requests all pass through here. A module must never be able
1097                // to make a frame look daemon-authored, so the origin bit a
1098                // module may have set is cleared; every other flag bit and
1099                // the body travel unchanged.
1100                frame.header.flags = frame.header.flags.without_daemon_origin();
1101                if let Err(err) = route.client_sink.send_to_client(frame).await {
1102                    if self
1103                        .forwarding
1104                        .escalate_client_delivery_failure_diagnosed(
1105                            route.client_connection_id,
1106                            route.client_channel,
1107                            route.client_epoch,
1108                            CloseReason::new(
1109                                "module_to_client_delivery_failed",
1110                                format!(
1111                                    "failed to enqueue module frame for client channel {} corr {corr}: {err}",
1112                                    route.client_channel
1113                                ),
1114                            ),
1115                            UndeliveredFrame {
1116                                module_id: Some(&route.module_id),
1117                                sink: &route.client_sink,
1118                            },
1119                            Some(&err.diagnosis),
1120                        )
1121                        .map_err(RouterError::Forwarding)?
1122                    {
1123                        self.counters
1124                            .increment_client_egress_close_delivery_failed();
1125                    }
1126                    return Ok(());
1127                }
1128                return Ok(());
1129            }
1130            DataRoute::Client(DataRouteState::EpochMismatch) => {
1131                if frame.header.ty == FrameType::Request {
1132                    self.counters.increment_client_frames_dropped_stale_route();
1133                    // Dropped before forwarding; a re-bind retry cannot double-execute this request.
1134                    let err = RouterError::StaleRouteEpoch {
1135                        channel,
1136                        epoch,
1137                        corr,
1138                    };
1139                    if let Some(error_frame) = err.to_error_frame() {
1140                        ctx.egress.send(error_frame).await?;
1141                    }
1142                }
1143                debug!(
1144                    connection_id = ctx.connection_id.get(),
1145                    channel, epoch, corr, "dropping client frame for stale route epoch"
1146                );
1147                return Ok(());
1148            }
1149            DataRoute::Client(DataRouteState::Reserved) => {
1150                if frame.header.ty == FrameType::Request {
1151                    let err = RouterError::UnknownChannel {
1152                        channel,
1153                        epoch,
1154                        corr,
1155                    };
1156                    if let Some(error_frame) = err.to_error_frame() {
1157                        ctx.egress.send(error_frame).await?;
1158                    }
1159                }
1160                return Ok(());
1161            }
1162            DataRoute::Client(DataRouteState::Bound(route)) => {
1163                if frame.header.ty == FrameType::Goodbye {
1164                    let _ = self
1165                        .control
1166                        .handle_route_goodbye(ctx.connection_id, channel, epoch)?;
1167                    return Ok(());
1168                }
1169                return self.forward_backend.handle_bound(frame, route).await;
1170            }
1171            DataRoute::Client(DataRouteState::Absent) => {}
1172        }
1173
1174        if let Some(backend) = self.backends.get(&channel) {
1175            return backend.handle(ctx.clone(), frame).await;
1176        }
1177        if frame.header.ty == FrameType::Request {
1178            let err = RouterError::UnknownChannel {
1179                channel,
1180                epoch,
1181                corr,
1182            };
1183            if let Some(error_frame) = err.to_error_frame() {
1184                ctx.egress.send(error_frame).await?;
1185            }
1186        }
1187        Ok(())
1188    }
1189
1190    async fn refuse_oversized_module_frame(
1191        &self,
1192        ctx: &RouteCtx,
1193        route: &RouteBinding,
1194        header: (FrameType, Flags),
1195        corr: u64,
1196        bytes: usize,
1197    ) -> Result<(), RouterError> {
1198        let (ty, flags) = header;
1199        let limit = route.client_sink.accounting.byte_budget;
1200        if ty == FrameType::Push {
1201            // Deployed Rust clients cannot decode a new route.closed reason.
1202            // Until that vocabulary can be extended compatibly, an oversized
1203            // unsolicited push still closes the connection, with its cause named.
1204            let diagnosis = ClientDeliveryDiagnosis {
1205                frame_type: ty,
1206                flags,
1207                cause: "frame_too_large",
1208            };
1209            if self
1210                .forwarding
1211                .escalate_client_delivery_failure_diagnosed(
1212                    route.client_connection_id,
1213                    route.client_channel,
1214                    route.client_epoch,
1215                    CloseReason::new(
1216                        error_codes::FRAME_TOO_LARGE,
1217                        format!(
1218                            "oversized push: size={bytes} limit={limit} (header plus body bytes)"
1219                        ),
1220                    ),
1221                    UndeliveredFrame {
1222                        module_id: Some(&route.module_id),
1223                        sink: &route.client_sink,
1224                    },
1225                    Some(&diagnosis),
1226                )
1227                .map_err(RouterError::Forwarding)?
1228            {
1229                self.counters
1230                    .increment_client_egress_close_delivery_failed();
1231            }
1232            return Ok(());
1233        }
1234
1235        warn!(
1236            connection_id = route.client_connection_id.get(),
1237            module_id = %route.module_id,
1238            client_principal = %principal_label(&route.principal),
1239            client_channel = route.client_channel,
1240            frame_type = ?ty,
1241            frame_bytes = bytes,
1242            byte_limit = limit,
1243            "refusing oversized module-to-client frame with ERROR frame_too_large"
1244        );
1245        if ty == FrameType::Request {
1246            // A reverse request belongs to the module, not to a waiting client.
1247            // Answer its sender without forwarding any part of it to the client.
1248            ctx.egress
1249                .send(frame_too_large_error(
1250                    route.module_channel,
1251                    route.module_epoch,
1252                    corr,
1253                    bytes,
1254                    limit,
1255                )?)
1256                .await?;
1257        } else {
1258            // A refused stream chunk also ends the waiting request. Releasing
1259            // its credit prevents a finished request from keeping a drain alive.
1260            route.flow.release_corr(corr);
1261            let error = frame_too_large_error(
1262                route.client_channel,
1263                route.client_epoch,
1264                corr,
1265                bytes,
1266                limit,
1267            )?;
1268            // The replacement shares the reply's bounded burst grace, so a
1269            // stopped client cannot indefinitely stall the module frame reader.
1270            if let Err(err) = route.client_sink.send_to_client(error).await {
1271                if self
1272                    .forwarding
1273                    .escalate_client_delivery_failure_diagnosed(
1274                        route.client_connection_id,
1275                        route.client_channel,
1276                        route.client_epoch,
1277                        CloseReason::new(
1278                            "module_to_client_delivery_failed",
1279                            format!(
1280                                "failed to enqueue frame_too_large for client channel {} corr {corr}: {err}",
1281                                route.client_channel
1282                            ),
1283                        ),
1284                        UndeliveredFrame {
1285                            module_id: Some(&route.module_id),
1286                            sink: &route.client_sink,
1287                        },
1288                        Some(&err.diagnosis),
1289                    )
1290                    .map_err(RouterError::Forwarding)?
1291                {
1292                    self.counters
1293                        .increment_client_egress_close_delivery_failed();
1294                }
1295            }
1296        }
1297        Ok(())
1298    }
1299}
1300
1301impl Default for Router {
1302    fn default() -> Self {
1303        Self::with_default_self_handler()
1304    }
1305}
1306
1307/// Connection-scoped cleanup guard returned by [`Router::begin_connection`].
1308#[must_use]
1309pub struct RouterConnection {
1310    id: ConnectionId,
1311    control_handler: Arc<ControlHandler>,
1312    forwarding: Arc<ForwardingTable>,
1313    close_receiver: Option<ConnectionCloseReceiver>,
1314    orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
1315}
1316
1317impl RouterConnection {
1318    pub fn id(&self) -> ConnectionId {
1319        self.id
1320    }
1321
1322    pub(crate) fn take_close_receiver(&mut self) -> ConnectionCloseReceiver {
1323        self.close_receiver
1324            .take()
1325            .expect("connection close receiver can only be taken once")
1326    }
1327}
1328
1329impl Drop for RouterConnection {
1330    fn drop(&mut self) {
1331        self.forwarding.unregister_connection_close(self.id);
1332        self.orphan_goodbyes.forget_connection(self.id);
1333        // GOODBYE (explicit) and connection-drop cleanup both call the same
1334        // idempotent deregistration path.
1335        let _ = self.control_handler.cleanup_connection(self.id);
1336    }
1337}
1338
1339/// Minimal in-memory backend used by tests and early wiring: it emits a
1340/// `RESPONSE` on the same channel/correlation id with the exact same body bytes.
1341#[derive(Debug, Default, Clone, Copy)]
1342pub struct EchoBackend;
1343
1344impl EchoBackend {
1345    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1346        let response = Frame::build_with_version(
1347            frame.header.ver,
1348            FrameType::Response,
1349            frame.header.flags,
1350            frame.header.channel,
1351            frame.header.epoch,
1352            frame.header.corr,
1353            frame.body,
1354        )
1355        .map_err(RouterError::FrameBuild)?;
1356        ctx.egress.send(response).await
1357    }
1358}
1359
1360/// Data-plane backend that splices client frames to the module connection bound at attach time.
1361#[derive(Debug, Clone)]
1362pub struct ForwardBackend {
1363    forwarding: Arc<ForwardingTable>,
1364}
1365
1366impl ForwardBackend {
1367    pub fn new(forwarding: Arc<ForwardingTable>) -> Self {
1368        Self { forwarding }
1369    }
1370
1371    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1372        let channel = frame.header.channel;
1373        let corr = frame.header.corr;
1374        let route = match self
1375            .forwarding
1376            .lookup_data_route(ctx.connection_id, channel, frame.header.epoch)
1377            .map_err(RouterError::Forwarding)?
1378        {
1379            DataRoute::Client(DataRouteState::Bound(route)) => route,
1380            DataRoute::Client(_) | DataRoute::Module(_) => {
1381                return Err(RouterError::UnknownChannel {
1382                    channel,
1383                    epoch: frame.header.epoch,
1384                    corr,
1385                });
1386            }
1387        };
1388        self.handle_bound(frame, route).await
1389    }
1390
1391    pub(crate) async fn handle_bound(
1392        &self,
1393        frame: Frame,
1394        route: Arc<RouteBinding>,
1395    ) -> Result<(), RouterError> {
1396        let channel = frame.header.channel;
1397        let corr = frame.header.corr;
1398        let frame_type = frame.header.ty;
1399
1400        // CANCEL and other non-REQUEST frames bypass the request-credit window;
1401        // the original request's credit is freed only by the module's terminal frame.
1402        let acquired_credit = frame_type == FrameType::Request;
1403        if acquired_credit {
1404            if let Err(err) = route
1405                .flow
1406                .acquire_tagged(corr, frame.header.flags.is_subscription())
1407                .await
1408            {
1409                // `module_reloading` here is answered BEFORE the frame is
1410                // forwarded, so the module never sees the request and callers
1411                // may re-dispatch it after reopening the route. That is a wire
1412                // guarantee documented on `error_codes::MODULE_RELOADING`; do
1413                // not emit this code for a request that already reached
1414                // `module_sink.send` below.
1415                if self
1416                    .forwarding
1417                    .endpoint_is_draining(route.module_endpoint)
1418                    .map_err(RouterError::Forwarding)?
1419                {
1420                    return Err(RouterError::route_error_with_epoch(
1421                        channel,
1422                        frame.header.epoch,
1423                        corr,
1424                        "module_reloading",
1425                        format!("module endpoint for route channel {channel} is reloading"),
1426                    ));
1427                }
1428                return Err(RouterError::backend_with_epoch(
1429                    channel,
1430                    frame.header.epoch,
1431                    corr,
1432                    format!("{err} for route channel {channel}"),
1433                ));
1434            }
1435        }
1436
1437        let mut frame = frame;
1438        frame.header.channel = route.module_channel;
1439        frame.header.epoch = route.module_epoch;
1440        // The origin bit is the daemon's own signal to clients. A client
1441        // frame carrying it must not reach a module appearing daemon-authored,
1442        // so it is cleared here; every other flag bit travels unchanged.
1443        frame.header.flags = frame.header.flags.without_daemon_origin();
1444        let result = route.module_sink.send(frame).await.map_err(|err| {
1445            RouterError::backend_with_epoch(channel, route.client_epoch, corr, err.to_string())
1446        });
1447        if acquired_credit && result.is_err() {
1448            route.flow.release_corr(corr);
1449        }
1450        result
1451    }
1452}
1453
1454fn is_terminal_frame(frame_type: FrameType) -> bool {
1455    matches!(
1456        frame_type,
1457        FrameType::Response | FrameType::Error | FrameType::StreamEnd
1458    )
1459}
1460
1461/// Typed router errors. Routable failures can be translated to canonical JSON
1462/// `ERROR` frames with [`RouterError::to_error_frame`].
1463#[derive(Debug, Clone, PartialEq, Eq)]
1464pub enum RouterError {
1465    ReservedChannelZero,
1466    DuplicateChannel {
1467        channel: u16,
1468    },
1469    UnknownChannel {
1470        channel: u16,
1471        epoch: u32,
1472        corr: u64,
1473    },
1474    StaleRouteEpoch {
1475        channel: u16,
1476        epoch: u32,
1477        corr: u64,
1478    },
1479    Backend {
1480        channel: u16,
1481        epoch: u32,
1482        corr: u64,
1483        message: String,
1484    },
1485    RouteError {
1486        channel: u16,
1487        epoch: u32,
1488        corr: u64,
1489        code: String,
1490        message: String,
1491    },
1492    FrameBuild(FrameBuildError),
1493    Forwarding(ForwardingError),
1494}
1495
1496impl RouterError {
1497    pub fn backend(channel: u16, corr: u64, message: impl Into<String>) -> Self {
1498        Self::backend_with_epoch(channel, 0, corr, message)
1499    }
1500
1501    pub fn backend_with_epoch(
1502        channel: u16,
1503        epoch: u32,
1504        corr: u64,
1505        message: impl Into<String>,
1506    ) -> Self {
1507        Self::Backend {
1508            channel,
1509            epoch,
1510            corr,
1511            message: message.into(),
1512        }
1513    }
1514
1515    pub fn route_error(
1516        channel: u16,
1517        corr: u64,
1518        code: impl Into<String>,
1519        message: impl Into<String>,
1520    ) -> Self {
1521        Self::route_error_with_epoch(channel, 0, corr, code, message)
1522    }
1523
1524    pub fn route_error_with_epoch(
1525        channel: u16,
1526        epoch: u32,
1527        corr: u64,
1528        code: impl Into<String>,
1529        message: impl Into<String>,
1530    ) -> Self {
1531        Self::RouteError {
1532            channel,
1533            epoch,
1534            corr,
1535            code: code.into(),
1536            message: message.into(),
1537        }
1538    }
1539
1540    /// Translate route failures that belong on the wire into an `ERROR` frame.
1541    pub fn to_error_frame(&self) -> Option<Frame> {
1542        match self {
1543            Self::UnknownChannel {
1544                channel,
1545                epoch,
1546                corr,
1547            } => error_frame(
1548                *channel,
1549                *epoch,
1550                *corr,
1551                error_codes::UNKNOWN_CHANNEL,
1552                format!("unknown channel {channel}"),
1553            ),
1554            Self::StaleRouteEpoch {
1555                channel,
1556                epoch,
1557                corr,
1558            } => error_frame(
1559                *channel,
1560                *epoch,
1561                *corr,
1562                error_codes::STALE_ROUTE_EPOCH,
1563                format!("stale route epoch for channel {channel}"),
1564            ),
1565            Self::Backend {
1566                channel,
1567                epoch,
1568                corr,
1569                message,
1570            } => error_frame(*channel, *epoch, *corr, "backend_error", message.clone()),
1571            Self::RouteError {
1572                channel,
1573                epoch,
1574                corr,
1575                code,
1576                message,
1577            } => error_frame(*channel, *epoch, *corr, code, message.clone()),
1578            Self::ReservedChannelZero
1579            | Self::DuplicateChannel { .. }
1580            | Self::FrameBuild(_)
1581            | Self::Forwarding(_) => None,
1582        }
1583    }
1584}
1585
1586/// Flags for an ERROR frame the daemon authors itself.
1587///
1588/// Bit 6 (`FLAG_DAEMON_ORIGIN`) tells the client the daemon wrote this
1589/// error, never a module. What a code then proves is per code: on a bound
1590/// route the daemon writes `module_reloading`, `unknown_channel` and
1591/// `stale_route_epoch` only before forwarding, so with the bit they mean the
1592/// request was not delivered, while `frame_too_large` replaces a reply the
1593/// module already produced. Every daemon-authored ERROR goes through here;
1594/// frames a module sent keep their own flags with the bit cleared at the
1595/// splice.
1596/// Nothing else about the flags differs: passive priority, not binary, not last.
1597pub(crate) fn daemon_error_flags() -> Flags {
1598    Flags::new(false, Priority::Passive, false).with_daemon_origin()
1599}
1600
1601fn error_frame(channel: u16, epoch: u32, corr: u64, code: &str, message: String) -> Option<Frame> {
1602    let body = serde_json::to_vec(&ErrorBody {
1603        code: code.to_string(),
1604        message,
1605        detail: None,
1606    })
1607    .ok()?;
1608
1609    Frame::build(
1610        FrameType::Error,
1611        daemon_error_flags(),
1612        channel,
1613        epoch,
1614        corr,
1615        body,
1616    )
1617    .ok()
1618}
1619
1620fn frame_too_large_error(
1621    channel: u16,
1622    epoch: u32,
1623    corr: u64,
1624    bytes: usize,
1625    limit: usize,
1626) -> Result<Frame, RouterError> {
1627    let body = serde_json::to_vec(&ErrorBody {
1628        code: error_codes::FRAME_TOO_LARGE.to_string(),
1629        message: format!(
1630            "module frame of {bytes} bytes exceeds client egress limit of {limit} bytes"
1631        ),
1632        detail: Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit })),
1633    })
1634    .expect("serializing a frame size error");
1635    Frame::build(
1636        FrameType::Error,
1637        daemon_error_flags(),
1638        channel,
1639        epoch,
1640        corr,
1641        body,
1642    )
1643    .map_err(RouterError::FrameBuild)
1644}
1645
1646impl fmt::Display for RouterError {
1647    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1648        match self {
1649            Self::ReservedChannelZero => write!(f, "channel 0 is reserved for subc"),
1650            Self::DuplicateChannel { channel } => {
1651                write!(f, "backend already registered for channel {channel}")
1652            }
1653            Self::UnknownChannel { channel, corr, .. } => {
1654                write!(f, "unknown channel {channel} for corr {corr}")
1655            }
1656            Self::StaleRouteEpoch { channel, corr, .. } => {
1657                write!(f, "stale route epoch for channel {channel} corr {corr}")
1658            }
1659            Self::Backend {
1660                channel,
1661                corr,
1662                message,
1663                ..
1664            } => write!(
1665                f,
1666                "backend error on channel {channel} corr {corr}: {message}"
1667            ),
1668            Self::RouteError {
1669                channel,
1670                corr,
1671                code,
1672                message,
1673                ..
1674            } => write!(
1675                f,
1676                "route error {code} on channel {channel} corr {corr}: {message}"
1677            ),
1678            Self::FrameBuild(err) => write!(f, "failed to build routed frame: {err}"),
1679            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
1680        }
1681    }
1682}
1683
1684impl Error for RouterError {
1685    fn source(&self) -> Option<&(dyn Error + 'static)> {
1686        match self {
1687            Self::FrameBuild(err) => Some(err),
1688            Self::Forwarding(err) => Some(err),
1689            Self::ReservedChannelZero
1690            | Self::DuplicateChannel { .. }
1691            | Self::UnknownChannel { .. }
1692            | Self::StaleRouteEpoch { .. }
1693            | Self::Backend { .. }
1694            | Self::RouteError { .. } => None,
1695        }
1696    }
1697}
1698
1699#[cfg(test)]
1700mod tests {
1701    use super::*;
1702    use crate::{
1703        forwarding::RouteBindRelayOutcome,
1704        supervise::{ModuleSpec, RestartPolicy, Supervisor, SupervisorHandle},
1705        ControlHandler, Registry,
1706    };
1707    use std::{
1708        sync::{mpsc as std_mpsc, Arc},
1709        time::Duration,
1710    };
1711    use subc_control::ModuleProtocol;
1712    use subc_protocol::{manifest::Concurrency, ErrorBody, Flags, FrameType, Priority};
1713    use tokio::sync::mpsc;
1714
1715    pub(crate) use crate::router::test_log::{captured_logs, log_capture};
1716
1717    fn logged_millis(logs: &str, field: &str) -> u64 {
1718        logs.split_whitespace()
1719            .find_map(|part| part.strip_prefix(field))
1720            .and_then(|value| value.parse().ok())
1721            .unwrap_or_else(|| panic!("missing numeric {field} in logs: {logs}"))
1722    }
1723
1724    fn request(channel: u16, corr: u64, body: &[u8]) -> Frame {
1725        Frame::build(
1726            FrameType::Request,
1727            Flags::new(true, Priority::Interactive, false),
1728            channel,
1729            0,
1730            corr,
1731            body.to_vec(),
1732        )
1733        .unwrap()
1734    }
1735
1736    fn ping(corr: u64) -> Frame {
1737        Frame::build(
1738            FrameType::Ping,
1739            Flags::new(false, Priority::Passive, false),
1740            0,
1741            0,
1742            corr,
1743            Vec::new(),
1744        )
1745        .unwrap()
1746    }
1747
1748    fn route_ctx() -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
1749        let (tx, rx) = mpsc::channel(8);
1750        (
1751            RouteCtx {
1752                connection_id: ConnectionId::LOCAL,
1753                egress: FrameSink::new(tx),
1754            },
1755            rx,
1756        )
1757    }
1758
1759    #[tokio::test]
1760    async fn echo_backend_returns_response_with_byte_identical_body() {
1761        let mut router = Router::with_default_self_handler();
1762        router.register_backend(7, EchoBackend).unwrap();
1763        let (ctx, mut rx) = route_ctx();
1764        let body = b"{not parsed}\0\xff";
1765
1766        router
1767            .route_for_connection(&ctx, request(7, 123, body))
1768            .await
1769            .unwrap();
1770        let response = rx.recv().await.unwrap();
1771
1772        assert_eq!(response.header.ty, FrameType::Response);
1773        assert_eq!(response.header.channel, 7);
1774        assert_eq!(response.header.corr, 123);
1775        assert_eq!(response.body, body);
1776        assert!(rx.try_recv().is_err());
1777    }
1778
1779    #[tokio::test]
1780    async fn unknown_channel_emits_canonical_error_frame() {
1781        let router = Router::with_default_self_handler();
1782        let (ctx, mut rx) = route_ctx();
1783
1784        router
1785            .route_for_connection(&ctx, request(99, 5, b"payload"))
1786            .await
1787            .unwrap();
1788        let error_frame = rx.recv().await.unwrap();
1789
1790        assert_eq!(error_frame.header.ty, FrameType::Error);
1791        assert_eq!(error_frame.header.channel, 99);
1792        assert_eq!(error_frame.header.corr, 5);
1793        let body: ErrorBody = serde_json::from_slice(&error_frame.body).unwrap();
1794        assert_eq!(body.code, "unknown_channel");
1795        assert_eq!(body.message, "unknown channel 99");
1796    }
1797
1798    /// Every ERROR the daemon authors carries the daemon-origin bit, so a
1799    /// client can tell a daemon refusal from a module's reply. Both the
1800    /// unknown-channel and the stale-epoch errors below are received on the
1801    /// client's own egress, not merely built.
1802    #[tokio::test]
1803    async fn daemon_authored_data_plane_errors_carry_daemon_origin() {
1804        let router = Router::with_default_self_handler();
1805        let (ctx, mut rx) = route_ctx();
1806        router
1807            .route_for_connection(&ctx, request(99, 5, b"payload"))
1808            .await
1809            .unwrap();
1810        let unknown = rx.recv().await.unwrap();
1811        assert_eq!(unknown.header.ty, FrameType::Error);
1812        let body: ErrorBody = serde_json::from_slice(&unknown.body).unwrap();
1813        assert_eq!(body.code, "unknown_channel");
1814        assert!(
1815            unknown.header.flags.is_daemon_origin(),
1816            "unknown_channel must carry daemon origin: {:?}",
1817            unknown.header.flags
1818        );
1819
1820        let (
1821            router,
1822            _forwarding,
1823            client_ctx,
1824            mut client_rx,
1825            _module_ctx,
1826            _module_egress_rx,
1827            _module_rx,
1828            pending,
1829        ) = dynamic_route_fixture(true);
1830        let _ = client_rx.recv().await;
1831        router
1832            .route_for_connection(
1833                &client_ctx,
1834                route_frame(
1835                    FrameType::Request,
1836                    pending.client_channel,
1837                    pending.client_epoch + 1,
1838                    702,
1839                ),
1840            )
1841            .await
1842            .unwrap();
1843        let stale = client_rx.recv().await.unwrap();
1844        assert_eq!(stale.header.ty, FrameType::Error);
1845        let body: ErrorBody = serde_json::from_slice(&stale.body).unwrap();
1846        assert_eq!(body.code, "stale_route_epoch");
1847        assert!(
1848            stale.header.flags.is_daemon_origin(),
1849            "stale_route_epoch must carry daemon origin: {:?}",
1850            stale.header.flags
1851        );
1852    }
1853
1854    /// A module ERROR that spoofs a daemon code with the daemon-origin bit
1855    /// set must reach the client with the bit cleared and the body
1856    /// byte-identical: the bit means the daemon wrote the frame, and a module
1857    /// must never be able to claim that.
1858    #[tokio::test]
1859    async fn module_error_with_daemon_bit_is_stripped_before_client_delivery() {
1860        let (
1861            router,
1862            _forwarding,
1863            client_ctx,
1864            mut client_rx,
1865            module_ctx,
1866            _module_egress_rx,
1867            mut module_rx,
1868            pending,
1869        ) = dynamic_route_fixture(true);
1870        let _ = client_rx.recv().await;
1871        // Open flow credit the way a real request would, so the terminal
1872        // error exercises the credit-release path too.
1873        router
1874            .route_for_connection(
1875                &client_ctx,
1876                route_frame(
1877                    FrameType::Request,
1878                    pending.client_channel,
1879                    pending.client_epoch,
1880                    701,
1881                ),
1882            )
1883            .await
1884            .unwrap();
1885        let _ = module_rx.recv().await;
1886
1887        let body = serde_json::to_vec(&ErrorBody::new("module_reloading", "try again")).unwrap();
1888        let spoof = Frame::build(
1889            FrameType::Error,
1890            Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
1891            pending.module_channel,
1892            pending.module_epoch,
1893            701,
1894            body.clone(),
1895        )
1896        .unwrap();
1897        router
1898            .route_for_connection(&module_ctx, spoof)
1899            .await
1900            .unwrap();
1901        let delivered = client_rx.recv().await.unwrap();
1902        assert_eq!(delivered.header.ty, FrameType::Error);
1903        assert_eq!(delivered.header.channel, pending.client_channel);
1904        assert_eq!(delivered.header.epoch, pending.client_epoch);
1905        assert_eq!(delivered.header.corr, 701);
1906        assert!(
1907            !delivered.header.flags.is_daemon_origin(),
1908            "module-set daemon origin must be cleared: {:?}",
1909            delivered.header.flags
1910        );
1911        assert_eq!(delivered.body, body, "the body travels byte-identical");
1912    }
1913
1914    /// Clearing the origin bit strips exactly that bit: priority, LAST and
1915    /// subscription survive the splice unchanged.
1916    #[tokio::test]
1917    async fn module_response_with_daemon_bit_is_stripped_preserving_other_flags() {
1918        use subc_protocol::FLAG_SUBSCRIPTION;
1919        let (
1920            router,
1921            _forwarding,
1922            client_ctx,
1923            mut client_rx,
1924            module_ctx,
1925            _module_egress_rx,
1926            mut module_rx,
1927            pending,
1928        ) = dynamic_route_fixture(true);
1929        let _ = client_rx.recv().await;
1930        router
1931            .route_for_connection(
1932                &client_ctx,
1933                route_frame(
1934                    FrameType::Request,
1935                    pending.client_channel,
1936                    pending.client_epoch,
1937                    701,
1938                ),
1939            )
1940            .await
1941            .unwrap();
1942        let _ = module_rx.recv().await;
1943
1944        let sent_flags = Flags(
1945            Flags::new(true, Priority::Background, true).0
1946                | FLAG_SUBSCRIPTION
1947                | subc_protocol::FLAG_DAEMON_ORIGIN,
1948        );
1949        assert!(sent_flags.is_daemon_origin());
1950        let response = Frame::build(
1951            FrameType::Response,
1952            sent_flags,
1953            pending.module_channel,
1954            pending.module_epoch,
1955            701,
1956            b"route-body".to_vec(),
1957        )
1958        .unwrap();
1959        router
1960            .route_for_connection(&module_ctx, response)
1961            .await
1962            .unwrap();
1963        let delivered = client_rx.recv().await.unwrap();
1964        assert_eq!(delivered.header.ty, FrameType::Response);
1965        assert!(
1966            !delivered.header.flags.is_daemon_origin(),
1967            "module-set daemon origin must be cleared: {:?}",
1968            delivered.header.flags
1969        );
1970        assert!(
1971            delivered.header.flags.is_binary(),
1972            "binary flag must survive: {:?}",
1973            delivered.header.flags
1974        );
1975        assert!(
1976            delivered.header.flags.is_last(),
1977            "LAST flag must survive: {:?}",
1978            delivered.header.flags
1979        );
1980        assert!(
1981            delivered.header.flags.is_subscription(),
1982            "subscription flag must survive: {:?}",
1983            delivered.header.flags
1984        );
1985        assert_eq!(
1986            delivered.header.flags.priority(),
1987            Some(Priority::Background),
1988            "priority must survive"
1989        );
1990    }
1991
1992    /// A client frame carrying the origin bit must not reach the module
1993    /// looking daemon-authored either: the splice clears it symmetrically.
1994    #[tokio::test]
1995    async fn client_frame_with_daemon_bit_is_stripped_before_module_delivery() {
1996        let (
1997            router,
1998            _forwarding,
1999            client_ctx,
2000            mut client_rx,
2001            _module_ctx,
2002            _module_egress_rx,
2003            mut module_rx,
2004            pending,
2005        ) = dynamic_route_fixture(true);
2006        let _ = client_rx.recv().await;
2007        let spoof = Frame::build(
2008            FrameType::Request,
2009            Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2010            pending.client_channel,
2011            pending.client_epoch,
2012            701,
2013            b"route-body".to_vec(),
2014        )
2015        .unwrap();
2016        router
2017            .route_for_connection(&client_ctx, spoof)
2018            .await
2019            .unwrap();
2020        let delivered = module_rx.recv().await.unwrap();
2021        assert_eq!(delivered.header.ty, FrameType::Request);
2022        assert!(
2023            !delivered.header.flags.is_daemon_origin(),
2024            "client-set daemon origin must be cleared: {:?}",
2025            delivered.header.flags
2026        );
2027    }
2028
2029    #[tokio::test]
2030    async fn channel_zero_uses_control_handler_not_backend_registry() {
2031        let mut router = Router::with_default_self_handler();
2032        router.register_backend(1, EchoBackend).unwrap();
2033        let (ctx, mut rx) = route_ctx();
2034
2035        router.route_for_connection(&ctx, ping(77)).await.unwrap();
2036        let response = rx.recv().await.unwrap();
2037
2038        assert_eq!(response.header.ty, FrameType::Pong);
2039        assert_eq!(response.header.channel, 0);
2040        assert_eq!(response.header.corr, 77);
2041        assert!(response.body.is_empty());
2042    }
2043
2044    #[tokio::test]
2045    async fn slow_control_dispatch_logs_decoded_op_and_elapsed_time() {
2046        let control = Arc::new(
2047            ControlHandler::new(Arc::new(Registry::default()))
2048                .with_control_dispatch_delay(Duration::from_millis(1050)),
2049        );
2050        let router = Router::with_control_handler(control);
2051        let (ctx, mut rx) = route_ctx();
2052        let (output, guard) = log_capture(tracing::Level::WARN);
2053
2054        router
2055            .route_for_connection(&ctx, request(0, 41, br#"{"op":"server.describe"}"#))
2056            .await
2057            .expect("slow request routes");
2058        assert!(rx.recv().await.is_some(), "request receives a response");
2059        drop(guard);
2060
2061        let logs = captured_logs(&output);
2062        assert!(logs.contains("slow control dispatch"));
2063        assert!(logs.contains("op=server.describe"));
2064        assert!(logs.contains("connection_id=0"));
2065        assert!(logs.contains("corr=41"));
2066        assert!(
2067            logged_millis(&logs, "elapsed_ms=") >= 1050,
2068            "elapsed must include the injected handler delay: {logs}"
2069        );
2070    }
2071
2072    #[tokio::test]
2073    async fn fast_control_dispatch_emits_arrival_without_slow_warning() {
2074        let router = Router::with_default_self_handler();
2075        let (ctx, mut rx) = route_ctx();
2076        let (output, guard) = log_capture(tracing::Level::DEBUG);
2077
2078        router
2079            .route_for_connection(&ctx, request(0, 42, br#"{"op":"server.describe"}"#))
2080            .await
2081            .expect("fast request routes");
2082        assert!(rx.recv().await.is_some(), "request receives a response");
2083        drop(guard);
2084
2085        let logs = captured_logs(&output);
2086        assert!(logs.contains("control dispatch op=server.describe connection_id=0 corr=42"));
2087        assert!(!logs.contains("slow control dispatch"));
2088    }
2089
2090    #[tokio::test]
2091    async fn control_dispatch_arrival_is_hidden_at_info() {
2092        let router = Router::with_default_self_handler();
2093        let (ctx, mut rx) = route_ctx();
2094        let (output, guard) = log_capture(tracing::Level::INFO);
2095
2096        router
2097            .route_for_connection(&ctx, request(0, 43, br#"{"op":"server.describe"}"#))
2098            .await
2099            .expect("fast request routes");
2100        assert!(rx.recv().await.is_some(), "request receives a response");
2101        drop(guard);
2102
2103        assert!(
2104            !captured_logs(&output).contains("control dispatch"),
2105            "arrival logging must stay hidden at INFO"
2106        );
2107    }
2108
2109    #[tokio::test]
2110    async fn supervisor_list_logs_contended_snapshot_lock_only() {
2111        let registry = Arc::new(Registry::default());
2112        let handle = SupervisorHandle::new();
2113        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
2114            .with_handle(handle.clone());
2115        let module = supervisor
2116            .supervise_configured(
2117                ModuleSpec {
2118                    module_id: "held-module".to_string(),
2119                    program: "test-module".into(),
2120                    args: Vec::new(),
2121                    env: Vec::new(),
2122                    reserved: false,
2123                    reserved_prefixes: Vec::new(),
2124                    protocol: ModuleProtocol::Subc,
2125                    overlap: Default::default(),
2126                },
2127                false,
2128            )
2129            .expect("disabled test module is supervised");
2130        let router = Router::with_control_handler(Arc::new(
2131            ControlHandler::new(Arc::clone(&registry)).with_supervisor(handle),
2132        ));
2133        let (ctx, mut rx) = route_ctx();
2134        let (acquired, ready) = std_mpsc::channel();
2135        let holder = module.hold_snapshot_for_test(acquired, Duration::from_millis(400));
2136        ready.recv().expect("holder acquired snapshot lock");
2137        let (output, guard) = log_capture(tracing::Level::WARN);
2138
2139        router
2140            .route_for_connection(&ctx, request(0, 44, br#"{"op":"supervisor.list"}"#))
2141            .await
2142            .expect("list request routes after the lock releases");
2143        assert!(
2144            rx.recv().await.is_some(),
2145            "list request receives a response"
2146        );
2147        holder.join().expect("snapshot holder exits cleanly");
2148        drop(guard);
2149
2150        let logs = captured_logs(&output);
2151        assert!(logs.contains("slow snapshot lock"));
2152        assert!(logs.contains("module_id=held-module"));
2153        assert!(logs.contains("caller=list"));
2154        assert!(
2155            logged_millis(&logs, "waited_ms=") >= 250,
2156            "wait must exceed the slow-lock threshold: {logs}"
2157        );
2158
2159        let (output, guard) = log_capture(tracing::Level::WARN);
2160        router
2161            .route_for_connection(&ctx, request(0, 45, br#"{"op":"supervisor.list"}"#))
2162            .await
2163            .expect("uncontended list request routes");
2164        assert!(
2165            rx.recv().await.is_some(),
2166            "uncontended list receives a response"
2167        );
2168        drop(guard);
2169        assert!(
2170            !captured_logs(&output).contains("slow snapshot lock"),
2171            "uncontended list acquisition must not warn"
2172        );
2173    }
2174
2175    #[tokio::test]
2176    async fn full_module_to_client_sink_requests_client_close_without_erroring_module() {
2177        let forwarding = Arc::new(ForwardingTable::default());
2178        let control = Arc::new(ControlHandler::with_forwarding(
2179            Arc::new(crate::Registry::default()),
2180            Arc::clone(&forwarding),
2181        ));
2182        let router = Router::with_control_handler(control);
2183        let module_connection = ConnectionId::new(10);
2184        let client_connection = ConnectionId::new(20);
2185        let mut close_receiver = forwarding.register_connection_close(client_connection);
2186        let (module_tx, _module_rx) = mpsc::channel(1);
2187        forwarding
2188            .register_module_connection(
2189                module_connection,
2190                "full-sink-provider".to_string(),
2191                1,
2192                Concurrency::ModuleManaged,
2193                FrameSink::new(module_tx),
2194            )
2195            .unwrap();
2196        let (client_tx, mut client_rx) = mpsc::channel(1);
2197        let pending = forwarding
2198            .begin_route_bind_relay_for_test(
2199                client_connection,
2200                FrameSink::new(client_tx),
2201                700,
2202                "full-sink-provider",
2203            )
2204            .unwrap();
2205        forwarding
2206            .complete_pending_relay(
2207                module_connection,
2208                pending.corr,
2209                RouteBindRelayOutcome::Accepted,
2210            )
2211            .unwrap();
2212
2213        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2214        let module_ctx = RouteCtx {
2215            connection_id: module_connection,
2216            egress: FrameSink::new(module_egress_tx),
2217        };
2218        let terminal = Frame::build(
2219            FrameType::Response,
2220            Flags::new(false, Priority::Interactive, true),
2221            pending.module_channel,
2222            pending.module_epoch,
2223            701,
2224            b"terminal".to_vec(),
2225        )
2226        .unwrap();
2227
2228        router
2229            .route_for_connection(&module_ctx, terminal)
2230            .await
2231            .unwrap();
2232        let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2233            .await
2234            .expect("close request should be sent for the full client sink")
2235            .expect("close sender should include a reason");
2236        assert!(
2237            reason
2238                .to_string()
2239                .contains("module_to_client_delivery_failed"),
2240            "unexpected close reason: {reason}"
2241        );
2242        assert_eq!(client_rx.try_recv().unwrap().header.corr, 700);
2243        assert!(client_rx.try_recv().is_err());
2244        assert_eq!(
2245            router.counters.snapshot()["client_egress_close_delivery_failed"],
2246            1
2247        );
2248    }
2249
2250    /// A terminal frame ends the request at the module even when the client
2251    /// cannot take it, so the drain must stop counting it at once rather than
2252    /// when the client connection's cleanup later removes the route.
2253    #[tokio::test]
2254    async fn terminal_frame_releases_its_credit_even_when_client_delivery_fails() {
2255        let forwarding = Arc::new(ForwardingTable::default());
2256        let control = Arc::new(ControlHandler::with_forwarding(
2257            Arc::new(crate::Registry::default()),
2258            Arc::clone(&forwarding),
2259        ));
2260        let router = Router::with_control_handler(control);
2261        let module_connection = ConnectionId::new(11);
2262        let client_connection = ConnectionId::new(21);
2263        let _close_receiver = forwarding.register_connection_close(client_connection);
2264        let (module_tx, _module_rx) = mpsc::channel(1);
2265        forwarding
2266            .register_module_connection(
2267                module_connection,
2268                "credit-provider".to_string(),
2269                1,
2270                Concurrency::ModuleManaged,
2271                FrameSink::new(module_tx),
2272            )
2273            .unwrap();
2274        // Capacity one, filled by the route.open response, so the terminal
2275        // frame below cannot be enqueued for the client.
2276        let (client_tx, _client_rx) = mpsc::channel(1);
2277        let pending = forwarding
2278            .begin_route_bind_relay_for_test(
2279                client_connection,
2280                FrameSink::new(client_tx),
2281                800,
2282                "credit-provider",
2283            )
2284            .unwrap();
2285        forwarding
2286            .complete_pending_relay(
2287                module_connection,
2288                pending.corr,
2289                RouteBindRelayOutcome::Accepted,
2290            )
2291            .unwrap();
2292        let DataRoute::Client(DataRouteState::Bound(route)) = forwarding
2293            .lookup_data_route(
2294                client_connection,
2295                pending.client_channel,
2296                pending.client_epoch,
2297            )
2298            .unwrap()
2299        else {
2300            panic!("expected a bound client route");
2301        };
2302        route.flow.acquire_tagged(801, false).await.unwrap();
2303        assert_eq!(route.flow.drain_in_flight(), 1);
2304
2305        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2306        let module_ctx = RouteCtx {
2307            connection_id: module_connection,
2308            egress: FrameSink::new(module_egress_tx),
2309        };
2310        let terminal = Frame::build(
2311            FrameType::Response,
2312            Flags::new(false, Priority::Interactive, true),
2313            pending.module_channel,
2314            pending.module_epoch,
2315            801,
2316            b"terminal".to_vec(),
2317        )
2318        .unwrap();
2319        router
2320            .route_for_connection(&module_ctx, terminal)
2321            .await
2322            .unwrap();
2323
2324        assert_eq!(
2325            router.counters.snapshot()["client_egress_close_delivery_failed"],
2326            1,
2327            "the client delivery must have failed for this test to mean anything"
2328        );
2329        assert_eq!(
2330            route.flow.drain_in_flight(),
2331            0,
2332            "the module's terminal frame must release its credit even though the client could not take it"
2333        );
2334    }
2335
2336    #[tokio::test]
2337    async fn full_route_goodbye_sink_requests_target_close_without_erroring_module() {
2338        let forwarding = Arc::new(ForwardingTable::default());
2339        let control = Arc::new(ControlHandler::with_forwarding(
2340            Arc::new(crate::Registry::default()),
2341            Arc::clone(&forwarding),
2342        ));
2343        let router = Router::with_control_handler(control);
2344        let module_connection = ConnectionId::new(30);
2345        let client_connection = ConnectionId::new(40);
2346        let mut close_receiver = forwarding.register_connection_close(client_connection);
2347        let (module_tx, _module_rx) = mpsc::channel(1);
2348        forwarding
2349            .register_module_connection(
2350                module_connection,
2351                "goodbye-full-provider".to_string(),
2352                1,
2353                Concurrency::ModuleManaged,
2354                FrameSink::new(module_tx),
2355            )
2356            .unwrap();
2357        let (client_tx, mut client_rx) = mpsc::channel(1);
2358        let pending = forwarding
2359            .begin_route_bind_relay_for_test(
2360                client_connection,
2361                FrameSink::new(client_tx),
2362                800,
2363                "goodbye-full-provider",
2364            )
2365            .unwrap();
2366        forwarding
2367            .complete_pending_relay(
2368                module_connection,
2369                pending.corr,
2370                RouteBindRelayOutcome::Accepted,
2371            )
2372            .unwrap();
2373
2374        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2375        let module_ctx = RouteCtx {
2376            connection_id: module_connection,
2377            egress: FrameSink::new(module_egress_tx),
2378        };
2379        let goodbye = Frame::build(
2380            FrameType::Goodbye,
2381            Flags::new(false, Priority::Passive, true),
2382            pending.module_channel,
2383            pending.module_epoch,
2384            801,
2385            Vec::new(),
2386        )
2387        .unwrap();
2388
2389        router
2390            .route_for_connection(&module_ctx, goodbye)
2391            .await
2392            .unwrap();
2393        let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2394            .await
2395            .expect("close request should be sent for the full GOODBYE sink")
2396            .expect("close sender should include a reason");
2397        assert!(
2398            reason.to_string().contains("route_goodbye_delivery_failed"),
2399            "unexpected close reason: {reason}"
2400        );
2401        assert_eq!(client_rx.try_recv().unwrap().header.corr, 800);
2402        assert!(client_rx.try_recv().is_err());
2403        assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 1);
2404        assert_eq!(router.counters.snapshot()["route_released_epoch_fenced"], 1);
2405    }
2406
2407    /// One module and one client connection with `routes` routes bound between
2408    /// them. The client uses a real connection egress queue (the same byte
2409    /// budget and frame-count backstop as a live connection), and its route.open
2410    /// responses are drained so the queue starts empty.
2411    async fn multi_route_client(
2412        module_id: &str,
2413        module_connection: ConnectionId,
2414        client_connection: ConnectionId,
2415        routes: usize,
2416    ) -> (
2417        Router,
2418        FrameSink,
2419        mpsc::Receiver<OutboundFrame>,
2420        RouteCtx,
2421        Vec<crate::forwarding::PendingRouteBindRelay>,
2422        ConnectionCloseReceiver,
2423    ) {
2424        let forwarding = Arc::new(ForwardingTable::default());
2425        let control = Arc::new(ControlHandler::with_forwarding(
2426            Arc::new(crate::Registry::default()),
2427            Arc::clone(&forwarding),
2428        ));
2429        let router = Router::with_control_handler(control);
2430        let close_receiver = forwarding.register_connection_close(client_connection);
2431        let (module_tx, _module_rx) = mpsc::channel(8);
2432        forwarding
2433            .register_module_connection(
2434                module_connection,
2435                module_id.to_string(),
2436                1,
2437                Concurrency::ModuleManaged,
2438                FrameSink::new(module_tx),
2439            )
2440            .unwrap();
2441        let (client_sink, mut client_rx) = crate::server::connection_egress();
2442        let mut bound = Vec::with_capacity(routes);
2443        for index in 0..routes {
2444            let pending = forwarding
2445                .begin_route_bind_relay_for_test(
2446                    client_connection,
2447                    client_sink.clone(),
2448                    900 + index as u64,
2449                    module_id,
2450                )
2451                .unwrap();
2452            forwarding
2453                .complete_pending_relay(
2454                    module_connection,
2455                    pending.corr,
2456                    RouteBindRelayOutcome::Accepted,
2457                )
2458                .unwrap();
2459            assert_eq!(
2460                client_rx.recv().await.unwrap().header.corr,
2461                900 + index as u64
2462            );
2463            bound.push(pending);
2464        }
2465        let (module_egress_tx, _module_egress_rx) = mpsc::channel(8);
2466        let module_ctx = RouteCtx {
2467            connection_id: module_connection,
2468            egress: FrameSink::new(module_egress_tx),
2469        };
2470        (
2471            router,
2472            client_sink,
2473            client_rx,
2474            module_ctx,
2475            bound,
2476            close_receiver,
2477        )
2478    }
2479
2480    /// Per-frame cost of the egress sink's admission and release accounting:
2481    /// one million 200-byte frames enqueued with `try_send` and taken off the
2482    /// queue the way the connection writer does, in batches of 1,000 so the
2483    /// queue stays well inside its budget. Prints nanoseconds per frame; it is
2484    /// a measurement, not a gate. Run with
2485    /// `cargo test --release -p subc-daemon --lib egress_sink_per_frame_cost -- --ignored --nocapture`.
2486    #[test]
2487    #[ignore = "timing measurement, run on demand"]
2488    fn egress_sink_per_frame_cost() {
2489        const FRAMES: usize = 1_000_000;
2490        const BATCH: usize = 1_000;
2491        let (sink, mut rx) = crate::server::connection_egress();
2492        let template = stream_frame(9, 1, 0, vec![b't'; 200]);
2493        let started = Instant::now();
2494        for _ in 0..FRAMES / BATCH {
2495            for _ in 0..BATCH {
2496                sink.try_send(template.clone()).unwrap();
2497            }
2498            for _ in 0..BATCH {
2499                let outbound = rx.try_recv().unwrap();
2500                if let Some(charge) = &outbound.charge {
2501                    charge.taken_by_writer();
2502                }
2503                drop(std::hint::black_box(outbound));
2504            }
2505        }
2506        let elapsed = started.elapsed();
2507        assert_eq!(sink.backlog().queued_bytes, 0);
2508        println!(
2509            "egress sink: {FRAMES} frames in {elapsed:?}, {:.1} ns/frame",
2510            elapsed.as_nanos() as f64 / FRAMES as f64
2511        );
2512    }
2513
2514    fn stream_frame(channel: u16, epoch: u32, corr: u64, body: Vec<u8>) -> Frame {
2515        Frame::build(
2516            FrameType::StreamData,
2517            Flags::new(false, Priority::Interactive, false),
2518            channel,
2519            epoch,
2520            corr,
2521            body,
2522        )
2523        .unwrap()
2524    }
2525
2526    fn module_route_for_test(
2527        router: &Router,
2528        ctx: &RouteCtx,
2529        route: &crate::forwarding::PendingRouteBindRelay,
2530    ) -> Arc<RouteBinding> {
2531        match router
2532            .forwarding
2533            .lookup_data_route(ctx.connection_id, route.module_channel, route.module_epoch)
2534            .unwrap()
2535        {
2536            DataRoute::Module(DataRouteState::Bound(binding)) => binding,
2537            other => panic!("expected a bound module route, got {other:?}"),
2538        }
2539    }
2540
2541    fn assert_frame_size_error(frame: &Frame, channel: u16, epoch: u32, corr: u64, bytes: usize) {
2542        assert_eq!(frame.header.ty, FrameType::Error);
2543        assert_eq!(frame.header.channel, channel);
2544        assert_eq!(frame.header.epoch, epoch);
2545        assert_eq!(frame.header.corr, corr);
2546        assert!(frame.header.flags.is_daemon_origin());
2547        let body: ErrorBody = serde_json::from_slice(&frame.body).unwrap();
2548        assert_eq!(body.code, error_codes::FRAME_TOO_LARGE);
2549        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2550        assert!(body.message.contains(&bytes.to_string()));
2551        assert!(body.message.contains(&limit.to_string()));
2552        assert_eq!(
2553            body.detail,
2554            Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit }))
2555        );
2556    }
2557
2558    #[tokio::test]
2559    async fn module_egress_response_oversized_reply_keeps_connection_and_other_route() {
2560        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
2561        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2562            "oversize-provider",
2563            ConnectionId::new(80),
2564            ConnectionId::new(81),
2565            2,
2566        )
2567        .await;
2568        let binding = module_route_for_test(&router, &ctx, &routes[0]);
2569        binding.flow.acquire_tagged(101, false).await.unwrap();
2570        module_route_for_test(&router, &ctx, &routes[1])
2571            .flow
2572            .acquire_tagged(102, false)
2573            .await
2574            .unwrap();
2575        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2576        let oversized = Frame::build(
2577            FrameType::Response,
2578            Flags::new(false, Priority::Interactive, true),
2579            routes[0].module_channel,
2580            routes[0].module_epoch,
2581            101,
2582            vec![b'x'; limit - subc_protocol::HEADER_LEN + 1],
2583        )
2584        .unwrap();
2585        assert_eq!(sink.backlog().queued_bytes, 0);
2586        router.route_for_connection(&ctx, oversized).await.unwrap();
2587
2588        // Reproduce a large reply followed by ordinary traffic on another route
2589        // before the client reads anything. The large reply must not monopolize
2590        // the queue and make that second delivery close the shared connection.
2591        router
2592            .route_for_connection(
2593                &ctx,
2594                route_frame(
2595                    FrameType::Response,
2596                    routes[1].module_channel,
2597                    routes[1].module_epoch,
2598                    102,
2599                ),
2600            )
2601            .await
2602            .unwrap();
2603        assert!(
2604            close.try_recv().is_err(),
2605            "an oversized reply must not close the multiplexed client"
2606        );
2607        let error = rx.try_recv().expect("the waiting client gets an ERROR");
2608        assert_frame_size_error(
2609            &error,
2610            routes[0].client_channel,
2611            routes[0].client_epoch,
2612            101,
2613            limit + 1,
2614        );
2615        let second = rx.try_recv().expect("the other route's reply arrives");
2616        assert_eq!(second.header.ty, FrameType::Response);
2617        assert_eq!(second.header.channel, routes[1].client_channel);
2618        assert_eq!(second.header.epoch, routes[1].client_epoch);
2619        assert_eq!(second.header.corr, 102);
2620        assert_eq!(second.body, b"route-body");
2621        assert_eq!(binding.flow.drain_in_flight(), 0);
2622        for route in &routes {
2623            module_route_for_test(&router, &ctx, route);
2624        }
2625        assert!(rx.try_recv().is_err());
2626        drop((error, second));
2627        assert_eq!(sink.backlog().queued_bytes, 0);
2628        assert_eq!(
2629            router.counters.snapshot()["client_egress_close_delivery_failed"],
2630            0
2631        );
2632
2633        let captured = test_log::captured_logs(&logs);
2634        let warnings: Vec<_> = captured
2635            .lines()
2636            .filter(|line| line.contains("WARN"))
2637            .collect();
2638        assert_eq!(warnings.len(), 1, "{captured}");
2639        let warning = warnings[0];
2640        for expected in [
2641            "module_id=oversize-provider".to_string(),
2642            "client_principal=direct".to_string(),
2643            format!("client_channel={}", routes[0].client_channel),
2644            format!("frame_bytes={}", limit + 1),
2645            format!("byte_limit={limit}"),
2646        ] {
2647            assert!(warning.contains(&expected), "missing {expected}: {warning}");
2648        }
2649    }
2650
2651    async fn poll_once<F: std::future::Future>(
2652        mut future: std::pin::Pin<&mut F>,
2653    ) -> std::task::Poll<F::Output> {
2654        std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
2655    }
2656
2657    fn fill_client_budget(sink: &FrameSink, route: &crate::forwarding::PendingRouteBindRelay) {
2658        sink.try_send(
2659            Frame::build(
2660                FrameType::Response,
2661                Flags::new(false, Priority::Interactive, true),
2662                route.client_channel,
2663                route.client_epoch,
2664                100,
2665                vec![
2666                    b'x';
2667                    crate::server::CONNECTION_EGRESS_BYTE_BUDGET - subc_protocol::HEADER_LEN
2668                ],
2669            )
2670            .unwrap(),
2671        )
2672        .unwrap();
2673        assert_eq!(
2674            sink.backlog().queued_bytes,
2675            crate::server::CONNECTION_EGRESS_BYTE_BUDGET
2676        );
2677    }
2678
2679    async fn assert_full_client_burst_survives(ty: FrameType, oversized: bool) {
2680        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2681            "draining-burst",
2682            ConnectionId::new(92),
2683            ConnectionId::new(93),
2684            2,
2685        )
2686        .await;
2687        fill_client_budget(&sink, &routes[0]);
2688        let frame = if oversized {
2689            Frame::build(
2690                ty,
2691                Flags::new(false, Priority::Interactive, false),
2692                routes[1].module_channel,
2693                routes[1].module_epoch,
2694                101,
2695                vec![b's'; crate::server::CONNECTION_EGRESS_BYTE_BUDGET],
2696            )
2697            .unwrap()
2698        } else {
2699            route_frame(ty, routes[1].module_channel, routes[1].module_epoch, 101)
2700        };
2701        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2702        assert!(poll_once(delivery.as_mut()).await.is_pending());
2703        assert!(
2704            close.try_recv().is_err(),
2705            "fresh bursts must not close: {ty:?}"
2706        );
2707        tokio::time::advance(Duration::from_millis(50)).await;
2708        let queued = rx.try_recv().unwrap();
2709        queued.charge.as_ref().unwrap().taken_by_writer();
2710        drop(queued);
2711        assert!(matches!(
2712            poll_once(delivery.as_mut()).await,
2713            std::task::Poll::Ready(Ok(()))
2714        ));
2715        let delivered = rx
2716            .try_recv()
2717            .expect("the burst frame must be delivered after draining");
2718        assert_eq!(delivered.header.channel, routes[1].client_channel);
2719        assert_eq!(delivered.header.epoch, routes[1].client_epoch);
2720        assert_eq!(delivered.header.corr, 101);
2721        if oversized {
2722            assert_frame_size_error(
2723                &delivered,
2724                routes[1].client_channel,
2725                routes[1].client_epoch,
2726                101,
2727                crate::server::CONNECTION_EGRESS_BYTE_BUDGET + subc_protocol::HEADER_LEN,
2728            );
2729        } else {
2730            assert_eq!(delivered.header.ty, ty);
2731        }
2732        assert!(rx.try_recv().is_err());
2733        assert!(close.try_recv().is_err());
2734        assert_eq!(
2735            router.counters.snapshot()["client_egress_close_delivery_failed"],
2736            0
2737        );
2738        assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 0);
2739    }
2740
2741    #[tokio::test(start_paused = true)]
2742    async fn module_egress_fresh_full_queue_burst_survives() {
2743        for ty in [
2744            FrameType::Response,
2745            FrameType::StreamData,
2746            FrameType::Request,
2747            FrameType::Push,
2748        ] {
2749            assert_full_client_burst_survives(ty, false).await;
2750        }
2751    }
2752
2753    #[tokio::test(start_paused = true)]
2754    async fn module_goodbye_fresh_full_queue_burst_survives() {
2755        assert_full_client_burst_survives(FrameType::Goodbye, false).await;
2756    }
2757
2758    #[tokio::test(start_paused = true)]
2759    async fn frame_count_backstop_also_allows_a_draining_burst() {
2760        let (tx, mut rx) = mpsc::channel(1);
2761        let sink = FrameSink::with_byte_budget(tx, 2_000);
2762        sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
2763            .unwrap();
2764        let mut delivery = Box::pin(sink.send_to_client(stream_frame(1, 1, 2, vec![b'y'; 100])));
2765        assert!(poll_once(delivery.as_mut()).await.is_pending());
2766        assert!(
2767            sink.backlog().queued_bytes < 2_000,
2768            "only frame capacity is exhausted"
2769        );
2770        tokio::time::advance(Duration::from_millis(50)).await;
2771        drop(rx.try_recv().unwrap());
2772        assert!(matches!(
2773            poll_once(delivery.as_mut()).await,
2774            std::task::Poll::Ready(Ok(()))
2775        ));
2776        assert_eq!(rx.try_recv().unwrap().header.corr, 2);
2777    }
2778
2779    #[tokio::test(start_paused = true)]
2780    async fn oversized_reply_replacement_fresh_full_queue_burst_survives() {
2781        for ty in [FrameType::Response, FrameType::StreamData] {
2782            assert_full_client_burst_survives(ty, true).await;
2783        }
2784    }
2785
2786    #[tokio::test(start_paused = true)]
2787    async fn module_egress_stopped_reader_closes_without_waiting() {
2788        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
2789        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2790            "oldest-stopped-reader",
2791            ConnectionId::new(94),
2792            ConnectionId::new(95),
2793            2,
2794        )
2795        .await;
2796        fill_client_budget(&sink, &routes[0]);
2797        // The writer has taken the head but cannot finish writing it. Its
2798        // bytes stay charged, and the atomic age still diagnoses that stall.
2799        let writing = rx.try_recv().unwrap();
2800        writing.charge.as_ref().unwrap().taken_by_writer();
2801        tokio::time::advance(Duration::from_millis(251)).await;
2802        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(251)));
2803        assert_eq!(sink.backlog().queued_bytes, queued_frame_bytes(&writing));
2804        let started = tokio::time::Instant::now();
2805        let frame = stream_frame(
2806            routes[0].module_channel,
2807            routes[0].module_epoch,
2808            101,
2809            b"secret-body-never-log".to_vec(),
2810        );
2811        let flags = frame.header.flags.0;
2812        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2813        assert!(
2814            matches!(
2815                poll_once(delivery.as_mut()).await,
2816                std::task::Poll::Ready(Ok(()))
2817            ),
2818            "an oldest frame older than 250 ms must close immediately, without a new wait"
2819        );
2820        assert_eq!(started.elapsed(), Duration::ZERO);
2821        assert!(close
2822            .try_recv()
2823            .unwrap()
2824            .to_string()
2825            .contains("module_to_client_delivery_failed"));
2826        let captured = test_log::captured_logs(&logs);
2827        for expected in [
2828            "frame_type=Some(StreamData)".to_string(),
2829            format!("frame_flags=Some({flags})"),
2830            "oldest_queued_ms=251".to_string(),
2831            "close_cause=\"oldest_frame_older_than_250_ms\"".to_string(),
2832        ] {
2833            assert!(
2834                captured.contains(&expected),
2835                "missing {expected}: {captured}"
2836            );
2837        }
2838        assert!(!captured.contains("secret-body-never-log"));
2839    }
2840
2841    #[tokio::test(start_paused = true)]
2842    async fn client_gone_mid_wait_resumes_module_reader_immediately() {
2843        for writer_gone in [false, true] {
2844            let (router, sink, mut rx, ctx, routes, _close) = multi_route_client(
2845                "gone-during-burst",
2846                ConnectionId::new(96),
2847                ConnectionId::new(97),
2848                1,
2849            )
2850            .await;
2851            fill_client_budget(&sink, &routes[0]);
2852            let frame = route_frame(
2853                FrameType::Response,
2854                routes[0].module_channel,
2855                routes[0].module_epoch,
2856                101,
2857            );
2858            let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2859            assert!(poll_once(delivery.as_mut()).await.is_pending());
2860            tokio::time::advance(Duration::from_millis(50)).await;
2861            let started = tokio::time::Instant::now();
2862            if writer_gone {
2863                rx.close();
2864            } else {
2865                sink.close_client_delivery();
2866            }
2867            assert!(
2868                matches!(
2869                    poll_once(delivery.as_mut()).await,
2870                    std::task::Poll::Ready(Ok(()))
2871                ),
2872                "client teardown must wake the module reader before the writer drains"
2873            );
2874            assert_eq!(started.elapsed(), Duration::ZERO);
2875            assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
2876            assert_eq!(rx.try_recv().unwrap().header.corr, 100);
2877            assert!(rx.try_recv().is_err());
2878        }
2879    }
2880
2881    #[tokio::test(start_paused = true)]
2882    async fn cancelled_module_delivery_releases_waiter_without_closing_client() {
2883        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2884            "cancelled-burst",
2885            ConnectionId::new(98),
2886            ConnectionId::new(99),
2887            1,
2888        )
2889        .await;
2890        fill_client_budget(&sink, &routes[0]);
2891        let frame = route_frame(
2892            FrameType::Response,
2893            routes[0].module_channel,
2894            routes[0].module_epoch,
2895            101,
2896        );
2897        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2898        assert!(poll_once(delivery.as_mut()).await.is_pending());
2899        assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 1);
2900        drop(delivery);
2901        assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
2902        tokio::time::advance(Duration::from_secs(1)).await;
2903        assert!(close.try_recv().is_err());
2904        drop(rx.try_recv().unwrap());
2905        assert_eq!(sink.backlog().queued_bytes, 0);
2906        assert!(sink.backlog().oldest_age.is_none());
2907    }
2908
2909    #[tokio::test(start_paused = true)]
2910    async fn egress_atomic_age_follows_writer_and_clears_when_drained() {
2911        let (sink, mut rx) = crate::server::connection_egress();
2912        sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
2913            .unwrap();
2914        tokio::time::advance(Duration::from_millis(200)).await;
2915        sink.try_send(stream_frame(2, 1, 2, vec![b'y'; 100]))
2916            .unwrap();
2917        let writing = rx.try_recv().unwrap();
2918        writing.charge.as_ref().unwrap().taken_by_writer();
2919        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
2920        drop(writing);
2921        // Between frames the lock-free marker can still name the previous
2922        // head. The next writer take advances it without scanning the queue.
2923        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
2924        tokio::time::advance(Duration::from_millis(50)).await;
2925        let next = rx.try_recv().unwrap();
2926        next.charge.as_ref().unwrap().taken_by_writer();
2927        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(50)));
2928        assert!(
2929            sink.backlog().queued_bytes > 0,
2930            "in-flight bytes remain charged until written"
2931        );
2932        drop(next);
2933        assert_eq!(sink.backlog().queued_bytes, 0);
2934        assert_eq!(sink.backlog().oldest_age, None);
2935    }
2936
2937    #[tokio::test(start_paused = true)]
2938    async fn module_egress_response_full_client_waits_only_remaining_grace() {
2939        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2940            "backpressure-provider",
2941            ConnectionId::new(82),
2942            ConnectionId::new(83),
2943            2,
2944        )
2945        .await;
2946        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2947        // Exactly the limit, including its header, must be admitted to an empty
2948        // queue. Leave it unread to deterministically exhaust only byte capacity.
2949        router
2950            .route_for_connection(
2951                &ctx,
2952                Frame::build(
2953                    FrameType::Response,
2954                    Flags::new(false, Priority::Interactive, true),
2955                    routes[0].module_channel,
2956                    routes[0].module_epoch,
2957                    201,
2958                    vec![b'x'; limit - subc_protocol::HEADER_LEN],
2959                )
2960                .unwrap(),
2961            )
2962            .await
2963            .unwrap();
2964        assert_eq!(sink.backlog().queued_bytes, limit);
2965        assert_eq!(sink.backlog().queued_frames, 1);
2966        let ordinary = route_frame(
2967            FrameType::Response,
2968            routes[1].module_channel,
2969            routes[1].module_epoch,
2970            202,
2971        );
2972        assert!(queued_frame_bytes(&ordinary) < limit);
2973        // A fitting reply gets only the unused portion of the oldest frame's
2974        // 250 ms grace, not an immediate close or a fresh 250 ms wait.
2975        tokio::time::advance(Duration::from_millis(100)).await;
2976        let mut delivery = Box::pin(router.route_for_connection(&ctx, ordinary));
2977        assert!(poll_once(delivery.as_mut()).await.is_pending());
2978        assert!(close.try_recv().is_err());
2979        tokio::time::advance(Duration::from_millis(149)).await;
2980        assert!(poll_once(delivery.as_mut()).await.is_pending());
2981        assert!(close.try_recv().is_err());
2982        tokio::time::advance(Duration::from_millis(1)).await;
2983        assert!(matches!(
2984            poll_once(delivery.as_mut()).await,
2985            std::task::Poll::Ready(Ok(()))
2986        ));
2987        drop(delivery);
2988        let reason = close.try_recv().expect("the full client must close");
2989        assert!(reason
2990            .to_string()
2991            .contains("module_to_client_delivery_failed"));
2992        assert!(!reason.to_string().contains("frame_too_large"));
2993        assert_eq!(
2994            router.counters.snapshot()["client_egress_close_delivery_failed"],
2995            1
2996        );
2997        let admitted = rx.try_recv().expect("the budget-sized frame was admitted");
2998        assert_eq!(admitted.header.ty, FrameType::Response);
2999        assert_eq!(admitted.header.corr, 201);
3000        assert!(
3001            rx.try_recv().is_err(),
3002            "no oversized-frame ERROR is substituted for backpressure"
3003        );
3004    }
3005
3006    #[tokio::test(start_paused = true)]
3007    async fn oversized_reply_to_a_full_client_waits_only_remaining_grace() {
3008        let (router, sink, _rx, ctx, routes, mut close) = multi_route_client(
3009            "full-oversize-provider",
3010            ConnectionId::new(90),
3011            ConnectionId::new(91),
3012            2,
3013        )
3014        .await;
3015        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3016        // Fill the client's queue to its budget and leave it unread.
3017        router
3018            .route_for_connection(
3019                &ctx,
3020                Frame::build(
3021                    FrameType::Response,
3022                    Flags::new(false, Priority::Interactive, true),
3023                    routes[0].module_channel,
3024                    routes[0].module_epoch,
3025                    501,
3026                    vec![b'x'; limit - subc_protocol::HEADER_LEN],
3027                )
3028                .unwrap(),
3029            )
3030            .await
3031            .unwrap();
3032        assert_eq!(sink.backlog().queued_bytes, limit);
3033        let oversized = Frame::build(
3034            FrameType::Response,
3035            Flags::new(false, Priority::Interactive, true),
3036            routes[1].module_channel,
3037            routes[1].module_epoch,
3038            502,
3039            vec![b'y'; limit - subc_protocol::HEADER_LEN + 1],
3040        )
3041        .unwrap();
3042        // The replacement ERROR gets the same bounded burst grace as the
3043        // ordinary reply; oversized payloads themselves are never queued.
3044        tokio::time::advance(Duration::from_millis(80)).await;
3045        let mut delivery = Box::pin(router.route_for_connection(&ctx, oversized));
3046        assert!(poll_once(delivery.as_mut()).await.is_pending());
3047        assert!(close.try_recv().is_err());
3048        tokio::time::advance(Duration::from_millis(169)).await;
3049        assert!(poll_once(delivery.as_mut()).await.is_pending());
3050        assert!(close.try_recv().is_err());
3051        tokio::time::advance(Duration::from_millis(1)).await;
3052        assert!(matches!(
3053            poll_once(delivery.as_mut()).await,
3054            std::task::Poll::Ready(Ok(()))
3055        ));
3056        drop(delivery);
3057        let reason = close.try_recv().expect("the full client must close");
3058        assert!(reason
3059            .to_string()
3060            .contains("module_to_client_delivery_failed"));
3061    }
3062
3063    #[tokio::test]
3064    async fn oversized_reverse_request_errors_module_without_reaching_client() {
3065        let (router, _sink, mut rx, mut ctx, routes, mut close) = multi_route_client(
3066            "reverse-oversize-provider",
3067            ConnectionId::new(84),
3068            ConnectionId::new(85),
3069            2,
3070        )
3071        .await;
3072        let (module_tx, mut module_rx) = mpsc::channel(8);
3073        ctx.egress = FrameSink::new(module_tx);
3074        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3075        router
3076            .route_for_connection(
3077                &ctx,
3078                Frame::build(
3079                    FrameType::Request,
3080                    Flags::new(false, Priority::Interactive, false),
3081                    routes[0].module_channel,
3082                    routes[0].module_epoch,
3083                    301,
3084                    vec![b'r'; limit - subc_protocol::HEADER_LEN + 1],
3085                )
3086                .unwrap(),
3087            )
3088            .await
3089            .unwrap();
3090        assert!(
3091            rx.try_recv().is_err(),
3092            "the reverse request never reaches the client"
3093        );
3094        let error = module_rx.try_recv().expect("the module gets the refusal");
3095        assert_frame_size_error(
3096            &error,
3097            routes[0].module_channel,
3098            routes[0].module_epoch,
3099            301,
3100            limit + 1,
3101        );
3102        assert!(module_rx.try_recv().is_err());
3103        for (index, route) in routes.iter().enumerate() {
3104            module_route_for_test(&router, &ctx, route);
3105            router
3106                .route_for_connection(
3107                    &ctx,
3108                    route_frame(
3109                        FrameType::Response,
3110                        route.module_channel,
3111                        route.module_epoch,
3112                        302 + index as u64,
3113                    ),
3114                )
3115                .await
3116                .unwrap();
3117            let reply = rx.try_recv().expect("both routes still deliver");
3118            assert_eq!(reply.header.ty, FrameType::Response);
3119            assert_eq!(reply.header.channel, route.client_channel);
3120        }
3121        assert!(close.try_recv().is_err());
3122        assert_eq!(
3123            router.counters.snapshot()["client_egress_close_delivery_failed"],
3124            0
3125        );
3126    }
3127
3128    #[tokio::test]
3129    async fn oversized_push_closes_client_with_precise_warn() {
3130        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3131        let (router, _sink, mut rx, ctx, routes, mut close) = multi_route_client(
3132            "push-oversize-provider",
3133            ConnectionId::new(86),
3134            ConnectionId::new(87),
3135            2,
3136        )
3137        .await;
3138        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3139        router
3140            .route_for_connection(
3141                &ctx,
3142                Frame::build(
3143                    FrameType::Push,
3144                    Flags::new(false, Priority::Interactive, false),
3145                    routes[0].module_channel,
3146                    routes[0].module_epoch,
3147                    0,
3148                    vec![b'p'; limit - subc_protocol::HEADER_LEN + 1],
3149                )
3150                .unwrap(),
3151            )
3152            .await
3153            .unwrap();
3154        let reason = close
3155            .try_recv()
3156            .expect("unsolicited oversize still closes the client");
3157        assert!(reason
3158            .to_string()
3159            .contains("frame_too_large: oversized push"));
3160        assert!(
3161            rx.try_recv().is_err(),
3162            "no part of the oversized push is forwarded"
3163        );
3164        let captured = test_log::captured_logs(&logs);
3165        let warnings: Vec<_> = captured
3166            .lines()
3167            .filter(|line| line.contains("WARN"))
3168            .collect();
3169        assert_eq!(warnings.len(), 1, "{captured}");
3170        let warning = warnings[0];
3171        for expected in [
3172            "oversized push".to_string(),
3173            "module_id=\"push-oversize-provider\"".to_string(),
3174            "principals=direct".to_string(),
3175            format!("client_channel={}", routes[0].client_channel),
3176            format!("size={}", limit + 1),
3177            format!("limit={limit}"),
3178        ] {
3179            assert!(warning.contains(&expected), "missing {expected}: {warning}");
3180        }
3181        assert_eq!(
3182            router.counters.snapshot()["client_egress_close_delivery_failed"],
3183            1
3184        );
3185    }
3186
3187    #[tokio::test]
3188    async fn oversized_stream_and_error_frames_end_request_even_with_queued_traffic() {
3189        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3190            "stream-oversize-provider",
3191            ConnectionId::new(88),
3192            ConnectionId::new(89),
3193            2,
3194        )
3195        .await;
3196        let binding = module_route_for_test(&router, &ctx, &routes[0]);
3197        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3198        for ty in [
3199            FrameType::StreamData,
3200            FrameType::StreamEnd,
3201            FrameType::Error,
3202        ] {
3203            binding.flow.acquire_tagged(401, false).await.unwrap();
3204            sink.try_send(route_frame(
3205                FrameType::Response,
3206                routes[1].client_channel,
3207                routes[1].client_epoch,
3208                402,
3209            ))
3210            .unwrap();
3211            router
3212                .route_for_connection(
3213                    &ctx,
3214                    Frame::build(
3215                        ty,
3216                        Flags::new(false, Priority::Interactive, false),
3217                        routes[0].module_channel,
3218                        routes[0].module_epoch,
3219                        401,
3220                        vec![b's'; limit - subc_protocol::HEADER_LEN + 1],
3221                    )
3222                    .unwrap(),
3223                )
3224                .await
3225                .unwrap();
3226            assert!(close.try_recv().is_err());
3227            assert_eq!(rx.try_recv().unwrap().header.corr, 402);
3228            let error = rx
3229                .try_recv()
3230                .expect("every correlated oversize frame becomes ERROR");
3231            assert_frame_size_error(
3232                &error,
3233                routes[0].client_channel,
3234                routes[0].client_epoch,
3235                401,
3236                limit + 1,
3237            );
3238            assert_eq!(binding.flow.drain_in_flight(), 0, "{ty:?} ends the request");
3239            assert!(rx.try_recv().is_err());
3240        }
3241    }
3242
3243    /// An awaited send (a control or error reply) is never refused by the byte
3244    /// budget: it parks until the writer frees bytes. If a release could miss a
3245    /// parked sender, that reply would hang for the connection's lifetime, so
3246    /// this pins the wakeup: the send stays parked while the queue is full and
3247    /// completes as soon as one frame leaves it.
3248    #[tokio::test]
3249    async fn awaited_send_parked_behind_a_full_byte_budget_wakes_when_bytes_free() {
3250        let (tx, mut rx) = mpsc::channel(1024);
3251        let sink = FrameSink::with_byte_budget(tx, 2_000);
3252        let mut queued = 0u64;
3253        while sink
3254            .try_send(stream_frame(7, 1, queued, vec![b'x'; 200]))
3255            .is_ok()
3256        {
3257            queued += 1;
3258        }
3259        assert!(queued > 0, "the budget admitted nothing");
3260
3261        let parked = tokio::spawn({
3262            let sink = sink.clone();
3263            async move { sink.send(stream_frame(0, 0, 999, vec![b'r'; 200])).await }
3264        });
3265        tokio::time::sleep(Duration::from_millis(100)).await;
3266        assert!(
3267            !parked.is_finished(),
3268            "the awaited send must wait while the byte budget is full"
3269        );
3270
3271        // The writer takes one frame and drops it, releasing its bytes.
3272        drop(rx.recv().await.expect("a queued frame"));
3273        tokio::time::timeout(Duration::from_secs(2), parked)
3274            .await
3275            .expect("the parked send was never woken after bytes were freed")
3276            .unwrap()
3277            .unwrap();
3278    }
3279
3280    /// A client multiplexing several token streams pauses its reader while the
3281    /// module keeps producing small frames. Far more frames than the old
3282    /// 64-frame queue allowed, but far fewer bytes than the budget, must all be
3283    /// held without closing the connection and delivered in order afterwards.
3284    #[tokio::test]
3285    async fn paused_client_reader_keeps_connection_through_small_frame_burst() {
3286        const ROUTES: usize = 4;
3287        const FRAMES_PER_ROUTE: usize = 250;
3288        let (router, client_sink, mut client_rx, module_ctx, routes, mut close_receiver) =
3289            multi_route_client(
3290                "burst-provider",
3291                ConnectionId::new(60),
3292                ConnectionId::new(61),
3293                ROUTES,
3294            )
3295            .await;
3296
3297        // The reader is paused: nothing is received until every frame is sent.
3298        for seq in 0..FRAMES_PER_ROUTE as u64 {
3299            for (index, route) in routes.iter().enumerate() {
3300                let body = format!("route-{index}-token-{seq:05}-{}", "t".repeat(170));
3301                router
3302                    .route_for_connection(
3303                        &module_ctx,
3304                        stream_frame(
3305                            route.module_channel,
3306                            route.module_epoch,
3307                            seq,
3308                            body.into_bytes(),
3309                        ),
3310                    )
3311                    .await
3312                    .unwrap();
3313            }
3314        }
3315
3316        let backlog = client_sink.backlog();
3317        assert_eq!(backlog.queued_frames, ROUTES * FRAMES_PER_ROUTE);
3318        assert!(backlog.queued_bytes < crate::server::CONNECTION_EGRESS_BYTE_BUDGET);
3319        assert!(
3320            close_receiver.try_recv().is_err(),
3321            "a paused reader under the byte budget must not be closed"
3322        );
3323        assert_eq!(
3324            router.counters.snapshot()["client_egress_close_delivery_failed"],
3325            0
3326        );
3327
3328        // The reader resumes: every frame arrives, in order within each route.
3329        let mut next_seq = vec![0u64; ROUTES];
3330        for _ in 0..ROUTES * FRAMES_PER_ROUTE {
3331            let frame = client_rx.try_recv().expect("every queued frame arrives");
3332            let index = routes
3333                .iter()
3334                .position(|route| route.client_channel == frame.header.channel)
3335                .expect("frame arrives on one of the bound client channels");
3336            assert_eq!(frame.header.corr, next_seq[index], "route {index} order");
3337            let expected_prefix = format!("route-{index}-token-{:05}-", next_seq[index]);
3338            assert!(frame.body.starts_with(expected_prefix.as_bytes()));
3339            next_seq[index] += 1;
3340        }
3341        assert!(client_rx.try_recv().is_err());
3342        assert_eq!(next_seq, vec![FRAMES_PER_ROUTE as u64; ROUTES]);
3343        assert_eq!(client_sink.backlog().queued_bytes, 0);
3344    }
3345
3346    /// A client that never reads exhausts its byte budget and the oldest
3347    /// frame's 250 ms grace, then closes with one header-only diagnosis.
3348    #[tokio::test(start_paused = true)]
3349    async fn never_reading_client_is_closed_at_byte_budget_with_warn_diagnosis() {
3350        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3351        const BODY: usize = 16 * 1024;
3352        let (router, client_sink, _client_rx, module_ctx, routes, mut close_receiver) =
3353            multi_route_client(
3354                "stuck-reader-provider",
3355                ConnectionId::new(70),
3356                ConnectionId::new(71),
3357                2,
3358            )
3359            .await;
3360
3361        let mut admitted = 0usize;
3362        let mut sent = 0u64;
3363        let started = tokio::time::Instant::now();
3364        while router.counters.snapshot()["client_egress_close_delivery_failed"] == 0 {
3365            assert!(sent < 1_000, "the byte budget never refused a frame");
3366            let route = &routes[(sent % 2) as usize];
3367            router
3368                .route_for_connection(
3369                    &module_ctx,
3370                    stream_frame(
3371                        route.module_channel,
3372                        route.module_epoch,
3373                        sent,
3374                        vec![b'z'; BODY],
3375                    ),
3376                )
3377                .await
3378                .unwrap();
3379            sent += 1;
3380            admitted = client_sink.backlog().queued_frames;
3381        }
3382        assert_eq!(
3383            started.elapsed(),
3384            crate::server::CONNECTION_EGRESS_STALL_GRACE
3385        );
3386        // The budget, not the frame-count backstop, did the refusing.
3387        let frame_bytes = subc_protocol::HEADER_LEN + BODY;
3388        assert_eq!(
3389            admitted,
3390            crate::server::CONNECTION_EGRESS_BYTE_BUDGET / frame_bytes
3391        );
3392        let reason = close_receiver
3393            .try_recv()
3394            .expect("the client connection must be asked to close");
3395        assert!(reason
3396            .to_string()
3397            .contains("module_to_client_delivery_failed"));
3398
3399        // A second refused frame for the same connection adds no second WARN.
3400        router
3401            .route_for_connection(
3402                &module_ctx,
3403                stream_frame(
3404                    routes[0].module_channel,
3405                    routes[0].module_epoch,
3406                    sent,
3407                    vec![b'z'; BODY],
3408                ),
3409            )
3410            .await
3411            .unwrap();
3412
3413        let captured = test_log::captured_logs(&logs);
3414        let warn_lines = captured
3415            .lines()
3416            .filter(|line| {
3417                line.contains("closing client connection: its egress queue could not take a frame")
3418            })
3419            .collect::<Vec<_>>();
3420        assert_eq!(warn_lines.len(), 1, "exactly one WARN, got: {captured}");
3421        let line = warn_lines[0];
3422        assert!(line.contains("WARN"), "{line}");
3423        assert!(line.contains("connection_id=71"), "{line}");
3424        assert!(
3425            line.contains("module_id=\"stuck-reader-provider\""),
3426            "{line}"
3427        );
3428        assert!(line.contains("client_channel="), "{line}");
3429        assert!(line.contains("principals=direct"), "{line}");
3430        let queued_bytes: usize = line
3431            .split("queued_bytes=")
3432            .nth(1)
3433            .and_then(|rest| rest.split_whitespace().next())
3434            .and_then(|value| value.parse().ok())
3435            .expect("queued_bytes is logged");
3436        assert_eq!(queued_bytes, admitted * frame_bytes);
3437        assert!(
3438            line.contains(&format!("queued_frames={admitted}")),
3439            "{line}"
3440        );
3441        assert!(line.contains("oldest_queued_ms="), "{line}");
3442        assert!(line.contains("oldest_queued_ms=250"), "{line}");
3443        assert!(line.contains("frame_type=Some(StreamData)"), "{line}");
3444        assert!(
3445            line.contains(&format!(
3446                "frame_flags=Some({})",
3447                Flags::new(false, Priority::Interactive, false).0
3448            )),
3449            "{line}"
3450        );
3451        assert!(
3452            line.contains("close_cause=\"waited_250_ms_without_room\""),
3453            "{line}"
3454        );
3455        assert!(
3456            !line.contains(&"z".repeat(20)),
3457            "body bytes must never be logged: {line}"
3458        );
3459    }
3460
3461    fn route_frame(ty: FrameType, channel: u16, epoch: u32, corr: u64) -> Frame {
3462        Frame::build(
3463            ty,
3464            Flags::new(false, Priority::Interactive, false),
3465            channel,
3466            epoch,
3467            corr,
3468            if ty == FrameType::Request || ty == FrameType::Response {
3469                b"route-body".to_vec()
3470            } else {
3471                Vec::new()
3472            },
3473        )
3474        .unwrap()
3475    }
3476
3477    type DynamicRouteFixture = (
3478        Router,
3479        Arc<ForwardingTable>,
3480        RouteCtx,
3481        mpsc::Receiver<crate::router::OutboundFrame>,
3482        RouteCtx,
3483        mpsc::Receiver<crate::router::OutboundFrame>,
3484        mpsc::Receiver<crate::router::OutboundFrame>,
3485        crate::forwarding::PendingRouteBindRelay,
3486    );
3487
3488    fn dynamic_route_fixture(commit: bool) -> DynamicRouteFixture {
3489        let forwarding = Arc::new(ForwardingTable::default());
3490        let control = Arc::new(crate::ControlHandler::with_forwarding(
3491            Arc::new(crate::Registry::default()),
3492            Arc::clone(&forwarding),
3493        ));
3494        let router = Router::with_control_handler(control);
3495        let module_connection = ConnectionId::new(500);
3496        let client_connection = ConnectionId::new(501);
3497        let (module_tx, module_rx) = mpsc::channel(8);
3498        forwarding
3499            .register_module_connection(
3500                module_connection,
3501                "epoch-router".into(),
3502                2,
3503                Concurrency::ModuleManaged,
3504                FrameSink::new(module_tx),
3505            )
3506            .unwrap();
3507        let (client_tx, client_rx) = mpsc::channel(8);
3508        let client_sink = FrameSink::new(client_tx);
3509        let pending = forwarding
3510            .begin_route_bind_relay_for_test(
3511                client_connection,
3512                client_sink.clone(),
3513                700,
3514                "epoch-router",
3515            )
3516            .unwrap();
3517        if commit {
3518            forwarding
3519                .complete_pending_relay(
3520                    module_connection,
3521                    pending.corr,
3522                    RouteBindRelayOutcome::Accepted,
3523                )
3524                .unwrap();
3525        }
3526        let (module_egress_tx, module_egress_rx) = mpsc::channel(8);
3527        (
3528            router,
3529            forwarding,
3530            RouteCtx {
3531                connection_id: client_connection,
3532                egress: client_sink,
3533            },
3534            client_rx,
3535            RouteCtx {
3536                connection_id: module_connection,
3537                egress: FrameSink::new(module_egress_tx),
3538            },
3539            module_egress_rx,
3540            module_rx,
3541            pending,
3542        )
3543    }
3544
3545    #[tokio::test]
3546    async fn route_epochs_validate_both_directions_and_rewrite_to_peer_handle() {
3547        let (
3548            router,
3549            _forwarding,
3550            client_ctx,
3551            mut client_rx,
3552            module_ctx,
3553            _module_egress_rx,
3554            mut module_rx,
3555            pending,
3556        ) = dynamic_route_fixture(true);
3557        let route_open = client_rx.recv().await.unwrap();
3558        assert_eq!(route_open.header.corr, 700);
3559
3560        router
3561            .route_for_connection(
3562                &client_ctx,
3563                route_frame(
3564                    FrameType::Request,
3565                    pending.client_channel,
3566                    pending.client_epoch,
3567                    701,
3568                ),
3569            )
3570            .await
3571            .unwrap();
3572        let forwarded = module_rx.recv().await.unwrap();
3573        assert_eq!(forwarded.header.channel, pending.module_channel);
3574        assert_eq!(forwarded.header.epoch, pending.module_epoch);
3575
3576        router
3577            .route_for_connection(
3578                &module_ctx,
3579                route_frame(
3580                    FrameType::Response,
3581                    pending.module_channel,
3582                    pending.module_epoch,
3583                    701,
3584                ),
3585            )
3586            .await
3587            .unwrap();
3588        let delivered = client_rx.recv().await.unwrap();
3589        assert_eq!(delivered.header.channel, pending.client_channel);
3590        assert_eq!(delivered.header.epoch, pending.client_epoch);
3591
3592        router
3593            .route_for_connection(
3594                &client_ctx,
3595                route_frame(
3596                    FrameType::Request,
3597                    pending.client_channel,
3598                    pending.client_epoch + 1,
3599                    702,
3600                ),
3601            )
3602            .await
3603            .unwrap();
3604        router
3605            .route_for_connection(
3606                &module_ctx,
3607                route_frame(
3608                    FrameType::Response,
3609                    pending.module_channel,
3610                    pending.module_epoch + 1,
3611                    703,
3612                ),
3613            )
3614            .await
3615            .unwrap();
3616        let stale_error = client_rx.recv().await.unwrap();
3617        assert_eq!(stale_error.header.ty, FrameType::Error);
3618        assert_eq!(stale_error.header.channel, pending.client_channel);
3619        assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
3620        assert_eq!(stale_error.header.corr, 702);
3621        let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
3622        assert_eq!(body.code, "stale_route_epoch");
3623        assert!(module_rx.try_recv().is_err());
3624        assert!(client_rx.try_recv().is_err());
3625        let counters = router.counters.snapshot();
3626        assert_eq!(counters["client_frames_dropped_stale_route"], 1);
3627        assert_eq!(counters["module_frames_dropped_no_route"], 1);
3628    }
3629
3630    #[tokio::test]
3631    async fn accepted_route_publishes_route_open_before_immediate_reverse_request() {
3632        let (
3633            router,
3634            _,
3635            _client_ctx,
3636            mut client_rx,
3637            module_ctx,
3638            _module_egress_rx,
3639            _module_rx,
3640            pending,
3641        ) = dynamic_route_fixture(true);
3642        router
3643            .route_for_connection(
3644                &module_ctx,
3645                route_frame(
3646                    FrameType::Request,
3647                    pending.module_channel,
3648                    pending.module_epoch,
3649                    800,
3650                ),
3651            )
3652            .await
3653            .unwrap();
3654
3655        let first = client_rx.recv().await.unwrap();
3656        let second = client_rx.recv().await.unwrap();
3657        assert_eq!(first.header.channel, 0);
3658        assert_eq!(first.header.corr, 700);
3659        assert_eq!(second.header.channel, pending.client_channel);
3660        assert_eq!(second.header.epoch, pending.client_epoch);
3661        assert_eq!(second.header.corr, 800);
3662    }
3663
3664    #[tokio::test]
3665    async fn reserved_slot_ingress_errors_only_matching_client_requests() {
3666        let (
3667            router,
3668            _forwarding,
3669            client_ctx,
3670            mut client_rx,
3671            _module_ctx,
3672            _module_egress_rx,
3673            mut module_rx,
3674            pending,
3675        ) = dynamic_route_fixture(false);
3676        router
3677            .route_for_connection(
3678                &client_ctx,
3679                route_frame(
3680                    FrameType::Request,
3681                    pending.client_channel,
3682                    pending.client_epoch,
3683                    900,
3684                ),
3685            )
3686            .await
3687            .unwrap();
3688        let error = client_rx.recv().await.unwrap();
3689        assert_eq!(error.header.ty, FrameType::Error);
3690        assert_eq!(error.header.channel, pending.client_channel);
3691        assert_eq!(error.header.epoch, pending.client_epoch);
3692        assert_eq!(error.header.corr, 900);
3693
3694        router
3695            .route_for_connection(
3696                &client_ctx,
3697                route_frame(
3698                    FrameType::Response,
3699                    pending.client_channel,
3700                    pending.client_epoch,
3701                    901,
3702                ),
3703            )
3704            .await
3705            .unwrap();
3706        router
3707            .route_for_connection(
3708                &client_ctx,
3709                route_frame(
3710                    FrameType::Request,
3711                    pending.client_channel,
3712                    pending.client_epoch + 1,
3713                    902,
3714                ),
3715            )
3716            .await
3717            .unwrap();
3718        let stale_error = client_rx.recv().await.unwrap();
3719        assert_eq!(stale_error.header.ty, FrameType::Error);
3720        assert_eq!(stale_error.header.channel, pending.client_channel);
3721        assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
3722        assert_eq!(stale_error.header.corr, 902);
3723        let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
3724        assert_eq!(body.code, "stale_route_epoch");
3725        assert!(module_rx.try_recv().is_err());
3726        let counters = router.counters.snapshot();
3727        assert_eq!(counters["client_frames_dropped_stale_route"], 1);
3728        assert_eq!(counters["module_frames_dropped_no_route"], 0);
3729    }
3730
3731    #[tokio::test]
3732    async fn dropped_module_route_goodbye_increments_counter() {
3733        let (
3734            router,
3735            _forwarding,
3736            client_ctx,
3737            mut client_rx,
3738            _module_ctx,
3739            _module_egress_rx,
3740            mut module_rx,
3741            pending,
3742        ) = dynamic_route_fixture(true);
3743        let _ = client_rx.recv().await;
3744        module_rx.close();
3745
3746        router
3747            .route_for_connection(
3748                &client_ctx,
3749                route_frame(
3750                    FrameType::Goodbye,
3751                    pending.client_channel,
3752                    pending.client_epoch,
3753                    999,
3754                ),
3755            )
3756            .await
3757            .unwrap();
3758
3759        let counters = router.counters.snapshot();
3760        assert_eq!(counters["goodbye_relay_module_dropped"], 1);
3761        assert_eq!(
3762            counters["goodbye_relay_module_dropped_by_module"],
3763            serde_json::json!({ "epoch-router": 1 })
3764        );
3765        assert_eq!(counters["route_released_epoch_fenced"], 1);
3766    }
3767
3768    #[tokio::test]
3769    async fn module_request_on_stale_epoch_receives_stale_route_epoch() {
3770        let (
3771            router,
3772            _forwarding,
3773            _client_ctx,
3774            _client_rx,
3775            module_ctx,
3776            mut module_egress_rx,
3777            mut module_rx,
3778            pending,
3779        ) = dynamic_route_fixture(true);
3780
3781        router
3782            .route_for_connection(
3783                &module_ctx,
3784                route_frame(
3785                    FrameType::Request,
3786                    pending.module_channel,
3787                    pending.module_epoch + 1,
3788                    1_000,
3789                ),
3790            )
3791            .await
3792            .unwrap();
3793
3794        let error = module_egress_rx.try_recv().unwrap();
3795        assert_eq!(error.header.ty, FrameType::Error);
3796        assert_eq!(error.header.channel, pending.module_channel);
3797        assert_eq!(error.header.epoch, pending.module_epoch + 1);
3798        assert_eq!(error.header.corr, 1_000);
3799        let body: ErrorBody = serde_json::from_slice(&error.body).unwrap();
3800        assert_eq!(body.code, "stale_route_epoch");
3801        assert!(module_rx.try_recv().is_err());
3802        let counters = router.counters.snapshot();
3803        assert_eq!(counters["module_requests_dropped_stale_route"], 1);
3804        assert_eq!(counters["module_frames_dropped_no_route"], 0);
3805    }
3806
3807    #[tokio::test]
3808    async fn module_request_on_reserved_or_absent_route_receives_unknown_channel() {
3809        let (
3810            reserved_router,
3811            _forwarding,
3812            _client_ctx,
3813            _client_rx,
3814            reserved_module_ctx,
3815            mut reserved_module_egress_rx,
3816            _module_rx,
3817            reserved,
3818        ) = dynamic_route_fixture(false);
3819        reserved_router
3820            .route_for_connection(
3821                &reserved_module_ctx,
3822                route_frame(
3823                    FrameType::Request,
3824                    reserved.module_channel,
3825                    reserved.module_epoch,
3826                    1_001,
3827                ),
3828            )
3829            .await
3830            .unwrap();
3831        let reserved_error = reserved_module_egress_rx.try_recv().unwrap();
3832        let reserved_body: ErrorBody = serde_json::from_slice(&reserved_error.body).unwrap();
3833        assert_eq!(reserved_error.header.ty, FrameType::Error);
3834        assert_eq!(reserved_error.header.channel, reserved.module_channel);
3835        assert_eq!(reserved_error.header.epoch, reserved.module_epoch);
3836        assert_eq!(reserved_error.header.corr, 1_001);
3837        assert_eq!(reserved_body.code, "unknown_channel");
3838        assert_eq!(
3839            reserved_router.counters.snapshot()["module_requests_dropped_stale_route"],
3840            1
3841        );
3842
3843        let (
3844            absent_router,
3845            _forwarding,
3846            _client_ctx,
3847            _client_rx,
3848            absent_module_ctx,
3849            mut absent_module_egress_rx,
3850            _module_rx,
3851            absent,
3852        ) = dynamic_route_fixture(false);
3853        absent_router
3854            .route_for_connection(
3855                &absent_module_ctx,
3856                route_frame(
3857                    FrameType::Request,
3858                    absent.module_channel + 1,
3859                    absent.module_epoch,
3860                    1_002,
3861                ),
3862            )
3863            .await
3864            .unwrap();
3865        let absent_error = absent_module_egress_rx.try_recv().unwrap();
3866        let absent_body: ErrorBody = serde_json::from_slice(&absent_error.body).unwrap();
3867        assert_eq!(absent_error.header.ty, FrameType::Error);
3868        assert_eq!(absent_error.header.channel, absent.module_channel + 1);
3869        assert_eq!(absent_error.header.epoch, absent.module_epoch);
3870        assert_eq!(absent_error.header.corr, 1_002);
3871        assert_eq!(absent_body.code, "unknown_channel");
3872        assert_eq!(
3873            absent_router.counters.snapshot()["module_requests_dropped_stale_route"],
3874            1
3875        );
3876    }
3877
3878    #[tokio::test]
3879    async fn non_request_module_frame_on_dead_route_is_counted_without_error() {
3880        let (
3881            router,
3882            forwarding,
3883            client_ctx,
3884            mut client_rx,
3885            module_ctx,
3886            mut module_egress_rx,
3887            mut module_rx,
3888            pending,
3889        ) = dynamic_route_fixture(true);
3890        let (other_module_tx, _other_module_rx) = mpsc::channel(8);
3891        forwarding
3892            .register_module_connection(
3893                ConnectionId::new(502),
3894                "other-module".into(),
3895                2,
3896                Concurrency::ModuleManaged,
3897                FrameSink::new(other_module_tx),
3898            )
3899            .unwrap();
3900        let _ = client_rx.recv().await.unwrap();
3901
3902        router
3903            .route_for_connection(
3904                &client_ctx,
3905                route_frame(
3906                    FrameType::Goodbye,
3907                    pending.client_channel,
3908                    pending.client_epoch,
3909                    1_003,
3910                ),
3911            )
3912            .await
3913            .unwrap();
3914        let _ = module_rx.recv().await.unwrap();
3915
3916        router
3917            .route_for_connection(
3918                &module_ctx,
3919                route_frame(
3920                    FrameType::StreamData,
3921                    pending.module_channel,
3922                    pending.module_epoch,
3923                    1_004,
3924                ),
3925            )
3926            .await
3927            .unwrap();
3928
3929        // No ERROR goes back for a non-request frame. The one reply is the
3930        // route GOODBYE telling the module to let go of the released route.
3931        let reply = module_egress_rx.try_recv().unwrap();
3932        assert_eq!(reply.header.ty, FrameType::Goodbye);
3933        assert!(module_egress_rx.try_recv().is_err());
3934        let counters = router.counters.snapshot();
3935        assert_eq!(counters["module_frames_dropped_no_route"], 1);
3936        assert_eq!(
3937            counters["module_frames_dropped_no_route_by_module"],
3938            serde_json::json!({ "epoch-router": 1 })
3939        );
3940        assert_eq!(counters["module_requests_dropped_stale_route"], 0);
3941    }
3942
3943    /// Drain whatever the module connection's queue holds right now, the way a
3944    /// module's reader would before it stalls.
3945    fn drain_now(rx: &mut mpsc::Receiver<OutboundFrame>) {
3946        while rx.try_recv().is_ok() {}
3947    }
3948
3949    /// A module that stops reading for a moment when a client closes one of
3950    /// its routes still learns the route is gone: the GOODBYE its full egress
3951    /// queue refused is delivered as soon as it reads again, rather than
3952    /// dropped, which would leave the module holding the route for the rest of
3953    /// its connection.
3954    #[tokio::test]
3955    async fn route_goodbye_refused_by_stalled_module_is_delivered_when_it_resumes_reading() {
3956        const BUDGET: usize = 4_096;
3957        let forwarding = Arc::new(ForwardingTable::default());
3958        let control = Arc::new(ControlHandler::with_forwarding(
3959            Arc::new(Registry::default()),
3960            Arc::clone(&forwarding),
3961        ));
3962        let router = Router::with_control_handler(control);
3963        let module_connection = ConnectionId::new(80);
3964        let client_connection = ConnectionId::new(81);
3965        let (module_tx, mut module_rx) = mpsc::channel(64);
3966        let module_sink = FrameSink::with_byte_budget(module_tx, BUDGET);
3967        forwarding
3968            .register_module_connection(
3969                module_connection,
3970                "stalled-provider".into(),
3971                2,
3972                Concurrency::ModuleManaged,
3973                module_sink.clone(),
3974            )
3975            .unwrap();
3976        let (client_tx, mut client_rx) = mpsc::channel(8);
3977        let client_sink = FrameSink::new(client_tx);
3978        let pending = forwarding
3979            .begin_route_bind_relay_for_test(
3980                client_connection,
3981                client_sink.clone(),
3982                1_100,
3983                "stalled-provider",
3984            )
3985            .unwrap();
3986        forwarding
3987            .complete_pending_relay(
3988                module_connection,
3989                pending.corr,
3990                RouteBindRelayOutcome::Accepted,
3991            )
3992            .unwrap();
3993        let _ = client_rx.recv().await.unwrap();
3994        drain_now(&mut module_rx);
3995
3996        // The module stalls: its queue holds more than the byte budget, so the
3997        // queue refuses anything further.
3998        module_sink
3999            .try_send(stream_frame(9, 1, 0, vec![b'f'; BUDGET]))
4000            .unwrap();
4001        assert!(module_sink
4002            .try_send(stream_frame(9, 1, 1, Vec::new()))
4003            .is_err());
4004
4005        let client_ctx = RouteCtx {
4006            connection_id: client_connection,
4007            egress: client_sink,
4008        };
4009        router
4010            .route_for_connection(
4011                &client_ctx,
4012                route_frame(
4013                    FrameType::Goodbye,
4014                    pending.client_channel,
4015                    pending.client_epoch,
4016                    1_101,
4017                ),
4018            )
4019            .await
4020            .unwrap();
4021        tokio::task::yield_now().await;
4022        assert_eq!(
4023            router.counters.snapshot()["goodbye_relay_module_dropped"],
4024            0,
4025            "a GOODBYE refused by a momentarily full module queue must not be dropped"
4026        );
4027
4028        // The module reads again: the filler comes off, then the GOODBYE.
4029        let filler = module_rx.recv().await.unwrap();
4030        assert_eq!(filler.header.ty, FrameType::StreamData);
4031        drop(filler);
4032        let goodbye = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
4033            .await
4034            .expect("the refused GOODBYE must be delivered once the module frees room")
4035            .unwrap();
4036        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
4037        assert_eq!(goodbye.header.channel, pending.module_channel);
4038        assert_eq!(goodbye.header.epoch, pending.module_epoch);
4039        assert_eq!(
4040            router.counters.snapshot()["goodbye_relay_module_dropped"],
4041            0
4042        );
4043    }
4044
4045    /// A module still sending on a route the daemon released is told, with a
4046    /// route GOODBYE for exactly the (channel, epoch) it sent on. The same
4047    /// happens for a stale epoch on a channel whose route moved on and for a
4048    /// channel that never had a route; only the first is counted as traffic on
4049    /// a released route.
4050    #[tokio::test]
4051    async fn module_frame_on_route_the_daemon_does_not_hold_is_answered_with_goodbye() {
4052        let (
4053            router,
4054            _forwarding,
4055            client_ctx,
4056            mut client_rx,
4057            module_ctx,
4058            mut module_egress_rx,
4059            mut module_rx,
4060            pending,
4061        ) = dynamic_route_fixture(true);
4062        let _ = client_rx.recv().await.unwrap();
4063        router
4064            .route_for_connection(
4065                &client_ctx,
4066                route_frame(
4067                    FrameType::Goodbye,
4068                    pending.client_channel,
4069                    pending.client_epoch,
4070                    1_200,
4071                ),
4072            )
4073            .await
4074            .unwrap();
4075        drain_now(&mut module_rx);
4076
4077        let cases = [
4078            // Released route: the daemon allocated this (channel, epoch).
4079            (pending.module_channel, pending.module_epoch),
4080            // A channel the daemon never allocated on this connection.
4081            (pending.module_channel + 1, 1),
4082        ];
4083        for (channel, epoch) in cases {
4084            router
4085                .route_for_connection(
4086                    &module_ctx,
4087                    route_frame(FrameType::StreamData, channel, epoch, 1_201),
4088                )
4089                .await
4090                .unwrap();
4091            let reply = module_egress_rx
4092                .try_recv()
4093                .expect("a frame on a route the daemon does not hold is answered");
4094            assert_eq!(reply.header.ty, FrameType::Goodbye);
4095            assert_eq!(reply.header.channel, channel);
4096            assert_eq!(reply.header.epoch, epoch);
4097            assert_eq!(reply.header.corr, 0);
4098            assert!(module_egress_rx.try_recv().is_err());
4099        }
4100
4101        // A GOODBYE from the module for a route the daemon already released
4102        // is not answered: the module is letting go already.
4103        router
4104            .route_for_connection(
4105                &module_ctx,
4106                route_frame(FrameType::Goodbye, pending.module_channel + 2, 1, 0),
4107            )
4108            .await
4109            .unwrap();
4110        assert!(module_egress_rx.try_recv().is_err());
4111
4112        let counters = router.counters.snapshot();
4113        assert_eq!(counters["module_frames_dropped_no_route"], 3);
4114        assert_eq!(counters["module_frames_dropped_released_route"], 1);
4115        assert_eq!(
4116            counters["module_frames_dropped_released_route_by_module"],
4117            serde_json::json!({ "epoch-router": 1 })
4118        );
4119        assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4120    }
4121
4122    /// A chatty orphan (a streaming module still producing on a route it
4123    /// missed the GOODBYE for) is answered once per interval, not once per
4124    /// frame, and answered again once the interval has passed.
4125    #[tokio::test(start_paused = true)]
4126    async fn burst_of_orphan_module_frames_is_answered_once_per_interval() {
4127        let (
4128            router,
4129            _forwarding,
4130            client_ctx,
4131            mut client_rx,
4132            module_ctx,
4133            mut module_egress_rx,
4134            mut module_rx,
4135            pending,
4136        ) = dynamic_route_fixture(true);
4137        let _ = client_rx.recv().await.unwrap();
4138        router
4139            .route_for_connection(
4140                &client_ctx,
4141                route_frame(
4142                    FrameType::Goodbye,
4143                    pending.client_channel,
4144                    pending.client_epoch,
4145                    1_300,
4146                ),
4147            )
4148            .await
4149            .unwrap();
4150        drain_now(&mut module_rx);
4151
4152        let send_burst = |corr: u64| {
4153            route_frame(
4154                FrameType::StreamData,
4155                pending.module_channel,
4156                pending.module_epoch,
4157                corr,
4158            )
4159        };
4160        for corr in 0..20 {
4161            router
4162                .route_for_connection(&module_ctx, send_burst(corr))
4163                .await
4164                .unwrap();
4165        }
4166        let mut replies = 0;
4167        while let Ok(reply) = module_egress_rx.try_recv() {
4168            assert_eq!(reply.header.ty, FrameType::Goodbye);
4169            replies += 1;
4170        }
4171        assert_eq!(replies, 1, "a burst within the interval gets one GOODBYE");
4172
4173        tokio::time::advance(ORPHAN_ROUTE_GOODBYE_INTERVAL).await;
4174        router
4175            .route_for_connection(&module_ctx, send_burst(20))
4176            .await
4177            .unwrap();
4178        let retry = module_egress_rx
4179            .try_recv()
4180            .expect("the first orphan frame after the interval is answered again");
4181        assert_eq!(retry.header.channel, pending.module_channel);
4182        assert_eq!(retry.header.epoch, pending.module_epoch);
4183
4184        let counters = router.counters.snapshot();
4185        assert_eq!(counters["module_frames_dropped_no_route"], 21);
4186        assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4187    }
4188
4189    /// The rate-limit memory is per connection and goes away with it.
4190    #[test]
4191    fn orphan_goodbye_rate_limit_state_is_released_with_the_connection() {
4192        let router = Router::with_default_self_handler();
4193        let connection = router.begin_connection();
4194        let id = connection.id();
4195        assert!(router.orphan_goodbyes.claim(id, 7));
4196        assert!(!router.orphan_goodbyes.claim(id, 7));
4197        assert!(router.orphan_goodbyes.claim(id, 8));
4198        drop(connection);
4199        assert!(router
4200            .orphan_goodbyes
4201            .last_sent
4202            .lock()
4203            .unwrap()
4204            .get(&id)
4205            .is_none());
4206    }
4207
4208    #[test]
4209    fn channel_zero_cannot_be_registered_as_backend() {
4210        let mut router = Router::with_default_self_handler();
4211
4212        let err = router.register_backend(0, EchoBackend).unwrap_err();
4213
4214        assert_eq!(err, RouterError::ReservedChannelZero);
4215    }
4216}