Skip to main content

subc_daemon/
router.rs

1use std::{
2    collections::HashMap,
3    error::Error,
4    fmt,
5    sync::{
6        atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
7        Arc, Mutex,
8    },
9    time::{Duration, Instant},
10};
11
12use subc_protocol::{error_codes, ErrorBody, Flags, FrameType, Priority};
13use tokio::sync::{mpsc, Notify};
14use tracing::{debug, warn};
15
16use crate::{
17    control::ControlHandler,
18    forwarding::{
19        principal_label, CloseReason, ConnectionCloseReceiver, DataRoute, DataRouteState,
20        ForwardingError, ForwardingTable, RouteBinding, RouteRelease, UndeliveredFrame,
21    },
22    registry::ConnectionId,
23    DaemonCounters, Frame, FrameBuildError,
24};
25
26/// One queued outbound frame plus the instant it entered the writer queue.
27///
28/// The stamp exists for the reply-path half of slow-control diagnosis: a
29/// handler can finish in microseconds while the reply sits in this queue
30/// waiting for the writer task to be scheduled, and without a per-item stamp
31/// that wait is invisible to every other timing point (the client's round
32/// trip is the only witness, and it cannot say which side ate the time).
33/// Constructed exclusively inside [`FrameSink`] so no caller can forget it.
34#[derive(Debug)]
35pub struct OutboundFrame {
36    pub frame: Frame,
37    pub enqueued_at: std::time::Instant,
38    pub(crate) flushed: Option<tokio::sync::oneshot::Sender<()>>,
39    /// This frame's share of the connection's queued-byte count. Dropping the
40    /// frame (after the writer has written it, or when the queue itself is
41    /// dropped with the frame still in it) gives the bytes back. `None` only for
42    /// frames built outside a [`FrameSink`], which exist in tests alone.
43    pub(crate) charge: Option<EgressCharge>,
44}
45
46impl OutboundFrame {
47    fn charged(frame: Frame, charge: EgressCharge) -> Self {
48        Self {
49            frame,
50            enqueued_at: charge.enqueued_at,
51            flushed: None,
52            charge: Some(charge),
53        }
54    }
55}
56
57/// Bytes a frame occupies in the connection's egress queue for budget
58/// purposes: the fixed envelope header plus the body.
59fn queued_frame_bytes(frame: &Frame) -> usize {
60    subc_protocol::HEADER_LEN + frame.body.len()
61}
62
63/// A point-in-time view of one connection's egress queue, for diagnosing why a
64/// frame did not fit.
65#[derive(Debug, Clone, Copy, PartialEq, Eq)]
66pub(crate) struct EgressBacklog {
67    pub queued_bytes: usize,
68    pub queued_frames: usize,
69    /// Now minus the enqueue time of the frame the writer most recently took
70    /// off the queue (the frame it is writing, or has just written), or, if
71    /// the writer has taken nothing since the queue was last empty, of the
72    /// frame that made the queue non-empty. While the writer is blocked on a
73    /// frame this is exactly the oldest frame's age; between frames it can be
74    /// one frame older than the true head. `None` when nothing is queued.
75    pub oldest_age: Option<Duration>,
76}
77
78/// Queued-byte accounting shared by every clone of one connection's
79/// [`FrameSink`] and by every frame that sink has admitted. Its per-frame
80/// accounting uses atomics: this is on the path of every frame a module sends
81/// to a client, so it takes no lock per frame.
82#[derive(Debug)]
83struct EgressAccounting {
84    byte_budget: usize,
85    queued_bytes: AtomicUsize,
86    queued_frames: AtomicUsize,
87    /// Origin for the nanosecond timestamps in `oldest_enqueued_nanos`.
88    time_base: Instant,
89    /// Enqueue time, as nanoseconds after `time_base` plus one, of the frame
90    /// described by [`EgressBacklog::oldest_age`]; 0 means none.
91    oldest_enqueued_nanos: AtomicU64,
92    /// Awaited senders currently parked on `freed`. A release only pays for a
93    /// notification when this is non-zero.
94    waiters: AtomicUsize,
95    /// Woken when a charged frame releases its bytes while a sender waits, so
96    /// the awaited send can re-check for room.
97    freed: Notify,
98    client_delivery_closed: AtomicBool,
99    client_delivery_stopped: Notify,
100}
101
102impl EgressAccounting {
103    fn new(byte_budget: usize) -> Self {
104        Self {
105            byte_budget,
106            queued_bytes: AtomicUsize::new(0),
107            queued_frames: AtomicUsize::new(0),
108            time_base: tokio::time::Instant::now().into_std(),
109            oldest_enqueued_nanos: AtomicU64::new(0),
110            waiters: AtomicUsize::new(0),
111            freed: Notify::new(),
112            client_delivery_closed: AtomicBool::new(false),
113            client_delivery_stopped: Notify::new(),
114        }
115    }
116
117    fn stamp(&self, at: Instant) -> u64 {
118        (at.saturating_duration_since(self.time_base).as_nanos() as u64).saturating_add(1)
119    }
120
121    /// Charge `bytes` if they fit in the budget. A frame larger than the whole
122    /// budget (bodies may be up to 64 MiB) is still admitted into an EMPTY
123    /// queue, since it could otherwise never be sent at all; it simply has the
124    /// queue to itself until it is written.
125    fn try_charge(self: &Arc<Self>, bytes: usize) -> Option<EgressCharge> {
126        // SeqCst pairs with `release`: either this load sees bytes a release
127        // just freed, or that release sees this sender counted in `waiters`.
128        let mut current = self.queued_bytes.load(Ordering::SeqCst);
129        loop {
130            if current != 0 && current.saturating_add(bytes) > self.byte_budget {
131                return None;
132            }
133            match self.queued_bytes.compare_exchange_weak(
134                current,
135                current + bytes,
136                Ordering::SeqCst,
137                Ordering::SeqCst,
138            ) {
139                Ok(_) => return Some(self.record(bytes)),
140                Err(actual) => current = actual,
141            }
142        }
143    }
144
145    /// Charge `bytes` regardless of the budget. Used only for a frame whose
146    /// queue slot was reserved in advance, which must be sendable.
147    fn charge_unconditionally(self: &Arc<Self>, bytes: usize) -> EgressCharge {
148        self.queued_bytes.fetch_add(bytes, Ordering::SeqCst);
149        self.record(bytes)
150    }
151
152    fn record(self: &Arc<Self>, bytes: usize) -> EgressCharge {
153        let enqueued_at = tokio::time::Instant::now().into_std();
154        let stamp = self.stamp(enqueued_at);
155        if self.queued_frames.fetch_add(1, Ordering::AcqRel) == 0 {
156            // This frame made the queue non-empty, so it is the head until the
157            // writer takes something.
158            self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
159        }
160        EgressCharge {
161            accounting: Arc::clone(self),
162            bytes,
163            stamp,
164            enqueued_at,
165        }
166    }
167
168    /// The writer has taken the frame stamped `stamp` off the queue.
169    fn taken_by_writer(&self, stamp: u64) {
170        self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
171    }
172
173    fn release(&self, bytes: usize, stamp: u64) {
174        self.queued_bytes.fetch_sub(bytes, Ordering::SeqCst);
175        if self.queued_frames.fetch_sub(1, Ordering::AcqRel) == 1 {
176            // The queue drained. Clear the marker only if it still names this
177            // frame, so a frame admitted in the meantime keeps its stamp.
178            let _ = self.oldest_enqueued_nanos.compare_exchange(
179                stamp,
180                0,
181                Ordering::AcqRel,
182                Ordering::Acquire,
183            );
184        }
185        if self.waiters.load(Ordering::SeqCst) != 0 {
186            self.freed.notify_waiters();
187        }
188    }
189
190    fn backlog(&self) -> EgressBacklog {
191        let queued_frames = self.queued_frames.load(Ordering::Acquire);
192        let stamp = self.oldest_enqueued_nanos.load(Ordering::Acquire);
193        let oldest_age = (queued_frames != 0 && stamp != 0).then(|| {
194            let enqueued = self.time_base + Duration::from_nanos(stamp - 1);
195            tokio::time::Instant::from_std(enqueued).elapsed()
196        });
197        EgressBacklog {
198            queued_bytes: self.queued_bytes.load(Ordering::Acquire),
199            queued_frames,
200            oldest_age,
201        }
202    }
203}
204
205/// One admitted frame's claim on its connection's egress byte budget,
206/// returned when the frame is dropped.
207#[derive(Debug)]
208pub(crate) struct EgressCharge {
209    accounting: Arc<EgressAccounting>,
210    bytes: usize,
211    stamp: u64,
212    enqueued_at: Instant,
213}
214
215impl EgressCharge {
216    /// Called by the connection writer when it takes this frame off the queue,
217    /// so the backlog's oldest-age figure follows the writer.
218    pub(crate) fn taken_by_writer(&self) {
219        self.accounting.taken_by_writer(self.stamp);
220    }
221}
222
223impl Drop for EgressCharge {
224    fn drop(&mut self) {
225        self.accounting.release(self.bytes, self.stamp);
226    }
227}
228
229/// A queue slot reserved ahead of time (a pending `route.open` holds one until
230/// its module answers). Sending through it never waits and never fails for
231/// lack of room: the slot is already held, and its frame is charged to the
232/// byte count outside the budget check. At most
233/// `MAX_PENDING_ROUTE_OPENS_PER_CONNECTION` such frames exist per connection.
234#[derive(Debug)]
235pub(crate) struct EgressPermit {
236    permit: mpsc::OwnedPermit<OutboundFrame>,
237    accounting: Arc<EgressAccounting>,
238}
239
240impl EgressPermit {
241    /// Enqueue `frame` in the reserved slot. Returns true when the connection's
242    /// writer had already gone away, meaning the frame will never be written.
243    pub(crate) fn send(self, frame: Frame) -> bool {
244        let charge = self
245            .accounting
246            .charge_unconditionally(queued_frame_bytes(&frame));
247        let sender = self.permit.send(OutboundFrame::charged(frame, charge));
248        sender.is_closed()
249    }
250}
251
252/// An `OutboundFrame` is a stamped `Frame`; deref keeps every existing frame
253/// read (headers, bodies, assertions) working on queued items unchanged.
254impl std::ops::Deref for OutboundFrame {
255    type Target = Frame;
256
257    fn deref(&self) -> &Frame {
258        &self.frame
259    }
260}
261
262/// Shared tracing-capture helpers for timing-observability tests across
263/// modules (router dispatch, server reply path). Test-only.
264#[cfg(test)]
265pub(crate) mod test_log {
266    use std::{
267        io::Write,
268        sync::{Arc, Mutex, OnceLock},
269    };
270
271    #[derive(Clone)]
272    struct TestLogWriter(Arc<Mutex<Vec<u8>>>);
273
274    impl Write for TestLogWriter {
275        fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
276            self.0
277                .lock()
278                .expect("test log capture is not poisoned")
279                .extend(buffer);
280            Ok(buffer.len())
281        }
282
283        fn flush(&mut self) -> std::io::Result<()> {
284            Ok(())
285        }
286    }
287
288    pub(crate) fn log_capture(
289        level: tracing::Level,
290    ) -> (Arc<Mutex<Vec<u8>>>, tracing::dispatcher::DefaultGuard) {
291        // tracing-core's single-dispatch fast path registers callsites using
292        // the emitting thread's default subscriber. A parallel test with no
293        // subscriber can therefore cache `never` for a callsite this capture
294        // needs. Keep a second dispatch alive so registration considers all
295        // live subscribers instead. This registry is never installed as a
296        // default: each test still records only into its own thread's writer.
297        static INTEREST_ANCHOR: OnceLock<tracing::Dispatch> = OnceLock::new();
298        INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
299        let output = Arc::new(Mutex::new(Vec::new()));
300        let writer = Arc::clone(&output);
301        let subscriber = tracing_subscriber::fmt()
302            .with_max_level(level)
303            .with_ansi(false)
304            .without_time()
305            .with_target(false)
306            .with_writer(move || TestLogWriter(Arc::clone(&writer)))
307            .finish();
308        let guard = tracing::subscriber::set_default(subscriber);
309        (output, guard)
310    }
311
312    pub(crate) fn captured_logs(output: &Arc<Mutex<Vec<u8>>>) -> String {
313        String::from_utf8(
314            output
315                .lock()
316                .expect("test log capture is not poisoned")
317                .clone(),
318        )
319        .expect("tracing output is UTF-8")
320    }
321
322    #[cfg(test)]
323    mod tests {
324        use super::*;
325
326        fn emit_capture_probe(source: &str) {
327            tracing::error!(source, "parallel capture probe");
328        }
329
330        #[test]
331        fn parallel_callsite_registration_keeps_the_test_capture_enabled() {
332            let (output, _guard) = log_capture(tracing::Level::ERROR);
333            // First register the very same callsite on a thread without a
334            // default subscriber, while the test's capture remains live.
335            std::thread::spawn(|| emit_capture_probe("other-thread"))
336                .join()
337                .expect("unsubscribed thread exits");
338            emit_capture_probe("captured-thread");
339
340            let logs = captured_logs(&output);
341            assert!(
342                logs.contains("captured-thread"),
343                "capture was disabled: {logs}"
344            );
345            assert!(!logs.contains("other-thread"), "capture leaked: {logs}");
346        }
347    }
348}
349
350/// Cheaply cloneable handle to one connection's bounded outbound frame queue.
351///
352/// Backends emit responses, streaming frames, and future PUSH frames through this
353/// single path. The queue is bounded twice: by queued BYTES (the budget that
354/// matters for memory and for how long a slow reader may pause), and by the
355/// `mpsc` channel's frame count (a backstop). The socket layer owns the sole
356/// receiver/writer.
357#[derive(Debug, Clone)]
358pub struct FrameSink {
359    tx: mpsc::Sender<OutboundFrame>,
360    accounting: Arc<EgressAccounting>,
361}
362
363impl FrameSink {
364    /// A sink with the standard per-connection byte budget
365    /// ([`crate::server::CONNECTION_EGRESS_BYTE_BUDGET`]); the frame-count bound
366    /// is whatever capacity `tx`'s channel was created with.
367    pub fn new(tx: mpsc::Sender<OutboundFrame>) -> Self {
368        Self::with_byte_budget(tx, crate::server::CONNECTION_EGRESS_BYTE_BUDGET)
369    }
370
371    pub(crate) fn with_byte_budget(tx: mpsc::Sender<OutboundFrame>, byte_budget: usize) -> Self {
372        Self {
373            tx,
374            accounting: Arc::new(EgressAccounting::new(byte_budget)),
375        }
376    }
377
378    /// Wait until the frame's bytes fit the budget, then charge them. Awaited
379    /// senders (control replies, error replies, shutdown notices) are never
380    /// refused by the byte budget: they wait for the writer to free bytes, just
381    /// as they wait for a free slot, which is the same backpressure they had
382    /// when the queue was bounded by frame count alone. Returns `None` if the
383    /// writer goes away while waiting.
384    async fn charge_waiting(&self, bytes: usize) -> Option<EgressCharge> {
385        if let Some(charge) = self.accounting.try_charge(bytes) {
386            return Some(charge);
387        }
388        // Counted as a waiter for as long as this future is parked, including
389        // when it is cancelled, so releases notify only while someone waits.
390        struct Waiting<'a>(&'a AtomicUsize);
391        impl Drop for Waiting<'_> {
392            fn drop(&mut self) {
393                self.0.fetch_sub(1, Ordering::SeqCst);
394            }
395        }
396        self.accounting.waiters.fetch_add(1, Ordering::SeqCst);
397        let _waiting = Waiting(&self.accounting.waiters);
398        loop {
399            let freed = self.accounting.freed.notified();
400            tokio::pin!(freed);
401            // Register interest before checking, so a release that happens
402            // between the check and the await still wakes this sender.
403            freed.as_mut().enable();
404            if let Some(charge) = self.accounting.try_charge(bytes) {
405                return Some(charge);
406            }
407            tokio::select! {
408                _ = &mut freed => {}
409                _ = self.tx.closed() => return None,
410            }
411        }
412    }
413
414    pub async fn send(&self, frame: Frame) -> Result<(), RouterError> {
415        let channel = frame.header.channel;
416        let epoch = frame.header.epoch;
417        let corr = frame.header.corr;
418        let closed =
419            || RouterError::backend_with_epoch(channel, epoch, corr, "connection writer closed");
420        let charge = self
421            .charge_waiting(queued_frame_bytes(&frame))
422            .await
423            .ok_or_else(closed)?;
424        self.tx
425            .send(OutboundFrame::charged(frame, charge))
426            .await
427            .map_err(|_| closed())
428    }
429
430    /// Shutdown notices must leave the socket writer before an idle daemon exits.
431    /// Queue admission alone does not prove this; the writer acknowledges flush.
432    ///
433    /// The local endpoint uses it the same way for its clients-only refusal: the
434    /// connection closes right after the refusal is sent, so the writer must
435    /// flush it first.
436    #[cfg(unix)]
437    pub async fn send_flushed(&self, frame: Frame) -> Result<(), RouterError> {
438        let (tx, rx) = tokio::sync::oneshot::channel();
439        let charge = self
440            .charge_waiting(queued_frame_bytes(&frame))
441            .await
442            .ok_or_else(|| RouterError::backend(0, 0, "connection writer closed"))?;
443        let mut outbound = OutboundFrame::charged(frame, charge);
444        outbound.flushed = Some(tx);
445        self.tx
446            .send(outbound)
447            .await
448            .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
449        rx.await
450            .map_err(|_| RouterError::backend(0, 0, "connection flush failed"))
451    }
452
453    pub(crate) async fn reserve_owned(&self) -> Result<EgressPermit, RouterError> {
454        let permit = self
455            .tx
456            .clone()
457            .reserve_owned()
458            .await
459            .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
460        Ok(EgressPermit {
461            permit,
462            accounting: Arc::clone(&self.accounting),
463        })
464    }
465
466    #[cfg(test)]
467    pub(crate) fn try_reserve_owned(&self) -> Result<EgressPermit, RouterError> {
468        let permit = self
469            .tx
470            .clone()
471            .try_reserve_owned()
472            .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
473        Ok(EgressPermit {
474            permit,
475            accounting: Arc::clone(&self.accounting),
476        })
477    }
478
479    pub(crate) fn is_closed(&self) -> bool {
480        self.tx.is_closed()
481    }
482
483    /// What the connection's egress queue holds right now.
484    pub(crate) fn backlog(&self) -> EgressBacklog {
485        self.accounting.backlog()
486    }
487
488    /// Stop module deliveries as soon as the client reader exits, without
489    /// waiting for the socket writer's separate teardown drain grace.
490    pub(crate) fn close_client_delivery(&self) {
491        self.accounting
492            .client_delivery_closed
493            .store(true, Ordering::Release);
494        self.accounting.client_delivery_stopped.notify_waiters();
495    }
496
497    /// A full queue can be a short fanout burst rather than a stopped reader.
498    /// Spend only the grace the atomic backlog age has not already used.
499    async fn send_to_client(&self, frame: Frame) -> Result<(), ClientDeliveryFailure> {
500        let frame_type = frame.header.ty;
501        let flags = frame.header.flags;
502        let stopped = self.accounting.client_delivery_stopped.notified();
503        tokio::pin!(stopped);
504        stopped.as_mut().enable();
505        let failure = |cause, error| ClientDeliveryFailure {
506            diagnosis: ClientDeliveryDiagnosis {
507                frame_type,
508                flags,
509                cause,
510            },
511            error,
512        };
513        let closed = || {
514            RouterError::backend_with_epoch(
515                frame.header.channel,
516                frame.header.epoch,
517                frame.header.corr,
518                "client connection closed",
519            )
520        };
521        if self
522            .accounting
523            .client_delivery_closed
524            .load(Ordering::Acquire)
525        {
526            return Err(failure("client_closed", closed()));
527        }
528        let (frame, error) = match self.try_send_owned(frame) {
529            Ok(()) => return Ok(()),
530            Err(refused) => refused,
531        };
532        if self.is_closed() {
533            return Err(failure("client_closed", error));
534        }
535        let grace = crate::server::CONNECTION_EGRESS_STALL_GRACE;
536        let age = self.backlog().oldest_age.unwrap_or_default();
537        if age > grace {
538            return Err(failure("oldest_frame_older_than_250_ms", error));
539        }
540        let result = tokio::select! {
541            biased;
542            _ = &mut stopped => return Err(failure("client_closed", error)),
543            result = tokio::time::timeout(grace.saturating_sub(age), self.send(frame)) => result,
544        };
545        match result {
546            Ok(Ok(())) => Ok(()),
547            Ok(Err(error)) => Err(failure("client_closed", error)),
548            Err(_) => Err(failure("waited_250_ms_without_room", error)),
549        }
550    }
551
552    /// Enqueue without waiting. Fails when the frame's bytes would take the
553    /// queue past its byte budget, when the channel's frame-count backstop is
554    /// full, or when the writer is gone.
555    pub(crate) fn try_send(&self, frame: Frame) -> Result<(), RouterError> {
556        self.try_send_owned(frame).map_err(|(_, error)| error)
557    }
558
559    fn try_send_owned(&self, frame: Frame) -> Result<(), (Frame, RouterError)> {
560        let channel = frame.header.channel;
561        let epoch = frame.header.epoch;
562        let corr = frame.header.corr;
563        let unavailable = |why: String| {
564            RouterError::backend_with_epoch(
565                channel,
566                epoch,
567                corr,
568                format!("connection writer unavailable: {why}"),
569            )
570        };
571        let bytes = queued_frame_bytes(&frame);
572        let Some(charge) = self.accounting.try_charge(bytes) else {
573            if self.tx.is_closed() {
574                return Err((frame, unavailable("channel closed".to_string())));
575            }
576            return Err((
577                frame,
578                unavailable(format!(
579                    "egress byte budget exhausted ({} queued bytes, frame of {bytes} bytes, budget {})",
580                    self.accounting.queued_bytes.load(Ordering::Acquire),
581                    self.accounting.byte_budget
582                )),
583            ));
584        };
585        // Return the refused frame for a bounded retry; dropping the rest of
586        // the outbound wrapper returns its charge.
587        self.tx
588            .try_send(OutboundFrame::charged(frame, charge))
589            .map_err(|err| {
590                let error = unavailable(err.to_string());
591                (err.into_inner().frame, error)
592            })
593    }
594}
595
596#[derive(Debug)]
597struct ClientDeliveryFailure {
598    diagnosis: ClientDeliveryDiagnosis,
599    error: RouterError,
600}
601
602/// Header facts only: payloads can contain secrets and must never enter logs.
603#[derive(Debug)]
604pub(crate) struct ClientDeliveryDiagnosis {
605    pub frame_type: FrameType,
606    pub flags: Flags,
607    pub cause: &'static str,
608}
609
610impl fmt::Display for ClientDeliveryFailure {
611    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
612        self.error.fmt(f)
613    }
614}
615
616/// Minimum time between two route GOODBYEs the daemon sends one module
617/// connection for one channel in answer to the module's frames on a (channel,
618/// epoch) the daemon holds no route for.
619///
620/// A module that missed a GOODBYE keeps sending on that route (a streaming
621/// module, many frames a second), and every one of those frames lands here. One
622/// answer is enough when it arrives, so answering each frame would only flood
623/// the module's egress queue at the moment it is already behind. Five seconds
624/// is long enough for an answer queued behind a deep backlog to reach the module
625/// and take effect before a second is sent, and short enough that an answer
626/// the queue refused is retried by the next orphan frame well within a minute.
627const ORPHAN_ROUTE_GOODBYE_INTERVAL: Duration = Duration::from_secs(5);
628
629/// Once one connection has this many remembered channels, entries older than
630/// [`ORPHAN_ROUTE_GOODBYE_INTERVAL`] are pruned before another is added. An
631/// expired entry has no effect on rate limiting, so pruning never changes a
632/// decision; it keeps a module that orphaned many channels long ago from
633/// pinning memory for all of them. The map is bounded in any case by the
634/// 16-bit channel space and is dropped whole when the connection ends.
635const ORPHAN_ROUTE_GOODBYE_PRUNE_AT: usize = 256;
636
637/// When each module connection was last sent an orphan-route GOODBYE, per
638/// channel. Shared by the [`Router`] (which consults it) and every
639/// [`RouterConnection`] (which removes its own entry when the connection ends).
640#[derive(Debug, Default)]
641struct OrphanGoodbyeLimiter {
642    last_sent: Mutex<HashMap<ConnectionId, HashMap<u16, tokio::time::Instant>>>,
643}
644
645impl OrphanGoodbyeLimiter {
646    /// True, and the send recorded, when `channel` on `connection_id` has not
647    /// been answered within the interval.
648    fn claim(&self, connection_id: ConnectionId, channel: u16) -> bool {
649        let now = tokio::time::Instant::now();
650        let mut last_sent = self
651            .last_sent
652            .lock()
653            .unwrap_or_else(std::sync::PoisonError::into_inner);
654        let channels = last_sent.entry(connection_id).or_default();
655        if channels.get(&channel).is_some_and(|sent| {
656            now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
657        }) {
658            return false;
659        }
660        if channels.len() >= ORPHAN_ROUTE_GOODBYE_PRUNE_AT {
661            channels.retain(|_, sent| {
662                now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
663            });
664        }
665        channels.insert(channel, now);
666        true
667    }
668
669    fn forget_connection(&self, connection_id: ConnectionId) {
670        self.last_sent
671            .lock()
672            .unwrap_or_else(std::sync::PoisonError::into_inner)
673            .remove(&connection_id);
674    }
675}
676
677/// Per-route context shared with backends besides the frame itself.
678#[derive(Debug, Clone)]
679pub struct RouteCtx {
680    pub connection_id: ConnectionId,
681    pub egress: FrameSink,
682}
683
684/// Closed set of data-plane backends for non-zero channels.
685///
686/// Channel 0 is structurally special and remains [`Router::control`], not an
687/// enum variant. Static/test backends live in [`Router::backends`]; the forwarding
688/// backend is selected dynamically only when a per-connection forwarding binding exists.
689#[derive(Debug, Clone)]
690pub enum Backend {
691    Echo(EchoBackend),
692    Forward(ForwardBackend),
693}
694
695impl From<EchoBackend> for Backend {
696    fn from(backend: EchoBackend) -> Self {
697        Self::Echo(backend)
698    }
699}
700
701impl From<ForwardBackend> for Backend {
702    fn from(backend: ForwardBackend) -> Self {
703        Self::Forward(backend)
704    }
705}
706
707impl Backend {
708    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
709        match self {
710            Self::Echo(backend) => backend.handle(ctx, frame).await,
711            Self::Forward(backend) => backend.handle(ctx, frame).await,
712        }
713    }
714}
715
716/// I/O-agnostic splice router keyed by envelope `channel`.
717///
718/// Channel 0 is reserved for subc itself and is always dispatched to the
719/// dedicated control handler. Other channels must be explicitly registered.
720/// Unknown client-originated non-zero channels are translated to canonical JSON `ERROR` frames on
721/// the connection sink so the peer can continue using the same socket. Module-originated frames for
722/// released route channels are logged and dropped as the channel-gone race backstop.
723///
724/// DATA-PLANE BODIES ARE NEVER DECODED HERE, and the consequence is worth stating
725/// because it looks like a guarantee and is not. Additive fields in a request or
726/// response body reach the far side untouched — not because anything permits them,
727/// but because routing reads only the 21-byte header and treats the body as opaque
728/// bytes. That is a performance property, so **nothing prevents it from changing**:
729/// a future reason to inspect a body would convert a wire-transparent path into a
730/// filtering one, and nobody would think of it as a contract change.
731///
732/// So when a peer asks whether subc sees an additive field, the answer is per-path
733/// and this path's zero means WE NEVER LOOK rather than WE LOOK AT EVERYTHING. The
734/// control plane (typed enums at the frame boundary) and the MCP gateway (envelope
735/// unwrap plus a named struct) both narrow; only this one does not.
736pub struct Router {
737    backends: HashMap<u16, Backend>,
738    control: Arc<ControlHandler>,
739    forwarding: Arc<ForwardingTable>,
740    forward_backend: ForwardBackend,
741    counters: DaemonCounters,
742    next_connection_id: AtomicU64,
743    orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
744}
745
746impl Router {
747    pub fn with_control_handler(control: Arc<ControlHandler>) -> Self {
748        // The handler the router serves is the one a swap must tell when it
749        // promotes a candidate; this is where it first sits behind an `Arc`.
750        control.install_swap_promotion_observer();
751        let forwarding = control.forwarding();
752        let counters = control.counters();
753        Self {
754            backends: HashMap::new(),
755            control,
756            forwarding: Arc::clone(&forwarding),
757            forward_backend: ForwardBackend::new(forwarding),
758            counters,
759            // ConnectionId::LOCAL is 0; real socket ids start at 1 and never collide.
760            next_connection_id: AtomicU64::new(1),
761            orphan_goodbyes: Arc::default(),
762        }
763    }
764
765    pub fn with_default_self_handler() -> Self {
766        Self::with_control_handler(Arc::new(ControlHandler::default()))
767    }
768
769    pub fn forwarding(&self) -> Arc<ForwardingTable> {
770        Arc::clone(&self.forwarding)
771    }
772
773    pub fn register_backend(
774        &mut self,
775        channel: u16,
776        backend: impl Into<Backend>,
777    ) -> Result<(), RouterError> {
778        self.register_backend_arc(channel, Arc::new(backend.into()))
779    }
780
781    pub(crate) fn register_backend_arc(
782        &mut self,
783        channel: u16,
784        backend: Arc<Backend>,
785    ) -> Result<(), RouterError> {
786        if channel == 0 {
787            return Err(RouterError::ReservedChannelZero);
788        }
789        if self.backends.contains_key(&channel) {
790            return Err(RouterError::DuplicateChannel { channel });
791        }
792        self.backends.insert(channel, backend.as_ref().clone());
793        Ok(())
794    }
795
796    /// Start a connection-scoped routing context. Dropping the guard releases
797    /// any control-plane registrations owned by the connection.
798    fn record_module_frame_drop(&self, connection_id: ConnectionId) -> Result<(), RouterError> {
799        let module_id = self
800            .forwarding
801            .module_id_for_connection(connection_id)
802            .map_err(RouterError::Forwarding)?;
803        self.counters
804            .increment_module_frames_dropped_no_route(module_id.as_deref());
805        Ok(())
806    }
807
808    /// A module sent a non-request frame on a (channel, epoch) the daemon holds
809    /// no route for: most often a route the daemon released whose GOODBYE the
810    /// module never received, so the module still believes it is open and keeps
811    /// sending. Count the drop, and tell the module to let go of exactly that
812    /// (channel, epoch) with a route GOODBYE, at most once per
813    /// [`ORPHAN_ROUTE_GOODBYE_INTERVAL`] per connection and channel.
814    ///
815    /// The answer uses `try_send`: it is a best-effort nudge, and if the queue
816    /// refuses it, the module's next frame on that route after the interval
817    /// asks again. A GOODBYE from the module is not answered, since the module
818    /// is already letting go of the route.
819    fn handle_orphan_module_frame(&self, ctx: &RouteCtx, frame: &Frame) -> Result<(), RouterError> {
820        let channel = frame.header.channel;
821        let epoch = frame.header.epoch;
822        let module_id = self
823            .forwarding
824            .module_id_for_connection(ctx.connection_id)
825            .map_err(RouterError::Forwarding)?;
826        self.counters
827            .increment_module_frames_dropped_no_route(module_id.as_deref());
828        if self
829            .forwarding
830            .module_route_epoch_was_allocated(ctx.connection_id, channel, epoch)
831            .map_err(RouterError::Forwarding)?
832        {
833            self.counters
834                .increment_module_frames_dropped_released_route(module_id.as_deref());
835        }
836        if frame.header.ty == FrameType::Goodbye
837            || !self.orphan_goodbyes.claim(ctx.connection_id, channel)
838        {
839            return Ok(());
840        }
841        let goodbye = Frame::build_with_version(
842            frame.header.ver,
843            FrameType::Goodbye,
844            Flags::new(false, Priority::Passive, false),
845            channel,
846            epoch,
847            0,
848            Vec::new(),
849        )
850        .map_err(RouterError::FrameBuild)?;
851        match ctx.egress.try_send(goodbye) {
852            Ok(()) => {
853                self.counters.increment_module_orphan_route_goodbyes_sent();
854                debug!(
855                    connection_id = ctx.connection_id.get(),
856                    module_id = module_id.as_deref().unwrap_or("unknown"),
857                    channel,
858                    epoch,
859                    "answered module frame on a route the daemon does not hold with a route GOODBYE"
860                );
861            }
862            Err(err) => debug!(
863                connection_id = ctx.connection_id.get(),
864                module_id = module_id.as_deref().unwrap_or("unknown"),
865                channel,
866                epoch,
867                error = %err,
868                "could not enqueue route GOODBYE for module frame on a route the daemon does not hold; the next such frame retries"
869            ),
870        }
871        Ok(())
872    }
873
874    pub fn begin_connection(&self) -> RouterConnection {
875        let raw = self.next_connection_id.fetch_add(1, Ordering::Relaxed);
876        let id = ConnectionId::new(raw);
877        let close_receiver = self.forwarding.register_connection_close(id);
878        RouterConnection {
879            id,
880            control_handler: Arc::clone(&self.control),
881            forwarding: Arc::clone(&self.forwarding),
882            close_receiver: Some(close_receiver),
883            orphan_goodbyes: Arc::clone(&self.orphan_goodbyes),
884        }
885    }
886
887    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
888        self.control.route_open_target(frame)
889    }
890
891    pub(crate) fn record_captured_peer(
892        &self,
893        connection: ConnectionId,
894        peer: subc_os::local_peer::CapturedPeer,
895    ) {
896        self.control.record_captured_peer(connection, peer);
897    }
898
899    pub(crate) fn record_caller_endpoints(
900        &self,
901        connection: ConnectionId,
902        endpoints: Option<(std::net::SocketAddr, std::net::SocketAddr)>,
903    ) {
904        self.control.record_caller_endpoints(connection, endpoints);
905    }
906
907    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
908        self.control.defers_privileged_control(ctx, frame)
909    }
910
911    pub(crate) fn route_open_capacity_refusal(
912        &self,
913        ctx: &RouteCtx,
914        frame: &Frame,
915        target_module_id: &str,
916        in_flight: usize,
917        limit: usize,
918    ) -> Result<Frame, RouterError> {
919        self.control
920            .route_open_capacity_refusal(ctx, frame, target_module_id, in_flight, limit)
921    }
922
923    pub async fn route_for_connection(
924        &self,
925        ctx: &RouteCtx,
926        frame: Frame,
927    ) -> Result<(), RouterError> {
928        self.route_for_connection_started(ctx, frame, None).await
929    }
930
931    pub(crate) async fn route_for_connection_started(
932        &self,
933        ctx: &RouteCtx,
934        frame: Frame,
935        dispatch_started_at: Option<Instant>,
936    ) -> Result<(), RouterError> {
937        let channel = frame.header.channel;
938        let epoch = frame.header.epoch;
939        let corr = frame.header.corr;
940        if channel == 0 {
941            debug!(
942                connection_id = ctx.connection_id.get(),
943                corr,
944                frame_type = ?frame.header.ty,
945                "routing control frame"
946            );
947            // The connection loop dispatches every control operation directly
948            // except `route.open`, activity writes and privileged requests. Each
949            // task passes a timestamp captured before
950            // spawn, so slow-dispatch timing includes scheduler delay but no
951            // wait in an application-owned queue.
952            let dispatch_started_at = (frame.header.ty == FrameType::Request)
953                .then(|| dispatch_started_at.unwrap_or_else(Instant::now));
954            let responses = self
955                .control
956                .handle_control_frame_timed(ctx, frame, dispatch_started_at)
957                .await?;
958            for response in responses {
959                ctx.egress.send(response).await?;
960            }
961            return Ok(());
962        }
963
964        let data_route = self
965            .forwarding
966            .lookup_data_route(ctx.connection_id, channel, epoch)
967            .map_err(RouterError::Forwarding)?;
968
969        match data_route {
970            DataRoute::Module(DataRouteState::EpochMismatch) => {
971                if frame.header.ty == FrameType::Request {
972                    self.counters
973                        .increment_module_requests_dropped_stale_route();
974                    let err = RouterError::StaleRouteEpoch {
975                        channel,
976                        epoch,
977                        corr,
978                    };
979                    if let Some(error_frame) = err.to_error_frame() {
980                        ctx.egress.send(error_frame).await?;
981                    }
982                } else {
983                    self.handle_orphan_module_frame(ctx, &frame)?;
984                }
985                debug!(
986                    connection_id = ctx.connection_id.get(),
987                    channel, epoch, corr, "dropping module frame for stale route epoch"
988                );
989                return Ok(());
990            }
991            DataRoute::Module(DataRouteState::Reserved) => {
992                if frame.header.ty == FrameType::Request {
993                    self.counters
994                        .increment_module_requests_dropped_stale_route();
995                    let err = RouterError::UnknownChannel {
996                        channel,
997                        epoch,
998                        corr,
999                    };
1000                    if let Some(error_frame) = err.to_error_frame() {
1001                        ctx.egress.send(error_frame).await?;
1002                    }
1003                } else {
1004                    self.record_module_frame_drop(ctx.connection_id)?;
1005                }
1006                debug!(
1007                    connection_id = ctx.connection_id.get(),
1008                    channel, epoch, corr, "dropping module frame for reserved route handle"
1009                );
1010                return Ok(());
1011            }
1012            DataRoute::Module(DataRouteState::Absent) => {
1013                if frame.header.ty == FrameType::Request {
1014                    self.counters
1015                        .increment_module_requests_dropped_stale_route();
1016                    let err = RouterError::UnknownChannel {
1017                        channel,
1018                        epoch,
1019                        corr,
1020                    };
1021                    if let Some(error_frame) = err.to_error_frame() {
1022                        ctx.egress.send(error_frame).await?;
1023                    }
1024                } else {
1025                    self.handle_orphan_module_frame(ctx, &frame)?;
1026                }
1027                debug!(
1028                    connection_id = ctx.connection_id.get(),
1029                    channel, epoch, corr, "dropping module frame for absent route handle"
1030                );
1031                return Ok(());
1032            }
1033            DataRoute::Module(DataRouteState::Bound(route)) => {
1034                if frame.header.ty == FrameType::Goodbye {
1035                    if let RouteRelease::Removed(target) = self
1036                        .forwarding
1037                        .release_module_route(ctx.connection_id, channel, epoch)
1038                        .map_err(RouterError::Forwarding)?
1039                    {
1040                        let mut goodbye = frame;
1041                        goodbye.header.channel = target.channel;
1042                        goodbye.header.epoch = target.epoch;
1043                        // A module must never be able to make a frame look
1044                        // daemon-authored, so the origin bit is cleared on
1045                        // every module frame relayed to a client.
1046                        goodbye.header.flags = goodbye.header.flags.without_daemon_origin();
1047                        if let Err(err) = target.sink.send_to_client(goodbye).await {
1048                            if target.close_on_delivery_failure()
1049                                && self
1050                                    .forwarding
1051                                    .escalate_client_delivery_failure_diagnosed(
1052                                        target.connection_id,
1053                                        target.channel,
1054                                        target.epoch,
1055                                        CloseReason::new(
1056                                            "route_goodbye_delivery_failed",
1057                                            format!(
1058                                                "failed to enqueue route GOODBYE for client channel {}: {err}",
1059                                                target.channel
1060                                            ),
1061                                        ),
1062                                        UndeliveredFrame {
1063                                            module_id: Some(&route.module_id),
1064                                            sink: &target.sink,
1065                                        },
1066                                        Some(&err.diagnosis),
1067                                    )
1068                                    .map_err(RouterError::Forwarding)?
1069                            {
1070                                self.counters.increment_goodbye_relay_client_failed();
1071                            }
1072                        }
1073                    }
1074                    return Ok(());
1075                }
1076
1077                // Accounting allows an oversized frame to occupy an empty queue
1078                // on other send paths. Refuse it at this splice before admission,
1079                // so the next ordinary reply cannot close a multiplexed client.
1080                let bytes = queued_frame_bytes(&frame);
1081                let limit = route.client_sink.accounting.byte_budget;
1082                if bytes > limit {
1083                    let ty = frame.header.ty;
1084                    let flags = frame.header.flags;
1085                    drop(frame);
1086                    return self
1087                        .refuse_oversized_module_frame(ctx, &route, (ty, flags), corr, bytes)
1088                        .await;
1089                }
1090
1091                // A terminal frame ends the request at the module whether or not
1092                // the client can still take it, so its credit is released before
1093                // delivery is attempted. Releasing only after a successful
1094                // enqueue would leave a drain counting a finished request until
1095                // the client connection's cleanup removes the route.
1096                let releases_credit = is_terminal_frame(frame.header.ty);
1097                if releases_credit {
1098                    route.flow.release_corr(corr);
1099                }
1100                let mut frame = frame;
1101                frame.header.channel = route.client_channel;
1102                frame.header.epoch = route.client_epoch;
1103                // Responses, errors, stream data and ends, and reverse
1104                // requests all pass through here. A module must never be able
1105                // to make a frame look daemon-authored, so the origin bit a
1106                // module may have set is cleared; every other flag bit and
1107                // the body travel unchanged.
1108                frame.header.flags = frame.header.flags.without_daemon_origin();
1109                if let Err(err) = route.client_sink.send_to_client(frame).await {
1110                    if self
1111                        .forwarding
1112                        .escalate_client_delivery_failure_diagnosed(
1113                            route.client_connection_id,
1114                            route.client_channel,
1115                            route.client_epoch,
1116                            CloseReason::new(
1117                                "module_to_client_delivery_failed",
1118                                format!(
1119                                    "failed to enqueue module frame for client channel {} corr {corr}: {err}",
1120                                    route.client_channel
1121                                ),
1122                            ),
1123                            UndeliveredFrame {
1124                                module_id: Some(&route.module_id),
1125                                sink: &route.client_sink,
1126                            },
1127                            Some(&err.diagnosis),
1128                        )
1129                        .map_err(RouterError::Forwarding)?
1130                    {
1131                        self.counters
1132                            .increment_client_egress_close_delivery_failed();
1133                    }
1134                    return Ok(());
1135                }
1136                return Ok(());
1137            }
1138            DataRoute::Client(DataRouteState::EpochMismatch) => {
1139                if frame.header.ty == FrameType::Request {
1140                    self.counters.increment_client_frames_dropped_stale_route();
1141                    // Dropped before forwarding; a re-bind retry cannot double-execute this request.
1142                    let err = RouterError::StaleRouteEpoch {
1143                        channel,
1144                        epoch,
1145                        corr,
1146                    };
1147                    if let Some(error_frame) = err.to_error_frame() {
1148                        ctx.egress.send(error_frame).await?;
1149                    }
1150                }
1151                debug!(
1152                    connection_id = ctx.connection_id.get(),
1153                    channel, epoch, corr, "dropping client frame for stale route epoch"
1154                );
1155                return Ok(());
1156            }
1157            DataRoute::Client(DataRouteState::Reserved) => {
1158                if frame.header.ty == FrameType::Request {
1159                    let err = RouterError::UnknownChannel {
1160                        channel,
1161                        epoch,
1162                        corr,
1163                    };
1164                    if let Some(error_frame) = err.to_error_frame() {
1165                        ctx.egress.send(error_frame).await?;
1166                    }
1167                }
1168                return Ok(());
1169            }
1170            DataRoute::Client(DataRouteState::Bound(route)) => {
1171                if frame.header.ty == FrameType::Goodbye {
1172                    let _ = self
1173                        .control
1174                        .handle_route_goodbye(ctx.connection_id, channel, epoch)?;
1175                    return Ok(());
1176                }
1177                return self.forward_backend.handle_bound(frame, route).await;
1178            }
1179            DataRoute::Client(DataRouteState::Absent) => {}
1180        }
1181
1182        if let Some(backend) = self.backends.get(&channel) {
1183            return backend.handle(ctx.clone(), frame).await;
1184        }
1185        if frame.header.ty == FrameType::Request {
1186            let err = RouterError::UnknownChannel {
1187                channel,
1188                epoch,
1189                corr,
1190            };
1191            if let Some(error_frame) = err.to_error_frame() {
1192                ctx.egress.send(error_frame).await?;
1193            }
1194        }
1195        Ok(())
1196    }
1197
1198    async fn refuse_oversized_module_frame(
1199        &self,
1200        ctx: &RouteCtx,
1201        route: &RouteBinding,
1202        header: (FrameType, Flags),
1203        corr: u64,
1204        bytes: usize,
1205    ) -> Result<(), RouterError> {
1206        let (ty, flags) = header;
1207        let limit = route.client_sink.accounting.byte_budget;
1208        if ty == FrameType::Push {
1209            // Deployed Rust clients cannot decode a new route.closed reason.
1210            // Until that vocabulary can be extended compatibly, an oversized
1211            // unsolicited push still closes the connection, with its cause named.
1212            let diagnosis = ClientDeliveryDiagnosis {
1213                frame_type: ty,
1214                flags,
1215                cause: "frame_too_large",
1216            };
1217            if self
1218                .forwarding
1219                .escalate_client_delivery_failure_diagnosed(
1220                    route.client_connection_id,
1221                    route.client_channel,
1222                    route.client_epoch,
1223                    CloseReason::new(
1224                        error_codes::FRAME_TOO_LARGE,
1225                        format!(
1226                            "oversized push: size={bytes} limit={limit} (header plus body bytes)"
1227                        ),
1228                    ),
1229                    UndeliveredFrame {
1230                        module_id: Some(&route.module_id),
1231                        sink: &route.client_sink,
1232                    },
1233                    Some(&diagnosis),
1234                )
1235                .map_err(RouterError::Forwarding)?
1236            {
1237                self.counters
1238                    .increment_client_egress_close_delivery_failed();
1239            }
1240            return Ok(());
1241        }
1242
1243        warn!(
1244            connection_id = route.client_connection_id.get(),
1245            module_id = %route.module_id,
1246            client_principal = %principal_label(&route.principal),
1247            client_channel = route.client_channel,
1248            frame_type = ?ty,
1249            frame_bytes = bytes,
1250            byte_limit = limit,
1251            "refusing oversized module-to-client frame with ERROR frame_too_large"
1252        );
1253        if ty == FrameType::Request {
1254            // A reverse request belongs to the module, not to a waiting client.
1255            // Answer its sender without forwarding any part of it to the client.
1256            ctx.egress
1257                .send(frame_too_large_error(
1258                    route.module_channel,
1259                    route.module_epoch,
1260                    corr,
1261                    bytes,
1262                    limit,
1263                )?)
1264                .await?;
1265        } else {
1266            // A refused stream chunk also ends the waiting request. Releasing
1267            // its credit prevents a finished request from keeping a drain alive.
1268            route.flow.release_corr(corr);
1269            let error = frame_too_large_error(
1270                route.client_channel,
1271                route.client_epoch,
1272                corr,
1273                bytes,
1274                limit,
1275            )?;
1276            // The replacement shares the reply's bounded burst grace, so a
1277            // stopped client cannot indefinitely stall the module frame reader.
1278            if let Err(err) = route.client_sink.send_to_client(error).await {
1279                if self
1280                    .forwarding
1281                    .escalate_client_delivery_failure_diagnosed(
1282                        route.client_connection_id,
1283                        route.client_channel,
1284                        route.client_epoch,
1285                        CloseReason::new(
1286                            "module_to_client_delivery_failed",
1287                            format!(
1288                                "failed to enqueue frame_too_large for client channel {} corr {corr}: {err}",
1289                                route.client_channel
1290                            ),
1291                        ),
1292                        UndeliveredFrame {
1293                            module_id: Some(&route.module_id),
1294                            sink: &route.client_sink,
1295                        },
1296                        Some(&err.diagnosis),
1297                    )
1298                    .map_err(RouterError::Forwarding)?
1299                {
1300                    self.counters
1301                        .increment_client_egress_close_delivery_failed();
1302                }
1303            }
1304        }
1305        Ok(())
1306    }
1307}
1308
1309impl Default for Router {
1310    fn default() -> Self {
1311        Self::with_default_self_handler()
1312    }
1313}
1314
1315/// Connection-scoped cleanup guard returned by [`Router::begin_connection`].
1316#[must_use]
1317pub struct RouterConnection {
1318    id: ConnectionId,
1319    control_handler: Arc<ControlHandler>,
1320    forwarding: Arc<ForwardingTable>,
1321    close_receiver: Option<ConnectionCloseReceiver>,
1322    orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
1323}
1324
1325impl RouterConnection {
1326    pub fn id(&self) -> ConnectionId {
1327        self.id
1328    }
1329
1330    pub(crate) fn take_close_receiver(&mut self) -> ConnectionCloseReceiver {
1331        self.close_receiver
1332            .take()
1333            .expect("connection close receiver can only be taken once")
1334    }
1335}
1336
1337impl Drop for RouterConnection {
1338    fn drop(&mut self) {
1339        self.forwarding.unregister_connection_close(self.id);
1340        self.orphan_goodbyes.forget_connection(self.id);
1341        // GOODBYE (explicit) and connection-drop cleanup both call the same
1342        // idempotent deregistration path.
1343        let _ = self.control_handler.cleanup_connection(self.id);
1344    }
1345}
1346
1347/// Minimal in-memory backend used by tests and early wiring: it emits a
1348/// `RESPONSE` on the same channel/correlation id with the exact same body bytes.
1349#[derive(Debug, Default, Clone, Copy)]
1350pub struct EchoBackend;
1351
1352impl EchoBackend {
1353    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1354        let response = Frame::build_with_version(
1355            frame.header.ver,
1356            FrameType::Response,
1357            frame.header.flags,
1358            frame.header.channel,
1359            frame.header.epoch,
1360            frame.header.corr,
1361            frame.body,
1362        )
1363        .map_err(RouterError::FrameBuild)?;
1364        ctx.egress.send(response).await
1365    }
1366}
1367
1368/// Data-plane backend that splices client frames to the module connection bound at attach time.
1369#[derive(Debug, Clone)]
1370pub struct ForwardBackend {
1371    forwarding: Arc<ForwardingTable>,
1372}
1373
1374impl ForwardBackend {
1375    pub fn new(forwarding: Arc<ForwardingTable>) -> Self {
1376        Self { forwarding }
1377    }
1378
1379    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1380        let channel = frame.header.channel;
1381        let corr = frame.header.corr;
1382        let route = match self
1383            .forwarding
1384            .lookup_data_route(ctx.connection_id, channel, frame.header.epoch)
1385            .map_err(RouterError::Forwarding)?
1386        {
1387            DataRoute::Client(DataRouteState::Bound(route)) => route,
1388            DataRoute::Client(_) | DataRoute::Module(_) => {
1389                return Err(RouterError::UnknownChannel {
1390                    channel,
1391                    epoch: frame.header.epoch,
1392                    corr,
1393                });
1394            }
1395        };
1396        self.handle_bound(frame, route).await
1397    }
1398
1399    pub(crate) async fn handle_bound(
1400        &self,
1401        frame: Frame,
1402        route: Arc<RouteBinding>,
1403    ) -> Result<(), RouterError> {
1404        let channel = frame.header.channel;
1405        let corr = frame.header.corr;
1406        let frame_type = frame.header.ty;
1407
1408        // CANCEL and other non-REQUEST frames bypass the request-credit window;
1409        // the original request's credit is freed only by the module's terminal frame.
1410        let acquired_credit = frame_type == FrameType::Request;
1411        if acquired_credit {
1412            if let Err(err) = route
1413                .flow
1414                .acquire_tagged(corr, frame.header.flags.is_subscription())
1415                .await
1416            {
1417                // `module_reloading` here is answered BEFORE the frame is
1418                // forwarded, so the module never sees the request and callers
1419                // may re-dispatch it after reopening the route. That is a wire
1420                // guarantee documented on `error_codes::MODULE_RELOADING`; do
1421                // not emit this code for a request that already reached
1422                // `module_sink.send` below.
1423                if self
1424                    .forwarding
1425                    .endpoint_is_draining(route.module_endpoint)
1426                    .map_err(RouterError::Forwarding)?
1427                {
1428                    return Err(RouterError::route_error_with_epoch(
1429                        channel,
1430                        frame.header.epoch,
1431                        corr,
1432                        "module_reloading",
1433                        format!("module endpoint for route channel {channel} is reloading"),
1434                    ));
1435                }
1436                return Err(RouterError::backend_with_epoch(
1437                    channel,
1438                    frame.header.epoch,
1439                    corr,
1440                    format!("{err} for route channel {channel}"),
1441                ));
1442            }
1443        }
1444
1445        let mut frame = frame;
1446        frame.header.channel = route.module_channel;
1447        frame.header.epoch = route.module_epoch;
1448        // The origin bit is the daemon's own signal to clients. A client
1449        // frame carrying it must not reach a module appearing daemon-authored,
1450        // so it is cleared here; every other flag bit travels unchanged.
1451        frame.header.flags = frame.header.flags.without_daemon_origin();
1452        let result = route.module_sink.send(frame).await.map_err(|err| {
1453            RouterError::backend_with_epoch(channel, route.client_epoch, corr, err.to_string())
1454        });
1455        if acquired_credit && result.is_err() {
1456            route.flow.release_corr(corr);
1457        }
1458        result
1459    }
1460}
1461
1462fn is_terminal_frame(frame_type: FrameType) -> bool {
1463    matches!(
1464        frame_type,
1465        FrameType::Response | FrameType::Error | FrameType::StreamEnd
1466    )
1467}
1468
1469/// Typed router errors. Routable failures can be translated to canonical JSON
1470/// `ERROR` frames with [`RouterError::to_error_frame`].
1471#[derive(Debug, Clone, PartialEq, Eq)]
1472pub enum RouterError {
1473    ReservedChannelZero,
1474    DuplicateChannel {
1475        channel: u16,
1476    },
1477    UnknownChannel {
1478        channel: u16,
1479        epoch: u32,
1480        corr: u64,
1481    },
1482    StaleRouteEpoch {
1483        channel: u16,
1484        epoch: u32,
1485        corr: u64,
1486    },
1487    Backend {
1488        channel: u16,
1489        epoch: u32,
1490        corr: u64,
1491        message: String,
1492    },
1493    RouteError {
1494        channel: u16,
1495        epoch: u32,
1496        corr: u64,
1497        code: String,
1498        message: String,
1499    },
1500    FrameBuild(FrameBuildError),
1501    Forwarding(ForwardingError),
1502}
1503
1504impl RouterError {
1505    pub fn backend(channel: u16, corr: u64, message: impl Into<String>) -> Self {
1506        Self::backend_with_epoch(channel, 0, corr, message)
1507    }
1508
1509    pub fn backend_with_epoch(
1510        channel: u16,
1511        epoch: u32,
1512        corr: u64,
1513        message: impl Into<String>,
1514    ) -> Self {
1515        Self::Backend {
1516            channel,
1517            epoch,
1518            corr,
1519            message: message.into(),
1520        }
1521    }
1522
1523    pub fn route_error(
1524        channel: u16,
1525        corr: u64,
1526        code: impl Into<String>,
1527        message: impl Into<String>,
1528    ) -> Self {
1529        Self::route_error_with_epoch(channel, 0, corr, code, message)
1530    }
1531
1532    pub fn route_error_with_epoch(
1533        channel: u16,
1534        epoch: u32,
1535        corr: u64,
1536        code: impl Into<String>,
1537        message: impl Into<String>,
1538    ) -> Self {
1539        Self::RouteError {
1540            channel,
1541            epoch,
1542            corr,
1543            code: code.into(),
1544            message: message.into(),
1545        }
1546    }
1547
1548    /// Translate route failures that belong on the wire into an `ERROR` frame.
1549    pub fn to_error_frame(&self) -> Option<Frame> {
1550        match self {
1551            Self::UnknownChannel {
1552                channel,
1553                epoch,
1554                corr,
1555            } => error_frame(
1556                *channel,
1557                *epoch,
1558                *corr,
1559                error_codes::UNKNOWN_CHANNEL,
1560                format!("unknown channel {channel}"),
1561            ),
1562            Self::StaleRouteEpoch {
1563                channel,
1564                epoch,
1565                corr,
1566            } => error_frame(
1567                *channel,
1568                *epoch,
1569                *corr,
1570                error_codes::STALE_ROUTE_EPOCH,
1571                format!("stale route epoch for channel {channel}"),
1572            ),
1573            Self::Backend {
1574                channel,
1575                epoch,
1576                corr,
1577                message,
1578            } => error_frame(*channel, *epoch, *corr, "backend_error", message.clone()),
1579            Self::RouteError {
1580                channel,
1581                epoch,
1582                corr,
1583                code,
1584                message,
1585            } => error_frame(*channel, *epoch, *corr, code, message.clone()),
1586            Self::ReservedChannelZero
1587            | Self::DuplicateChannel { .. }
1588            | Self::FrameBuild(_)
1589            | Self::Forwarding(_) => None,
1590        }
1591    }
1592}
1593
1594/// Flags for an ERROR frame the daemon authors itself.
1595///
1596/// Bit 6 (`FLAG_DAEMON_ORIGIN`) tells the client the daemon wrote this
1597/// error, never a module. What a code then proves is per code: on a bound
1598/// route the daemon writes `module_reloading`, `unknown_channel` and
1599/// `stale_route_epoch` only before forwarding, so with the bit they mean the
1600/// request was not delivered, while `frame_too_large` replaces a reply the
1601/// module already produced. Every daemon-authored ERROR goes through here;
1602/// frames a module sent keep their own flags with the bit cleared at the
1603/// splice.
1604/// Nothing else about the flags differs: passive priority, not binary, not last.
1605pub(crate) fn daemon_error_flags() -> Flags {
1606    Flags::new(false, Priority::Passive, false).with_daemon_origin()
1607}
1608
1609fn error_frame(channel: u16, epoch: u32, corr: u64, code: &str, message: String) -> Option<Frame> {
1610    let body = serde_json::to_vec(&ErrorBody {
1611        code: code.to_string(),
1612        message,
1613        detail: None,
1614    })
1615    .ok()?;
1616
1617    Frame::build(
1618        FrameType::Error,
1619        daemon_error_flags(),
1620        channel,
1621        epoch,
1622        corr,
1623        body,
1624    )
1625    .ok()
1626}
1627
1628fn frame_too_large_error(
1629    channel: u16,
1630    epoch: u32,
1631    corr: u64,
1632    bytes: usize,
1633    limit: usize,
1634) -> Result<Frame, RouterError> {
1635    let body = serde_json::to_vec(&ErrorBody {
1636        code: error_codes::FRAME_TOO_LARGE.to_string(),
1637        message: format!(
1638            "module frame of {bytes} bytes exceeds client egress limit of {limit} bytes"
1639        ),
1640        detail: Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit })),
1641    })
1642    .expect("serializing a frame size error");
1643    Frame::build(
1644        FrameType::Error,
1645        daemon_error_flags(),
1646        channel,
1647        epoch,
1648        corr,
1649        body,
1650    )
1651    .map_err(RouterError::FrameBuild)
1652}
1653
1654impl fmt::Display for RouterError {
1655    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1656        match self {
1657            Self::ReservedChannelZero => write!(f, "channel 0 is reserved for subc"),
1658            Self::DuplicateChannel { channel } => {
1659                write!(f, "backend already registered for channel {channel}")
1660            }
1661            Self::UnknownChannel { channel, corr, .. } => {
1662                write!(f, "unknown channel {channel} for corr {corr}")
1663            }
1664            Self::StaleRouteEpoch { channel, corr, .. } => {
1665                write!(f, "stale route epoch for channel {channel} corr {corr}")
1666            }
1667            Self::Backend {
1668                channel,
1669                corr,
1670                message,
1671                ..
1672            } => write!(
1673                f,
1674                "backend error on channel {channel} corr {corr}: {message}"
1675            ),
1676            Self::RouteError {
1677                channel,
1678                corr,
1679                code,
1680                message,
1681                ..
1682            } => write!(
1683                f,
1684                "route error {code} on channel {channel} corr {corr}: {message}"
1685            ),
1686            Self::FrameBuild(err) => write!(f, "failed to build routed frame: {err}"),
1687            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
1688        }
1689    }
1690}
1691
1692impl Error for RouterError {
1693    fn source(&self) -> Option<&(dyn Error + 'static)> {
1694        match self {
1695            Self::FrameBuild(err) => Some(err),
1696            Self::Forwarding(err) => Some(err),
1697            Self::ReservedChannelZero
1698            | Self::DuplicateChannel { .. }
1699            | Self::UnknownChannel { .. }
1700            | Self::StaleRouteEpoch { .. }
1701            | Self::Backend { .. }
1702            | Self::RouteError { .. } => None,
1703        }
1704    }
1705}
1706
1707#[cfg(test)]
1708mod tests {
1709    use super::*;
1710    use crate::{
1711        forwarding::RouteBindRelayOutcome,
1712        supervise::{ModuleSpec, RestartPolicy, Supervisor, SupervisorHandle},
1713        ControlHandler, Registry,
1714    };
1715    use std::{
1716        sync::{mpsc as std_mpsc, Arc},
1717        time::Duration,
1718    };
1719    use subc_control::ModuleProtocol;
1720    use subc_protocol::{manifest::Concurrency, ErrorBody, Flags, FrameType, Priority};
1721    use tokio::sync::mpsc;
1722
1723    pub(crate) use crate::router::test_log::{captured_logs, log_capture};
1724
1725    fn logged_millis(logs: &str, field: &str) -> u64 {
1726        logs.split_whitespace()
1727            .find_map(|part| part.strip_prefix(field))
1728            .and_then(|value| value.parse().ok())
1729            .unwrap_or_else(|| panic!("missing numeric {field} in logs: {logs}"))
1730    }
1731
1732    fn request(channel: u16, corr: u64, body: &[u8]) -> Frame {
1733        Frame::build(
1734            FrameType::Request,
1735            Flags::new(true, Priority::Interactive, false),
1736            channel,
1737            0,
1738            corr,
1739            body.to_vec(),
1740        )
1741        .unwrap()
1742    }
1743
1744    fn ping(corr: u64) -> Frame {
1745        Frame::build(
1746            FrameType::Ping,
1747            Flags::new(false, Priority::Passive, false),
1748            0,
1749            0,
1750            corr,
1751            Vec::new(),
1752        )
1753        .unwrap()
1754    }
1755
1756    fn route_ctx() -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
1757        let (tx, rx) = mpsc::channel(8);
1758        (
1759            RouteCtx {
1760                connection_id: ConnectionId::LOCAL,
1761                egress: FrameSink::new(tx),
1762            },
1763            rx,
1764        )
1765    }
1766
1767    #[tokio::test]
1768    async fn echo_backend_returns_response_with_byte_identical_body() {
1769        let mut router = Router::with_default_self_handler();
1770        router.register_backend(7, EchoBackend).unwrap();
1771        let (ctx, mut rx) = route_ctx();
1772        let body = b"{not parsed}\0\xff";
1773
1774        router
1775            .route_for_connection(&ctx, request(7, 123, body))
1776            .await
1777            .unwrap();
1778        let response = rx.recv().await.unwrap();
1779
1780        assert_eq!(response.header.ty, FrameType::Response);
1781        assert_eq!(response.header.channel, 7);
1782        assert_eq!(response.header.corr, 123);
1783        assert_eq!(response.body, body);
1784        assert!(rx.try_recv().is_err());
1785    }
1786
1787    #[tokio::test]
1788    async fn unknown_channel_emits_canonical_error_frame() {
1789        let router = Router::with_default_self_handler();
1790        let (ctx, mut rx) = route_ctx();
1791
1792        router
1793            .route_for_connection(&ctx, request(99, 5, b"payload"))
1794            .await
1795            .unwrap();
1796        let error_frame = rx.recv().await.unwrap();
1797
1798        assert_eq!(error_frame.header.ty, FrameType::Error);
1799        assert_eq!(error_frame.header.channel, 99);
1800        assert_eq!(error_frame.header.corr, 5);
1801        let body: ErrorBody = serde_json::from_slice(&error_frame.body).unwrap();
1802        assert_eq!(body.code, "unknown_channel");
1803        assert_eq!(body.message, "unknown channel 99");
1804    }
1805
1806    /// Every ERROR the daemon authors carries the daemon-origin bit, so a
1807    /// client can tell a daemon refusal from a module's reply. Both the
1808    /// unknown-channel and the stale-epoch errors below are received on the
1809    /// client's own egress, not merely built.
1810    #[tokio::test]
1811    async fn daemon_authored_data_plane_errors_carry_daemon_origin() {
1812        let router = Router::with_default_self_handler();
1813        let (ctx, mut rx) = route_ctx();
1814        router
1815            .route_for_connection(&ctx, request(99, 5, b"payload"))
1816            .await
1817            .unwrap();
1818        let unknown = rx.recv().await.unwrap();
1819        assert_eq!(unknown.header.ty, FrameType::Error);
1820        let body: ErrorBody = serde_json::from_slice(&unknown.body).unwrap();
1821        assert_eq!(body.code, "unknown_channel");
1822        assert!(
1823            unknown.header.flags.is_daemon_origin(),
1824            "unknown_channel must carry daemon origin: {:?}",
1825            unknown.header.flags
1826        );
1827
1828        let (
1829            router,
1830            _forwarding,
1831            client_ctx,
1832            mut client_rx,
1833            _module_ctx,
1834            _module_egress_rx,
1835            _module_rx,
1836            pending,
1837        ) = dynamic_route_fixture(true);
1838        let _ = client_rx.recv().await;
1839        router
1840            .route_for_connection(
1841                &client_ctx,
1842                route_frame(
1843                    FrameType::Request,
1844                    pending.client_channel,
1845                    pending.client_epoch + 1,
1846                    702,
1847                ),
1848            )
1849            .await
1850            .unwrap();
1851        let stale = client_rx.recv().await.unwrap();
1852        assert_eq!(stale.header.ty, FrameType::Error);
1853        let body: ErrorBody = serde_json::from_slice(&stale.body).unwrap();
1854        assert_eq!(body.code, "stale_route_epoch");
1855        assert!(
1856            stale.header.flags.is_daemon_origin(),
1857            "stale_route_epoch must carry daemon origin: {:?}",
1858            stale.header.flags
1859        );
1860    }
1861
1862    /// A module ERROR that spoofs a daemon code with the daemon-origin bit
1863    /// set must reach the client with the bit cleared and the body
1864    /// byte-identical: the bit means the daemon wrote the frame, and a module
1865    /// must never be able to claim that.
1866    #[tokio::test]
1867    async fn module_error_with_daemon_bit_is_stripped_before_client_delivery() {
1868        let (
1869            router,
1870            _forwarding,
1871            client_ctx,
1872            mut client_rx,
1873            module_ctx,
1874            _module_egress_rx,
1875            mut module_rx,
1876            pending,
1877        ) = dynamic_route_fixture(true);
1878        let _ = client_rx.recv().await;
1879        // Open flow credit the way a real request would, so the terminal
1880        // error exercises the credit-release path too.
1881        router
1882            .route_for_connection(
1883                &client_ctx,
1884                route_frame(
1885                    FrameType::Request,
1886                    pending.client_channel,
1887                    pending.client_epoch,
1888                    701,
1889                ),
1890            )
1891            .await
1892            .unwrap();
1893        let _ = module_rx.recv().await;
1894
1895        let body = serde_json::to_vec(&ErrorBody::new("module_reloading", "try again")).unwrap();
1896        let spoof = Frame::build(
1897            FrameType::Error,
1898            Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
1899            pending.module_channel,
1900            pending.module_epoch,
1901            701,
1902            body.clone(),
1903        )
1904        .unwrap();
1905        router
1906            .route_for_connection(&module_ctx, spoof)
1907            .await
1908            .unwrap();
1909        let delivered = client_rx.recv().await.unwrap();
1910        assert_eq!(delivered.header.ty, FrameType::Error);
1911        assert_eq!(delivered.header.channel, pending.client_channel);
1912        assert_eq!(delivered.header.epoch, pending.client_epoch);
1913        assert_eq!(delivered.header.corr, 701);
1914        assert!(
1915            !delivered.header.flags.is_daemon_origin(),
1916            "module-set daemon origin must be cleared: {:?}",
1917            delivered.header.flags
1918        );
1919        assert_eq!(delivered.body, body, "the body travels byte-identical");
1920    }
1921
1922    /// Clearing the origin bit strips exactly that bit: priority, LAST and
1923    /// subscription survive the splice unchanged.
1924    #[tokio::test]
1925    async fn module_response_with_daemon_bit_is_stripped_preserving_other_flags() {
1926        use subc_protocol::FLAG_SUBSCRIPTION;
1927        let (
1928            router,
1929            _forwarding,
1930            client_ctx,
1931            mut client_rx,
1932            module_ctx,
1933            _module_egress_rx,
1934            mut module_rx,
1935            pending,
1936        ) = dynamic_route_fixture(true);
1937        let _ = client_rx.recv().await;
1938        router
1939            .route_for_connection(
1940                &client_ctx,
1941                route_frame(
1942                    FrameType::Request,
1943                    pending.client_channel,
1944                    pending.client_epoch,
1945                    701,
1946                ),
1947            )
1948            .await
1949            .unwrap();
1950        let _ = module_rx.recv().await;
1951
1952        let sent_flags = Flags(
1953            Flags::new(true, Priority::Background, true).0
1954                | FLAG_SUBSCRIPTION
1955                | subc_protocol::FLAG_DAEMON_ORIGIN,
1956        );
1957        assert!(sent_flags.is_daemon_origin());
1958        let response = Frame::build(
1959            FrameType::Response,
1960            sent_flags,
1961            pending.module_channel,
1962            pending.module_epoch,
1963            701,
1964            b"route-body".to_vec(),
1965        )
1966        .unwrap();
1967        router
1968            .route_for_connection(&module_ctx, response)
1969            .await
1970            .unwrap();
1971        let delivered = client_rx.recv().await.unwrap();
1972        assert_eq!(delivered.header.ty, FrameType::Response);
1973        assert!(
1974            !delivered.header.flags.is_daemon_origin(),
1975            "module-set daemon origin must be cleared: {:?}",
1976            delivered.header.flags
1977        );
1978        assert!(
1979            delivered.header.flags.is_binary(),
1980            "binary flag must survive: {:?}",
1981            delivered.header.flags
1982        );
1983        assert!(
1984            delivered.header.flags.is_last(),
1985            "LAST flag must survive: {:?}",
1986            delivered.header.flags
1987        );
1988        assert!(
1989            delivered.header.flags.is_subscription(),
1990            "subscription flag must survive: {:?}",
1991            delivered.header.flags
1992        );
1993        assert_eq!(
1994            delivered.header.flags.priority(),
1995            Some(Priority::Background),
1996            "priority must survive"
1997        );
1998    }
1999
2000    /// A client frame carrying the origin bit must not reach the module
2001    /// looking daemon-authored either: the splice clears it symmetrically.
2002    #[tokio::test]
2003    async fn client_frame_with_daemon_bit_is_stripped_before_module_delivery() {
2004        let (
2005            router,
2006            _forwarding,
2007            client_ctx,
2008            mut client_rx,
2009            _module_ctx,
2010            _module_egress_rx,
2011            mut module_rx,
2012            pending,
2013        ) = dynamic_route_fixture(true);
2014        let _ = client_rx.recv().await;
2015        let spoof = Frame::build(
2016            FrameType::Request,
2017            Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2018            pending.client_channel,
2019            pending.client_epoch,
2020            701,
2021            b"route-body".to_vec(),
2022        )
2023        .unwrap();
2024        router
2025            .route_for_connection(&client_ctx, spoof)
2026            .await
2027            .unwrap();
2028        let delivered = module_rx.recv().await.unwrap();
2029        assert_eq!(delivered.header.ty, FrameType::Request);
2030        assert!(
2031            !delivered.header.flags.is_daemon_origin(),
2032            "client-set daemon origin must be cleared: {:?}",
2033            delivered.header.flags
2034        );
2035    }
2036
2037    #[tokio::test]
2038    async fn channel_zero_uses_control_handler_not_backend_registry() {
2039        let mut router = Router::with_default_self_handler();
2040        router.register_backend(1, EchoBackend).unwrap();
2041        let (ctx, mut rx) = route_ctx();
2042
2043        router.route_for_connection(&ctx, ping(77)).await.unwrap();
2044        let response = rx.recv().await.unwrap();
2045
2046        assert_eq!(response.header.ty, FrameType::Pong);
2047        assert_eq!(response.header.channel, 0);
2048        assert_eq!(response.header.corr, 77);
2049        assert!(response.body.is_empty());
2050    }
2051
2052    #[tokio::test]
2053    async fn slow_control_dispatch_logs_decoded_op_and_elapsed_time() {
2054        let control = Arc::new(
2055            ControlHandler::new(Arc::new(Registry::default()))
2056                .with_control_dispatch_delay(Duration::from_millis(1050)),
2057        );
2058        let router = Router::with_control_handler(control);
2059        let (ctx, mut rx) = route_ctx();
2060        let (output, guard) = log_capture(tracing::Level::WARN);
2061
2062        router
2063            .route_for_connection(&ctx, request(0, 41, br#"{"op":"server.describe"}"#))
2064            .await
2065            .expect("slow request routes");
2066        assert!(rx.recv().await.is_some(), "request receives a response");
2067        drop(guard);
2068
2069        let logs = captured_logs(&output);
2070        assert!(logs.contains("slow control dispatch"));
2071        assert!(logs.contains("op=server.describe"));
2072        assert!(logs.contains("connection_id=0"));
2073        assert!(logs.contains("corr=41"));
2074        assert!(
2075            logged_millis(&logs, "elapsed_ms=") >= 1050,
2076            "elapsed must include the injected handler delay: {logs}"
2077        );
2078    }
2079
2080    #[tokio::test]
2081    async fn fast_control_dispatch_emits_arrival_without_slow_warning() {
2082        let router = Router::with_default_self_handler();
2083        let (ctx, mut rx) = route_ctx();
2084        let (output, guard) = log_capture(tracing::Level::DEBUG);
2085
2086        router
2087            .route_for_connection(&ctx, request(0, 42, br#"{"op":"server.describe"}"#))
2088            .await
2089            .expect("fast request routes");
2090        assert!(rx.recv().await.is_some(), "request receives a response");
2091        drop(guard);
2092
2093        let logs = captured_logs(&output);
2094        assert!(logs.contains("control dispatch op=server.describe connection_id=0 corr=42"));
2095        assert!(!logs.contains("slow control dispatch"));
2096    }
2097
2098    #[tokio::test]
2099    async fn control_dispatch_arrival_is_hidden_at_info() {
2100        let router = Router::with_default_self_handler();
2101        let (ctx, mut rx) = route_ctx();
2102        let (output, guard) = log_capture(tracing::Level::INFO);
2103
2104        router
2105            .route_for_connection(&ctx, request(0, 43, br#"{"op":"server.describe"}"#))
2106            .await
2107            .expect("fast request routes");
2108        assert!(rx.recv().await.is_some(), "request receives a response");
2109        drop(guard);
2110
2111        assert!(
2112            !captured_logs(&output).contains("control dispatch"),
2113            "arrival logging must stay hidden at INFO"
2114        );
2115    }
2116
2117    #[tokio::test]
2118    async fn supervisor_list_logs_contended_snapshot_lock_only() {
2119        let registry = Arc::new(Registry::default());
2120        let handle = SupervisorHandle::new();
2121        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
2122            .with_handle(handle.clone());
2123        let module = supervisor
2124            .supervise_configured(
2125                ModuleSpec {
2126                    module_id: "held-module".to_string(),
2127                    program: "test-module".into(),
2128                    args: Vec::new(),
2129                    env: Vec::new(),
2130                    reserved: false,
2131                    reserved_prefixes: Vec::new(),
2132                    protocol: ModuleProtocol::Subc,
2133                    overlap: Default::default(),
2134                },
2135                false,
2136            )
2137            .expect("disabled test module is supervised");
2138        let router = Router::with_control_handler(Arc::new(
2139            ControlHandler::new(Arc::clone(&registry)).with_supervisor(handle),
2140        ));
2141        let (ctx, mut rx) = route_ctx();
2142        let (acquired, ready) = std_mpsc::channel();
2143        let holder = module.hold_snapshot_for_test(acquired, Duration::from_millis(400));
2144        ready.recv().expect("holder acquired snapshot lock");
2145        let (output, guard) = log_capture(tracing::Level::WARN);
2146
2147        router
2148            .route_for_connection(&ctx, request(0, 44, br#"{"op":"supervisor.list"}"#))
2149            .await
2150            .expect("list request routes after the lock releases");
2151        assert!(
2152            rx.recv().await.is_some(),
2153            "list request receives a response"
2154        );
2155        holder.join().expect("snapshot holder exits cleanly");
2156        drop(guard);
2157
2158        let logs = captured_logs(&output);
2159        assert!(logs.contains("slow snapshot lock"));
2160        assert!(logs.contains("module_id=held-module"));
2161        assert!(logs.contains("caller=list"));
2162        assert!(
2163            logged_millis(&logs, "waited_ms=") >= 250,
2164            "wait must exceed the slow-lock threshold: {logs}"
2165        );
2166
2167        let (output, guard) = log_capture(tracing::Level::WARN);
2168        router
2169            .route_for_connection(&ctx, request(0, 45, br#"{"op":"supervisor.list"}"#))
2170            .await
2171            .expect("uncontended list request routes");
2172        assert!(
2173            rx.recv().await.is_some(),
2174            "uncontended list receives a response"
2175        );
2176        drop(guard);
2177        assert!(
2178            !captured_logs(&output).contains("slow snapshot lock"),
2179            "uncontended list acquisition must not warn"
2180        );
2181    }
2182
2183    #[tokio::test]
2184    async fn full_module_to_client_sink_requests_client_close_without_erroring_module() {
2185        let forwarding = Arc::new(ForwardingTable::default());
2186        let control = Arc::new(ControlHandler::with_forwarding(
2187            Arc::new(crate::Registry::default()),
2188            Arc::clone(&forwarding),
2189        ));
2190        let router = Router::with_control_handler(control);
2191        let module_connection = ConnectionId::new(10);
2192        let client_connection = ConnectionId::new(20);
2193        let mut close_receiver = forwarding.register_connection_close(client_connection);
2194        let (module_tx, _module_rx) = mpsc::channel(1);
2195        forwarding
2196            .register_module_connection(
2197                module_connection,
2198                "full-sink-provider".to_string(),
2199                1,
2200                Concurrency::ModuleManaged,
2201                FrameSink::new(module_tx),
2202            )
2203            .unwrap();
2204        let (client_tx, mut client_rx) = mpsc::channel(1);
2205        let pending = forwarding
2206            .begin_route_bind_relay_for_test(
2207                client_connection,
2208                FrameSink::new(client_tx),
2209                700,
2210                "full-sink-provider",
2211            )
2212            .unwrap();
2213        forwarding
2214            .complete_pending_relay(
2215                module_connection,
2216                pending.corr,
2217                RouteBindRelayOutcome::Accepted,
2218            )
2219            .unwrap();
2220
2221        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2222        let module_ctx = RouteCtx {
2223            connection_id: module_connection,
2224            egress: FrameSink::new(module_egress_tx),
2225        };
2226        let terminal = Frame::build(
2227            FrameType::Response,
2228            Flags::new(false, Priority::Interactive, true),
2229            pending.module_channel,
2230            pending.module_epoch,
2231            701,
2232            b"terminal".to_vec(),
2233        )
2234        .unwrap();
2235
2236        router
2237            .route_for_connection(&module_ctx, terminal)
2238            .await
2239            .unwrap();
2240        let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2241            .await
2242            .expect("close request should be sent for the full client sink")
2243            .expect("close sender should include a reason");
2244        assert!(
2245            reason
2246                .to_string()
2247                .contains("module_to_client_delivery_failed"),
2248            "unexpected close reason: {reason}"
2249        );
2250        assert_eq!(client_rx.try_recv().unwrap().header.corr, 700);
2251        assert!(client_rx.try_recv().is_err());
2252        assert_eq!(
2253            router.counters.snapshot()["client_egress_close_delivery_failed"],
2254            1
2255        );
2256    }
2257
2258    /// A terminal frame ends the request at the module even when the client
2259    /// cannot take it, so the drain must stop counting it at once rather than
2260    /// when the client connection's cleanup later removes the route.
2261    #[tokio::test]
2262    async fn terminal_frame_releases_its_credit_even_when_client_delivery_fails() {
2263        let forwarding = Arc::new(ForwardingTable::default());
2264        let control = Arc::new(ControlHandler::with_forwarding(
2265            Arc::new(crate::Registry::default()),
2266            Arc::clone(&forwarding),
2267        ));
2268        let router = Router::with_control_handler(control);
2269        let module_connection = ConnectionId::new(11);
2270        let client_connection = ConnectionId::new(21);
2271        let _close_receiver = forwarding.register_connection_close(client_connection);
2272        let (module_tx, _module_rx) = mpsc::channel(1);
2273        forwarding
2274            .register_module_connection(
2275                module_connection,
2276                "credit-provider".to_string(),
2277                1,
2278                Concurrency::ModuleManaged,
2279                FrameSink::new(module_tx),
2280            )
2281            .unwrap();
2282        // Capacity one, filled by the route.open response, so the terminal
2283        // frame below cannot be enqueued for the client.
2284        let (client_tx, _client_rx) = mpsc::channel(1);
2285        let pending = forwarding
2286            .begin_route_bind_relay_for_test(
2287                client_connection,
2288                FrameSink::new(client_tx),
2289                800,
2290                "credit-provider",
2291            )
2292            .unwrap();
2293        forwarding
2294            .complete_pending_relay(
2295                module_connection,
2296                pending.corr,
2297                RouteBindRelayOutcome::Accepted,
2298            )
2299            .unwrap();
2300        let DataRoute::Client(DataRouteState::Bound(route)) = forwarding
2301            .lookup_data_route(
2302                client_connection,
2303                pending.client_channel,
2304                pending.client_epoch,
2305            )
2306            .unwrap()
2307        else {
2308            panic!("expected a bound client route");
2309        };
2310        route.flow.acquire_tagged(801, false).await.unwrap();
2311        assert_eq!(route.flow.drain_in_flight(), 1);
2312
2313        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2314        let module_ctx = RouteCtx {
2315            connection_id: module_connection,
2316            egress: FrameSink::new(module_egress_tx),
2317        };
2318        let terminal = Frame::build(
2319            FrameType::Response,
2320            Flags::new(false, Priority::Interactive, true),
2321            pending.module_channel,
2322            pending.module_epoch,
2323            801,
2324            b"terminal".to_vec(),
2325        )
2326        .unwrap();
2327        router
2328            .route_for_connection(&module_ctx, terminal)
2329            .await
2330            .unwrap();
2331
2332        assert_eq!(
2333            router.counters.snapshot()["client_egress_close_delivery_failed"],
2334            1,
2335            "the client delivery must have failed for this test to mean anything"
2336        );
2337        assert_eq!(
2338            route.flow.drain_in_flight(),
2339            0,
2340            "the module's terminal frame must release its credit even though the client could not take it"
2341        );
2342    }
2343
2344    #[tokio::test]
2345    async fn full_route_goodbye_sink_requests_target_close_without_erroring_module() {
2346        let forwarding = Arc::new(ForwardingTable::default());
2347        let control = Arc::new(ControlHandler::with_forwarding(
2348            Arc::new(crate::Registry::default()),
2349            Arc::clone(&forwarding),
2350        ));
2351        let router = Router::with_control_handler(control);
2352        let module_connection = ConnectionId::new(30);
2353        let client_connection = ConnectionId::new(40);
2354        let mut close_receiver = forwarding.register_connection_close(client_connection);
2355        let (module_tx, _module_rx) = mpsc::channel(1);
2356        forwarding
2357            .register_module_connection(
2358                module_connection,
2359                "goodbye-full-provider".to_string(),
2360                1,
2361                Concurrency::ModuleManaged,
2362                FrameSink::new(module_tx),
2363            )
2364            .unwrap();
2365        let (client_tx, mut client_rx) = mpsc::channel(1);
2366        let pending = forwarding
2367            .begin_route_bind_relay_for_test(
2368                client_connection,
2369                FrameSink::new(client_tx),
2370                800,
2371                "goodbye-full-provider",
2372            )
2373            .unwrap();
2374        forwarding
2375            .complete_pending_relay(
2376                module_connection,
2377                pending.corr,
2378                RouteBindRelayOutcome::Accepted,
2379            )
2380            .unwrap();
2381
2382        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2383        let module_ctx = RouteCtx {
2384            connection_id: module_connection,
2385            egress: FrameSink::new(module_egress_tx),
2386        };
2387        let goodbye = Frame::build(
2388            FrameType::Goodbye,
2389            Flags::new(false, Priority::Passive, true),
2390            pending.module_channel,
2391            pending.module_epoch,
2392            801,
2393            Vec::new(),
2394        )
2395        .unwrap();
2396
2397        router
2398            .route_for_connection(&module_ctx, goodbye)
2399            .await
2400            .unwrap();
2401        let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2402            .await
2403            .expect("close request should be sent for the full GOODBYE sink")
2404            .expect("close sender should include a reason");
2405        assert!(
2406            reason.to_string().contains("route_goodbye_delivery_failed"),
2407            "unexpected close reason: {reason}"
2408        );
2409        assert_eq!(client_rx.try_recv().unwrap().header.corr, 800);
2410        assert!(client_rx.try_recv().is_err());
2411        assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 1);
2412        assert_eq!(router.counters.snapshot()["route_released_epoch_fenced"], 1);
2413    }
2414
2415    /// One module and one client connection with `routes` routes bound between
2416    /// them. The client uses a real connection egress queue (the same byte
2417    /// budget and frame-count backstop as a live connection), and its route.open
2418    /// responses are drained so the queue starts empty.
2419    async fn multi_route_client(
2420        module_id: &str,
2421        module_connection: ConnectionId,
2422        client_connection: ConnectionId,
2423        routes: usize,
2424    ) -> (
2425        Router,
2426        FrameSink,
2427        mpsc::Receiver<OutboundFrame>,
2428        RouteCtx,
2429        Vec<crate::forwarding::PendingRouteBindRelay>,
2430        ConnectionCloseReceiver,
2431    ) {
2432        let forwarding = Arc::new(ForwardingTable::default());
2433        let control = Arc::new(ControlHandler::with_forwarding(
2434            Arc::new(crate::Registry::default()),
2435            Arc::clone(&forwarding),
2436        ));
2437        let router = Router::with_control_handler(control);
2438        let close_receiver = forwarding.register_connection_close(client_connection);
2439        let (module_tx, _module_rx) = mpsc::channel(8);
2440        forwarding
2441            .register_module_connection(
2442                module_connection,
2443                module_id.to_string(),
2444                1,
2445                Concurrency::ModuleManaged,
2446                FrameSink::new(module_tx),
2447            )
2448            .unwrap();
2449        let (client_sink, mut client_rx) = crate::server::connection_egress();
2450        let mut bound = Vec::with_capacity(routes);
2451        for index in 0..routes {
2452            let pending = forwarding
2453                .begin_route_bind_relay_for_test(
2454                    client_connection,
2455                    client_sink.clone(),
2456                    900 + index as u64,
2457                    module_id,
2458                )
2459                .unwrap();
2460            forwarding
2461                .complete_pending_relay(
2462                    module_connection,
2463                    pending.corr,
2464                    RouteBindRelayOutcome::Accepted,
2465                )
2466                .unwrap();
2467            assert_eq!(
2468                client_rx.recv().await.unwrap().header.corr,
2469                900 + index as u64
2470            );
2471            bound.push(pending);
2472        }
2473        let (module_egress_tx, _module_egress_rx) = mpsc::channel(8);
2474        let module_ctx = RouteCtx {
2475            connection_id: module_connection,
2476            egress: FrameSink::new(module_egress_tx),
2477        };
2478        (
2479            router,
2480            client_sink,
2481            client_rx,
2482            module_ctx,
2483            bound,
2484            close_receiver,
2485        )
2486    }
2487
2488    /// Per-frame cost of the egress sink's admission and release accounting:
2489    /// one million 200-byte frames enqueued with `try_send` and taken off the
2490    /// queue the way the connection writer does, in batches of 1,000 so the
2491    /// queue stays well inside its budget. Prints nanoseconds per frame; it is
2492    /// a measurement, not a gate. Run with
2493    /// `cargo test --release -p subc-daemon --lib egress_sink_per_frame_cost -- --ignored --nocapture`.
2494    #[test]
2495    #[ignore = "timing measurement, run on demand"]
2496    fn egress_sink_per_frame_cost() {
2497        const FRAMES: usize = 1_000_000;
2498        const BATCH: usize = 1_000;
2499        let (sink, mut rx) = crate::server::connection_egress();
2500        let template = stream_frame(9, 1, 0, vec![b't'; 200]);
2501        let started = Instant::now();
2502        for _ in 0..FRAMES / BATCH {
2503            for _ in 0..BATCH {
2504                sink.try_send(template.clone()).unwrap();
2505            }
2506            for _ in 0..BATCH {
2507                let outbound = rx.try_recv().unwrap();
2508                if let Some(charge) = &outbound.charge {
2509                    charge.taken_by_writer();
2510                }
2511                drop(std::hint::black_box(outbound));
2512            }
2513        }
2514        let elapsed = started.elapsed();
2515        assert_eq!(sink.backlog().queued_bytes, 0);
2516        println!(
2517            "egress sink: {FRAMES} frames in {elapsed:?}, {:.1} ns/frame",
2518            elapsed.as_nanos() as f64 / FRAMES as f64
2519        );
2520    }
2521
2522    fn stream_frame(channel: u16, epoch: u32, corr: u64, body: Vec<u8>) -> Frame {
2523        Frame::build(
2524            FrameType::StreamData,
2525            Flags::new(false, Priority::Interactive, false),
2526            channel,
2527            epoch,
2528            corr,
2529            body,
2530        )
2531        .unwrap()
2532    }
2533
2534    fn module_route_for_test(
2535        router: &Router,
2536        ctx: &RouteCtx,
2537        route: &crate::forwarding::PendingRouteBindRelay,
2538    ) -> Arc<RouteBinding> {
2539        match router
2540            .forwarding
2541            .lookup_data_route(ctx.connection_id, route.module_channel, route.module_epoch)
2542            .unwrap()
2543        {
2544            DataRoute::Module(DataRouteState::Bound(binding)) => binding,
2545            other => panic!("expected a bound module route, got {other:?}"),
2546        }
2547    }
2548
2549    fn assert_frame_size_error(frame: &Frame, channel: u16, epoch: u32, corr: u64, bytes: usize) {
2550        assert_eq!(frame.header.ty, FrameType::Error);
2551        assert_eq!(frame.header.channel, channel);
2552        assert_eq!(frame.header.epoch, epoch);
2553        assert_eq!(frame.header.corr, corr);
2554        assert!(frame.header.flags.is_daemon_origin());
2555        let body: ErrorBody = serde_json::from_slice(&frame.body).unwrap();
2556        assert_eq!(body.code, error_codes::FRAME_TOO_LARGE);
2557        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2558        assert!(body.message.contains(&bytes.to_string()));
2559        assert!(body.message.contains(&limit.to_string()));
2560        assert_eq!(
2561            body.detail,
2562            Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit }))
2563        );
2564    }
2565
2566    #[tokio::test]
2567    async fn module_egress_response_oversized_reply_keeps_connection_and_other_route() {
2568        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
2569        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2570            "oversize-provider",
2571            ConnectionId::new(80),
2572            ConnectionId::new(81),
2573            2,
2574        )
2575        .await;
2576        let binding = module_route_for_test(&router, &ctx, &routes[0]);
2577        binding.flow.acquire_tagged(101, false).await.unwrap();
2578        module_route_for_test(&router, &ctx, &routes[1])
2579            .flow
2580            .acquire_tagged(102, false)
2581            .await
2582            .unwrap();
2583        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2584        let oversized = Frame::build(
2585            FrameType::Response,
2586            Flags::new(false, Priority::Interactive, true),
2587            routes[0].module_channel,
2588            routes[0].module_epoch,
2589            101,
2590            vec![b'x'; limit - subc_protocol::HEADER_LEN + 1],
2591        )
2592        .unwrap();
2593        assert_eq!(sink.backlog().queued_bytes, 0);
2594        router.route_for_connection(&ctx, oversized).await.unwrap();
2595
2596        // Reproduce a large reply followed by ordinary traffic on another route
2597        // before the client reads anything. The large reply must not monopolize
2598        // the queue and make that second delivery close the shared connection.
2599        router
2600            .route_for_connection(
2601                &ctx,
2602                route_frame(
2603                    FrameType::Response,
2604                    routes[1].module_channel,
2605                    routes[1].module_epoch,
2606                    102,
2607                ),
2608            )
2609            .await
2610            .unwrap();
2611        assert!(
2612            close.try_recv().is_err(),
2613            "an oversized reply must not close the multiplexed client"
2614        );
2615        let error = rx.try_recv().expect("the waiting client gets an ERROR");
2616        assert_frame_size_error(
2617            &error,
2618            routes[0].client_channel,
2619            routes[0].client_epoch,
2620            101,
2621            limit + 1,
2622        );
2623        let second = rx.try_recv().expect("the other route's reply arrives");
2624        assert_eq!(second.header.ty, FrameType::Response);
2625        assert_eq!(second.header.channel, routes[1].client_channel);
2626        assert_eq!(second.header.epoch, routes[1].client_epoch);
2627        assert_eq!(second.header.corr, 102);
2628        assert_eq!(second.body, b"route-body");
2629        assert_eq!(binding.flow.drain_in_flight(), 0);
2630        for route in &routes {
2631            module_route_for_test(&router, &ctx, route);
2632        }
2633        assert!(rx.try_recv().is_err());
2634        drop((error, second));
2635        assert_eq!(sink.backlog().queued_bytes, 0);
2636        assert_eq!(
2637            router.counters.snapshot()["client_egress_close_delivery_failed"],
2638            0
2639        );
2640
2641        let captured = test_log::captured_logs(&logs);
2642        let warnings: Vec<_> = captured
2643            .lines()
2644            .filter(|line| line.contains("WARN"))
2645            .collect();
2646        assert_eq!(warnings.len(), 1, "{captured}");
2647        let warning = warnings[0];
2648        for expected in [
2649            "module_id=oversize-provider".to_string(),
2650            "client_principal=direct".to_string(),
2651            format!("client_channel={}", routes[0].client_channel),
2652            format!("frame_bytes={}", limit + 1),
2653            format!("byte_limit={limit}"),
2654        ] {
2655            assert!(warning.contains(&expected), "missing {expected}: {warning}");
2656        }
2657    }
2658
2659    async fn poll_once<F: std::future::Future>(
2660        mut future: std::pin::Pin<&mut F>,
2661    ) -> std::task::Poll<F::Output> {
2662        std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
2663    }
2664
2665    fn fill_client_budget(sink: &FrameSink, route: &crate::forwarding::PendingRouteBindRelay) {
2666        sink.try_send(
2667            Frame::build(
2668                FrameType::Response,
2669                Flags::new(false, Priority::Interactive, true),
2670                route.client_channel,
2671                route.client_epoch,
2672                100,
2673                vec![
2674                    b'x';
2675                    crate::server::CONNECTION_EGRESS_BYTE_BUDGET - subc_protocol::HEADER_LEN
2676                ],
2677            )
2678            .unwrap(),
2679        )
2680        .unwrap();
2681        assert_eq!(
2682            sink.backlog().queued_bytes,
2683            crate::server::CONNECTION_EGRESS_BYTE_BUDGET
2684        );
2685    }
2686
2687    async fn assert_full_client_burst_survives(ty: FrameType, oversized: bool) {
2688        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2689            "draining-burst",
2690            ConnectionId::new(92),
2691            ConnectionId::new(93),
2692            2,
2693        )
2694        .await;
2695        fill_client_budget(&sink, &routes[0]);
2696        let frame = if oversized {
2697            Frame::build(
2698                ty,
2699                Flags::new(false, Priority::Interactive, false),
2700                routes[1].module_channel,
2701                routes[1].module_epoch,
2702                101,
2703                vec![b's'; crate::server::CONNECTION_EGRESS_BYTE_BUDGET],
2704            )
2705            .unwrap()
2706        } else {
2707            route_frame(ty, routes[1].module_channel, routes[1].module_epoch, 101)
2708        };
2709        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2710        assert!(poll_once(delivery.as_mut()).await.is_pending());
2711        assert!(
2712            close.try_recv().is_err(),
2713            "fresh bursts must not close: {ty:?}"
2714        );
2715        tokio::time::advance(Duration::from_millis(50)).await;
2716        let queued = rx.try_recv().unwrap();
2717        queued.charge.as_ref().unwrap().taken_by_writer();
2718        drop(queued);
2719        assert!(matches!(
2720            poll_once(delivery.as_mut()).await,
2721            std::task::Poll::Ready(Ok(()))
2722        ));
2723        let delivered = rx
2724            .try_recv()
2725            .expect("the burst frame must be delivered after draining");
2726        assert_eq!(delivered.header.channel, routes[1].client_channel);
2727        assert_eq!(delivered.header.epoch, routes[1].client_epoch);
2728        assert_eq!(delivered.header.corr, 101);
2729        if oversized {
2730            assert_frame_size_error(
2731                &delivered,
2732                routes[1].client_channel,
2733                routes[1].client_epoch,
2734                101,
2735                crate::server::CONNECTION_EGRESS_BYTE_BUDGET + subc_protocol::HEADER_LEN,
2736            );
2737        } else {
2738            assert_eq!(delivered.header.ty, ty);
2739        }
2740        assert!(rx.try_recv().is_err());
2741        assert!(close.try_recv().is_err());
2742        assert_eq!(
2743            router.counters.snapshot()["client_egress_close_delivery_failed"],
2744            0
2745        );
2746        assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 0);
2747    }
2748
2749    #[tokio::test(start_paused = true)]
2750    async fn module_egress_fresh_full_queue_burst_survives() {
2751        for ty in [
2752            FrameType::Response,
2753            FrameType::StreamData,
2754            FrameType::Request,
2755            FrameType::Push,
2756        ] {
2757            assert_full_client_burst_survives(ty, false).await;
2758        }
2759    }
2760
2761    #[tokio::test(start_paused = true)]
2762    async fn module_goodbye_fresh_full_queue_burst_survives() {
2763        assert_full_client_burst_survives(FrameType::Goodbye, false).await;
2764    }
2765
2766    #[tokio::test(start_paused = true)]
2767    async fn frame_count_backstop_also_allows_a_draining_burst() {
2768        let (tx, mut rx) = mpsc::channel(1);
2769        let sink = FrameSink::with_byte_budget(tx, 2_000);
2770        sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
2771            .unwrap();
2772        let mut delivery = Box::pin(sink.send_to_client(stream_frame(1, 1, 2, vec![b'y'; 100])));
2773        assert!(poll_once(delivery.as_mut()).await.is_pending());
2774        assert!(
2775            sink.backlog().queued_bytes < 2_000,
2776            "only frame capacity is exhausted"
2777        );
2778        tokio::time::advance(Duration::from_millis(50)).await;
2779        drop(rx.try_recv().unwrap());
2780        assert!(matches!(
2781            poll_once(delivery.as_mut()).await,
2782            std::task::Poll::Ready(Ok(()))
2783        ));
2784        assert_eq!(rx.try_recv().unwrap().header.corr, 2);
2785    }
2786
2787    #[tokio::test(start_paused = true)]
2788    async fn oversized_reply_replacement_fresh_full_queue_burst_survives() {
2789        for ty in [FrameType::Response, FrameType::StreamData] {
2790            assert_full_client_burst_survives(ty, true).await;
2791        }
2792    }
2793
2794    #[tokio::test(start_paused = true)]
2795    async fn module_egress_stopped_reader_closes_without_waiting() {
2796        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
2797        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2798            "oldest-stopped-reader",
2799            ConnectionId::new(94),
2800            ConnectionId::new(95),
2801            2,
2802        )
2803        .await;
2804        fill_client_budget(&sink, &routes[0]);
2805        // The writer has taken the head but cannot finish writing it. Its
2806        // bytes stay charged, and the atomic age still diagnoses that stall.
2807        let writing = rx.try_recv().unwrap();
2808        writing.charge.as_ref().unwrap().taken_by_writer();
2809        tokio::time::advance(Duration::from_millis(251)).await;
2810        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(251)));
2811        assert_eq!(sink.backlog().queued_bytes, queued_frame_bytes(&writing));
2812        let started = tokio::time::Instant::now();
2813        let frame = stream_frame(
2814            routes[0].module_channel,
2815            routes[0].module_epoch,
2816            101,
2817            b"secret-body-never-log".to_vec(),
2818        );
2819        let flags = frame.header.flags.0;
2820        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2821        assert!(
2822            matches!(
2823                poll_once(delivery.as_mut()).await,
2824                std::task::Poll::Ready(Ok(()))
2825            ),
2826            "an oldest frame older than 250 ms must close immediately, without a new wait"
2827        );
2828        assert_eq!(started.elapsed(), Duration::ZERO);
2829        assert!(close
2830            .try_recv()
2831            .unwrap()
2832            .to_string()
2833            .contains("module_to_client_delivery_failed"));
2834        let captured = test_log::captured_logs(&logs);
2835        for expected in [
2836            "frame_type=Some(StreamData)".to_string(),
2837            format!("frame_flags=Some({flags})"),
2838            "oldest_queued_ms=251".to_string(),
2839            "close_cause=\"oldest_frame_older_than_250_ms\"".to_string(),
2840        ] {
2841            assert!(
2842                captured.contains(&expected),
2843                "missing {expected}: {captured}"
2844            );
2845        }
2846        assert!(!captured.contains("secret-body-never-log"));
2847    }
2848
2849    #[tokio::test(start_paused = true)]
2850    async fn client_gone_mid_wait_resumes_module_reader_immediately() {
2851        for writer_gone in [false, true] {
2852            let (router, sink, mut rx, ctx, routes, _close) = multi_route_client(
2853                "gone-during-burst",
2854                ConnectionId::new(96),
2855                ConnectionId::new(97),
2856                1,
2857            )
2858            .await;
2859            fill_client_budget(&sink, &routes[0]);
2860            let frame = route_frame(
2861                FrameType::Response,
2862                routes[0].module_channel,
2863                routes[0].module_epoch,
2864                101,
2865            );
2866            let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2867            assert!(poll_once(delivery.as_mut()).await.is_pending());
2868            tokio::time::advance(Duration::from_millis(50)).await;
2869            let started = tokio::time::Instant::now();
2870            if writer_gone {
2871                rx.close();
2872            } else {
2873                sink.close_client_delivery();
2874            }
2875            assert!(
2876                matches!(
2877                    poll_once(delivery.as_mut()).await,
2878                    std::task::Poll::Ready(Ok(()))
2879                ),
2880                "client teardown must wake the module reader before the writer drains"
2881            );
2882            assert_eq!(started.elapsed(), Duration::ZERO);
2883            assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
2884            assert_eq!(rx.try_recv().unwrap().header.corr, 100);
2885            assert!(rx.try_recv().is_err());
2886        }
2887    }
2888
2889    #[tokio::test(start_paused = true)]
2890    async fn cancelled_module_delivery_releases_waiter_without_closing_client() {
2891        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2892            "cancelled-burst",
2893            ConnectionId::new(98),
2894            ConnectionId::new(99),
2895            1,
2896        )
2897        .await;
2898        fill_client_budget(&sink, &routes[0]);
2899        let frame = route_frame(
2900            FrameType::Response,
2901            routes[0].module_channel,
2902            routes[0].module_epoch,
2903            101,
2904        );
2905        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2906        assert!(poll_once(delivery.as_mut()).await.is_pending());
2907        assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 1);
2908        drop(delivery);
2909        assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
2910        tokio::time::advance(Duration::from_secs(1)).await;
2911        assert!(close.try_recv().is_err());
2912        drop(rx.try_recv().unwrap());
2913        assert_eq!(sink.backlog().queued_bytes, 0);
2914        assert!(sink.backlog().oldest_age.is_none());
2915    }
2916
2917    #[tokio::test(start_paused = true)]
2918    async fn egress_atomic_age_follows_writer_and_clears_when_drained() {
2919        let (sink, mut rx) = crate::server::connection_egress();
2920        sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
2921            .unwrap();
2922        tokio::time::advance(Duration::from_millis(200)).await;
2923        sink.try_send(stream_frame(2, 1, 2, vec![b'y'; 100]))
2924            .unwrap();
2925        let writing = rx.try_recv().unwrap();
2926        writing.charge.as_ref().unwrap().taken_by_writer();
2927        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
2928        drop(writing);
2929        // Between frames the lock-free marker can still name the previous
2930        // head. The next writer take advances it without scanning the queue.
2931        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
2932        tokio::time::advance(Duration::from_millis(50)).await;
2933        let next = rx.try_recv().unwrap();
2934        next.charge.as_ref().unwrap().taken_by_writer();
2935        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(50)));
2936        assert!(
2937            sink.backlog().queued_bytes > 0,
2938            "in-flight bytes remain charged until written"
2939        );
2940        drop(next);
2941        assert_eq!(sink.backlog().queued_bytes, 0);
2942        assert_eq!(sink.backlog().oldest_age, None);
2943    }
2944
2945    #[tokio::test(start_paused = true)]
2946    async fn module_egress_response_full_client_waits_only_remaining_grace() {
2947        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2948            "backpressure-provider",
2949            ConnectionId::new(82),
2950            ConnectionId::new(83),
2951            2,
2952        )
2953        .await;
2954        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2955        // Exactly the limit, including its header, must be admitted to an empty
2956        // queue. Leave it unread to deterministically exhaust only byte capacity.
2957        router
2958            .route_for_connection(
2959                &ctx,
2960                Frame::build(
2961                    FrameType::Response,
2962                    Flags::new(false, Priority::Interactive, true),
2963                    routes[0].module_channel,
2964                    routes[0].module_epoch,
2965                    201,
2966                    vec![b'x'; limit - subc_protocol::HEADER_LEN],
2967                )
2968                .unwrap(),
2969            )
2970            .await
2971            .unwrap();
2972        assert_eq!(sink.backlog().queued_bytes, limit);
2973        assert_eq!(sink.backlog().queued_frames, 1);
2974        let ordinary = route_frame(
2975            FrameType::Response,
2976            routes[1].module_channel,
2977            routes[1].module_epoch,
2978            202,
2979        );
2980        assert!(queued_frame_bytes(&ordinary) < limit);
2981        // A fitting reply gets only the unused portion of the oldest frame's
2982        // 250 ms grace, not an immediate close or a fresh 250 ms wait.
2983        tokio::time::advance(Duration::from_millis(100)).await;
2984        let mut delivery = Box::pin(router.route_for_connection(&ctx, ordinary));
2985        assert!(poll_once(delivery.as_mut()).await.is_pending());
2986        assert!(close.try_recv().is_err());
2987        tokio::time::advance(Duration::from_millis(149)).await;
2988        assert!(poll_once(delivery.as_mut()).await.is_pending());
2989        assert!(close.try_recv().is_err());
2990        tokio::time::advance(Duration::from_millis(1)).await;
2991        assert!(matches!(
2992            poll_once(delivery.as_mut()).await,
2993            std::task::Poll::Ready(Ok(()))
2994        ));
2995        drop(delivery);
2996        let reason = close.try_recv().expect("the full client must close");
2997        assert!(reason
2998            .to_string()
2999            .contains("module_to_client_delivery_failed"));
3000        assert!(!reason.to_string().contains("frame_too_large"));
3001        assert_eq!(
3002            router.counters.snapshot()["client_egress_close_delivery_failed"],
3003            1
3004        );
3005        let admitted = rx.try_recv().expect("the budget-sized frame was admitted");
3006        assert_eq!(admitted.header.ty, FrameType::Response);
3007        assert_eq!(admitted.header.corr, 201);
3008        assert!(
3009            rx.try_recv().is_err(),
3010            "no oversized-frame ERROR is substituted for backpressure"
3011        );
3012    }
3013
3014    #[tokio::test(start_paused = true)]
3015    async fn oversized_reply_to_a_full_client_waits_only_remaining_grace() {
3016        let (router, sink, _rx, ctx, routes, mut close) = multi_route_client(
3017            "full-oversize-provider",
3018            ConnectionId::new(90),
3019            ConnectionId::new(91),
3020            2,
3021        )
3022        .await;
3023        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3024        // Fill the client's queue to its budget and leave it unread.
3025        router
3026            .route_for_connection(
3027                &ctx,
3028                Frame::build(
3029                    FrameType::Response,
3030                    Flags::new(false, Priority::Interactive, true),
3031                    routes[0].module_channel,
3032                    routes[0].module_epoch,
3033                    501,
3034                    vec![b'x'; limit - subc_protocol::HEADER_LEN],
3035                )
3036                .unwrap(),
3037            )
3038            .await
3039            .unwrap();
3040        assert_eq!(sink.backlog().queued_bytes, limit);
3041        let oversized = Frame::build(
3042            FrameType::Response,
3043            Flags::new(false, Priority::Interactive, true),
3044            routes[1].module_channel,
3045            routes[1].module_epoch,
3046            502,
3047            vec![b'y'; limit - subc_protocol::HEADER_LEN + 1],
3048        )
3049        .unwrap();
3050        // The replacement ERROR gets the same bounded burst grace as the
3051        // ordinary reply; oversized payloads themselves are never queued.
3052        tokio::time::advance(Duration::from_millis(80)).await;
3053        let mut delivery = Box::pin(router.route_for_connection(&ctx, oversized));
3054        assert!(poll_once(delivery.as_mut()).await.is_pending());
3055        assert!(close.try_recv().is_err());
3056        tokio::time::advance(Duration::from_millis(169)).await;
3057        assert!(poll_once(delivery.as_mut()).await.is_pending());
3058        assert!(close.try_recv().is_err());
3059        tokio::time::advance(Duration::from_millis(1)).await;
3060        assert!(matches!(
3061            poll_once(delivery.as_mut()).await,
3062            std::task::Poll::Ready(Ok(()))
3063        ));
3064        drop(delivery);
3065        let reason = close.try_recv().expect("the full client must close");
3066        assert!(reason
3067            .to_string()
3068            .contains("module_to_client_delivery_failed"));
3069    }
3070
3071    #[tokio::test]
3072    async fn oversized_reverse_request_errors_module_without_reaching_client() {
3073        let (router, _sink, mut rx, mut ctx, routes, mut close) = multi_route_client(
3074            "reverse-oversize-provider",
3075            ConnectionId::new(84),
3076            ConnectionId::new(85),
3077            2,
3078        )
3079        .await;
3080        let (module_tx, mut module_rx) = mpsc::channel(8);
3081        ctx.egress = FrameSink::new(module_tx);
3082        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3083        router
3084            .route_for_connection(
3085                &ctx,
3086                Frame::build(
3087                    FrameType::Request,
3088                    Flags::new(false, Priority::Interactive, false),
3089                    routes[0].module_channel,
3090                    routes[0].module_epoch,
3091                    301,
3092                    vec![b'r'; limit - subc_protocol::HEADER_LEN + 1],
3093                )
3094                .unwrap(),
3095            )
3096            .await
3097            .unwrap();
3098        assert!(
3099            rx.try_recv().is_err(),
3100            "the reverse request never reaches the client"
3101        );
3102        let error = module_rx.try_recv().expect("the module gets the refusal");
3103        assert_frame_size_error(
3104            &error,
3105            routes[0].module_channel,
3106            routes[0].module_epoch,
3107            301,
3108            limit + 1,
3109        );
3110        assert!(module_rx.try_recv().is_err());
3111        for (index, route) in routes.iter().enumerate() {
3112            module_route_for_test(&router, &ctx, route);
3113            router
3114                .route_for_connection(
3115                    &ctx,
3116                    route_frame(
3117                        FrameType::Response,
3118                        route.module_channel,
3119                        route.module_epoch,
3120                        302 + index as u64,
3121                    ),
3122                )
3123                .await
3124                .unwrap();
3125            let reply = rx.try_recv().expect("both routes still deliver");
3126            assert_eq!(reply.header.ty, FrameType::Response);
3127            assert_eq!(reply.header.channel, route.client_channel);
3128        }
3129        assert!(close.try_recv().is_err());
3130        assert_eq!(
3131            router.counters.snapshot()["client_egress_close_delivery_failed"],
3132            0
3133        );
3134    }
3135
3136    #[tokio::test]
3137    async fn oversized_push_closes_client_with_precise_warn() {
3138        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3139        let (router, _sink, mut rx, ctx, routes, mut close) = multi_route_client(
3140            "push-oversize-provider",
3141            ConnectionId::new(86),
3142            ConnectionId::new(87),
3143            2,
3144        )
3145        .await;
3146        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3147        router
3148            .route_for_connection(
3149                &ctx,
3150                Frame::build(
3151                    FrameType::Push,
3152                    Flags::new(false, Priority::Interactive, false),
3153                    routes[0].module_channel,
3154                    routes[0].module_epoch,
3155                    0,
3156                    vec![b'p'; limit - subc_protocol::HEADER_LEN + 1],
3157                )
3158                .unwrap(),
3159            )
3160            .await
3161            .unwrap();
3162        let reason = close
3163            .try_recv()
3164            .expect("unsolicited oversize still closes the client");
3165        assert!(reason
3166            .to_string()
3167            .contains("frame_too_large: oversized push"));
3168        assert!(
3169            rx.try_recv().is_err(),
3170            "no part of the oversized push is forwarded"
3171        );
3172        let captured = test_log::captured_logs(&logs);
3173        let warnings: Vec<_> = captured
3174            .lines()
3175            .filter(|line| line.contains("WARN"))
3176            .collect();
3177        assert_eq!(warnings.len(), 1, "{captured}");
3178        let warning = warnings[0];
3179        for expected in [
3180            "oversized push".to_string(),
3181            "module_id=\"push-oversize-provider\"".to_string(),
3182            "principals=direct".to_string(),
3183            format!("client_channel={}", routes[0].client_channel),
3184            format!("size={}", limit + 1),
3185            format!("limit={limit}"),
3186        ] {
3187            assert!(warning.contains(&expected), "missing {expected}: {warning}");
3188        }
3189        assert_eq!(
3190            router.counters.snapshot()["client_egress_close_delivery_failed"],
3191            1
3192        );
3193    }
3194
3195    #[tokio::test]
3196    async fn oversized_stream_and_error_frames_end_request_even_with_queued_traffic() {
3197        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3198            "stream-oversize-provider",
3199            ConnectionId::new(88),
3200            ConnectionId::new(89),
3201            2,
3202        )
3203        .await;
3204        let binding = module_route_for_test(&router, &ctx, &routes[0]);
3205        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3206        for ty in [
3207            FrameType::StreamData,
3208            FrameType::StreamEnd,
3209            FrameType::Error,
3210        ] {
3211            binding.flow.acquire_tagged(401, false).await.unwrap();
3212            sink.try_send(route_frame(
3213                FrameType::Response,
3214                routes[1].client_channel,
3215                routes[1].client_epoch,
3216                402,
3217            ))
3218            .unwrap();
3219            router
3220                .route_for_connection(
3221                    &ctx,
3222                    Frame::build(
3223                        ty,
3224                        Flags::new(false, Priority::Interactive, false),
3225                        routes[0].module_channel,
3226                        routes[0].module_epoch,
3227                        401,
3228                        vec![b's'; limit - subc_protocol::HEADER_LEN + 1],
3229                    )
3230                    .unwrap(),
3231                )
3232                .await
3233                .unwrap();
3234            assert!(close.try_recv().is_err());
3235            assert_eq!(rx.try_recv().unwrap().header.corr, 402);
3236            let error = rx
3237                .try_recv()
3238                .expect("every correlated oversize frame becomes ERROR");
3239            assert_frame_size_error(
3240                &error,
3241                routes[0].client_channel,
3242                routes[0].client_epoch,
3243                401,
3244                limit + 1,
3245            );
3246            assert_eq!(binding.flow.drain_in_flight(), 0, "{ty:?} ends the request");
3247            assert!(rx.try_recv().is_err());
3248        }
3249    }
3250
3251    /// An awaited send (a control or error reply) is never refused by the byte
3252    /// budget: it parks until the writer frees bytes. If a release could miss a
3253    /// parked sender, that reply would hang for the connection's lifetime, so
3254    /// this pins the wakeup: the send stays parked while the queue is full and
3255    /// completes as soon as one frame leaves it.
3256    #[tokio::test]
3257    async fn awaited_send_parked_behind_a_full_byte_budget_wakes_when_bytes_free() {
3258        let (tx, mut rx) = mpsc::channel(1024);
3259        let sink = FrameSink::with_byte_budget(tx, 2_000);
3260        let mut queued = 0u64;
3261        while sink
3262            .try_send(stream_frame(7, 1, queued, vec![b'x'; 200]))
3263            .is_ok()
3264        {
3265            queued += 1;
3266        }
3267        assert!(queued > 0, "the budget admitted nothing");
3268
3269        let parked = tokio::spawn({
3270            let sink = sink.clone();
3271            async move { sink.send(stream_frame(0, 0, 999, vec![b'r'; 200])).await }
3272        });
3273        tokio::time::sleep(Duration::from_millis(100)).await;
3274        assert!(
3275            !parked.is_finished(),
3276            "the awaited send must wait while the byte budget is full"
3277        );
3278
3279        // The writer takes one frame and drops it, releasing its bytes.
3280        drop(rx.recv().await.expect("a queued frame"));
3281        tokio::time::timeout(Duration::from_secs(2), parked)
3282            .await
3283            .expect("the parked send was never woken after bytes were freed")
3284            .unwrap()
3285            .unwrap();
3286    }
3287
3288    /// A client multiplexing several token streams pauses its reader while the
3289    /// module keeps producing small frames. Far more frames than the old
3290    /// 64-frame queue allowed, but far fewer bytes than the budget, must all be
3291    /// held without closing the connection and delivered in order afterwards.
3292    #[tokio::test]
3293    async fn paused_client_reader_keeps_connection_through_small_frame_burst() {
3294        const ROUTES: usize = 4;
3295        const FRAMES_PER_ROUTE: usize = 250;
3296        let (router, client_sink, mut client_rx, module_ctx, routes, mut close_receiver) =
3297            multi_route_client(
3298                "burst-provider",
3299                ConnectionId::new(60),
3300                ConnectionId::new(61),
3301                ROUTES,
3302            )
3303            .await;
3304
3305        // The reader is paused: nothing is received until every frame is sent.
3306        for seq in 0..FRAMES_PER_ROUTE as u64 {
3307            for (index, route) in routes.iter().enumerate() {
3308                let body = format!("route-{index}-token-{seq:05}-{}", "t".repeat(170));
3309                router
3310                    .route_for_connection(
3311                        &module_ctx,
3312                        stream_frame(
3313                            route.module_channel,
3314                            route.module_epoch,
3315                            seq,
3316                            body.into_bytes(),
3317                        ),
3318                    )
3319                    .await
3320                    .unwrap();
3321            }
3322        }
3323
3324        let backlog = client_sink.backlog();
3325        assert_eq!(backlog.queued_frames, ROUTES * FRAMES_PER_ROUTE);
3326        assert!(backlog.queued_bytes < crate::server::CONNECTION_EGRESS_BYTE_BUDGET);
3327        assert!(
3328            close_receiver.try_recv().is_err(),
3329            "a paused reader under the byte budget must not be closed"
3330        );
3331        assert_eq!(
3332            router.counters.snapshot()["client_egress_close_delivery_failed"],
3333            0
3334        );
3335
3336        // The reader resumes: every frame arrives, in order within each route.
3337        let mut next_seq = vec![0u64; ROUTES];
3338        for _ in 0..ROUTES * FRAMES_PER_ROUTE {
3339            let frame = client_rx.try_recv().expect("every queued frame arrives");
3340            let index = routes
3341                .iter()
3342                .position(|route| route.client_channel == frame.header.channel)
3343                .expect("frame arrives on one of the bound client channels");
3344            assert_eq!(frame.header.corr, next_seq[index], "route {index} order");
3345            let expected_prefix = format!("route-{index}-token-{:05}-", next_seq[index]);
3346            assert!(frame.body.starts_with(expected_prefix.as_bytes()));
3347            next_seq[index] += 1;
3348        }
3349        assert!(client_rx.try_recv().is_err());
3350        assert_eq!(next_seq, vec![FRAMES_PER_ROUTE as u64; ROUTES]);
3351        assert_eq!(client_sink.backlog().queued_bytes, 0);
3352    }
3353
3354    /// A client that never reads exhausts its byte budget and the oldest
3355    /// frame's 250 ms grace, then closes with one header-only diagnosis.
3356    #[tokio::test(start_paused = true)]
3357    async fn never_reading_client_is_closed_at_byte_budget_with_warn_diagnosis() {
3358        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3359        const BODY: usize = 16 * 1024;
3360        let (router, client_sink, _client_rx, module_ctx, routes, mut close_receiver) =
3361            multi_route_client(
3362                "stuck-reader-provider",
3363                ConnectionId::new(70),
3364                ConnectionId::new(71),
3365                2,
3366            )
3367            .await;
3368
3369        let mut admitted = 0usize;
3370        let mut sent = 0u64;
3371        let started = tokio::time::Instant::now();
3372        while router.counters.snapshot()["client_egress_close_delivery_failed"] == 0 {
3373            assert!(sent < 1_000, "the byte budget never refused a frame");
3374            let route = &routes[(sent % 2) as usize];
3375            router
3376                .route_for_connection(
3377                    &module_ctx,
3378                    stream_frame(
3379                        route.module_channel,
3380                        route.module_epoch,
3381                        sent,
3382                        vec![b'z'; BODY],
3383                    ),
3384                )
3385                .await
3386                .unwrap();
3387            sent += 1;
3388            admitted = client_sink.backlog().queued_frames;
3389        }
3390        assert_eq!(
3391            started.elapsed(),
3392            crate::server::CONNECTION_EGRESS_STALL_GRACE
3393        );
3394        // The budget, not the frame-count backstop, did the refusing.
3395        let frame_bytes = subc_protocol::HEADER_LEN + BODY;
3396        assert_eq!(
3397            admitted,
3398            crate::server::CONNECTION_EGRESS_BYTE_BUDGET / frame_bytes
3399        );
3400        let reason = close_receiver
3401            .try_recv()
3402            .expect("the client connection must be asked to close");
3403        assert!(reason
3404            .to_string()
3405            .contains("module_to_client_delivery_failed"));
3406
3407        // A second refused frame for the same connection adds no second WARN.
3408        router
3409            .route_for_connection(
3410                &module_ctx,
3411                stream_frame(
3412                    routes[0].module_channel,
3413                    routes[0].module_epoch,
3414                    sent,
3415                    vec![b'z'; BODY],
3416                ),
3417            )
3418            .await
3419            .unwrap();
3420
3421        let captured = test_log::captured_logs(&logs);
3422        let warn_lines = captured
3423            .lines()
3424            .filter(|line| {
3425                line.contains("closing client connection: its egress queue could not take a frame")
3426            })
3427            .collect::<Vec<_>>();
3428        assert_eq!(warn_lines.len(), 1, "exactly one WARN, got: {captured}");
3429        let line = warn_lines[0];
3430        assert!(line.contains("WARN"), "{line}");
3431        assert!(line.contains("connection_id=71"), "{line}");
3432        assert!(
3433            line.contains("module_id=\"stuck-reader-provider\""),
3434            "{line}"
3435        );
3436        assert!(line.contains("client_channel="), "{line}");
3437        assert!(line.contains("principals=direct"), "{line}");
3438        let queued_bytes: usize = line
3439            .split("queued_bytes=")
3440            .nth(1)
3441            .and_then(|rest| rest.split_whitespace().next())
3442            .and_then(|value| value.parse().ok())
3443            .expect("queued_bytes is logged");
3444        assert_eq!(queued_bytes, admitted * frame_bytes);
3445        assert!(
3446            line.contains(&format!("queued_frames={admitted}")),
3447            "{line}"
3448        );
3449        assert!(line.contains("oldest_queued_ms="), "{line}");
3450        assert!(line.contains("oldest_queued_ms=250"), "{line}");
3451        assert!(line.contains("frame_type=Some(StreamData)"), "{line}");
3452        assert!(
3453            line.contains(&format!(
3454                "frame_flags=Some({})",
3455                Flags::new(false, Priority::Interactive, false).0
3456            )),
3457            "{line}"
3458        );
3459        assert!(
3460            line.contains("close_cause=\"waited_250_ms_without_room\""),
3461            "{line}"
3462        );
3463        assert!(
3464            !line.contains(&"z".repeat(20)),
3465            "body bytes must never be logged: {line}"
3466        );
3467    }
3468
3469    fn route_frame(ty: FrameType, channel: u16, epoch: u32, corr: u64) -> Frame {
3470        Frame::build(
3471            ty,
3472            Flags::new(false, Priority::Interactive, false),
3473            channel,
3474            epoch,
3475            corr,
3476            if ty == FrameType::Request || ty == FrameType::Response {
3477                b"route-body".to_vec()
3478            } else {
3479                Vec::new()
3480            },
3481        )
3482        .unwrap()
3483    }
3484
3485    type DynamicRouteFixture = (
3486        Router,
3487        Arc<ForwardingTable>,
3488        RouteCtx,
3489        mpsc::Receiver<crate::router::OutboundFrame>,
3490        RouteCtx,
3491        mpsc::Receiver<crate::router::OutboundFrame>,
3492        mpsc::Receiver<crate::router::OutboundFrame>,
3493        crate::forwarding::PendingRouteBindRelay,
3494    );
3495
3496    fn dynamic_route_fixture(commit: bool) -> DynamicRouteFixture {
3497        let forwarding = Arc::new(ForwardingTable::default());
3498        let control = Arc::new(crate::ControlHandler::with_forwarding(
3499            Arc::new(crate::Registry::default()),
3500            Arc::clone(&forwarding),
3501        ));
3502        let router = Router::with_control_handler(control);
3503        let module_connection = ConnectionId::new(500);
3504        let client_connection = ConnectionId::new(501);
3505        let (module_tx, module_rx) = mpsc::channel(8);
3506        forwarding
3507            .register_module_connection(
3508                module_connection,
3509                "epoch-router".into(),
3510                2,
3511                Concurrency::ModuleManaged,
3512                FrameSink::new(module_tx),
3513            )
3514            .unwrap();
3515        let (client_tx, client_rx) = mpsc::channel(8);
3516        let client_sink = FrameSink::new(client_tx);
3517        let pending = forwarding
3518            .begin_route_bind_relay_for_test(
3519                client_connection,
3520                client_sink.clone(),
3521                700,
3522                "epoch-router",
3523            )
3524            .unwrap();
3525        if commit {
3526            forwarding
3527                .complete_pending_relay(
3528                    module_connection,
3529                    pending.corr,
3530                    RouteBindRelayOutcome::Accepted,
3531                )
3532                .unwrap();
3533        }
3534        let (module_egress_tx, module_egress_rx) = mpsc::channel(8);
3535        (
3536            router,
3537            forwarding,
3538            RouteCtx {
3539                connection_id: client_connection,
3540                egress: client_sink,
3541            },
3542            client_rx,
3543            RouteCtx {
3544                connection_id: module_connection,
3545                egress: FrameSink::new(module_egress_tx),
3546            },
3547            module_egress_rx,
3548            module_rx,
3549            pending,
3550        )
3551    }
3552
3553    #[tokio::test]
3554    async fn route_epochs_validate_both_directions_and_rewrite_to_peer_handle() {
3555        let (
3556            router,
3557            _forwarding,
3558            client_ctx,
3559            mut client_rx,
3560            module_ctx,
3561            _module_egress_rx,
3562            mut module_rx,
3563            pending,
3564        ) = dynamic_route_fixture(true);
3565        let route_open = client_rx.recv().await.unwrap();
3566        assert_eq!(route_open.header.corr, 700);
3567
3568        router
3569            .route_for_connection(
3570                &client_ctx,
3571                route_frame(
3572                    FrameType::Request,
3573                    pending.client_channel,
3574                    pending.client_epoch,
3575                    701,
3576                ),
3577            )
3578            .await
3579            .unwrap();
3580        let forwarded = module_rx.recv().await.unwrap();
3581        assert_eq!(forwarded.header.channel, pending.module_channel);
3582        assert_eq!(forwarded.header.epoch, pending.module_epoch);
3583
3584        router
3585            .route_for_connection(
3586                &module_ctx,
3587                route_frame(
3588                    FrameType::Response,
3589                    pending.module_channel,
3590                    pending.module_epoch,
3591                    701,
3592                ),
3593            )
3594            .await
3595            .unwrap();
3596        let delivered = client_rx.recv().await.unwrap();
3597        assert_eq!(delivered.header.channel, pending.client_channel);
3598        assert_eq!(delivered.header.epoch, pending.client_epoch);
3599
3600        router
3601            .route_for_connection(
3602                &client_ctx,
3603                route_frame(
3604                    FrameType::Request,
3605                    pending.client_channel,
3606                    pending.client_epoch + 1,
3607                    702,
3608                ),
3609            )
3610            .await
3611            .unwrap();
3612        router
3613            .route_for_connection(
3614                &module_ctx,
3615                route_frame(
3616                    FrameType::Response,
3617                    pending.module_channel,
3618                    pending.module_epoch + 1,
3619                    703,
3620                ),
3621            )
3622            .await
3623            .unwrap();
3624        let stale_error = client_rx.recv().await.unwrap();
3625        assert_eq!(stale_error.header.ty, FrameType::Error);
3626        assert_eq!(stale_error.header.channel, pending.client_channel);
3627        assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
3628        assert_eq!(stale_error.header.corr, 702);
3629        let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
3630        assert_eq!(body.code, "stale_route_epoch");
3631        assert!(module_rx.try_recv().is_err());
3632        assert!(client_rx.try_recv().is_err());
3633        let counters = router.counters.snapshot();
3634        assert_eq!(counters["client_frames_dropped_stale_route"], 1);
3635        assert_eq!(counters["module_frames_dropped_no_route"], 1);
3636    }
3637
3638    #[tokio::test]
3639    async fn accepted_route_publishes_route_open_before_immediate_reverse_request() {
3640        let (
3641            router,
3642            _,
3643            _client_ctx,
3644            mut client_rx,
3645            module_ctx,
3646            _module_egress_rx,
3647            _module_rx,
3648            pending,
3649        ) = dynamic_route_fixture(true);
3650        router
3651            .route_for_connection(
3652                &module_ctx,
3653                route_frame(
3654                    FrameType::Request,
3655                    pending.module_channel,
3656                    pending.module_epoch,
3657                    800,
3658                ),
3659            )
3660            .await
3661            .unwrap();
3662
3663        let first = client_rx.recv().await.unwrap();
3664        let second = client_rx.recv().await.unwrap();
3665        assert_eq!(first.header.channel, 0);
3666        assert_eq!(first.header.corr, 700);
3667        assert_eq!(second.header.channel, pending.client_channel);
3668        assert_eq!(second.header.epoch, pending.client_epoch);
3669        assert_eq!(second.header.corr, 800);
3670    }
3671
3672    #[tokio::test]
3673    async fn reserved_slot_ingress_errors_only_matching_client_requests() {
3674        let (
3675            router,
3676            _forwarding,
3677            client_ctx,
3678            mut client_rx,
3679            _module_ctx,
3680            _module_egress_rx,
3681            mut module_rx,
3682            pending,
3683        ) = dynamic_route_fixture(false);
3684        router
3685            .route_for_connection(
3686                &client_ctx,
3687                route_frame(
3688                    FrameType::Request,
3689                    pending.client_channel,
3690                    pending.client_epoch,
3691                    900,
3692                ),
3693            )
3694            .await
3695            .unwrap();
3696        let error = client_rx.recv().await.unwrap();
3697        assert_eq!(error.header.ty, FrameType::Error);
3698        assert_eq!(error.header.channel, pending.client_channel);
3699        assert_eq!(error.header.epoch, pending.client_epoch);
3700        assert_eq!(error.header.corr, 900);
3701
3702        router
3703            .route_for_connection(
3704                &client_ctx,
3705                route_frame(
3706                    FrameType::Response,
3707                    pending.client_channel,
3708                    pending.client_epoch,
3709                    901,
3710                ),
3711            )
3712            .await
3713            .unwrap();
3714        router
3715            .route_for_connection(
3716                &client_ctx,
3717                route_frame(
3718                    FrameType::Request,
3719                    pending.client_channel,
3720                    pending.client_epoch + 1,
3721                    902,
3722                ),
3723            )
3724            .await
3725            .unwrap();
3726        let stale_error = client_rx.recv().await.unwrap();
3727        assert_eq!(stale_error.header.ty, FrameType::Error);
3728        assert_eq!(stale_error.header.channel, pending.client_channel);
3729        assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
3730        assert_eq!(stale_error.header.corr, 902);
3731        let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
3732        assert_eq!(body.code, "stale_route_epoch");
3733        assert!(module_rx.try_recv().is_err());
3734        let counters = router.counters.snapshot();
3735        assert_eq!(counters["client_frames_dropped_stale_route"], 1);
3736        assert_eq!(counters["module_frames_dropped_no_route"], 0);
3737    }
3738
3739    #[tokio::test]
3740    async fn dropped_module_route_goodbye_increments_counter() {
3741        let (
3742            router,
3743            _forwarding,
3744            client_ctx,
3745            mut client_rx,
3746            _module_ctx,
3747            _module_egress_rx,
3748            mut module_rx,
3749            pending,
3750        ) = dynamic_route_fixture(true);
3751        let _ = client_rx.recv().await;
3752        module_rx.close();
3753
3754        router
3755            .route_for_connection(
3756                &client_ctx,
3757                route_frame(
3758                    FrameType::Goodbye,
3759                    pending.client_channel,
3760                    pending.client_epoch,
3761                    999,
3762                ),
3763            )
3764            .await
3765            .unwrap();
3766
3767        let counters = router.counters.snapshot();
3768        assert_eq!(counters["goodbye_relay_module_dropped"], 1);
3769        assert_eq!(
3770            counters["goodbye_relay_module_dropped_by_module"],
3771            serde_json::json!({ "epoch-router": 1 })
3772        );
3773        assert_eq!(counters["route_released_epoch_fenced"], 1);
3774    }
3775
3776    #[tokio::test]
3777    async fn module_request_on_stale_epoch_receives_stale_route_epoch() {
3778        let (
3779            router,
3780            _forwarding,
3781            _client_ctx,
3782            _client_rx,
3783            module_ctx,
3784            mut module_egress_rx,
3785            mut module_rx,
3786            pending,
3787        ) = dynamic_route_fixture(true);
3788
3789        router
3790            .route_for_connection(
3791                &module_ctx,
3792                route_frame(
3793                    FrameType::Request,
3794                    pending.module_channel,
3795                    pending.module_epoch + 1,
3796                    1_000,
3797                ),
3798            )
3799            .await
3800            .unwrap();
3801
3802        let error = module_egress_rx.try_recv().unwrap();
3803        assert_eq!(error.header.ty, FrameType::Error);
3804        assert_eq!(error.header.channel, pending.module_channel);
3805        assert_eq!(error.header.epoch, pending.module_epoch + 1);
3806        assert_eq!(error.header.corr, 1_000);
3807        let body: ErrorBody = serde_json::from_slice(&error.body).unwrap();
3808        assert_eq!(body.code, "stale_route_epoch");
3809        assert!(module_rx.try_recv().is_err());
3810        let counters = router.counters.snapshot();
3811        assert_eq!(counters["module_requests_dropped_stale_route"], 1);
3812        assert_eq!(counters["module_frames_dropped_no_route"], 0);
3813    }
3814
3815    #[tokio::test]
3816    async fn module_request_on_reserved_or_absent_route_receives_unknown_channel() {
3817        let (
3818            reserved_router,
3819            _forwarding,
3820            _client_ctx,
3821            _client_rx,
3822            reserved_module_ctx,
3823            mut reserved_module_egress_rx,
3824            _module_rx,
3825            reserved,
3826        ) = dynamic_route_fixture(false);
3827        reserved_router
3828            .route_for_connection(
3829                &reserved_module_ctx,
3830                route_frame(
3831                    FrameType::Request,
3832                    reserved.module_channel,
3833                    reserved.module_epoch,
3834                    1_001,
3835                ),
3836            )
3837            .await
3838            .unwrap();
3839        let reserved_error = reserved_module_egress_rx.try_recv().unwrap();
3840        let reserved_body: ErrorBody = serde_json::from_slice(&reserved_error.body).unwrap();
3841        assert_eq!(reserved_error.header.ty, FrameType::Error);
3842        assert_eq!(reserved_error.header.channel, reserved.module_channel);
3843        assert_eq!(reserved_error.header.epoch, reserved.module_epoch);
3844        assert_eq!(reserved_error.header.corr, 1_001);
3845        assert_eq!(reserved_body.code, "unknown_channel");
3846        assert_eq!(
3847            reserved_router.counters.snapshot()["module_requests_dropped_stale_route"],
3848            1
3849        );
3850
3851        let (
3852            absent_router,
3853            _forwarding,
3854            _client_ctx,
3855            _client_rx,
3856            absent_module_ctx,
3857            mut absent_module_egress_rx,
3858            _module_rx,
3859            absent,
3860        ) = dynamic_route_fixture(false);
3861        absent_router
3862            .route_for_connection(
3863                &absent_module_ctx,
3864                route_frame(
3865                    FrameType::Request,
3866                    absent.module_channel + 1,
3867                    absent.module_epoch,
3868                    1_002,
3869                ),
3870            )
3871            .await
3872            .unwrap();
3873        let absent_error = absent_module_egress_rx.try_recv().unwrap();
3874        let absent_body: ErrorBody = serde_json::from_slice(&absent_error.body).unwrap();
3875        assert_eq!(absent_error.header.ty, FrameType::Error);
3876        assert_eq!(absent_error.header.channel, absent.module_channel + 1);
3877        assert_eq!(absent_error.header.epoch, absent.module_epoch);
3878        assert_eq!(absent_error.header.corr, 1_002);
3879        assert_eq!(absent_body.code, "unknown_channel");
3880        assert_eq!(
3881            absent_router.counters.snapshot()["module_requests_dropped_stale_route"],
3882            1
3883        );
3884    }
3885
3886    #[tokio::test]
3887    async fn non_request_module_frame_on_dead_route_is_counted_without_error() {
3888        let (
3889            router,
3890            forwarding,
3891            client_ctx,
3892            mut client_rx,
3893            module_ctx,
3894            mut module_egress_rx,
3895            mut module_rx,
3896            pending,
3897        ) = dynamic_route_fixture(true);
3898        let (other_module_tx, _other_module_rx) = mpsc::channel(8);
3899        forwarding
3900            .register_module_connection(
3901                ConnectionId::new(502),
3902                "other-module".into(),
3903                2,
3904                Concurrency::ModuleManaged,
3905                FrameSink::new(other_module_tx),
3906            )
3907            .unwrap();
3908        let _ = client_rx.recv().await.unwrap();
3909
3910        router
3911            .route_for_connection(
3912                &client_ctx,
3913                route_frame(
3914                    FrameType::Goodbye,
3915                    pending.client_channel,
3916                    pending.client_epoch,
3917                    1_003,
3918                ),
3919            )
3920            .await
3921            .unwrap();
3922        let _ = module_rx.recv().await.unwrap();
3923
3924        router
3925            .route_for_connection(
3926                &module_ctx,
3927                route_frame(
3928                    FrameType::StreamData,
3929                    pending.module_channel,
3930                    pending.module_epoch,
3931                    1_004,
3932                ),
3933            )
3934            .await
3935            .unwrap();
3936
3937        // No ERROR goes back for a non-request frame. The one reply is the
3938        // route GOODBYE telling the module to let go of the released route.
3939        let reply = module_egress_rx.try_recv().unwrap();
3940        assert_eq!(reply.header.ty, FrameType::Goodbye);
3941        assert!(module_egress_rx.try_recv().is_err());
3942        let counters = router.counters.snapshot();
3943        assert_eq!(counters["module_frames_dropped_no_route"], 1);
3944        assert_eq!(
3945            counters["module_frames_dropped_no_route_by_module"],
3946            serde_json::json!({ "epoch-router": 1 })
3947        );
3948        assert_eq!(counters["module_requests_dropped_stale_route"], 0);
3949    }
3950
3951    /// Drain whatever the module connection's queue holds right now, the way a
3952    /// module's reader would before it stalls.
3953    fn drain_now(rx: &mut mpsc::Receiver<OutboundFrame>) {
3954        while rx.try_recv().is_ok() {}
3955    }
3956
3957    /// A module that stops reading for a moment when a client closes one of
3958    /// its routes still learns the route is gone: the GOODBYE its full egress
3959    /// queue refused is delivered as soon as it reads again, rather than
3960    /// dropped, which would leave the module holding the route for the rest of
3961    /// its connection.
3962    #[tokio::test]
3963    async fn route_goodbye_refused_by_stalled_module_is_delivered_when_it_resumes_reading() {
3964        const BUDGET: usize = 4_096;
3965        let forwarding = Arc::new(ForwardingTable::default());
3966        let control = Arc::new(ControlHandler::with_forwarding(
3967            Arc::new(Registry::default()),
3968            Arc::clone(&forwarding),
3969        ));
3970        let router = Router::with_control_handler(control);
3971        let module_connection = ConnectionId::new(80);
3972        let client_connection = ConnectionId::new(81);
3973        let (module_tx, mut module_rx) = mpsc::channel(64);
3974        let module_sink = FrameSink::with_byte_budget(module_tx, BUDGET);
3975        forwarding
3976            .register_module_connection(
3977                module_connection,
3978                "stalled-provider".into(),
3979                2,
3980                Concurrency::ModuleManaged,
3981                module_sink.clone(),
3982            )
3983            .unwrap();
3984        let (client_tx, mut client_rx) = mpsc::channel(8);
3985        let client_sink = FrameSink::new(client_tx);
3986        let pending = forwarding
3987            .begin_route_bind_relay_for_test(
3988                client_connection,
3989                client_sink.clone(),
3990                1_100,
3991                "stalled-provider",
3992            )
3993            .unwrap();
3994        forwarding
3995            .complete_pending_relay(
3996                module_connection,
3997                pending.corr,
3998                RouteBindRelayOutcome::Accepted,
3999            )
4000            .unwrap();
4001        let _ = client_rx.recv().await.unwrap();
4002        drain_now(&mut module_rx);
4003
4004        // The module stalls: its queue holds more than the byte budget, so the
4005        // queue refuses anything further.
4006        module_sink
4007            .try_send(stream_frame(9, 1, 0, vec![b'f'; BUDGET]))
4008            .unwrap();
4009        assert!(module_sink
4010            .try_send(stream_frame(9, 1, 1, Vec::new()))
4011            .is_err());
4012
4013        let client_ctx = RouteCtx {
4014            connection_id: client_connection,
4015            egress: client_sink,
4016        };
4017        router
4018            .route_for_connection(
4019                &client_ctx,
4020                route_frame(
4021                    FrameType::Goodbye,
4022                    pending.client_channel,
4023                    pending.client_epoch,
4024                    1_101,
4025                ),
4026            )
4027            .await
4028            .unwrap();
4029        tokio::task::yield_now().await;
4030        assert_eq!(
4031            router.counters.snapshot()["goodbye_relay_module_dropped"],
4032            0,
4033            "a GOODBYE refused by a momentarily full module queue must not be dropped"
4034        );
4035
4036        // The module reads again: the filler comes off, then the GOODBYE.
4037        let filler = module_rx.recv().await.unwrap();
4038        assert_eq!(filler.header.ty, FrameType::StreamData);
4039        drop(filler);
4040        let goodbye = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
4041            .await
4042            .expect("the refused GOODBYE must be delivered once the module frees room")
4043            .unwrap();
4044        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
4045        assert_eq!(goodbye.header.channel, pending.module_channel);
4046        assert_eq!(goodbye.header.epoch, pending.module_epoch);
4047        assert_eq!(
4048            router.counters.snapshot()["goodbye_relay_module_dropped"],
4049            0
4050        );
4051    }
4052
4053    /// A module still sending on a route the daemon released is told, with a
4054    /// route GOODBYE for exactly the (channel, epoch) it sent on. The same
4055    /// happens for a stale epoch on a channel whose route moved on and for a
4056    /// channel that never had a route; only the first is counted as traffic on
4057    /// a released route.
4058    #[tokio::test]
4059    async fn module_frame_on_route_the_daemon_does_not_hold_is_answered_with_goodbye() {
4060        let (
4061            router,
4062            _forwarding,
4063            client_ctx,
4064            mut client_rx,
4065            module_ctx,
4066            mut module_egress_rx,
4067            mut module_rx,
4068            pending,
4069        ) = dynamic_route_fixture(true);
4070        let _ = client_rx.recv().await.unwrap();
4071        router
4072            .route_for_connection(
4073                &client_ctx,
4074                route_frame(
4075                    FrameType::Goodbye,
4076                    pending.client_channel,
4077                    pending.client_epoch,
4078                    1_200,
4079                ),
4080            )
4081            .await
4082            .unwrap();
4083        drain_now(&mut module_rx);
4084
4085        let cases = [
4086            // Released route: the daemon allocated this (channel, epoch).
4087            (pending.module_channel, pending.module_epoch),
4088            // A channel the daemon never allocated on this connection.
4089            (pending.module_channel + 1, 1),
4090        ];
4091        for (channel, epoch) in cases {
4092            router
4093                .route_for_connection(
4094                    &module_ctx,
4095                    route_frame(FrameType::StreamData, channel, epoch, 1_201),
4096                )
4097                .await
4098                .unwrap();
4099            let reply = module_egress_rx
4100                .try_recv()
4101                .expect("a frame on a route the daemon does not hold is answered");
4102            assert_eq!(reply.header.ty, FrameType::Goodbye);
4103            assert_eq!(reply.header.channel, channel);
4104            assert_eq!(reply.header.epoch, epoch);
4105            assert_eq!(reply.header.corr, 0);
4106            assert!(module_egress_rx.try_recv().is_err());
4107        }
4108
4109        // A GOODBYE from the module for a route the daemon already released
4110        // is not answered: the module is letting go already.
4111        router
4112            .route_for_connection(
4113                &module_ctx,
4114                route_frame(FrameType::Goodbye, pending.module_channel + 2, 1, 0),
4115            )
4116            .await
4117            .unwrap();
4118        assert!(module_egress_rx.try_recv().is_err());
4119
4120        let counters = router.counters.snapshot();
4121        assert_eq!(counters["module_frames_dropped_no_route"], 3);
4122        assert_eq!(counters["module_frames_dropped_released_route"], 1);
4123        assert_eq!(
4124            counters["module_frames_dropped_released_route_by_module"],
4125            serde_json::json!({ "epoch-router": 1 })
4126        );
4127        assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4128    }
4129
4130    /// A chatty orphan (a streaming module still producing on a route it
4131    /// missed the GOODBYE for) is answered once per interval, not once per
4132    /// frame, and answered again once the interval has passed.
4133    #[tokio::test(start_paused = true)]
4134    async fn burst_of_orphan_module_frames_is_answered_once_per_interval() {
4135        let (
4136            router,
4137            _forwarding,
4138            client_ctx,
4139            mut client_rx,
4140            module_ctx,
4141            mut module_egress_rx,
4142            mut module_rx,
4143            pending,
4144        ) = dynamic_route_fixture(true);
4145        let _ = client_rx.recv().await.unwrap();
4146        router
4147            .route_for_connection(
4148                &client_ctx,
4149                route_frame(
4150                    FrameType::Goodbye,
4151                    pending.client_channel,
4152                    pending.client_epoch,
4153                    1_300,
4154                ),
4155            )
4156            .await
4157            .unwrap();
4158        drain_now(&mut module_rx);
4159
4160        let send_burst = |corr: u64| {
4161            route_frame(
4162                FrameType::StreamData,
4163                pending.module_channel,
4164                pending.module_epoch,
4165                corr,
4166            )
4167        };
4168        for corr in 0..20 {
4169            router
4170                .route_for_connection(&module_ctx, send_burst(corr))
4171                .await
4172                .unwrap();
4173        }
4174        let mut replies = 0;
4175        while let Ok(reply) = module_egress_rx.try_recv() {
4176            assert_eq!(reply.header.ty, FrameType::Goodbye);
4177            replies += 1;
4178        }
4179        assert_eq!(replies, 1, "a burst within the interval gets one GOODBYE");
4180
4181        tokio::time::advance(ORPHAN_ROUTE_GOODBYE_INTERVAL).await;
4182        router
4183            .route_for_connection(&module_ctx, send_burst(20))
4184            .await
4185            .unwrap();
4186        let retry = module_egress_rx
4187            .try_recv()
4188            .expect("the first orphan frame after the interval is answered again");
4189        assert_eq!(retry.header.channel, pending.module_channel);
4190        assert_eq!(retry.header.epoch, pending.module_epoch);
4191
4192        let counters = router.counters.snapshot();
4193        assert_eq!(counters["module_frames_dropped_no_route"], 21);
4194        assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4195    }
4196
4197    /// The rate-limit memory is per connection and goes away with it.
4198    #[test]
4199    fn orphan_goodbye_rate_limit_state_is_released_with_the_connection() {
4200        let router = Router::with_default_self_handler();
4201        let connection = router.begin_connection();
4202        let id = connection.id();
4203        assert!(router.orphan_goodbyes.claim(id, 7));
4204        assert!(!router.orphan_goodbyes.claim(id, 7));
4205        assert!(router.orphan_goodbyes.claim(id, 8));
4206        drop(connection);
4207        assert!(router
4208            .orphan_goodbyes
4209            .last_sent
4210            .lock()
4211            .unwrap()
4212            .get(&id)
4213            .is_none());
4214    }
4215
4216    #[test]
4217    fn channel_zero_cannot_be_registered_as_backend() {
4218        let mut router = Router::with_default_self_handler();
4219
4220        let err = router.register_backend(0, EchoBackend).unwrap_err();
4221
4222        assert_eq!(err, RouterError::ReservedChannelZero);
4223    }
4224}