Skip to main content

subc_daemon/
router.rs

1use std::{
2    collections::HashMap,
3    error::Error,
4    fmt,
5    sync::{
6        atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
7        Arc, Mutex,
8    },
9    time::{Duration, Instant},
10};
11
12use subc_protocol::{error_codes, ErrorBody, Flags, FrameType, Priority};
13use tokio::sync::{mpsc, Notify};
14use tracing::{debug, warn};
15
16use crate::{
17    control::ControlHandler,
18    forwarding::{
19        principal_label, CloseReason, ConnectionCloseReceiver, DataRoute, DataRouteState,
20        ForwardingError, ForwardingTable, RouteBinding, RouteRelease, UndeliveredFrame,
21    },
22    registry::ConnectionId,
23    DaemonCounters, Frame, FrameBuildError,
24};
25
26/// One queued outbound frame plus the instant it entered the writer queue.
27///
28/// The stamp exists for the reply-path half of slow-control diagnosis: a
29/// handler can finish in microseconds while the reply sits in this queue
30/// waiting for the writer task to be scheduled, and without a per-item stamp
31/// that wait is invisible to every other timing point (the client's round
32/// trip is the only witness, and it cannot say which side ate the time).
33/// Constructed exclusively inside [`FrameSink`] so no caller can forget it.
34#[derive(Debug)]
35pub struct OutboundFrame {
36    pub frame: Frame,
37    pub enqueued_at: std::time::Instant,
38    pub(crate) flushed: Option<tokio::sync::oneshot::Sender<()>>,
39    /// This frame's share of the connection's queued-byte count. Dropping the
40    /// frame (after the writer has written it, or when the queue itself is
41    /// dropped with the frame still in it) gives the bytes back. `None` only for
42    /// frames built outside a [`FrameSink`], which exist in tests alone.
43    pub(crate) charge: Option<EgressCharge>,
44}
45
46impl OutboundFrame {
47    fn charged(frame: Frame, charge: EgressCharge) -> Self {
48        Self {
49            frame,
50            enqueued_at: charge.enqueued_at,
51            flushed: None,
52            charge: Some(charge),
53        }
54    }
55}
56
57/// Bytes a frame occupies in the connection's egress queue for budget
58/// purposes: the fixed envelope header plus the body.
59fn queued_frame_bytes(frame: &Frame) -> usize {
60    subc_protocol::HEADER_LEN + frame.body.len()
61}
62
63/// A point-in-time view of one connection's egress queue, for diagnosing why a
64/// frame did not fit.
65#[derive(Debug, Clone, Copy, PartialEq, Eq)]
66pub(crate) struct EgressBacklog {
67    pub queued_bytes: usize,
68    pub queued_frames: usize,
69    /// Now minus the enqueue time of the frame the writer most recently took
70    /// off the queue (the frame it is writing, or has just written), or, if
71    /// the writer has taken nothing since the queue was last empty, of the
72    /// frame that made the queue non-empty. While the writer is blocked on a
73    /// frame this is exactly the oldest frame's age; between frames it can be
74    /// one frame older than the true head. `None` when nothing is queued.
75    pub oldest_age: Option<Duration>,
76}
77
78/// Queued-byte accounting shared by every clone of one connection's
79/// [`FrameSink`] and by every frame that sink has admitted. Its per-frame
80/// accounting uses atomics: this is on the path of every frame a module sends
81/// to a client, so it takes no lock per frame.
82#[derive(Debug)]
83struct EgressAccounting {
84    byte_budget: usize,
85    queued_bytes: AtomicUsize,
86    queued_frames: AtomicUsize,
87    /// Origin for the nanosecond timestamps in `oldest_enqueued_nanos`.
88    time_base: Instant,
89    /// Enqueue time, as nanoseconds after `time_base` plus one, of the frame
90    /// described by [`EgressBacklog::oldest_age`]; 0 means none.
91    oldest_enqueued_nanos: AtomicU64,
92    /// Awaited senders currently parked on `freed`. A release only pays for a
93    /// notification when this is non-zero.
94    waiters: AtomicUsize,
95    /// Woken when a charged frame releases its bytes while a sender waits, so
96    /// the awaited send can re-check for room.
97    freed: Notify,
98    client_delivery_closed: AtomicBool,
99    client_delivery_stopped: Notify,
100}
101
102impl EgressAccounting {
103    fn new(byte_budget: usize) -> Self {
104        Self {
105            byte_budget,
106            queued_bytes: AtomicUsize::new(0),
107            queued_frames: AtomicUsize::new(0),
108            time_base: tokio::time::Instant::now().into_std(),
109            oldest_enqueued_nanos: AtomicU64::new(0),
110            waiters: AtomicUsize::new(0),
111            freed: Notify::new(),
112            client_delivery_closed: AtomicBool::new(false),
113            client_delivery_stopped: Notify::new(),
114        }
115    }
116
117    fn stamp(&self, at: Instant) -> u64 {
118        (at.saturating_duration_since(self.time_base).as_nanos() as u64).saturating_add(1)
119    }
120
121    /// Charge `bytes` if they fit in the budget. A frame larger than the whole
122    /// budget (bodies may be up to 64 MiB) is still admitted into an EMPTY
123    /// queue, since it could otherwise never be sent at all; it simply has the
124    /// queue to itself until it is written.
125    fn try_charge(self: &Arc<Self>, bytes: usize) -> Option<EgressCharge> {
126        // SeqCst pairs with `release`: either this load sees bytes a release
127        // just freed, or that release sees this sender counted in `waiters`.
128        let mut current = self.queued_bytes.load(Ordering::SeqCst);
129        loop {
130            if current != 0 && current.saturating_add(bytes) > self.byte_budget {
131                return None;
132            }
133            match self.queued_bytes.compare_exchange_weak(
134                current,
135                current + bytes,
136                Ordering::SeqCst,
137                Ordering::SeqCst,
138            ) {
139                Ok(_) => return Some(self.record(bytes)),
140                Err(actual) => current = actual,
141            }
142        }
143    }
144
145    /// Charge `bytes` regardless of the budget. Used only for a frame whose
146    /// queue slot was reserved in advance, which must be sendable.
147    fn charge_unconditionally(self: &Arc<Self>, bytes: usize) -> EgressCharge {
148        self.queued_bytes.fetch_add(bytes, Ordering::SeqCst);
149        self.record(bytes)
150    }
151
152    fn record(self: &Arc<Self>, bytes: usize) -> EgressCharge {
153        let enqueued_at = tokio::time::Instant::now().into_std();
154        let stamp = self.stamp(enqueued_at);
155        if self.queued_frames.fetch_add(1, Ordering::AcqRel) == 0 {
156            // This frame made the queue non-empty, so it is the head until the
157            // writer takes something.
158            self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
159        }
160        EgressCharge {
161            accounting: Arc::clone(self),
162            bytes,
163            stamp,
164            enqueued_at,
165        }
166    }
167
168    /// The writer has taken the frame stamped `stamp` off the queue.
169    fn taken_by_writer(&self, stamp: u64) {
170        self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
171    }
172
173    fn release(&self, bytes: usize, stamp: u64) {
174        self.queued_bytes.fetch_sub(bytes, Ordering::SeqCst);
175        if self.queued_frames.fetch_sub(1, Ordering::AcqRel) == 1 {
176            // The queue drained. Clear the marker only if it still names this
177            // frame, so a frame admitted in the meantime keeps its stamp.
178            let _ = self.oldest_enqueued_nanos.compare_exchange(
179                stamp,
180                0,
181                Ordering::AcqRel,
182                Ordering::Acquire,
183            );
184        }
185        if self.waiters.load(Ordering::SeqCst) != 0 {
186            self.freed.notify_waiters();
187        }
188    }
189
190    fn backlog(&self) -> EgressBacklog {
191        let queued_frames = self.queued_frames.load(Ordering::Acquire);
192        let stamp = self.oldest_enqueued_nanos.load(Ordering::Acquire);
193        let oldest_age = (queued_frames != 0 && stamp != 0).then(|| {
194            let enqueued = self.time_base + Duration::from_nanos(stamp - 1);
195            tokio::time::Instant::from_std(enqueued).elapsed()
196        });
197        EgressBacklog {
198            queued_bytes: self.queued_bytes.load(Ordering::Acquire),
199            queued_frames,
200            oldest_age,
201        }
202    }
203}
204
205/// One admitted frame's claim on its connection's egress byte budget,
206/// returned when the frame is dropped.
207#[derive(Debug)]
208pub(crate) struct EgressCharge {
209    accounting: Arc<EgressAccounting>,
210    bytes: usize,
211    stamp: u64,
212    enqueued_at: Instant,
213}
214
215impl EgressCharge {
216    /// Called by the connection writer when it takes this frame off the queue,
217    /// so the backlog's oldest-age figure follows the writer.
218    pub(crate) fn taken_by_writer(&self) {
219        self.accounting.taken_by_writer(self.stamp);
220    }
221}
222
223impl Drop for EgressCharge {
224    fn drop(&mut self) {
225        self.accounting.release(self.bytes, self.stamp);
226    }
227}
228
229/// A queue slot reserved ahead of time (a pending `route.open` holds one until
230/// its module answers). Sending through it never waits and never fails for
231/// lack of room: the slot is already held, and its frame is charged to the
232/// byte count outside the budget check. At most
233/// `MAX_PENDING_ROUTE_OPENS_PER_CONNECTION` such frames exist per connection.
234#[derive(Debug)]
235pub(crate) struct EgressPermit {
236    permit: mpsc::OwnedPermit<OutboundFrame>,
237    accounting: Arc<EgressAccounting>,
238}
239
240impl EgressPermit {
241    /// Enqueue `frame` in the reserved slot. Returns true when the connection's
242    /// writer had already gone away, meaning the frame will never be written.
243    pub(crate) fn send(self, frame: Frame) -> bool {
244        let charge = self
245            .accounting
246            .charge_unconditionally(queued_frame_bytes(&frame));
247        let sender = self.permit.send(OutboundFrame::charged(frame, charge));
248        sender.is_closed()
249    }
250}
251
252/// An `OutboundFrame` is a stamped `Frame`; deref keeps every existing frame
253/// read (headers, bodies, assertions) working on queued items unchanged.
254impl std::ops::Deref for OutboundFrame {
255    type Target = Frame;
256
257    fn deref(&self) -> &Frame {
258        &self.frame
259    }
260}
261
262/// Shared tracing-capture helpers for timing-observability tests across
263/// modules (router dispatch, server reply path). Test-only.
264#[cfg(test)]
265pub(crate) mod test_log {
266    use std::{
267        io::Write,
268        sync::{Arc, Mutex, OnceLock},
269    };
270
271    #[derive(Clone)]
272    struct TestLogWriter(Arc<Mutex<Vec<u8>>>);
273
274    impl Write for TestLogWriter {
275        fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
276            self.0
277                .lock()
278                .expect("test log capture is not poisoned")
279                .extend(buffer);
280            Ok(buffer.len())
281        }
282
283        fn flush(&mut self) -> std::io::Result<()> {
284            Ok(())
285        }
286    }
287
288    pub(crate) fn log_capture(
289        level: tracing::Level,
290    ) -> (Arc<Mutex<Vec<u8>>>, tracing::dispatcher::DefaultGuard) {
291        // tracing-core's single-dispatch fast path registers callsites using
292        // the emitting thread's default subscriber. A parallel test with no
293        // subscriber can therefore cache `never` for a callsite this capture
294        // needs. Keep a second dispatch alive so registration considers all
295        // live subscribers instead. This registry is never installed as a
296        // default: each test still records only into its own thread's writer.
297        static INTEREST_ANCHOR: OnceLock<tracing::Dispatch> = OnceLock::new();
298        INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
299        let output = Arc::new(Mutex::new(Vec::new()));
300        let writer = Arc::clone(&output);
301        let subscriber = tracing_subscriber::fmt()
302            .with_max_level(level)
303            .with_ansi(false)
304            .without_time()
305            .with_target(false)
306            .with_writer(move || TestLogWriter(Arc::clone(&writer)))
307            .finish();
308        let guard = tracing::subscriber::set_default(subscriber);
309        (output, guard)
310    }
311
312    pub(crate) fn captured_logs(output: &Arc<Mutex<Vec<u8>>>) -> String {
313        String::from_utf8(
314            output
315                .lock()
316                .expect("test log capture is not poisoned")
317                .clone(),
318        )
319        .expect("tracing output is UTF-8")
320    }
321
322    #[cfg(test)]
323    mod tests {
324        use super::*;
325
326        fn emit_capture_probe(source: &str) {
327            tracing::error!(source, "parallel capture probe");
328        }
329
330        #[test]
331        fn parallel_callsite_registration_keeps_the_test_capture_enabled() {
332            let (output, _guard) = log_capture(tracing::Level::ERROR);
333            // First register the very same callsite on a thread without a
334            // default subscriber, while the test's capture remains live.
335            std::thread::spawn(|| emit_capture_probe("other-thread"))
336                .join()
337                .expect("unsubscribed thread exits");
338            emit_capture_probe("captured-thread");
339
340            let logs = captured_logs(&output);
341            assert!(
342                logs.contains("captured-thread"),
343                "capture was disabled: {logs}"
344            );
345            assert!(!logs.contains("other-thread"), "capture leaked: {logs}");
346        }
347    }
348}
349
350/// Cheaply cloneable handle to one connection's bounded outbound frame queue.
351///
352/// Backends emit responses, streaming frames, and future PUSH frames through this
353/// single path. The queue is bounded twice: by queued BYTES (the budget that
354/// matters for memory and for how long a slow reader may pause), and by the
355/// `mpsc` channel's frame count (a backstop). The socket layer owns the sole
356/// receiver/writer.
357#[derive(Debug, Clone)]
358pub struct FrameSink {
359    tx: mpsc::Sender<OutboundFrame>,
360    accounting: Arc<EgressAccounting>,
361}
362
363impl FrameSink {
364    /// A sink with the standard per-connection byte budget
365    /// ([`crate::server::CONNECTION_EGRESS_BYTE_BUDGET`]); the frame-count bound
366    /// is whatever capacity `tx`'s channel was created with.
367    pub fn new(tx: mpsc::Sender<OutboundFrame>) -> Self {
368        Self::with_byte_budget(tx, crate::server::CONNECTION_EGRESS_BYTE_BUDGET)
369    }
370
371    pub(crate) fn with_byte_budget(tx: mpsc::Sender<OutboundFrame>, byte_budget: usize) -> Self {
372        Self {
373            tx,
374            accounting: Arc::new(EgressAccounting::new(byte_budget)),
375        }
376    }
377
378    /// Wait until the frame's bytes fit the budget, then charge them. Awaited
379    /// senders (control replies, error replies, shutdown notices) are never
380    /// refused by the byte budget: they wait for the writer to free bytes, just
381    /// as they wait for a free slot, which is the same backpressure they had
382    /// when the queue was bounded by frame count alone. Returns `None` if the
383    /// writer goes away while waiting.
384    async fn charge_waiting(&self, bytes: usize) -> Option<EgressCharge> {
385        if let Some(charge) = self.accounting.try_charge(bytes) {
386            return Some(charge);
387        }
388        // Counted as a waiter for as long as this future is parked, including
389        // when it is cancelled, so releases notify only while someone waits.
390        struct Waiting<'a>(&'a AtomicUsize);
391        impl Drop for Waiting<'_> {
392            fn drop(&mut self) {
393                self.0.fetch_sub(1, Ordering::SeqCst);
394            }
395        }
396        self.accounting.waiters.fetch_add(1, Ordering::SeqCst);
397        let _waiting = Waiting(&self.accounting.waiters);
398        loop {
399            let freed = self.accounting.freed.notified();
400            tokio::pin!(freed);
401            // Register interest before checking, so a release that happens
402            // between the check and the await still wakes this sender.
403            freed.as_mut().enable();
404            if let Some(charge) = self.accounting.try_charge(bytes) {
405                return Some(charge);
406            }
407            tokio::select! {
408                _ = &mut freed => {}
409                _ = self.tx.closed() => return None,
410            }
411        }
412    }
413
414    pub async fn send(&self, frame: Frame) -> Result<(), RouterError> {
415        let channel = frame.header.channel;
416        let epoch = frame.header.epoch;
417        let corr = frame.header.corr;
418        let closed =
419            || RouterError::backend_with_epoch(channel, epoch, corr, "connection writer closed");
420        let charge = self
421            .charge_waiting(queued_frame_bytes(&frame))
422            .await
423            .ok_or_else(closed)?;
424        self.tx
425            .send(OutboundFrame::charged(frame, charge))
426            .await
427            .map_err(|_| closed())
428    }
429
430    /// Shutdown notices must leave the socket writer before an idle daemon exits.
431    /// Queue admission alone does not prove this; the writer acknowledges flush.
432    ///
433    /// The local endpoint uses it the same way for its clients-only refusal: the
434    /// connection closes right after the refusal is sent, so the writer must
435    /// flush it first.
436    #[cfg(unix)]
437    pub async fn send_flushed(&self, frame: Frame) -> Result<(), RouterError> {
438        let (tx, rx) = tokio::sync::oneshot::channel();
439        let charge = self
440            .charge_waiting(queued_frame_bytes(&frame))
441            .await
442            .ok_or_else(|| RouterError::backend(0, 0, "connection writer closed"))?;
443        let mut outbound = OutboundFrame::charged(frame, charge);
444        outbound.flushed = Some(tx);
445        self.tx
446            .send(outbound)
447            .await
448            .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
449        rx.await
450            .map_err(|_| RouterError::backend(0, 0, "connection flush failed"))
451    }
452
453    pub(crate) async fn reserve_owned(&self) -> Result<EgressPermit, RouterError> {
454        let permit = self
455            .tx
456            .clone()
457            .reserve_owned()
458            .await
459            .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
460        Ok(EgressPermit {
461            permit,
462            accounting: Arc::clone(&self.accounting),
463        })
464    }
465
466    #[cfg(test)]
467    pub(crate) fn try_reserve_owned(&self) -> Result<EgressPermit, RouterError> {
468        let permit = self
469            .tx
470            .clone()
471            .try_reserve_owned()
472            .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
473        Ok(EgressPermit {
474            permit,
475            accounting: Arc::clone(&self.accounting),
476        })
477    }
478
479    pub(crate) fn is_closed(&self) -> bool {
480        self.tx.is_closed()
481    }
482
483    /// What the connection's egress queue holds right now.
484    pub(crate) fn backlog(&self) -> EgressBacklog {
485        self.accounting.backlog()
486    }
487
488    /// Stop module deliveries as soon as the client reader exits, without
489    /// waiting for the socket writer's separate teardown drain grace.
490    pub(crate) fn close_client_delivery(&self) {
491        self.accounting
492            .client_delivery_closed
493            .store(true, Ordering::Release);
494        self.accounting.client_delivery_stopped.notify_waiters();
495    }
496
497    /// A full queue can be a short fanout burst rather than a stopped reader.
498    /// Spend only the grace the atomic backlog age has not already used.
499    async fn send_to_client(&self, frame: Frame) -> Result<(), ClientDeliveryFailure> {
500        let frame_type = frame.header.ty;
501        let flags = frame.header.flags;
502        let stopped = self.accounting.client_delivery_stopped.notified();
503        tokio::pin!(stopped);
504        stopped.as_mut().enable();
505        let failure = |cause, error| ClientDeliveryFailure {
506            diagnosis: ClientDeliveryDiagnosis {
507                frame_type,
508                flags,
509                cause,
510            },
511            error,
512        };
513        let closed = || {
514            RouterError::backend_with_epoch(
515                frame.header.channel,
516                frame.header.epoch,
517                frame.header.corr,
518                "client connection closed",
519            )
520        };
521        if self
522            .accounting
523            .client_delivery_closed
524            .load(Ordering::Acquire)
525        {
526            return Err(failure("client_closed", closed()));
527        }
528        let (frame, error) = match self.try_send_owned(frame) {
529            Ok(()) => return Ok(()),
530            Err(refused) => refused,
531        };
532        if self.is_closed() {
533            return Err(failure("client_closed", error));
534        }
535        let grace = crate::server::CONNECTION_EGRESS_STALL_GRACE;
536        let age = self.backlog().oldest_age.unwrap_or_default();
537        if age > grace {
538            return Err(failure("oldest_frame_older_than_250_ms", error));
539        }
540        let result = tokio::select! {
541            biased;
542            _ = &mut stopped => return Err(failure("client_closed", error)),
543            result = tokio::time::timeout(grace.saturating_sub(age), self.send(frame)) => result,
544        };
545        match result {
546            Ok(Ok(())) => Ok(()),
547            Ok(Err(error)) => Err(failure("client_closed", error)),
548            Err(_) => Err(failure("waited_250_ms_without_room", error)),
549        }
550    }
551
552    /// Enqueue without waiting. Fails when the frame's bytes would take the
553    /// queue past its byte budget, when the channel's frame-count backstop is
554    /// full, or when the writer is gone.
555    pub(crate) fn try_send(&self, frame: Frame) -> Result<(), RouterError> {
556        self.try_send_owned(frame).map_err(|(_, error)| error)
557    }
558
559    fn try_send_owned(&self, frame: Frame) -> Result<(), (Frame, RouterError)> {
560        let channel = frame.header.channel;
561        let epoch = frame.header.epoch;
562        let corr = frame.header.corr;
563        let unavailable = |why: String| {
564            RouterError::backend_with_epoch(
565                channel,
566                epoch,
567                corr,
568                format!("connection writer unavailable: {why}"),
569            )
570        };
571        let bytes = queued_frame_bytes(&frame);
572        let Some(charge) = self.accounting.try_charge(bytes) else {
573            if self.tx.is_closed() {
574                return Err((frame, unavailable("channel closed".to_string())));
575            }
576            return Err((
577                frame,
578                unavailable(format!(
579                    "egress byte budget exhausted ({} queued bytes, frame of {bytes} bytes, budget {})",
580                    self.accounting.queued_bytes.load(Ordering::Acquire),
581                    self.accounting.byte_budget
582                )),
583            ));
584        };
585        // Return the refused frame for a bounded retry; dropping the rest of
586        // the outbound wrapper returns its charge.
587        self.tx
588            .try_send(OutboundFrame::charged(frame, charge))
589            .map_err(|err| {
590                let error = unavailable(err.to_string());
591                (err.into_inner().frame, error)
592            })
593    }
594}
595
596#[derive(Debug)]
597struct ClientDeliveryFailure {
598    diagnosis: ClientDeliveryDiagnosis,
599    error: RouterError,
600}
601
602/// Header facts only: payloads can contain secrets and must never enter logs.
603#[derive(Debug)]
604pub(crate) struct ClientDeliveryDiagnosis {
605    pub frame_type: FrameType,
606    pub flags: Flags,
607    pub cause: &'static str,
608}
609
610impl fmt::Display for ClientDeliveryFailure {
611    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
612        self.error.fmt(f)
613    }
614}
615
616/// Minimum time between two route GOODBYEs the daemon sends one module
617/// connection for one channel in answer to the module's frames on a (channel,
618/// epoch) the daemon holds no route for.
619///
620/// A module that missed a GOODBYE keeps sending on that route (a streaming
621/// module, many frames a second), and every one of those frames lands here. One
622/// answer is enough when it arrives, so answering each frame would only flood
623/// the module's egress queue at the moment it is already behind. Five seconds
624/// is long enough for an answer queued behind a deep backlog to reach the module
625/// and take effect before a second is sent, and short enough that an answer
626/// the queue refused is retried by the next orphan frame well within a minute.
627const ORPHAN_ROUTE_GOODBYE_INTERVAL: Duration = Duration::from_secs(5);
628
629/// Once one connection has this many remembered channels, entries older than
630/// [`ORPHAN_ROUTE_GOODBYE_INTERVAL`] are pruned before another is added. An
631/// expired entry has no effect on rate limiting, so pruning never changes a
632/// decision; it keeps a module that orphaned many channels long ago from
633/// pinning memory for all of them. The map is bounded in any case by the
634/// 16-bit channel space and is dropped whole when the connection ends.
635const ORPHAN_ROUTE_GOODBYE_PRUNE_AT: usize = 256;
636
637/// When each module connection was last sent an orphan-route GOODBYE, per
638/// channel. Shared by the [`Router`] (which consults it) and every
639/// [`RouterConnection`] (which removes its own entry when the connection ends).
640#[derive(Debug, Default)]
641struct OrphanGoodbyeLimiter {
642    last_sent: Mutex<HashMap<ConnectionId, HashMap<u16, tokio::time::Instant>>>,
643}
644
645impl OrphanGoodbyeLimiter {
646    /// True, and the send recorded, when `channel` on `connection_id` has not
647    /// been answered within the interval.
648    fn claim(&self, connection_id: ConnectionId, channel: u16) -> bool {
649        let now = tokio::time::Instant::now();
650        let mut last_sent = self
651            .last_sent
652            .lock()
653            .unwrap_or_else(std::sync::PoisonError::into_inner);
654        let channels = last_sent.entry(connection_id).or_default();
655        if channels.get(&channel).is_some_and(|sent| {
656            now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
657        }) {
658            return false;
659        }
660        if channels.len() >= ORPHAN_ROUTE_GOODBYE_PRUNE_AT {
661            channels.retain(|_, sent| {
662                now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
663            });
664        }
665        channels.insert(channel, now);
666        true
667    }
668
669    fn forget_connection(&self, connection_id: ConnectionId) {
670        self.last_sent
671            .lock()
672            .unwrap_or_else(std::sync::PoisonError::into_inner)
673            .remove(&connection_id);
674    }
675}
676
677/// Per-route context shared with backends besides the frame itself.
678#[derive(Debug, Clone)]
679pub struct RouteCtx {
680    pub connection_id: ConnectionId,
681    pub egress: FrameSink,
682}
683
684/// Closed set of data-plane backends for non-zero channels.
685///
686/// Channel 0 is structurally special and remains [`Router::control`], not an
687/// enum variant. Static/test backends live in [`Router::backends`]; the forwarding
688/// backend is selected dynamically only when a per-connection forwarding binding exists.
689#[derive(Debug, Clone)]
690pub enum Backend {
691    Echo(EchoBackend),
692    Forward(ForwardBackend),
693}
694
695impl From<EchoBackend> for Backend {
696    fn from(backend: EchoBackend) -> Self {
697        Self::Echo(backend)
698    }
699}
700
701impl From<ForwardBackend> for Backend {
702    fn from(backend: ForwardBackend) -> Self {
703        Self::Forward(backend)
704    }
705}
706
707impl Backend {
708    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
709        match self {
710            Self::Echo(backend) => backend.handle(ctx, frame).await,
711            Self::Forward(backend) => backend.handle(ctx, frame).await,
712        }
713    }
714}
715
716/// I/O-agnostic splice router keyed by envelope `channel`.
717///
718/// Channel 0 is reserved for subc itself and is always dispatched to the
719/// dedicated control handler. Other channels must be explicitly registered.
720/// Unknown client-originated non-zero channels are translated to canonical JSON `ERROR` frames on
721/// the connection sink so the peer can continue using the same socket. Module-originated frames for
722/// released route channels are logged and dropped as the channel-gone race backstop.
723///
724/// DATA-PLANE BODIES ARE NEVER DECODED HERE, and the consequence is worth stating
725/// because it looks like a guarantee and is not. Additive fields in a request or
726/// response body reach the far side untouched — not because anything permits them,
727/// but because routing reads only the 21-byte header and treats the body as opaque
728/// bytes. That is a performance property, so **nothing prevents it from changing**:
729/// a future reason to inspect a body would convert a wire-transparent path into a
730/// filtering one, and nobody would think of it as a contract change.
731///
732/// So when a peer asks whether subc sees an additive field, the answer is per-path
733/// and this path's zero means WE NEVER LOOK rather than WE LOOK AT EVERYTHING. The
734/// control plane (typed enums at the frame boundary) and the MCP gateway (envelope
735/// unwrap plus a named struct) both narrow; only this one does not.
736pub struct Router {
737    backends: HashMap<u16, Backend>,
738    control: Arc<ControlHandler>,
739    forwarding: Arc<ForwardingTable>,
740    forward_backend: ForwardBackend,
741    counters: DaemonCounters,
742    next_connection_id: AtomicU64,
743    orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
744}
745
746impl Router {
747    pub fn with_control_handler(control: Arc<ControlHandler>) -> Self {
748        // The handler the router serves is the one a swap must tell when it
749        // promotes a candidate; this is where it first sits behind an `Arc`.
750        control.install_swap_promotion_observer();
751        let forwarding = control.forwarding();
752        let counters = control.counters();
753        Self {
754            backends: HashMap::new(),
755            control,
756            forwarding: Arc::clone(&forwarding),
757            forward_backend: ForwardBackend::new(forwarding),
758            counters,
759            // ConnectionId::LOCAL is 0; real socket ids start at 1 and never collide.
760            next_connection_id: AtomicU64::new(1),
761            orphan_goodbyes: Arc::default(),
762        }
763    }
764
765    pub fn with_default_self_handler() -> Self {
766        Self::with_control_handler(Arc::new(ControlHandler::default()))
767    }
768
769    pub fn forwarding(&self) -> Arc<ForwardingTable> {
770        Arc::clone(&self.forwarding)
771    }
772
773    pub fn register_backend(
774        &mut self,
775        channel: u16,
776        backend: impl Into<Backend>,
777    ) -> Result<(), RouterError> {
778        self.register_backend_arc(channel, Arc::new(backend.into()))
779    }
780
781    pub(crate) fn register_backend_arc(
782        &mut self,
783        channel: u16,
784        backend: Arc<Backend>,
785    ) -> Result<(), RouterError> {
786        if channel == 0 {
787            return Err(RouterError::ReservedChannelZero);
788        }
789        if self.backends.contains_key(&channel) {
790            return Err(RouterError::DuplicateChannel { channel });
791        }
792        self.backends.insert(channel, backend.as_ref().clone());
793        Ok(())
794    }
795
796    /// Start a connection-scoped routing context. Dropping the guard releases
797    /// any control-plane registrations owned by the connection.
798    fn record_module_frame_drop(&self, connection_id: ConnectionId) -> Result<(), RouterError> {
799        let module_id = self
800            .forwarding
801            .module_id_for_connection(connection_id)
802            .map_err(RouterError::Forwarding)?;
803        self.counters
804            .increment_module_frames_dropped_no_route(module_id.as_deref());
805        Ok(())
806    }
807
808    /// A module sent a non-request frame on a (channel, epoch) the daemon holds
809    /// no route for: most often a route the daemon released whose GOODBYE the
810    /// module never received, so the module still believes it is open and keeps
811    /// sending. Count the drop, and tell the module to let go of exactly that
812    /// (channel, epoch) with a route GOODBYE, at most once per
813    /// [`ORPHAN_ROUTE_GOODBYE_INTERVAL`] per connection and channel.
814    ///
815    /// The answer uses `try_send`: it is a best-effort nudge, and if the queue
816    /// refuses it, the module's next frame on that route after the interval
817    /// asks again. A GOODBYE from the module is not answered, since the module
818    /// is already letting go of the route.
819    fn handle_orphan_module_frame(&self, ctx: &RouteCtx, frame: &Frame) -> Result<(), RouterError> {
820        let channel = frame.header.channel;
821        let epoch = frame.header.epoch;
822        let module_id = self
823            .forwarding
824            .module_id_for_connection(ctx.connection_id)
825            .map_err(RouterError::Forwarding)?;
826        self.counters
827            .increment_module_frames_dropped_no_route(module_id.as_deref());
828        if self
829            .forwarding
830            .module_route_epoch_was_allocated(ctx.connection_id, channel, epoch)
831            .map_err(RouterError::Forwarding)?
832        {
833            self.counters
834                .increment_module_frames_dropped_released_route(module_id.as_deref());
835        }
836        if frame.header.ty == FrameType::Goodbye
837            || !self.orphan_goodbyes.claim(ctx.connection_id, channel)
838        {
839            return Ok(());
840        }
841        let goodbye = Frame::build_with_version(
842            frame.header.ver,
843            FrameType::Goodbye,
844            Flags::new(false, Priority::Passive, false),
845            channel,
846            epoch,
847            0,
848            Vec::new(),
849        )
850        .map_err(RouterError::FrameBuild)?;
851        match ctx.egress.try_send(goodbye) {
852            Ok(()) => {
853                self.counters.increment_module_orphan_route_goodbyes_sent();
854                debug!(
855                    connection_id = ctx.connection_id.get(),
856                    module_id = module_id.as_deref().unwrap_or("unknown"),
857                    channel,
858                    epoch,
859                    "answered module frame on a route the daemon does not hold with a route GOODBYE"
860                );
861            }
862            Err(err) => {
863                self.forwarding.warn_frame_drop(
864                    ctx.connection_id,
865                    channel,
866                    epoch,
867                    "daemon_to_module",
868                    if ctx.egress.is_closed() {
869                        "endpoint_gone"
870                    } else {
871                        "egress_budget"
872                    },
873                );
874                debug!(
875                connection_id = ctx.connection_id.get(),
876                module_id = module_id.as_deref().unwrap_or("unknown"),
877                channel,
878                epoch,
879                error = %err,
880                "could not enqueue route GOODBYE for module frame on a route the daemon does not hold; the next such frame retries"
881                );
882            }
883        }
884        Ok(())
885    }
886
887    pub fn begin_connection(&self) -> RouterConnection {
888        let raw = self.next_connection_id.fetch_add(1, Ordering::Relaxed);
889        let id = ConnectionId::new(raw);
890        let close_receiver = self.forwarding.register_connection_close(id);
891        RouterConnection {
892            id,
893            control_handler: Arc::clone(&self.control),
894            forwarding: Arc::clone(&self.forwarding),
895            close_receiver: Some(close_receiver),
896            orphan_goodbyes: Arc::clone(&self.orphan_goodbyes),
897        }
898    }
899
900    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
901        self.control.route_open_target(frame)
902    }
903
904    pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
905        self.control.record_local_connection(connection);
906    }
907
908    pub(crate) fn record_captured_peer(
909        &self,
910        connection: ConnectionId,
911        peer: subc_os::local_peer::CapturedPeer,
912    ) {
913        self.control.record_captured_peer(connection, peer);
914    }
915
916    pub(crate) fn record_caller_endpoints(
917        &self,
918        connection: ConnectionId,
919        endpoints: Option<(std::net::SocketAddr, std::net::SocketAddr)>,
920    ) {
921        self.control.record_caller_endpoints(connection, endpoints);
922    }
923
924    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
925        self.control.defers_privileged_control(ctx, frame)
926    }
927
928    pub(crate) fn route_open_capacity_refusal(
929        &self,
930        ctx: &RouteCtx,
931        frame: &Frame,
932        target_module_id: &str,
933        in_flight: usize,
934        limit: usize,
935    ) -> Result<Frame, RouterError> {
936        self.control
937            .route_open_capacity_refusal(ctx, frame, target_module_id, in_flight, limit)
938    }
939
940    pub async fn route_for_connection(
941        &self,
942        ctx: &RouteCtx,
943        frame: Frame,
944    ) -> Result<(), RouterError> {
945        self.route_for_connection_started(ctx, frame, None).await
946    }
947
948    pub(crate) async fn route_for_connection_started(
949        &self,
950        ctx: &RouteCtx,
951        frame: Frame,
952        dispatch_started_at: Option<Instant>,
953    ) -> Result<(), RouterError> {
954        let channel = frame.header.channel;
955        let epoch = frame.header.epoch;
956        let corr = frame.header.corr;
957        if channel == 0 {
958            debug!(
959                connection_id = ctx.connection_id.get(),
960                corr,
961                frame_type = ?frame.header.ty,
962                "routing control frame"
963            );
964            // The connection loop dispatches every control operation directly
965            // except `route.open`, activity writes and privileged requests. Each
966            // task passes a timestamp captured before
967            // spawn, so slow-dispatch timing includes scheduler delay but no
968            // wait in an application-owned queue.
969            let dispatch_started_at = (frame.header.ty == FrameType::Request)
970                .then(|| dispatch_started_at.unwrap_or_else(Instant::now));
971            let responses = self
972                .control
973                .handle_control_frame_timed(ctx, frame, dispatch_started_at)
974                .await?;
975            for response in responses {
976                ctx.egress.send(response).await?;
977            }
978            return Ok(());
979        }
980
981        let data_route = self
982            .forwarding
983            .lookup_data_route(ctx.connection_id, channel, epoch)
984            .map_err(RouterError::Forwarding)?;
985
986        match data_route {
987            DataRoute::Module(DataRouteState::EpochMismatch) => {
988                self.forwarding.warn_frame_drop(
989                    ctx.connection_id,
990                    channel,
991                    epoch,
992                    "module_to_client",
993                    "stale_epoch",
994                );
995                if frame.header.ty == FrameType::Request {
996                    self.counters
997                        .increment_module_requests_dropped_stale_route();
998                    let err = RouterError::StaleRouteEpoch {
999                        channel,
1000                        epoch,
1001                        corr,
1002                    };
1003                    if let Some(error_frame) = err.to_error_frame() {
1004                        ctx.egress.send(error_frame).await?;
1005                    }
1006                } else {
1007                    self.handle_orphan_module_frame(ctx, &frame)?;
1008                }
1009                debug!(
1010                    connection_id = ctx.connection_id.get(),
1011                    channel, epoch, corr, "dropping module frame for stale route epoch"
1012                );
1013                return Ok(());
1014            }
1015            DataRoute::Module(DataRouteState::Reserved) => {
1016                self.forwarding.warn_frame_drop(
1017                    ctx.connection_id,
1018                    channel,
1019                    epoch,
1020                    "module_to_client",
1021                    "missing_binding",
1022                );
1023                if frame.header.ty == FrameType::Request {
1024                    self.counters
1025                        .increment_module_requests_dropped_stale_route();
1026                    let err = RouterError::UnknownChannel {
1027                        channel,
1028                        epoch,
1029                        corr,
1030                    };
1031                    if let Some(error_frame) = err.to_error_frame() {
1032                        ctx.egress.send(error_frame).await?;
1033                    }
1034                } else {
1035                    self.record_module_frame_drop(ctx.connection_id)?;
1036                }
1037                debug!(
1038                    connection_id = ctx.connection_id.get(),
1039                    channel, epoch, corr, "dropping module frame for reserved route handle"
1040                );
1041                return Ok(());
1042            }
1043            DataRoute::Module(DataRouteState::Absent) => {
1044                self.forwarding.warn_frame_drop(
1045                    ctx.connection_id,
1046                    channel,
1047                    epoch,
1048                    "module_to_client",
1049                    "unknown_channel",
1050                );
1051                if frame.header.ty == FrameType::Request {
1052                    self.counters
1053                        .increment_module_requests_dropped_stale_route();
1054                    let err = RouterError::UnknownChannel {
1055                        channel,
1056                        epoch,
1057                        corr,
1058                    };
1059                    if let Some(error_frame) = err.to_error_frame() {
1060                        ctx.egress.send(error_frame).await?;
1061                    }
1062                } else {
1063                    self.handle_orphan_module_frame(ctx, &frame)?;
1064                }
1065                debug!(
1066                    connection_id = ctx.connection_id.get(),
1067                    channel, epoch, corr, "dropping module frame for absent route handle"
1068                );
1069                return Ok(());
1070            }
1071            DataRoute::Module(DataRouteState::Bound(route)) => {
1072                if frame.header.ty == FrameType::Goodbye {
1073                    if let RouteRelease::Removed(target) = self
1074                        .forwarding
1075                        .release_module_route(ctx.connection_id, channel, epoch)
1076                        .map_err(RouterError::Forwarding)?
1077                    {
1078                        let mut goodbye = frame;
1079                        goodbye.header.channel = target.channel;
1080                        goodbye.header.epoch = target.epoch;
1081                        // A module must never be able to make a frame look
1082                        // daemon-authored, so the origin bit is cleared on
1083                        // every module frame relayed to a client.
1084                        goodbye.header.flags = goodbye.header.flags.without_daemon_origin();
1085                        if let Err(err) = target.sink.send_to_client(goodbye).await {
1086                            if !target.close_on_delivery_failure() {
1087                                self.forwarding.warn_frame_drop(
1088                                    target.connection_id,
1089                                    target.channel,
1090                                    target.epoch,
1091                                    "module_to_client",
1092                                    err.diagnosis.cause,
1093                                );
1094                            }
1095                            if target.close_on_delivery_failure()
1096                                && self
1097                                    .forwarding
1098                                    .escalate_client_delivery_failure_diagnosed(
1099                                        target.connection_id,
1100                                        target.channel,
1101                                        target.epoch,
1102                                        CloseReason::new(
1103                                            "route_goodbye_delivery_failed",
1104                                            format!(
1105                                                "failed to enqueue route GOODBYE for client channel {}: {err}",
1106                                                target.channel
1107                                            ),
1108                                        ),
1109                                        UndeliveredFrame {
1110                                            module_id: Some(&route.module_id),
1111                                            sink: &target.sink,
1112                                        },
1113                                        Some(&err.diagnosis),
1114                                    )
1115                                    .map_err(RouterError::Forwarding)?
1116                            {
1117                                self.counters.increment_goodbye_relay_client_failed();
1118                            }
1119                        }
1120                    } else {
1121                        self.forwarding.warn_frame_drop(
1122                            ctx.connection_id,
1123                            channel,
1124                            epoch,
1125                            "module_to_client",
1126                            "missing_binding",
1127                        );
1128                    }
1129                    return Ok(());
1130                }
1131
1132                // Accounting allows an oversized frame to occupy an empty queue
1133                // on other send paths. Refuse it at this splice before admission,
1134                // so the next ordinary reply cannot close a multiplexed client.
1135                let bytes = queued_frame_bytes(&frame);
1136                let limit = route.client_sink.accounting.byte_budget;
1137                if bytes > limit {
1138                    let ty = frame.header.ty;
1139                    let flags = frame.header.flags;
1140                    drop(frame);
1141                    return self
1142                        .refuse_oversized_module_frame(ctx, &route, (ty, flags), corr, bytes)
1143                        .await;
1144                }
1145
1146                // A terminal frame ends the request at the module whether or not
1147                // the client can still take it, so its credit is released before
1148                // delivery is attempted. Releasing only after a successful
1149                // enqueue would leave a drain counting a finished request until
1150                // the client connection's cleanup removes the route.
1151                let releases_credit = is_terminal_frame(frame.header.ty);
1152                if releases_credit {
1153                    route.flow.release_corr(corr);
1154                }
1155                let mut frame = frame;
1156                frame.header.channel = route.client_channel;
1157                frame.header.epoch = route.client_epoch;
1158                // Responses, errors, stream data and ends, and reverse
1159                // requests all pass through here. A module must never be able
1160                // to make a frame look daemon-authored, so the origin bit a
1161                // module may have set is cleared; every other flag bit and
1162                // the body travel unchanged.
1163                frame.header.flags = frame.header.flags.without_daemon_origin();
1164                if let Err(err) = route.client_sink.send_to_client(frame).await {
1165                    if self
1166                        .forwarding
1167                        .escalate_client_delivery_failure_diagnosed(
1168                            route.client_connection_id,
1169                            route.client_channel,
1170                            route.client_epoch,
1171                            CloseReason::new(
1172                                "module_to_client_delivery_failed",
1173                                format!(
1174                                    "failed to enqueue module frame for client channel {} corr {corr}: {err}",
1175                                    route.client_channel
1176                                ),
1177                            ),
1178                            UndeliveredFrame {
1179                                module_id: Some(&route.module_id),
1180                                sink: &route.client_sink,
1181                            },
1182                            Some(&err.diagnosis),
1183                        )
1184                        .map_err(RouterError::Forwarding)?
1185                    {
1186                        self.counters
1187                            .increment_client_egress_close_delivery_failed();
1188                    }
1189                    return Ok(());
1190                }
1191                return Ok(());
1192            }
1193            DataRoute::Client(DataRouteState::EpochMismatch) => {
1194                self.forwarding.warn_frame_drop(
1195                    ctx.connection_id,
1196                    channel,
1197                    epoch,
1198                    "client_to_module",
1199                    "stale_epoch",
1200                );
1201                if frame.header.ty == FrameType::Request {
1202                    self.counters.increment_client_frames_dropped_stale_route();
1203                    // Dropped before forwarding; a re-bind retry cannot double-execute this request.
1204                    let err = RouterError::StaleRouteEpoch {
1205                        channel,
1206                        epoch,
1207                        corr,
1208                    };
1209                    if let Some(error_frame) = err.to_error_frame() {
1210                        ctx.egress.send(error_frame).await?;
1211                    }
1212                }
1213                debug!(
1214                    connection_id = ctx.connection_id.get(),
1215                    channel, epoch, corr, "dropping client frame for stale route epoch"
1216                );
1217                return Ok(());
1218            }
1219            DataRoute::Client(DataRouteState::Reserved) => {
1220                self.forwarding.warn_frame_drop(
1221                    ctx.connection_id,
1222                    channel,
1223                    epoch,
1224                    "client_to_module",
1225                    "missing_binding",
1226                );
1227                if frame.header.ty == FrameType::Request {
1228                    let err = RouterError::UnknownChannel {
1229                        channel,
1230                        epoch,
1231                        corr,
1232                    };
1233                    if let Some(error_frame) = err.to_error_frame() {
1234                        ctx.egress.send(error_frame).await?;
1235                    }
1236                }
1237                return Ok(());
1238            }
1239            DataRoute::Client(DataRouteState::Bound(route)) => {
1240                if frame.header.ty == FrameType::Goodbye {
1241                    let _ = self
1242                        .control
1243                        .handle_route_goodbye(ctx.connection_id, channel, epoch)?;
1244                    return Ok(());
1245                }
1246                return self.forward_backend.handle_bound(frame, route).await;
1247            }
1248            DataRoute::Client(DataRouteState::Absent) => {}
1249        }
1250
1251        if let Some(backend) = self.backends.get(&channel) {
1252            return backend.handle(ctx.clone(), frame).await.inspect_err(|_| {
1253                self.forwarding.warn_frame_drop(
1254                    ctx.connection_id,
1255                    channel,
1256                    epoch,
1257                    "client_to_backend",
1258                    "endpoint_gone",
1259                );
1260            });
1261        }
1262        self.forwarding.warn_frame_drop(
1263            ctx.connection_id,
1264            channel,
1265            epoch,
1266            "client_to_module",
1267            "unknown_channel",
1268        );
1269        if frame.header.ty == FrameType::Request {
1270            let err = RouterError::UnknownChannel {
1271                channel,
1272                epoch,
1273                corr,
1274            };
1275            if let Some(error_frame) = err.to_error_frame() {
1276                ctx.egress.send(error_frame).await?;
1277            }
1278        }
1279        Ok(())
1280    }
1281
1282    async fn refuse_oversized_module_frame(
1283        &self,
1284        ctx: &RouteCtx,
1285        route: &RouteBinding,
1286        header: (FrameType, Flags),
1287        corr: u64,
1288        bytes: usize,
1289    ) -> Result<(), RouterError> {
1290        let (ty, flags) = header;
1291        let limit = route.client_sink.accounting.byte_budget;
1292        if ty == FrameType::Push {
1293            // Deployed Rust clients cannot decode a new route.closed reason.
1294            // Until that vocabulary can be extended compatibly, an oversized
1295            // unsolicited push still closes the connection, with its cause named.
1296            let diagnosis = ClientDeliveryDiagnosis {
1297                frame_type: ty,
1298                flags,
1299                cause: "frame_too_large",
1300            };
1301            if self
1302                .forwarding
1303                .escalate_client_delivery_failure_diagnosed(
1304                    route.client_connection_id,
1305                    route.client_channel,
1306                    route.client_epoch,
1307                    CloseReason::new(
1308                        error_codes::FRAME_TOO_LARGE,
1309                        format!(
1310                            "oversized push: size={bytes} limit={limit} (header plus body bytes)"
1311                        ),
1312                    ),
1313                    UndeliveredFrame {
1314                        module_id: Some(&route.module_id),
1315                        sink: &route.client_sink,
1316                    },
1317                    Some(&diagnosis),
1318                )
1319                .map_err(RouterError::Forwarding)?
1320            {
1321                self.counters
1322                    .increment_client_egress_close_delivery_failed();
1323            }
1324            return Ok(());
1325        }
1326
1327        warn!(
1328            connection_id = route.client_connection_id.get(),
1329            module_id = %route.module_id,
1330            client_principal = %principal_label(&route.principal),
1331            client_channel = route.client_channel,
1332            frame_type = ?ty,
1333            frame_bytes = bytes,
1334            byte_limit = limit,
1335            "refusing oversized module-to-client frame with ERROR frame_too_large"
1336        );
1337        if ty == FrameType::Request {
1338            // A reverse request belongs to the module, not to a waiting client.
1339            // Answer its sender without forwarding any part of it to the client.
1340            ctx.egress
1341                .send(frame_too_large_error(
1342                    route.module_channel,
1343                    route.module_epoch,
1344                    corr,
1345                    bytes,
1346                    limit,
1347                )?)
1348                .await?;
1349        } else {
1350            // A refused stream chunk also ends the waiting request. Releasing
1351            // its credit prevents a finished request from keeping a drain alive.
1352            route.flow.release_corr(corr);
1353            let error = frame_too_large_error(
1354                route.client_channel,
1355                route.client_epoch,
1356                corr,
1357                bytes,
1358                limit,
1359            )?;
1360            // The replacement shares the reply's bounded burst grace, so a
1361            // stopped client cannot indefinitely stall the module frame reader.
1362            if let Err(err) = route.client_sink.send_to_client(error).await {
1363                if self
1364                    .forwarding
1365                    .escalate_client_delivery_failure_diagnosed(
1366                        route.client_connection_id,
1367                        route.client_channel,
1368                        route.client_epoch,
1369                        CloseReason::new(
1370                            "module_to_client_delivery_failed",
1371                            format!(
1372                                "failed to enqueue frame_too_large for client channel {} corr {corr}: {err}",
1373                                route.client_channel
1374                            ),
1375                        ),
1376                        UndeliveredFrame {
1377                            module_id: Some(&route.module_id),
1378                            sink: &route.client_sink,
1379                        },
1380                        Some(&err.diagnosis),
1381                    )
1382                    .map_err(RouterError::Forwarding)?
1383                {
1384                    self.counters
1385                        .increment_client_egress_close_delivery_failed();
1386                }
1387            }
1388        }
1389        Ok(())
1390    }
1391}
1392
1393impl Default for Router {
1394    fn default() -> Self {
1395        Self::with_default_self_handler()
1396    }
1397}
1398
1399/// Connection-scoped cleanup guard returned by [`Router::begin_connection`].
1400#[must_use]
1401pub struct RouterConnection {
1402    id: ConnectionId,
1403    control_handler: Arc<ControlHandler>,
1404    forwarding: Arc<ForwardingTable>,
1405    close_receiver: Option<ConnectionCloseReceiver>,
1406    orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
1407}
1408
1409impl RouterConnection {
1410    pub fn id(&self) -> ConnectionId {
1411        self.id
1412    }
1413
1414    pub(crate) fn take_close_receiver(&mut self) -> ConnectionCloseReceiver {
1415        self.close_receiver
1416            .take()
1417            .expect("connection close receiver can only be taken once")
1418    }
1419}
1420
1421impl Drop for RouterConnection {
1422    fn drop(&mut self) {
1423        self.forwarding.unregister_connection_close(self.id);
1424        self.orphan_goodbyes.forget_connection(self.id);
1425        // GOODBYE (explicit) and connection-drop cleanup both call the same
1426        // idempotent deregistration path.
1427        let _ = self.control_handler.cleanup_connection(self.id);
1428    }
1429}
1430
1431/// Minimal in-memory backend used by tests and early wiring: it emits a
1432/// `RESPONSE` on the same channel/correlation id with the exact same body bytes.
1433#[derive(Debug, Default, Clone, Copy)]
1434pub struct EchoBackend;
1435
1436impl EchoBackend {
1437    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1438        let response = Frame::build_with_version(
1439            frame.header.ver,
1440            FrameType::Response,
1441            frame.header.flags,
1442            frame.header.channel,
1443            frame.header.epoch,
1444            frame.header.corr,
1445            frame.body,
1446        )
1447        .map_err(RouterError::FrameBuild)?;
1448        ctx.egress.send(response).await
1449    }
1450}
1451
1452/// Data-plane backend that splices client frames to the module connection bound at attach time.
1453#[derive(Debug, Clone)]
1454pub struct ForwardBackend {
1455    forwarding: Arc<ForwardingTable>,
1456}
1457
1458impl ForwardBackend {
1459    pub fn new(forwarding: Arc<ForwardingTable>) -> Self {
1460        Self { forwarding }
1461    }
1462
1463    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1464        let channel = frame.header.channel;
1465        let corr = frame.header.corr;
1466        let route = match self
1467            .forwarding
1468            .lookup_data_route(ctx.connection_id, channel, frame.header.epoch)
1469            .map_err(RouterError::Forwarding)?
1470        {
1471            DataRoute::Client(DataRouteState::Bound(route)) => route,
1472            DataRoute::Client(_) | DataRoute::Module(_) => {
1473                self.forwarding.warn_frame_drop(
1474                    ctx.connection_id,
1475                    channel,
1476                    frame.header.epoch,
1477                    "client_to_module",
1478                    "missing_binding",
1479                );
1480                return Err(RouterError::UnknownChannel {
1481                    channel,
1482                    epoch: frame.header.epoch,
1483                    corr,
1484                });
1485            }
1486        };
1487        self.handle_bound(frame, route).await
1488    }
1489
1490    pub(crate) async fn handle_bound(
1491        &self,
1492        frame: Frame,
1493        route: Arc<RouteBinding>,
1494    ) -> Result<(), RouterError> {
1495        let channel = frame.header.channel;
1496        let corr = frame.header.corr;
1497        let frame_type = frame.header.ty;
1498
1499        // CANCEL and other non-REQUEST frames bypass the request-credit window;
1500        // the original request's credit is freed only by the module's terminal frame.
1501        let acquired_credit = frame_type == FrameType::Request;
1502        if acquired_credit {
1503            if let Err(err) = route
1504                .flow
1505                .acquire_tagged(corr, frame.header.flags.is_subscription())
1506                .await
1507            {
1508                self.forwarding.warn_frame_drop(
1509                    route.client_connection_id,
1510                    channel,
1511                    frame.header.epoch,
1512                    "client_to_module",
1513                    "endpoint_gone",
1514                );
1515                // `module_reloading` here is answered BEFORE the frame is
1516                // forwarded, so the module never sees the request and callers
1517                // may re-dispatch it after reopening the route. That is a wire
1518                // guarantee documented on `error_codes::MODULE_RELOADING`; do
1519                // not emit this code for a request that already reached
1520                // `module_sink.send` below.
1521                if self
1522                    .forwarding
1523                    .endpoint_is_draining(route.module_endpoint)
1524                    .map_err(RouterError::Forwarding)?
1525                {
1526                    return Err(RouterError::route_error_with_epoch(
1527                        channel,
1528                        frame.header.epoch,
1529                        corr,
1530                        "module_reloading",
1531                        format!("module endpoint for route channel {channel} is reloading"),
1532                    ));
1533                }
1534                return Err(RouterError::backend_with_epoch(
1535                    channel,
1536                    frame.header.epoch,
1537                    corr,
1538                    format!("{err} for route channel {channel}"),
1539                ));
1540            }
1541        }
1542
1543        let mut frame = frame;
1544        frame.header.channel = route.module_channel;
1545        frame.header.epoch = route.module_epoch;
1546        // The origin bit is the daemon's own signal to clients. A client
1547        // frame carrying it must not reach a module appearing daemon-authored,
1548        // so it is cleared here; every other flag bit travels unchanged.
1549        frame.header.flags = frame.header.flags.without_daemon_origin();
1550        let result = route.module_sink.send(frame).await.map_err(|err| {
1551            self.forwarding.warn_frame_drop(
1552                route.client_connection_id,
1553                channel,
1554                route.client_epoch,
1555                "client_to_module",
1556                "endpoint_gone",
1557            );
1558            RouterError::backend_with_epoch(channel, route.client_epoch, corr, err.to_string())
1559        });
1560        if acquired_credit && result.is_err() {
1561            route.flow.release_corr(corr);
1562        }
1563        result
1564    }
1565}
1566
1567fn is_terminal_frame(frame_type: FrameType) -> bool {
1568    matches!(
1569        frame_type,
1570        FrameType::Response | FrameType::Error | FrameType::StreamEnd
1571    )
1572}
1573
1574/// Typed router errors. Routable failures can be translated to canonical JSON
1575/// `ERROR` frames with [`RouterError::to_error_frame`].
1576#[derive(Debug, Clone, PartialEq, Eq)]
1577pub enum RouterError {
1578    ReservedChannelZero,
1579    DuplicateChannel {
1580        channel: u16,
1581    },
1582    UnknownChannel {
1583        channel: u16,
1584        epoch: u32,
1585        corr: u64,
1586    },
1587    StaleRouteEpoch {
1588        channel: u16,
1589        epoch: u32,
1590        corr: u64,
1591    },
1592    Backend {
1593        channel: u16,
1594        epoch: u32,
1595        corr: u64,
1596        message: String,
1597    },
1598    RouteError {
1599        channel: u16,
1600        epoch: u32,
1601        corr: u64,
1602        code: String,
1603        message: String,
1604    },
1605    FrameBuild(FrameBuildError),
1606    Forwarding(ForwardingError),
1607}
1608
1609impl RouterError {
1610    pub fn backend(channel: u16, corr: u64, message: impl Into<String>) -> Self {
1611        Self::backend_with_epoch(channel, 0, corr, message)
1612    }
1613
1614    pub fn backend_with_epoch(
1615        channel: u16,
1616        epoch: u32,
1617        corr: u64,
1618        message: impl Into<String>,
1619    ) -> Self {
1620        Self::Backend {
1621            channel,
1622            epoch,
1623            corr,
1624            message: message.into(),
1625        }
1626    }
1627
1628    pub fn route_error(
1629        channel: u16,
1630        corr: u64,
1631        code: impl Into<String>,
1632        message: impl Into<String>,
1633    ) -> Self {
1634        Self::route_error_with_epoch(channel, 0, corr, code, message)
1635    }
1636
1637    pub fn route_error_with_epoch(
1638        channel: u16,
1639        epoch: u32,
1640        corr: u64,
1641        code: impl Into<String>,
1642        message: impl Into<String>,
1643    ) -> Self {
1644        Self::RouteError {
1645            channel,
1646            epoch,
1647            corr,
1648            code: code.into(),
1649            message: message.into(),
1650        }
1651    }
1652
1653    /// Translate route failures that belong on the wire into an `ERROR` frame.
1654    pub fn to_error_frame(&self) -> Option<Frame> {
1655        match self {
1656            Self::UnknownChannel {
1657                channel,
1658                epoch,
1659                corr,
1660            } => error_frame(
1661                *channel,
1662                *epoch,
1663                *corr,
1664                error_codes::UNKNOWN_CHANNEL,
1665                format!("unknown channel {channel}"),
1666            ),
1667            Self::StaleRouteEpoch {
1668                channel,
1669                epoch,
1670                corr,
1671            } => error_frame(
1672                *channel,
1673                *epoch,
1674                *corr,
1675                error_codes::STALE_ROUTE_EPOCH,
1676                format!("stale route epoch for channel {channel}"),
1677            ),
1678            Self::Backend {
1679                channel,
1680                epoch,
1681                corr,
1682                message,
1683            } => error_frame(*channel, *epoch, *corr, "backend_error", message.clone()),
1684            Self::RouteError {
1685                channel,
1686                epoch,
1687                corr,
1688                code,
1689                message,
1690            } => error_frame(*channel, *epoch, *corr, code, message.clone()),
1691            Self::ReservedChannelZero
1692            | Self::DuplicateChannel { .. }
1693            | Self::FrameBuild(_)
1694            | Self::Forwarding(_) => None,
1695        }
1696    }
1697}
1698
1699/// Flags for an ERROR frame the daemon authors itself.
1700///
1701/// Bit 6 (`FLAG_DAEMON_ORIGIN`) tells the client the daemon wrote this
1702/// error, never a module. What a code then proves is per code: on a bound
1703/// route the daemon writes `module_reloading`, `unknown_channel` and
1704/// `stale_route_epoch` only before forwarding, so with the bit they mean the
1705/// request was not delivered, while `frame_too_large` replaces a reply the
1706/// module already produced. Every daemon-authored ERROR goes through here;
1707/// frames a module sent keep their own flags with the bit cleared at the
1708/// splice.
1709/// Nothing else about the flags differs: passive priority, not binary, not last.
1710pub(crate) fn daemon_error_flags() -> Flags {
1711    Flags::new(false, Priority::Passive, false).with_daemon_origin()
1712}
1713
1714fn error_frame(channel: u16, epoch: u32, corr: u64, code: &str, message: String) -> Option<Frame> {
1715    let body = serde_json::to_vec(&ErrorBody {
1716        code: code.to_string(),
1717        message,
1718        detail: None,
1719    })
1720    .ok()?;
1721
1722    Frame::build(
1723        FrameType::Error,
1724        daemon_error_flags(),
1725        channel,
1726        epoch,
1727        corr,
1728        body,
1729    )
1730    .ok()
1731}
1732
1733fn frame_too_large_error(
1734    channel: u16,
1735    epoch: u32,
1736    corr: u64,
1737    bytes: usize,
1738    limit: usize,
1739) -> Result<Frame, RouterError> {
1740    let body = serde_json::to_vec(&ErrorBody {
1741        code: error_codes::FRAME_TOO_LARGE.to_string(),
1742        message: format!(
1743            "module frame of {bytes} bytes exceeds client egress limit of {limit} bytes"
1744        ),
1745        detail: Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit })),
1746    })
1747    .expect("serializing a frame size error");
1748    Frame::build(
1749        FrameType::Error,
1750        daemon_error_flags(),
1751        channel,
1752        epoch,
1753        corr,
1754        body,
1755    )
1756    .map_err(RouterError::FrameBuild)
1757}
1758
1759impl fmt::Display for RouterError {
1760    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1761        match self {
1762            Self::ReservedChannelZero => write!(f, "channel 0 is reserved for subc"),
1763            Self::DuplicateChannel { channel } => {
1764                write!(f, "backend already registered for channel {channel}")
1765            }
1766            Self::UnknownChannel { channel, corr, .. } => {
1767                write!(f, "unknown channel {channel} for corr {corr}")
1768            }
1769            Self::StaleRouteEpoch { channel, corr, .. } => {
1770                write!(f, "stale route epoch for channel {channel} corr {corr}")
1771            }
1772            Self::Backend {
1773                channel,
1774                corr,
1775                message,
1776                ..
1777            } => write!(
1778                f,
1779                "backend error on channel {channel} corr {corr}: {message}"
1780            ),
1781            Self::RouteError {
1782                channel,
1783                corr,
1784                code,
1785                message,
1786                ..
1787            } => write!(
1788                f,
1789                "route error {code} on channel {channel} corr {corr}: {message}"
1790            ),
1791            Self::FrameBuild(err) => write!(f, "failed to build routed frame: {err}"),
1792            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
1793        }
1794    }
1795}
1796
1797impl Error for RouterError {
1798    fn source(&self) -> Option<&(dyn Error + 'static)> {
1799        match self {
1800            Self::FrameBuild(err) => Some(err),
1801            Self::Forwarding(err) => Some(err),
1802            Self::ReservedChannelZero
1803            | Self::DuplicateChannel { .. }
1804            | Self::UnknownChannel { .. }
1805            | Self::StaleRouteEpoch { .. }
1806            | Self::Backend { .. }
1807            | Self::RouteError { .. } => None,
1808        }
1809    }
1810}
1811
1812#[cfg(test)]
1813mod tests {
1814    use super::*;
1815    use crate::{
1816        forwarding::RouteBindRelayOutcome,
1817        supervise::{ModuleSpec, RestartPolicy, Supervisor, SupervisorHandle},
1818        ControlHandler, Registry,
1819    };
1820    use std::{
1821        sync::{mpsc as std_mpsc, Arc},
1822        time::Duration,
1823    };
1824    use subc_control::ModuleProtocol;
1825    use subc_protocol::{manifest::Concurrency, ErrorBody, Flags, FrameType, Priority};
1826    use tokio::sync::mpsc;
1827
1828    pub(crate) use crate::router::test_log::{captured_logs, log_capture};
1829
1830    fn logged_millis(logs: &str, field: &str) -> u64 {
1831        logs.split_whitespace()
1832            .find_map(|part| part.strip_prefix(field))
1833            .and_then(|value| value.parse().ok())
1834            .unwrap_or_else(|| panic!("missing numeric {field} in logs: {logs}"))
1835    }
1836
1837    fn request(channel: u16, corr: u64, body: &[u8]) -> Frame {
1838        Frame::build(
1839            FrameType::Request,
1840            Flags::new(true, Priority::Interactive, false),
1841            channel,
1842            0,
1843            corr,
1844            body.to_vec(),
1845        )
1846        .unwrap()
1847    }
1848
1849    fn ping(corr: u64) -> Frame {
1850        Frame::build(
1851            FrameType::Ping,
1852            Flags::new(false, Priority::Passive, false),
1853            0,
1854            0,
1855            corr,
1856            Vec::new(),
1857        )
1858        .unwrap()
1859    }
1860
1861    fn route_ctx() -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
1862        let (tx, rx) = mpsc::channel(8);
1863        (
1864            RouteCtx {
1865                connection_id: ConnectionId::LOCAL,
1866                egress: FrameSink::new(tx),
1867            },
1868            rx,
1869        )
1870    }
1871
1872    #[tokio::test]
1873    async fn echo_backend_returns_response_with_byte_identical_body() {
1874        let mut router = Router::with_default_self_handler();
1875        router.register_backend(7, EchoBackend).unwrap();
1876        let (ctx, mut rx) = route_ctx();
1877        let body = b"{not parsed}\0\xff";
1878
1879        router
1880            .route_for_connection(&ctx, request(7, 123, body))
1881            .await
1882            .unwrap();
1883        let response = rx.recv().await.unwrap();
1884
1885        assert_eq!(response.header.ty, FrameType::Response);
1886        assert_eq!(response.header.channel, 7);
1887        assert_eq!(response.header.corr, 123);
1888        assert_eq!(response.body, body);
1889        assert!(rx.try_recv().is_err());
1890    }
1891
1892    #[tokio::test]
1893    async fn unknown_channel_emits_canonical_error_frame() {
1894        let router = Router::with_default_self_handler();
1895        let (ctx, mut rx) = route_ctx();
1896
1897        router
1898            .route_for_connection(&ctx, request(99, 5, b"payload"))
1899            .await
1900            .unwrap();
1901        let error_frame = rx.recv().await.unwrap();
1902
1903        assert_eq!(error_frame.header.ty, FrameType::Error);
1904        assert_eq!(error_frame.header.channel, 99);
1905        assert_eq!(error_frame.header.corr, 5);
1906        let body: ErrorBody = serde_json::from_slice(&error_frame.body).unwrap();
1907        assert_eq!(body.code, "unknown_channel");
1908        assert_eq!(body.message, "unknown channel 99");
1909    }
1910
1911    /// Every ERROR the daemon authors carries the daemon-origin bit, so a
1912    /// client can tell a daemon refusal from a module's reply. Both the
1913    /// unknown-channel and the stale-epoch errors below are received on the
1914    /// client's own egress, not merely built.
1915    #[tokio::test]
1916    async fn daemon_authored_data_plane_errors_carry_daemon_origin() {
1917        let router = Router::with_default_self_handler();
1918        let (ctx, mut rx) = route_ctx();
1919        router
1920            .route_for_connection(&ctx, request(99, 5, b"payload"))
1921            .await
1922            .unwrap();
1923        let unknown = rx.recv().await.unwrap();
1924        assert_eq!(unknown.header.ty, FrameType::Error);
1925        let body: ErrorBody = serde_json::from_slice(&unknown.body).unwrap();
1926        assert_eq!(body.code, "unknown_channel");
1927        assert!(
1928            unknown.header.flags.is_daemon_origin(),
1929            "unknown_channel must carry daemon origin: {:?}",
1930            unknown.header.flags
1931        );
1932
1933        let (
1934            router,
1935            _forwarding,
1936            client_ctx,
1937            mut client_rx,
1938            _module_ctx,
1939            _module_egress_rx,
1940            _module_rx,
1941            pending,
1942        ) = dynamic_route_fixture(true);
1943        let _ = client_rx.recv().await;
1944        router
1945            .route_for_connection(
1946                &client_ctx,
1947                route_frame(
1948                    FrameType::Request,
1949                    pending.client_channel,
1950                    pending.client_epoch + 1,
1951                    702,
1952                ),
1953            )
1954            .await
1955            .unwrap();
1956        let stale = client_rx.recv().await.unwrap();
1957        assert_eq!(stale.header.ty, FrameType::Error);
1958        let body: ErrorBody = serde_json::from_slice(&stale.body).unwrap();
1959        assert_eq!(body.code, "stale_route_epoch");
1960        assert!(
1961            stale.header.flags.is_daemon_origin(),
1962            "stale_route_epoch must carry daemon origin: {:?}",
1963            stale.header.flags
1964        );
1965    }
1966
1967    /// A module ERROR that spoofs a daemon code with the daemon-origin bit
1968    /// set must reach the client with the bit cleared and the body
1969    /// byte-identical: the bit means the daemon wrote the frame, and a module
1970    /// must never be able to claim that.
1971    #[tokio::test]
1972    async fn module_error_with_daemon_bit_is_stripped_before_client_delivery() {
1973        let (
1974            router,
1975            _forwarding,
1976            client_ctx,
1977            mut client_rx,
1978            module_ctx,
1979            _module_egress_rx,
1980            mut module_rx,
1981            pending,
1982        ) = dynamic_route_fixture(true);
1983        let _ = client_rx.recv().await;
1984        // Open flow credit the way a real request would, so the terminal
1985        // error exercises the credit-release path too.
1986        router
1987            .route_for_connection(
1988                &client_ctx,
1989                route_frame(
1990                    FrameType::Request,
1991                    pending.client_channel,
1992                    pending.client_epoch,
1993                    701,
1994                ),
1995            )
1996            .await
1997            .unwrap();
1998        let _ = module_rx.recv().await;
1999
2000        let body = serde_json::to_vec(&ErrorBody::new("module_reloading", "try again")).unwrap();
2001        let spoof = Frame::build(
2002            FrameType::Error,
2003            Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2004            pending.module_channel,
2005            pending.module_epoch,
2006            701,
2007            body.clone(),
2008        )
2009        .unwrap();
2010        router
2011            .route_for_connection(&module_ctx, spoof)
2012            .await
2013            .unwrap();
2014        let delivered = client_rx.recv().await.unwrap();
2015        assert_eq!(delivered.header.ty, FrameType::Error);
2016        assert_eq!(delivered.header.channel, pending.client_channel);
2017        assert_eq!(delivered.header.epoch, pending.client_epoch);
2018        assert_eq!(delivered.header.corr, 701);
2019        assert!(
2020            !delivered.header.flags.is_daemon_origin(),
2021            "module-set daemon origin must be cleared: {:?}",
2022            delivered.header.flags
2023        );
2024        assert_eq!(delivered.body, body, "the body travels byte-identical");
2025    }
2026
2027    /// Clearing the origin bit strips exactly that bit: priority, LAST and
2028    /// subscription survive the splice unchanged.
2029    #[tokio::test]
2030    async fn module_response_with_daemon_bit_is_stripped_preserving_other_flags() {
2031        use subc_protocol::FLAG_SUBSCRIPTION;
2032        let (
2033            router,
2034            _forwarding,
2035            client_ctx,
2036            mut client_rx,
2037            module_ctx,
2038            _module_egress_rx,
2039            mut module_rx,
2040            pending,
2041        ) = dynamic_route_fixture(true);
2042        let _ = client_rx.recv().await;
2043        router
2044            .route_for_connection(
2045                &client_ctx,
2046                route_frame(
2047                    FrameType::Request,
2048                    pending.client_channel,
2049                    pending.client_epoch,
2050                    701,
2051                ),
2052            )
2053            .await
2054            .unwrap();
2055        let _ = module_rx.recv().await;
2056
2057        let sent_flags = Flags(
2058            Flags::new(true, Priority::Background, true).0
2059                | FLAG_SUBSCRIPTION
2060                | subc_protocol::FLAG_DAEMON_ORIGIN,
2061        );
2062        assert!(sent_flags.is_daemon_origin());
2063        let response = Frame::build(
2064            FrameType::Response,
2065            sent_flags,
2066            pending.module_channel,
2067            pending.module_epoch,
2068            701,
2069            b"route-body".to_vec(),
2070        )
2071        .unwrap();
2072        router
2073            .route_for_connection(&module_ctx, response)
2074            .await
2075            .unwrap();
2076        let delivered = client_rx.recv().await.unwrap();
2077        assert_eq!(delivered.header.ty, FrameType::Response);
2078        assert!(
2079            !delivered.header.flags.is_daemon_origin(),
2080            "module-set daemon origin must be cleared: {:?}",
2081            delivered.header.flags
2082        );
2083        assert!(
2084            delivered.header.flags.is_binary(),
2085            "binary flag must survive: {:?}",
2086            delivered.header.flags
2087        );
2088        assert!(
2089            delivered.header.flags.is_last(),
2090            "LAST flag must survive: {:?}",
2091            delivered.header.flags
2092        );
2093        assert!(
2094            delivered.header.flags.is_subscription(),
2095            "subscription flag must survive: {:?}",
2096            delivered.header.flags
2097        );
2098        assert_eq!(
2099            delivered.header.flags.priority(),
2100            Some(Priority::Background),
2101            "priority must survive"
2102        );
2103    }
2104
2105    /// A client frame carrying the origin bit must not reach the module
2106    /// looking daemon-authored either: the splice clears it symmetrically.
2107    #[tokio::test]
2108    async fn client_frame_with_daemon_bit_is_stripped_before_module_delivery() {
2109        let (
2110            router,
2111            _forwarding,
2112            client_ctx,
2113            mut client_rx,
2114            _module_ctx,
2115            _module_egress_rx,
2116            mut module_rx,
2117            pending,
2118        ) = dynamic_route_fixture(true);
2119        let _ = client_rx.recv().await;
2120        let spoof = Frame::build(
2121            FrameType::Request,
2122            Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2123            pending.client_channel,
2124            pending.client_epoch,
2125            701,
2126            b"route-body".to_vec(),
2127        )
2128        .unwrap();
2129        router
2130            .route_for_connection(&client_ctx, spoof)
2131            .await
2132            .unwrap();
2133        let delivered = module_rx.recv().await.unwrap();
2134        assert_eq!(delivered.header.ty, FrameType::Request);
2135        assert!(
2136            !delivered.header.flags.is_daemon_origin(),
2137            "client-set daemon origin must be cleared: {:?}",
2138            delivered.header.flags
2139        );
2140    }
2141
2142    #[tokio::test]
2143    async fn channel_zero_uses_control_handler_not_backend_registry() {
2144        let mut router = Router::with_default_self_handler();
2145        router.register_backend(1, EchoBackend).unwrap();
2146        let (ctx, mut rx) = route_ctx();
2147
2148        router.route_for_connection(&ctx, ping(77)).await.unwrap();
2149        let response = rx.recv().await.unwrap();
2150
2151        assert_eq!(response.header.ty, FrameType::Pong);
2152        assert_eq!(response.header.channel, 0);
2153        assert_eq!(response.header.corr, 77);
2154        assert!(response.body.is_empty());
2155    }
2156
2157    #[tokio::test]
2158    async fn slow_control_dispatch_logs_decoded_op_and_elapsed_time() {
2159        let control = Arc::new(
2160            ControlHandler::new(Arc::new(Registry::default()))
2161                .with_control_dispatch_delay(Duration::from_millis(1050)),
2162        );
2163        let router = Router::with_control_handler(control);
2164        let (ctx, mut rx) = route_ctx();
2165        let (output, guard) = log_capture(tracing::Level::WARN);
2166
2167        router
2168            .route_for_connection(&ctx, request(0, 41, br#"{"op":"server.describe"}"#))
2169            .await
2170            .expect("slow request routes");
2171        assert!(rx.recv().await.is_some(), "request receives a response");
2172        drop(guard);
2173
2174        let logs = captured_logs(&output);
2175        assert!(logs.contains("slow control dispatch"));
2176        assert!(logs.contains("op=server.describe"));
2177        assert!(logs.contains("connection_id=0"));
2178        assert!(logs.contains("corr=41"));
2179        assert!(
2180            logged_millis(&logs, "elapsed_ms=") >= 1050,
2181            "elapsed must include the injected handler delay: {logs}"
2182        );
2183    }
2184
2185    #[tokio::test]
2186    async fn fast_control_dispatch_emits_arrival_without_slow_warning() {
2187        let router = Router::with_default_self_handler();
2188        let (ctx, mut rx) = route_ctx();
2189        let (output, guard) = log_capture(tracing::Level::DEBUG);
2190
2191        router
2192            .route_for_connection(&ctx, request(0, 42, br#"{"op":"server.describe"}"#))
2193            .await
2194            .expect("fast request routes");
2195        assert!(rx.recv().await.is_some(), "request receives a response");
2196        drop(guard);
2197
2198        let logs = captured_logs(&output);
2199        assert!(logs.contains("control dispatch op=server.describe connection_id=0 corr=42"));
2200        assert!(!logs.contains("slow control dispatch"));
2201    }
2202
2203    #[tokio::test]
2204    async fn control_dispatch_arrival_is_hidden_at_info() {
2205        let router = Router::with_default_self_handler();
2206        let (ctx, mut rx) = route_ctx();
2207        let (output, guard) = log_capture(tracing::Level::INFO);
2208
2209        router
2210            .route_for_connection(&ctx, request(0, 43, br#"{"op":"server.describe"}"#))
2211            .await
2212            .expect("fast request routes");
2213        assert!(rx.recv().await.is_some(), "request receives a response");
2214        drop(guard);
2215
2216        assert!(
2217            !captured_logs(&output).contains("control dispatch"),
2218            "arrival logging must stay hidden at INFO"
2219        );
2220    }
2221
2222    #[tokio::test]
2223    async fn supervisor_list_logs_contended_snapshot_lock_only() {
2224        let registry = Arc::new(Registry::default());
2225        let handle = SupervisorHandle::new();
2226        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
2227            .with_handle(handle.clone());
2228        let module = supervisor
2229            .supervise_configured(
2230                ModuleSpec {
2231                    module_id: "held-module".to_string(),
2232                    program: "test-module".into(),
2233                    args: Vec::new(),
2234                    env: Vec::new(),
2235                    reserved: false,
2236                    reserved_prefixes: Vec::new(),
2237                    protocol: ModuleProtocol::Subc,
2238                    overlap: Default::default(),
2239                },
2240                false,
2241            )
2242            .expect("disabled test module is supervised");
2243        let router = Router::with_control_handler(Arc::new(
2244            ControlHandler::new(Arc::clone(&registry)).with_supervisor(handle),
2245        ));
2246        let (ctx, mut rx) = route_ctx();
2247        let (acquired, ready) = std_mpsc::channel();
2248        let holder = module.hold_snapshot_for_test(acquired, Duration::from_millis(400));
2249        ready.recv().expect("holder acquired snapshot lock");
2250        let (output, guard) = log_capture(tracing::Level::WARN);
2251
2252        router
2253            .route_for_connection(&ctx, request(0, 44, br#"{"op":"supervisor.list"}"#))
2254            .await
2255            .expect("list request routes after the lock releases");
2256        assert!(
2257            rx.recv().await.is_some(),
2258            "list request receives a response"
2259        );
2260        holder.join().expect("snapshot holder exits cleanly");
2261        drop(guard);
2262
2263        let logs = captured_logs(&output);
2264        assert!(logs.contains("slow snapshot lock"));
2265        assert!(logs.contains("module_id=held-module"));
2266        assert!(logs.contains("caller=list"));
2267        assert!(
2268            logged_millis(&logs, "waited_ms=") >= 250,
2269            "wait must exceed the slow-lock threshold: {logs}"
2270        );
2271
2272        let (output, guard) = log_capture(tracing::Level::WARN);
2273        router
2274            .route_for_connection(&ctx, request(0, 45, br#"{"op":"supervisor.list"}"#))
2275            .await
2276            .expect("uncontended list request routes");
2277        assert!(
2278            rx.recv().await.is_some(),
2279            "uncontended list receives a response"
2280        );
2281        drop(guard);
2282        assert!(
2283            !captured_logs(&output).contains("slow snapshot lock"),
2284            "uncontended list acquisition must not warn"
2285        );
2286    }
2287
2288    #[tokio::test]
2289    async fn full_module_to_client_sink_requests_client_close_without_erroring_module() {
2290        let forwarding = Arc::new(ForwardingTable::default());
2291        let control = Arc::new(ControlHandler::with_forwarding(
2292            Arc::new(crate::Registry::default()),
2293            Arc::clone(&forwarding),
2294        ));
2295        let router = Router::with_control_handler(control);
2296        let module_connection = ConnectionId::new(10);
2297        let client_connection = ConnectionId::new(20);
2298        let mut close_receiver = forwarding.register_connection_close(client_connection);
2299        let (module_tx, _module_rx) = mpsc::channel(1);
2300        forwarding
2301            .register_module_connection(
2302                module_connection,
2303                "full-sink-provider".to_string(),
2304                1,
2305                Concurrency::ModuleManaged,
2306                FrameSink::new(module_tx),
2307            )
2308            .unwrap();
2309        let (client_tx, mut client_rx) = mpsc::channel(1);
2310        let pending = forwarding
2311            .begin_route_bind_relay_for_test(
2312                client_connection,
2313                FrameSink::new(client_tx),
2314                700,
2315                "full-sink-provider",
2316            )
2317            .unwrap();
2318        forwarding
2319            .complete_pending_relay(
2320                module_connection,
2321                pending.corr,
2322                RouteBindRelayOutcome::Accepted,
2323            )
2324            .unwrap();
2325
2326        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2327        let module_ctx = RouteCtx {
2328            connection_id: module_connection,
2329            egress: FrameSink::new(module_egress_tx),
2330        };
2331        let terminal = Frame::build(
2332            FrameType::Response,
2333            Flags::new(false, Priority::Interactive, true),
2334            pending.module_channel,
2335            pending.module_epoch,
2336            701,
2337            b"terminal".to_vec(),
2338        )
2339        .unwrap();
2340
2341        router
2342            .route_for_connection(&module_ctx, terminal)
2343            .await
2344            .unwrap();
2345        let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2346            .await
2347            .expect("close request should be sent for the full client sink")
2348            .expect("close sender should include a reason");
2349        assert!(
2350            reason
2351                .to_string()
2352                .contains("module_to_client_delivery_failed"),
2353            "unexpected close reason: {reason}"
2354        );
2355        assert_eq!(client_rx.try_recv().unwrap().header.corr, 700);
2356        assert!(client_rx.try_recv().is_err());
2357        assert_eq!(
2358            router.counters.snapshot()["client_egress_close_delivery_failed"],
2359            1
2360        );
2361    }
2362
2363    /// A terminal frame ends the request at the module even when the client
2364    /// cannot take it, so the drain must stop counting it at once rather than
2365    /// when the client connection's cleanup later removes the route.
2366    #[tokio::test]
2367    async fn terminal_frame_releases_its_credit_even_when_client_delivery_fails() {
2368        let forwarding = Arc::new(ForwardingTable::default());
2369        let control = Arc::new(ControlHandler::with_forwarding(
2370            Arc::new(crate::Registry::default()),
2371            Arc::clone(&forwarding),
2372        ));
2373        let router = Router::with_control_handler(control);
2374        let module_connection = ConnectionId::new(11);
2375        let client_connection = ConnectionId::new(21);
2376        let _close_receiver = forwarding.register_connection_close(client_connection);
2377        let (module_tx, _module_rx) = mpsc::channel(1);
2378        forwarding
2379            .register_module_connection(
2380                module_connection,
2381                "credit-provider".to_string(),
2382                1,
2383                Concurrency::ModuleManaged,
2384                FrameSink::new(module_tx),
2385            )
2386            .unwrap();
2387        // Capacity one, filled by the route.open response, so the terminal
2388        // frame below cannot be enqueued for the client.
2389        let (client_tx, _client_rx) = mpsc::channel(1);
2390        let pending = forwarding
2391            .begin_route_bind_relay_for_test(
2392                client_connection,
2393                FrameSink::new(client_tx),
2394                800,
2395                "credit-provider",
2396            )
2397            .unwrap();
2398        forwarding
2399            .complete_pending_relay(
2400                module_connection,
2401                pending.corr,
2402                RouteBindRelayOutcome::Accepted,
2403            )
2404            .unwrap();
2405        let DataRoute::Client(DataRouteState::Bound(route)) = forwarding
2406            .lookup_data_route(
2407                client_connection,
2408                pending.client_channel,
2409                pending.client_epoch,
2410            )
2411            .unwrap()
2412        else {
2413            panic!("expected a bound client route");
2414        };
2415        route.flow.acquire_tagged(801, false).await.unwrap();
2416        assert_eq!(route.flow.drain_in_flight(), 1);
2417
2418        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2419        let module_ctx = RouteCtx {
2420            connection_id: module_connection,
2421            egress: FrameSink::new(module_egress_tx),
2422        };
2423        let terminal = Frame::build(
2424            FrameType::Response,
2425            Flags::new(false, Priority::Interactive, true),
2426            pending.module_channel,
2427            pending.module_epoch,
2428            801,
2429            b"terminal".to_vec(),
2430        )
2431        .unwrap();
2432        router
2433            .route_for_connection(&module_ctx, terminal)
2434            .await
2435            .unwrap();
2436
2437        assert_eq!(
2438            router.counters.snapshot()["client_egress_close_delivery_failed"],
2439            1,
2440            "the client delivery must have failed for this test to mean anything"
2441        );
2442        assert_eq!(
2443            route.flow.drain_in_flight(),
2444            0,
2445            "the module's terminal frame must release its credit even though the client could not take it"
2446        );
2447    }
2448
2449    #[tokio::test]
2450    async fn full_route_goodbye_sink_requests_target_close_without_erroring_module() {
2451        let forwarding = Arc::new(ForwardingTable::default());
2452        let control = Arc::new(ControlHandler::with_forwarding(
2453            Arc::new(crate::Registry::default()),
2454            Arc::clone(&forwarding),
2455        ));
2456        let router = Router::with_control_handler(control);
2457        let module_connection = ConnectionId::new(30);
2458        let client_connection = ConnectionId::new(40);
2459        let mut close_receiver = forwarding.register_connection_close(client_connection);
2460        let (module_tx, _module_rx) = mpsc::channel(1);
2461        forwarding
2462            .register_module_connection(
2463                module_connection,
2464                "goodbye-full-provider".to_string(),
2465                1,
2466                Concurrency::ModuleManaged,
2467                FrameSink::new(module_tx),
2468            )
2469            .unwrap();
2470        let (client_tx, mut client_rx) = mpsc::channel(1);
2471        let pending = forwarding
2472            .begin_route_bind_relay_for_test(
2473                client_connection,
2474                FrameSink::new(client_tx),
2475                800,
2476                "goodbye-full-provider",
2477            )
2478            .unwrap();
2479        forwarding
2480            .complete_pending_relay(
2481                module_connection,
2482                pending.corr,
2483                RouteBindRelayOutcome::Accepted,
2484            )
2485            .unwrap();
2486
2487        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2488        let module_ctx = RouteCtx {
2489            connection_id: module_connection,
2490            egress: FrameSink::new(module_egress_tx),
2491        };
2492        let goodbye = Frame::build(
2493            FrameType::Goodbye,
2494            Flags::new(false, Priority::Passive, true),
2495            pending.module_channel,
2496            pending.module_epoch,
2497            801,
2498            Vec::new(),
2499        )
2500        .unwrap();
2501
2502        router
2503            .route_for_connection(&module_ctx, goodbye)
2504            .await
2505            .unwrap();
2506        let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2507            .await
2508            .expect("close request should be sent for the full GOODBYE sink")
2509            .expect("close sender should include a reason");
2510        assert!(
2511            reason.to_string().contains("route_goodbye_delivery_failed"),
2512            "unexpected close reason: {reason}"
2513        );
2514        assert_eq!(client_rx.try_recv().unwrap().header.corr, 800);
2515        assert!(client_rx.try_recv().is_err());
2516        assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 1);
2517        assert_eq!(router.counters.snapshot()["route_released_epoch_fenced"], 1);
2518    }
2519
2520    /// One module and one client connection with `routes` routes bound between
2521    /// them. The client uses a real connection egress queue (the same byte
2522    /// budget and frame-count backstop as a live connection), and its route.open
2523    /// responses are drained so the queue starts empty.
2524    async fn multi_route_client(
2525        module_id: &str,
2526        module_connection: ConnectionId,
2527        client_connection: ConnectionId,
2528        routes: usize,
2529    ) -> (
2530        Router,
2531        FrameSink,
2532        mpsc::Receiver<OutboundFrame>,
2533        RouteCtx,
2534        Vec<crate::forwarding::PendingRouteBindRelay>,
2535        ConnectionCloseReceiver,
2536    ) {
2537        let forwarding = Arc::new(ForwardingTable::default());
2538        let control = Arc::new(ControlHandler::with_forwarding(
2539            Arc::new(crate::Registry::default()),
2540            Arc::clone(&forwarding),
2541        ));
2542        let router = Router::with_control_handler(control);
2543        let close_receiver = forwarding.register_connection_close(client_connection);
2544        let (module_tx, _module_rx) = mpsc::channel(8);
2545        forwarding
2546            .register_module_connection(
2547                module_connection,
2548                module_id.to_string(),
2549                1,
2550                Concurrency::ModuleManaged,
2551                FrameSink::new(module_tx),
2552            )
2553            .unwrap();
2554        let (client_sink, mut client_rx) = crate::server::connection_egress();
2555        let mut bound = Vec::with_capacity(routes);
2556        for index in 0..routes {
2557            let pending = forwarding
2558                .begin_route_bind_relay_for_test(
2559                    client_connection,
2560                    client_sink.clone(),
2561                    900 + index as u64,
2562                    module_id,
2563                )
2564                .unwrap();
2565            forwarding
2566                .complete_pending_relay(
2567                    module_connection,
2568                    pending.corr,
2569                    RouteBindRelayOutcome::Accepted,
2570                )
2571                .unwrap();
2572            assert_eq!(
2573                client_rx.recv().await.unwrap().header.corr,
2574                900 + index as u64
2575            );
2576            bound.push(pending);
2577        }
2578        let (module_egress_tx, _module_egress_rx) = mpsc::channel(8);
2579        let module_ctx = RouteCtx {
2580            connection_id: module_connection,
2581            egress: FrameSink::new(module_egress_tx),
2582        };
2583        (
2584            router,
2585            client_sink,
2586            client_rx,
2587            module_ctx,
2588            bound,
2589            close_receiver,
2590        )
2591    }
2592
2593    /// Per-frame cost of the egress sink's admission and release accounting:
2594    /// one million 200-byte frames enqueued with `try_send` and taken off the
2595    /// queue the way the connection writer does, in batches of 1,000 so the
2596    /// queue stays well inside its budget. Prints nanoseconds per frame; it is
2597    /// a measurement, not a gate. Run with
2598    /// `cargo test --release -p subc-daemon --lib egress_sink_per_frame_cost -- --ignored --nocapture`.
2599    #[test]
2600    #[ignore = "timing measurement, run on demand"]
2601    fn egress_sink_per_frame_cost() {
2602        const FRAMES: usize = 1_000_000;
2603        const BATCH: usize = 1_000;
2604        let (sink, mut rx) = crate::server::connection_egress();
2605        let template = stream_frame(9, 1, 0, vec![b't'; 200]);
2606        let started = Instant::now();
2607        for _ in 0..FRAMES / BATCH {
2608            for _ in 0..BATCH {
2609                sink.try_send(template.clone()).unwrap();
2610            }
2611            for _ in 0..BATCH {
2612                let outbound = rx.try_recv().unwrap();
2613                if let Some(charge) = &outbound.charge {
2614                    charge.taken_by_writer();
2615                }
2616                drop(std::hint::black_box(outbound));
2617            }
2618        }
2619        let elapsed = started.elapsed();
2620        assert_eq!(sink.backlog().queued_bytes, 0);
2621        println!(
2622            "egress sink: {FRAMES} frames in {elapsed:?}, {:.1} ns/frame",
2623            elapsed.as_nanos() as f64 / FRAMES as f64
2624        );
2625    }
2626
2627    fn stream_frame(channel: u16, epoch: u32, corr: u64, body: Vec<u8>) -> Frame {
2628        Frame::build(
2629            FrameType::StreamData,
2630            Flags::new(false, Priority::Interactive, false),
2631            channel,
2632            epoch,
2633            corr,
2634            body,
2635        )
2636        .unwrap()
2637    }
2638
2639    fn module_route_for_test(
2640        router: &Router,
2641        ctx: &RouteCtx,
2642        route: &crate::forwarding::PendingRouteBindRelay,
2643    ) -> Arc<RouteBinding> {
2644        match router
2645            .forwarding
2646            .lookup_data_route(ctx.connection_id, route.module_channel, route.module_epoch)
2647            .unwrap()
2648        {
2649            DataRoute::Module(DataRouteState::Bound(binding)) => binding,
2650            other => panic!("expected a bound module route, got {other:?}"),
2651        }
2652    }
2653
2654    fn assert_frame_size_error(frame: &Frame, channel: u16, epoch: u32, corr: u64, bytes: usize) {
2655        assert_eq!(frame.header.ty, FrameType::Error);
2656        assert_eq!(frame.header.channel, channel);
2657        assert_eq!(frame.header.epoch, epoch);
2658        assert_eq!(frame.header.corr, corr);
2659        assert!(frame.header.flags.is_daemon_origin());
2660        let body: ErrorBody = serde_json::from_slice(&frame.body).unwrap();
2661        assert_eq!(body.code, error_codes::FRAME_TOO_LARGE);
2662        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2663        assert!(body.message.contains(&bytes.to_string()));
2664        assert!(body.message.contains(&limit.to_string()));
2665        assert_eq!(
2666            body.detail,
2667            Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit }))
2668        );
2669    }
2670
2671    #[tokio::test]
2672    async fn module_egress_response_oversized_reply_keeps_connection_and_other_route() {
2673        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
2674        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2675            "oversize-provider",
2676            ConnectionId::new(80),
2677            ConnectionId::new(81),
2678            2,
2679        )
2680        .await;
2681        let binding = module_route_for_test(&router, &ctx, &routes[0]);
2682        binding.flow.acquire_tagged(101, false).await.unwrap();
2683        module_route_for_test(&router, &ctx, &routes[1])
2684            .flow
2685            .acquire_tagged(102, false)
2686            .await
2687            .unwrap();
2688        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2689        let oversized = Frame::build(
2690            FrameType::Response,
2691            Flags::new(false, Priority::Interactive, true),
2692            routes[0].module_channel,
2693            routes[0].module_epoch,
2694            101,
2695            vec![b'x'; limit - subc_protocol::HEADER_LEN + 1],
2696        )
2697        .unwrap();
2698        assert_eq!(sink.backlog().queued_bytes, 0);
2699        router.route_for_connection(&ctx, oversized).await.unwrap();
2700
2701        // Reproduce a large reply followed by ordinary traffic on another route
2702        // before the client reads anything. The large reply must not monopolize
2703        // the queue and make that second delivery close the shared connection.
2704        router
2705            .route_for_connection(
2706                &ctx,
2707                route_frame(
2708                    FrameType::Response,
2709                    routes[1].module_channel,
2710                    routes[1].module_epoch,
2711                    102,
2712                ),
2713            )
2714            .await
2715            .unwrap();
2716        assert!(
2717            close.try_recv().is_err(),
2718            "an oversized reply must not close the multiplexed client"
2719        );
2720        let error = rx.try_recv().expect("the waiting client gets an ERROR");
2721        assert_frame_size_error(
2722            &error,
2723            routes[0].client_channel,
2724            routes[0].client_epoch,
2725            101,
2726            limit + 1,
2727        );
2728        let second = rx.try_recv().expect("the other route's reply arrives");
2729        assert_eq!(second.header.ty, FrameType::Response);
2730        assert_eq!(second.header.channel, routes[1].client_channel);
2731        assert_eq!(second.header.epoch, routes[1].client_epoch);
2732        assert_eq!(second.header.corr, 102);
2733        assert_eq!(second.body, b"route-body");
2734        assert_eq!(binding.flow.drain_in_flight(), 0);
2735        for route in &routes {
2736            module_route_for_test(&router, &ctx, route);
2737        }
2738        assert!(rx.try_recv().is_err());
2739        drop((error, second));
2740        assert_eq!(sink.backlog().queued_bytes, 0);
2741        assert_eq!(
2742            router.counters.snapshot()["client_egress_close_delivery_failed"],
2743            0
2744        );
2745
2746        let captured = test_log::captured_logs(&logs);
2747        let warnings: Vec<_> = captured
2748            .lines()
2749            .filter(|line| line.contains("WARN"))
2750            .collect();
2751        assert_eq!(warnings.len(), 1, "{captured}");
2752        let warning = warnings[0];
2753        for expected in [
2754            "module_id=oversize-provider".to_string(),
2755            "client_principal=direct".to_string(),
2756            format!("client_channel={}", routes[0].client_channel),
2757            format!("frame_bytes={}", limit + 1),
2758            format!("byte_limit={limit}"),
2759        ] {
2760            assert!(warning.contains(&expected), "missing {expected}: {warning}");
2761        }
2762    }
2763
2764    async fn poll_once<F: std::future::Future>(
2765        mut future: std::pin::Pin<&mut F>,
2766    ) -> std::task::Poll<F::Output> {
2767        std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
2768    }
2769
2770    fn fill_client_budget(sink: &FrameSink, route: &crate::forwarding::PendingRouteBindRelay) {
2771        sink.try_send(
2772            Frame::build(
2773                FrameType::Response,
2774                Flags::new(false, Priority::Interactive, true),
2775                route.client_channel,
2776                route.client_epoch,
2777                100,
2778                vec![
2779                    b'x';
2780                    crate::server::CONNECTION_EGRESS_BYTE_BUDGET - subc_protocol::HEADER_LEN
2781                ],
2782            )
2783            .unwrap(),
2784        )
2785        .unwrap();
2786        assert_eq!(
2787            sink.backlog().queued_bytes,
2788            crate::server::CONNECTION_EGRESS_BYTE_BUDGET
2789        );
2790    }
2791
2792    async fn assert_full_client_burst_survives(ty: FrameType, oversized: bool) {
2793        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2794            "draining-burst",
2795            ConnectionId::new(92),
2796            ConnectionId::new(93),
2797            2,
2798        )
2799        .await;
2800        fill_client_budget(&sink, &routes[0]);
2801        let frame = if oversized {
2802            Frame::build(
2803                ty,
2804                Flags::new(false, Priority::Interactive, false),
2805                routes[1].module_channel,
2806                routes[1].module_epoch,
2807                101,
2808                vec![b's'; crate::server::CONNECTION_EGRESS_BYTE_BUDGET],
2809            )
2810            .unwrap()
2811        } else {
2812            route_frame(ty, routes[1].module_channel, routes[1].module_epoch, 101)
2813        };
2814        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2815        assert!(poll_once(delivery.as_mut()).await.is_pending());
2816        assert!(
2817            close.try_recv().is_err(),
2818            "fresh bursts must not close: {ty:?}"
2819        );
2820        tokio::time::advance(Duration::from_millis(50)).await;
2821        let queued = rx.try_recv().unwrap();
2822        queued.charge.as_ref().unwrap().taken_by_writer();
2823        drop(queued);
2824        assert!(matches!(
2825            poll_once(delivery.as_mut()).await,
2826            std::task::Poll::Ready(Ok(()))
2827        ));
2828        let delivered = rx
2829            .try_recv()
2830            .expect("the burst frame must be delivered after draining");
2831        assert_eq!(delivered.header.channel, routes[1].client_channel);
2832        assert_eq!(delivered.header.epoch, routes[1].client_epoch);
2833        assert_eq!(delivered.header.corr, 101);
2834        if oversized {
2835            assert_frame_size_error(
2836                &delivered,
2837                routes[1].client_channel,
2838                routes[1].client_epoch,
2839                101,
2840                crate::server::CONNECTION_EGRESS_BYTE_BUDGET + subc_protocol::HEADER_LEN,
2841            );
2842        } else {
2843            assert_eq!(delivered.header.ty, ty);
2844        }
2845        assert!(rx.try_recv().is_err());
2846        assert!(close.try_recv().is_err());
2847        assert_eq!(
2848            router.counters.snapshot()["client_egress_close_delivery_failed"],
2849            0
2850        );
2851        assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 0);
2852    }
2853
2854    #[tokio::test(start_paused = true)]
2855    async fn module_egress_fresh_full_queue_burst_survives() {
2856        for ty in [
2857            FrameType::Response,
2858            FrameType::StreamData,
2859            FrameType::Request,
2860            FrameType::Push,
2861        ] {
2862            assert_full_client_burst_survives(ty, false).await;
2863        }
2864    }
2865
2866    #[tokio::test(start_paused = true)]
2867    async fn module_goodbye_fresh_full_queue_burst_survives() {
2868        assert_full_client_burst_survives(FrameType::Goodbye, false).await;
2869    }
2870
2871    #[tokio::test(start_paused = true)]
2872    async fn frame_count_backstop_also_allows_a_draining_burst() {
2873        let (tx, mut rx) = mpsc::channel(1);
2874        let sink = FrameSink::with_byte_budget(tx, 2_000);
2875        sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
2876            .unwrap();
2877        let mut delivery = Box::pin(sink.send_to_client(stream_frame(1, 1, 2, vec![b'y'; 100])));
2878        assert!(poll_once(delivery.as_mut()).await.is_pending());
2879        assert!(
2880            sink.backlog().queued_bytes < 2_000,
2881            "only frame capacity is exhausted"
2882        );
2883        tokio::time::advance(Duration::from_millis(50)).await;
2884        drop(rx.try_recv().unwrap());
2885        assert!(matches!(
2886            poll_once(delivery.as_mut()).await,
2887            std::task::Poll::Ready(Ok(()))
2888        ));
2889        assert_eq!(rx.try_recv().unwrap().header.corr, 2);
2890    }
2891
2892    #[tokio::test(start_paused = true)]
2893    async fn oversized_reply_replacement_fresh_full_queue_burst_survives() {
2894        for ty in [FrameType::Response, FrameType::StreamData] {
2895            assert_full_client_burst_survives(ty, true).await;
2896        }
2897    }
2898
2899    #[tokio::test(start_paused = true)]
2900    async fn module_egress_stopped_reader_closes_without_waiting() {
2901        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
2902        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2903            "oldest-stopped-reader",
2904            ConnectionId::new(94),
2905            ConnectionId::new(95),
2906            2,
2907        )
2908        .await;
2909        fill_client_budget(&sink, &routes[0]);
2910        // The writer has taken the head but cannot finish writing it. Its
2911        // bytes stay charged, and the atomic age still diagnoses that stall.
2912        let writing = rx.try_recv().unwrap();
2913        writing.charge.as_ref().unwrap().taken_by_writer();
2914        tokio::time::advance(Duration::from_millis(251)).await;
2915        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(251)));
2916        assert_eq!(sink.backlog().queued_bytes, queued_frame_bytes(&writing));
2917        let started = tokio::time::Instant::now();
2918        let frame = stream_frame(
2919            routes[0].module_channel,
2920            routes[0].module_epoch,
2921            101,
2922            b"secret-body-never-log".to_vec(),
2923        );
2924        let flags = frame.header.flags.0;
2925        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2926        assert!(
2927            matches!(
2928                poll_once(delivery.as_mut()).await,
2929                std::task::Poll::Ready(Ok(()))
2930            ),
2931            "an oldest frame older than 250 ms must close immediately, without a new wait"
2932        );
2933        assert_eq!(started.elapsed(), Duration::ZERO);
2934        assert!(close
2935            .try_recv()
2936            .unwrap()
2937            .to_string()
2938            .contains("module_to_client_delivery_failed"));
2939        let captured = test_log::captured_logs(&logs);
2940        for expected in [
2941            "frame_type=Some(StreamData)".to_string(),
2942            format!("frame_flags=Some({flags})"),
2943            "oldest_queued_ms=251".to_string(),
2944            "close_cause=\"oldest_frame_older_than_250_ms\"".to_string(),
2945        ] {
2946            assert!(
2947                captured.contains(&expected),
2948                "missing {expected}: {captured}"
2949            );
2950        }
2951        assert!(!captured.contains("secret-body-never-log"));
2952    }
2953
2954    #[tokio::test(start_paused = true)]
2955    async fn client_gone_mid_wait_resumes_module_reader_immediately() {
2956        for writer_gone in [false, true] {
2957            let (router, sink, mut rx, ctx, routes, _close) = multi_route_client(
2958                "gone-during-burst",
2959                ConnectionId::new(96),
2960                ConnectionId::new(97),
2961                1,
2962            )
2963            .await;
2964            fill_client_budget(&sink, &routes[0]);
2965            let frame = route_frame(
2966                FrameType::Response,
2967                routes[0].module_channel,
2968                routes[0].module_epoch,
2969                101,
2970            );
2971            let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2972            assert!(poll_once(delivery.as_mut()).await.is_pending());
2973            tokio::time::advance(Duration::from_millis(50)).await;
2974            let started = tokio::time::Instant::now();
2975            if writer_gone {
2976                rx.close();
2977            } else {
2978                sink.close_client_delivery();
2979            }
2980            assert!(
2981                matches!(
2982                    poll_once(delivery.as_mut()).await,
2983                    std::task::Poll::Ready(Ok(()))
2984                ),
2985                "client teardown must wake the module reader before the writer drains"
2986            );
2987            assert_eq!(started.elapsed(), Duration::ZERO);
2988            assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
2989            assert_eq!(rx.try_recv().unwrap().header.corr, 100);
2990            assert!(rx.try_recv().is_err());
2991        }
2992    }
2993
2994    #[tokio::test(start_paused = true)]
2995    async fn cancelled_module_delivery_releases_waiter_without_closing_client() {
2996        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2997            "cancelled-burst",
2998            ConnectionId::new(98),
2999            ConnectionId::new(99),
3000            1,
3001        )
3002        .await;
3003        fill_client_budget(&sink, &routes[0]);
3004        let frame = route_frame(
3005            FrameType::Response,
3006            routes[0].module_channel,
3007            routes[0].module_epoch,
3008            101,
3009        );
3010        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3011        assert!(poll_once(delivery.as_mut()).await.is_pending());
3012        assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 1);
3013        drop(delivery);
3014        assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
3015        tokio::time::advance(Duration::from_secs(1)).await;
3016        assert!(close.try_recv().is_err());
3017        drop(rx.try_recv().unwrap());
3018        assert_eq!(sink.backlog().queued_bytes, 0);
3019        assert!(sink.backlog().oldest_age.is_none());
3020    }
3021
3022    #[tokio::test(start_paused = true)]
3023    async fn egress_atomic_age_follows_writer_and_clears_when_drained() {
3024        let (sink, mut rx) = crate::server::connection_egress();
3025        sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
3026            .unwrap();
3027        tokio::time::advance(Duration::from_millis(200)).await;
3028        sink.try_send(stream_frame(2, 1, 2, vec![b'y'; 100]))
3029            .unwrap();
3030        let writing = rx.try_recv().unwrap();
3031        writing.charge.as_ref().unwrap().taken_by_writer();
3032        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3033        drop(writing);
3034        // Between frames the lock-free marker can still name the previous
3035        // head. The next writer take advances it without scanning the queue.
3036        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3037        tokio::time::advance(Duration::from_millis(50)).await;
3038        let next = rx.try_recv().unwrap();
3039        next.charge.as_ref().unwrap().taken_by_writer();
3040        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(50)));
3041        assert!(
3042            sink.backlog().queued_bytes > 0,
3043            "in-flight bytes remain charged until written"
3044        );
3045        drop(next);
3046        assert_eq!(sink.backlog().queued_bytes, 0);
3047        assert_eq!(sink.backlog().oldest_age, None);
3048    }
3049
3050    #[tokio::test(start_paused = true)]
3051    async fn module_egress_response_full_client_waits_only_remaining_grace() {
3052        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3053            "backpressure-provider",
3054            ConnectionId::new(82),
3055            ConnectionId::new(83),
3056            2,
3057        )
3058        .await;
3059        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3060        // Exactly the limit, including its header, must be admitted to an empty
3061        // queue. Leave it unread to deterministically exhaust only byte capacity.
3062        router
3063            .route_for_connection(
3064                &ctx,
3065                Frame::build(
3066                    FrameType::Response,
3067                    Flags::new(false, Priority::Interactive, true),
3068                    routes[0].module_channel,
3069                    routes[0].module_epoch,
3070                    201,
3071                    vec![b'x'; limit - subc_protocol::HEADER_LEN],
3072                )
3073                .unwrap(),
3074            )
3075            .await
3076            .unwrap();
3077        assert_eq!(sink.backlog().queued_bytes, limit);
3078        assert_eq!(sink.backlog().queued_frames, 1);
3079        let ordinary = route_frame(
3080            FrameType::Response,
3081            routes[1].module_channel,
3082            routes[1].module_epoch,
3083            202,
3084        );
3085        assert!(queued_frame_bytes(&ordinary) < limit);
3086        // A fitting reply gets only the unused portion of the oldest frame's
3087        // 250 ms grace, not an immediate close or a fresh 250 ms wait.
3088        tokio::time::advance(Duration::from_millis(100)).await;
3089        let mut delivery = Box::pin(router.route_for_connection(&ctx, ordinary));
3090        assert!(poll_once(delivery.as_mut()).await.is_pending());
3091        assert!(close.try_recv().is_err());
3092        tokio::time::advance(Duration::from_millis(149)).await;
3093        assert!(poll_once(delivery.as_mut()).await.is_pending());
3094        assert!(close.try_recv().is_err());
3095        tokio::time::advance(Duration::from_millis(1)).await;
3096        assert!(matches!(
3097            poll_once(delivery.as_mut()).await,
3098            std::task::Poll::Ready(Ok(()))
3099        ));
3100        drop(delivery);
3101        let reason = close.try_recv().expect("the full client must close");
3102        assert!(reason
3103            .to_string()
3104            .contains("module_to_client_delivery_failed"));
3105        assert!(!reason.to_string().contains("frame_too_large"));
3106        assert_eq!(
3107            router.counters.snapshot()["client_egress_close_delivery_failed"],
3108            1
3109        );
3110        let admitted = rx.try_recv().expect("the budget-sized frame was admitted");
3111        assert_eq!(admitted.header.ty, FrameType::Response);
3112        assert_eq!(admitted.header.corr, 201);
3113        assert!(
3114            rx.try_recv().is_err(),
3115            "no oversized-frame ERROR is substituted for backpressure"
3116        );
3117    }
3118
3119    #[tokio::test(start_paused = true)]
3120    async fn oversized_reply_to_a_full_client_waits_only_remaining_grace() {
3121        let (router, sink, _rx, ctx, routes, mut close) = multi_route_client(
3122            "full-oversize-provider",
3123            ConnectionId::new(90),
3124            ConnectionId::new(91),
3125            2,
3126        )
3127        .await;
3128        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3129        // Fill the client's queue to its budget and leave it unread.
3130        router
3131            .route_for_connection(
3132                &ctx,
3133                Frame::build(
3134                    FrameType::Response,
3135                    Flags::new(false, Priority::Interactive, true),
3136                    routes[0].module_channel,
3137                    routes[0].module_epoch,
3138                    501,
3139                    vec![b'x'; limit - subc_protocol::HEADER_LEN],
3140                )
3141                .unwrap(),
3142            )
3143            .await
3144            .unwrap();
3145        assert_eq!(sink.backlog().queued_bytes, limit);
3146        let oversized = Frame::build(
3147            FrameType::Response,
3148            Flags::new(false, Priority::Interactive, true),
3149            routes[1].module_channel,
3150            routes[1].module_epoch,
3151            502,
3152            vec![b'y'; limit - subc_protocol::HEADER_LEN + 1],
3153        )
3154        .unwrap();
3155        // The replacement ERROR gets the same bounded burst grace as the
3156        // ordinary reply; oversized payloads themselves are never queued.
3157        tokio::time::advance(Duration::from_millis(80)).await;
3158        let mut delivery = Box::pin(router.route_for_connection(&ctx, oversized));
3159        assert!(poll_once(delivery.as_mut()).await.is_pending());
3160        assert!(close.try_recv().is_err());
3161        tokio::time::advance(Duration::from_millis(169)).await;
3162        assert!(poll_once(delivery.as_mut()).await.is_pending());
3163        assert!(close.try_recv().is_err());
3164        tokio::time::advance(Duration::from_millis(1)).await;
3165        assert!(matches!(
3166            poll_once(delivery.as_mut()).await,
3167            std::task::Poll::Ready(Ok(()))
3168        ));
3169        drop(delivery);
3170        let reason = close.try_recv().expect("the full client must close");
3171        assert!(reason
3172            .to_string()
3173            .contains("module_to_client_delivery_failed"));
3174    }
3175
3176    #[tokio::test]
3177    async fn oversized_reverse_request_errors_module_without_reaching_client() {
3178        let (router, _sink, mut rx, mut ctx, routes, mut close) = multi_route_client(
3179            "reverse-oversize-provider",
3180            ConnectionId::new(84),
3181            ConnectionId::new(85),
3182            2,
3183        )
3184        .await;
3185        let (module_tx, mut module_rx) = mpsc::channel(8);
3186        ctx.egress = FrameSink::new(module_tx);
3187        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3188        router
3189            .route_for_connection(
3190                &ctx,
3191                Frame::build(
3192                    FrameType::Request,
3193                    Flags::new(false, Priority::Interactive, false),
3194                    routes[0].module_channel,
3195                    routes[0].module_epoch,
3196                    301,
3197                    vec![b'r'; limit - subc_protocol::HEADER_LEN + 1],
3198                )
3199                .unwrap(),
3200            )
3201            .await
3202            .unwrap();
3203        assert!(
3204            rx.try_recv().is_err(),
3205            "the reverse request never reaches the client"
3206        );
3207        let error = module_rx.try_recv().expect("the module gets the refusal");
3208        assert_frame_size_error(
3209            &error,
3210            routes[0].module_channel,
3211            routes[0].module_epoch,
3212            301,
3213            limit + 1,
3214        );
3215        assert!(module_rx.try_recv().is_err());
3216        for (index, route) in routes.iter().enumerate() {
3217            module_route_for_test(&router, &ctx, route);
3218            router
3219                .route_for_connection(
3220                    &ctx,
3221                    route_frame(
3222                        FrameType::Response,
3223                        route.module_channel,
3224                        route.module_epoch,
3225                        302 + index as u64,
3226                    ),
3227                )
3228                .await
3229                .unwrap();
3230            let reply = rx.try_recv().expect("both routes still deliver");
3231            assert_eq!(reply.header.ty, FrameType::Response);
3232            assert_eq!(reply.header.channel, route.client_channel);
3233        }
3234        assert!(close.try_recv().is_err());
3235        assert_eq!(
3236            router.counters.snapshot()["client_egress_close_delivery_failed"],
3237            0
3238        );
3239    }
3240
3241    #[tokio::test]
3242    async fn oversized_push_closes_client_with_precise_warn() {
3243        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3244        let (router, _sink, mut rx, ctx, routes, mut close) = multi_route_client(
3245            "push-oversize-provider",
3246            ConnectionId::new(86),
3247            ConnectionId::new(87),
3248            2,
3249        )
3250        .await;
3251        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3252        router
3253            .route_for_connection(
3254                &ctx,
3255                Frame::build(
3256                    FrameType::Push,
3257                    Flags::new(false, Priority::Interactive, false),
3258                    routes[0].module_channel,
3259                    routes[0].module_epoch,
3260                    0,
3261                    vec![b'p'; limit - subc_protocol::HEADER_LEN + 1],
3262                )
3263                .unwrap(),
3264            )
3265            .await
3266            .unwrap();
3267        let reason = close
3268            .try_recv()
3269            .expect("unsolicited oversize still closes the client");
3270        assert!(reason
3271            .to_string()
3272            .contains("frame_too_large: oversized push"));
3273        assert!(
3274            rx.try_recv().is_err(),
3275            "no part of the oversized push is forwarded"
3276        );
3277        let captured = test_log::captured_logs(&logs);
3278        let warnings: Vec<_> = captured
3279            .lines()
3280            .filter(|line| line.contains("WARN"))
3281            .collect();
3282        assert_eq!(warnings.len(), 1, "{captured}");
3283        let warning = warnings[0];
3284        for expected in [
3285            "oversized push".to_string(),
3286            "module_id=\"push-oversize-provider\"".to_string(),
3287            "principals=direct".to_string(),
3288            format!("client_channel={}", routes[0].client_channel),
3289            format!("size={}", limit + 1),
3290            format!("limit={limit}"),
3291        ] {
3292            assert!(warning.contains(&expected), "missing {expected}: {warning}");
3293        }
3294        assert_eq!(
3295            router.counters.snapshot()["client_egress_close_delivery_failed"],
3296            1
3297        );
3298    }
3299
3300    #[tokio::test]
3301    async fn oversized_stream_and_error_frames_end_request_even_with_queued_traffic() {
3302        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3303            "stream-oversize-provider",
3304            ConnectionId::new(88),
3305            ConnectionId::new(89),
3306            2,
3307        )
3308        .await;
3309        let binding = module_route_for_test(&router, &ctx, &routes[0]);
3310        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3311        for ty in [
3312            FrameType::StreamData,
3313            FrameType::StreamEnd,
3314            FrameType::Error,
3315        ] {
3316            binding.flow.acquire_tagged(401, false).await.unwrap();
3317            sink.try_send(route_frame(
3318                FrameType::Response,
3319                routes[1].client_channel,
3320                routes[1].client_epoch,
3321                402,
3322            ))
3323            .unwrap();
3324            router
3325                .route_for_connection(
3326                    &ctx,
3327                    Frame::build(
3328                        ty,
3329                        Flags::new(false, Priority::Interactive, false),
3330                        routes[0].module_channel,
3331                        routes[0].module_epoch,
3332                        401,
3333                        vec![b's'; limit - subc_protocol::HEADER_LEN + 1],
3334                    )
3335                    .unwrap(),
3336                )
3337                .await
3338                .unwrap();
3339            assert!(close.try_recv().is_err());
3340            assert_eq!(rx.try_recv().unwrap().header.corr, 402);
3341            let error = rx
3342                .try_recv()
3343                .expect("every correlated oversize frame becomes ERROR");
3344            assert_frame_size_error(
3345                &error,
3346                routes[0].client_channel,
3347                routes[0].client_epoch,
3348                401,
3349                limit + 1,
3350            );
3351            assert_eq!(binding.flow.drain_in_flight(), 0, "{ty:?} ends the request");
3352            assert!(rx.try_recv().is_err());
3353        }
3354    }
3355
3356    /// An awaited send (a control or error reply) is never refused by the byte
3357    /// budget: it parks until the writer frees bytes. If a release could miss a
3358    /// parked sender, that reply would hang for the connection's lifetime, so
3359    /// this pins the wakeup: the send stays parked while the queue is full and
3360    /// completes as soon as one frame leaves it.
3361    #[tokio::test]
3362    async fn awaited_send_parked_behind_a_full_byte_budget_wakes_when_bytes_free() {
3363        let (tx, mut rx) = mpsc::channel(1024);
3364        let sink = FrameSink::with_byte_budget(tx, 2_000);
3365        let mut queued = 0u64;
3366        while sink
3367            .try_send(stream_frame(7, 1, queued, vec![b'x'; 200]))
3368            .is_ok()
3369        {
3370            queued += 1;
3371        }
3372        assert!(queued > 0, "the budget admitted nothing");
3373
3374        let parked = tokio::spawn({
3375            let sink = sink.clone();
3376            async move { sink.send(stream_frame(0, 0, 999, vec![b'r'; 200])).await }
3377        });
3378        tokio::time::sleep(Duration::from_millis(100)).await;
3379        assert!(
3380            !parked.is_finished(),
3381            "the awaited send must wait while the byte budget is full"
3382        );
3383
3384        // The writer takes one frame and drops it, releasing its bytes.
3385        drop(rx.recv().await.expect("a queued frame"));
3386        tokio::time::timeout(Duration::from_secs(2), parked)
3387            .await
3388            .expect("the parked send was never woken after bytes were freed")
3389            .unwrap()
3390            .unwrap();
3391    }
3392
3393    /// A client multiplexing several token streams pauses its reader while the
3394    /// module keeps producing small frames. Far more frames than the old
3395    /// 64-frame queue allowed, but far fewer bytes than the budget, must all be
3396    /// held without closing the connection and delivered in order afterwards.
3397    #[tokio::test]
3398    async fn paused_client_reader_keeps_connection_through_small_frame_burst() {
3399        const ROUTES: usize = 4;
3400        const FRAMES_PER_ROUTE: usize = 250;
3401        let (router, client_sink, mut client_rx, module_ctx, routes, mut close_receiver) =
3402            multi_route_client(
3403                "burst-provider",
3404                ConnectionId::new(60),
3405                ConnectionId::new(61),
3406                ROUTES,
3407            )
3408            .await;
3409
3410        // The reader is paused: nothing is received until every frame is sent.
3411        for seq in 0..FRAMES_PER_ROUTE as u64 {
3412            for (index, route) in routes.iter().enumerate() {
3413                let body = format!("route-{index}-token-{seq:05}-{}", "t".repeat(170));
3414                router
3415                    .route_for_connection(
3416                        &module_ctx,
3417                        stream_frame(
3418                            route.module_channel,
3419                            route.module_epoch,
3420                            seq,
3421                            body.into_bytes(),
3422                        ),
3423                    )
3424                    .await
3425                    .unwrap();
3426            }
3427        }
3428
3429        let backlog = client_sink.backlog();
3430        assert_eq!(backlog.queued_frames, ROUTES * FRAMES_PER_ROUTE);
3431        assert!(backlog.queued_bytes < crate::server::CONNECTION_EGRESS_BYTE_BUDGET);
3432        assert!(
3433            close_receiver.try_recv().is_err(),
3434            "a paused reader under the byte budget must not be closed"
3435        );
3436        assert_eq!(
3437            router.counters.snapshot()["client_egress_close_delivery_failed"],
3438            0
3439        );
3440
3441        // The reader resumes: every frame arrives, in order within each route.
3442        let mut next_seq = vec![0u64; ROUTES];
3443        for _ in 0..ROUTES * FRAMES_PER_ROUTE {
3444            let frame = client_rx.try_recv().expect("every queued frame arrives");
3445            let index = routes
3446                .iter()
3447                .position(|route| route.client_channel == frame.header.channel)
3448                .expect("frame arrives on one of the bound client channels");
3449            assert_eq!(frame.header.corr, next_seq[index], "route {index} order");
3450            let expected_prefix = format!("route-{index}-token-{:05}-", next_seq[index]);
3451            assert!(frame.body.starts_with(expected_prefix.as_bytes()));
3452            next_seq[index] += 1;
3453        }
3454        assert!(client_rx.try_recv().is_err());
3455        assert_eq!(next_seq, vec![FRAMES_PER_ROUTE as u64; ROUTES]);
3456        assert_eq!(client_sink.backlog().queued_bytes, 0);
3457    }
3458
3459    /// A client that never reads exhausts its byte budget and the oldest
3460    /// frame's 250 ms grace, then closes with one header-only diagnosis.
3461    #[tokio::test(start_paused = true)]
3462    async fn never_reading_client_is_closed_at_byte_budget_with_warn_diagnosis() {
3463        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3464        const BODY: usize = 16 * 1024;
3465        let (router, client_sink, _client_rx, module_ctx, routes, mut close_receiver) =
3466            multi_route_client(
3467                "stuck-reader-provider",
3468                ConnectionId::new(70),
3469                ConnectionId::new(71),
3470                2,
3471            )
3472            .await;
3473
3474        let mut admitted = 0usize;
3475        let mut sent = 0u64;
3476        let started = tokio::time::Instant::now();
3477        while router.counters.snapshot()["client_egress_close_delivery_failed"] == 0 {
3478            assert!(sent < 1_000, "the byte budget never refused a frame");
3479            let route = &routes[(sent % 2) as usize];
3480            router
3481                .route_for_connection(
3482                    &module_ctx,
3483                    stream_frame(
3484                        route.module_channel,
3485                        route.module_epoch,
3486                        sent,
3487                        vec![b'z'; BODY],
3488                    ),
3489                )
3490                .await
3491                .unwrap();
3492            sent += 1;
3493            admitted = client_sink.backlog().queued_frames;
3494        }
3495        assert_eq!(
3496            started.elapsed(),
3497            crate::server::CONNECTION_EGRESS_STALL_GRACE
3498        );
3499        // The budget, not the frame-count backstop, did the refusing.
3500        let frame_bytes = subc_protocol::HEADER_LEN + BODY;
3501        assert_eq!(
3502            admitted,
3503            crate::server::CONNECTION_EGRESS_BYTE_BUDGET / frame_bytes
3504        );
3505        let reason = close_receiver
3506            .try_recv()
3507            .expect("the client connection must be asked to close");
3508        assert!(reason
3509            .to_string()
3510            .contains("module_to_client_delivery_failed"));
3511
3512        // A second refused frame for the same connection adds no second WARN.
3513        router
3514            .route_for_connection(
3515                &module_ctx,
3516                stream_frame(
3517                    routes[0].module_channel,
3518                    routes[0].module_epoch,
3519                    sent,
3520                    vec![b'z'; BODY],
3521                ),
3522            )
3523            .await
3524            .unwrap();
3525
3526        let captured = test_log::captured_logs(&logs);
3527        let warn_lines = captured
3528            .lines()
3529            .filter(|line| {
3530                line.contains("closing client connection: its egress queue could not take a frame")
3531            })
3532            .collect::<Vec<_>>();
3533        assert_eq!(warn_lines.len(), 1, "exactly one WARN, got: {captured}");
3534        let line = warn_lines[0];
3535        assert!(line.contains("WARN"), "{line}");
3536        assert!(line.contains("connection_id=71"), "{line}");
3537        assert!(
3538            line.contains("module_id=\"stuck-reader-provider\""),
3539            "{line}"
3540        );
3541        assert!(line.contains("client_channel="), "{line}");
3542        assert!(line.contains("principals=direct"), "{line}");
3543        let queued_bytes: usize = line
3544            .split("queued_bytes=")
3545            .nth(1)
3546            .and_then(|rest| rest.split_whitespace().next())
3547            .and_then(|value| value.parse().ok())
3548            .expect("queued_bytes is logged");
3549        assert_eq!(queued_bytes, admitted * frame_bytes);
3550        assert!(
3551            line.contains(&format!("queued_frames={admitted}")),
3552            "{line}"
3553        );
3554        assert!(line.contains("oldest_queued_ms="), "{line}");
3555        assert!(line.contains("oldest_queued_ms=250"), "{line}");
3556        assert!(line.contains("frame_type=Some(StreamData)"), "{line}");
3557        assert!(
3558            line.contains(&format!(
3559                "frame_flags=Some({})",
3560                Flags::new(false, Priority::Interactive, false).0
3561            )),
3562            "{line}"
3563        );
3564        assert!(
3565            line.contains("close_cause=\"waited_250_ms_without_room\""),
3566            "{line}"
3567        );
3568        assert!(
3569            !line.contains(&"z".repeat(20)),
3570            "body bytes must never be logged: {line}"
3571        );
3572    }
3573
3574    fn route_frame(ty: FrameType, channel: u16, epoch: u32, corr: u64) -> Frame {
3575        Frame::build(
3576            ty,
3577            Flags::new(false, Priority::Interactive, false),
3578            channel,
3579            epoch,
3580            corr,
3581            if ty == FrameType::Request || ty == FrameType::Response {
3582                b"route-body".to_vec()
3583            } else {
3584                Vec::new()
3585            },
3586        )
3587        .unwrap()
3588    }
3589
3590    type DynamicRouteFixture = (
3591        Router,
3592        Arc<ForwardingTable>,
3593        RouteCtx,
3594        mpsc::Receiver<crate::router::OutboundFrame>,
3595        RouteCtx,
3596        mpsc::Receiver<crate::router::OutboundFrame>,
3597        mpsc::Receiver<crate::router::OutboundFrame>,
3598        crate::forwarding::PendingRouteBindRelay,
3599    );
3600
3601    fn dynamic_route_fixture(commit: bool) -> DynamicRouteFixture {
3602        let forwarding = Arc::new(ForwardingTable::default());
3603        let control = Arc::new(crate::ControlHandler::with_forwarding(
3604            Arc::new(crate::Registry::default()),
3605            Arc::clone(&forwarding),
3606        ));
3607        let router = Router::with_control_handler(control);
3608        let module_connection = ConnectionId::new(500);
3609        let client_connection = ConnectionId::new(501);
3610        let (module_tx, module_rx) = mpsc::channel(8);
3611        forwarding
3612            .register_module_connection(
3613                module_connection,
3614                "epoch-router".into(),
3615                2,
3616                Concurrency::ModuleManaged,
3617                FrameSink::new(module_tx),
3618            )
3619            .unwrap();
3620        let (client_tx, client_rx) = mpsc::channel(8);
3621        let client_sink = FrameSink::new(client_tx);
3622        let pending = forwarding
3623            .begin_route_bind_relay_for_test(
3624                client_connection,
3625                client_sink.clone(),
3626                700,
3627                "epoch-router",
3628            )
3629            .unwrap();
3630        if commit {
3631            forwarding
3632                .complete_pending_relay(
3633                    module_connection,
3634                    pending.corr,
3635                    RouteBindRelayOutcome::Accepted,
3636                )
3637                .unwrap();
3638        }
3639        let (module_egress_tx, module_egress_rx) = mpsc::channel(8);
3640        (
3641            router,
3642            forwarding,
3643            RouteCtx {
3644                connection_id: client_connection,
3645                egress: client_sink,
3646            },
3647            client_rx,
3648            RouteCtx {
3649                connection_id: module_connection,
3650                egress: FrameSink::new(module_egress_tx),
3651            },
3652            module_egress_rx,
3653            module_rx,
3654            pending,
3655        )
3656    }
3657
3658    #[tokio::test]
3659    async fn route_epochs_validate_both_directions_and_rewrite_to_peer_handle() {
3660        let (
3661            router,
3662            _forwarding,
3663            client_ctx,
3664            mut client_rx,
3665            module_ctx,
3666            _module_egress_rx,
3667            mut module_rx,
3668            pending,
3669        ) = dynamic_route_fixture(true);
3670        let route_open = client_rx.recv().await.unwrap();
3671        assert_eq!(route_open.header.corr, 700);
3672
3673        router
3674            .route_for_connection(
3675                &client_ctx,
3676                route_frame(
3677                    FrameType::Request,
3678                    pending.client_channel,
3679                    pending.client_epoch,
3680                    701,
3681                ),
3682            )
3683            .await
3684            .unwrap();
3685        let forwarded = module_rx.recv().await.unwrap();
3686        assert_eq!(forwarded.header.channel, pending.module_channel);
3687        assert_eq!(forwarded.header.epoch, pending.module_epoch);
3688
3689        router
3690            .route_for_connection(
3691                &module_ctx,
3692                route_frame(
3693                    FrameType::Response,
3694                    pending.module_channel,
3695                    pending.module_epoch,
3696                    701,
3697                ),
3698            )
3699            .await
3700            .unwrap();
3701        let delivered = client_rx.recv().await.unwrap();
3702        assert_eq!(delivered.header.channel, pending.client_channel);
3703        assert_eq!(delivered.header.epoch, pending.client_epoch);
3704
3705        router
3706            .route_for_connection(
3707                &client_ctx,
3708                route_frame(
3709                    FrameType::Request,
3710                    pending.client_channel,
3711                    pending.client_epoch + 1,
3712                    702,
3713                ),
3714            )
3715            .await
3716            .unwrap();
3717        router
3718            .route_for_connection(
3719                &module_ctx,
3720                route_frame(
3721                    FrameType::Response,
3722                    pending.module_channel,
3723                    pending.module_epoch + 1,
3724                    703,
3725                ),
3726            )
3727            .await
3728            .unwrap();
3729        let stale_error = client_rx.recv().await.unwrap();
3730        assert_eq!(stale_error.header.ty, FrameType::Error);
3731        assert_eq!(stale_error.header.channel, pending.client_channel);
3732        assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
3733        assert_eq!(stale_error.header.corr, 702);
3734        let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
3735        assert_eq!(body.code, "stale_route_epoch");
3736        assert!(module_rx.try_recv().is_err());
3737        assert!(client_rx.try_recv().is_err());
3738        let counters = router.counters.snapshot();
3739        assert_eq!(counters["client_frames_dropped_stale_route"], 1);
3740        assert_eq!(counters["module_frames_dropped_no_route"], 1);
3741    }
3742
3743    #[tokio::test]
3744    async fn accepted_route_publishes_route_open_before_immediate_reverse_request() {
3745        let (
3746            router,
3747            _,
3748            _client_ctx,
3749            mut client_rx,
3750            module_ctx,
3751            _module_egress_rx,
3752            _module_rx,
3753            pending,
3754        ) = dynamic_route_fixture(true);
3755        router
3756            .route_for_connection(
3757                &module_ctx,
3758                route_frame(
3759                    FrameType::Request,
3760                    pending.module_channel,
3761                    pending.module_epoch,
3762                    800,
3763                ),
3764            )
3765            .await
3766            .unwrap();
3767
3768        let first = client_rx.recv().await.unwrap();
3769        let second = client_rx.recv().await.unwrap();
3770        assert_eq!(first.header.channel, 0);
3771        assert_eq!(first.header.corr, 700);
3772        assert_eq!(second.header.channel, pending.client_channel);
3773        assert_eq!(second.header.epoch, pending.client_epoch);
3774        assert_eq!(second.header.corr, 800);
3775    }
3776
3777    #[tokio::test]
3778    async fn reserved_slot_ingress_errors_only_matching_client_requests() {
3779        let (
3780            router,
3781            _forwarding,
3782            client_ctx,
3783            mut client_rx,
3784            _module_ctx,
3785            _module_egress_rx,
3786            mut module_rx,
3787            pending,
3788        ) = dynamic_route_fixture(false);
3789        router
3790            .route_for_connection(
3791                &client_ctx,
3792                route_frame(
3793                    FrameType::Request,
3794                    pending.client_channel,
3795                    pending.client_epoch,
3796                    900,
3797                ),
3798            )
3799            .await
3800            .unwrap();
3801        let error = client_rx.recv().await.unwrap();
3802        assert_eq!(error.header.ty, FrameType::Error);
3803        assert_eq!(error.header.channel, pending.client_channel);
3804        assert_eq!(error.header.epoch, pending.client_epoch);
3805        assert_eq!(error.header.corr, 900);
3806
3807        router
3808            .route_for_connection(
3809                &client_ctx,
3810                route_frame(
3811                    FrameType::Response,
3812                    pending.client_channel,
3813                    pending.client_epoch,
3814                    901,
3815                ),
3816            )
3817            .await
3818            .unwrap();
3819        router
3820            .route_for_connection(
3821                &client_ctx,
3822                route_frame(
3823                    FrameType::Request,
3824                    pending.client_channel,
3825                    pending.client_epoch + 1,
3826                    902,
3827                ),
3828            )
3829            .await
3830            .unwrap();
3831        let stale_error = client_rx.recv().await.unwrap();
3832        assert_eq!(stale_error.header.ty, FrameType::Error);
3833        assert_eq!(stale_error.header.channel, pending.client_channel);
3834        assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
3835        assert_eq!(stale_error.header.corr, 902);
3836        let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
3837        assert_eq!(body.code, "stale_route_epoch");
3838        assert!(module_rx.try_recv().is_err());
3839        let counters = router.counters.snapshot();
3840        assert_eq!(counters["client_frames_dropped_stale_route"], 1);
3841        assert_eq!(counters["module_frames_dropped_no_route"], 0);
3842    }
3843
3844    #[tokio::test]
3845    async fn dropped_module_route_goodbye_increments_counter() {
3846        let (
3847            router,
3848            _forwarding,
3849            client_ctx,
3850            mut client_rx,
3851            _module_ctx,
3852            _module_egress_rx,
3853            mut module_rx,
3854            pending,
3855        ) = dynamic_route_fixture(true);
3856        let _ = client_rx.recv().await;
3857        module_rx.close();
3858
3859        router
3860            .route_for_connection(
3861                &client_ctx,
3862                route_frame(
3863                    FrameType::Goodbye,
3864                    pending.client_channel,
3865                    pending.client_epoch,
3866                    999,
3867                ),
3868            )
3869            .await
3870            .unwrap();
3871
3872        let counters = router.counters.snapshot();
3873        assert_eq!(counters["goodbye_relay_module_dropped"], 1);
3874        assert_eq!(
3875            counters["goodbye_relay_module_dropped_by_module"],
3876            serde_json::json!({ "epoch-router": 1 })
3877        );
3878        assert_eq!(counters["route_released_epoch_fenced"], 1);
3879    }
3880
3881    #[tokio::test]
3882    async fn module_request_on_stale_epoch_receives_stale_route_epoch() {
3883        let (
3884            router,
3885            _forwarding,
3886            _client_ctx,
3887            _client_rx,
3888            module_ctx,
3889            mut module_egress_rx,
3890            mut module_rx,
3891            pending,
3892        ) = dynamic_route_fixture(true);
3893
3894        router
3895            .route_for_connection(
3896                &module_ctx,
3897                route_frame(
3898                    FrameType::Request,
3899                    pending.module_channel,
3900                    pending.module_epoch + 1,
3901                    1_000,
3902                ),
3903            )
3904            .await
3905            .unwrap();
3906
3907        let error = module_egress_rx.try_recv().unwrap();
3908        assert_eq!(error.header.ty, FrameType::Error);
3909        assert_eq!(error.header.channel, pending.module_channel);
3910        assert_eq!(error.header.epoch, pending.module_epoch + 1);
3911        assert_eq!(error.header.corr, 1_000);
3912        let body: ErrorBody = serde_json::from_slice(&error.body).unwrap();
3913        assert_eq!(body.code, "stale_route_epoch");
3914        assert!(module_rx.try_recv().is_err());
3915        let counters = router.counters.snapshot();
3916        assert_eq!(counters["module_requests_dropped_stale_route"], 1);
3917        assert_eq!(counters["module_frames_dropped_no_route"], 0);
3918    }
3919
3920    #[tokio::test(start_paused = true)]
3921    async fn stale_epoch_drop_warns_once_per_minute_with_suppressed_count() {
3922        let (output, _guard) = test_log::log_capture(tracing::Level::WARN);
3923        let (
3924            router,
3925            _forwarding,
3926            client_ctx,
3927            mut client_rx,
3928            _module_ctx,
3929            _module_egress_rx,
3930            mut module_rx,
3931            pending,
3932        ) = dynamic_route_fixture(true);
3933        let _ = client_rx.recv().await;
3934        for corr in 1..=3 {
3935            router
3936                .route_for_connection(
3937                    &client_ctx,
3938                    route_frame(
3939                        FrameType::Request,
3940                        pending.client_channel,
3941                        pending.client_epoch + 1,
3942                        corr,
3943                    ),
3944                )
3945                .await
3946                .unwrap();
3947            assert_eq!(client_rx.recv().await.unwrap().header.ty, FrameType::Error);
3948        }
3949        assert!(module_rx.try_recv().is_err());
3950        let logs = test_log::captured_logs(&output);
3951        assert_eq!(
3952            logs.matches("dropping data-plane frame").count(),
3953            1,
3954            "{logs}"
3955        );
3956        assert!(logs.contains("WARN"), "{logs}");
3957        assert!(
3958            logs.contains(&format!("connection_id={}", client_ctx.connection_id.get())),
3959            "{logs}"
3960        );
3961        assert!(
3962            logs.contains(&format!("channel={}", pending.client_channel)),
3963            "{logs}"
3964        );
3965        assert!(
3966            logs.contains(&format!("epoch={}", pending.client_epoch + 1)),
3967            "{logs}"
3968        );
3969        assert!(
3970            logs.contains("client_to_module") && logs.contains("stale_epoch"),
3971            "{logs}"
3972        );
3973        assert!(logs.contains("suppressed=0"), "{logs}");
3974
3975        tokio::time::advance(Duration::from_secs(60)).await;
3976        router
3977            .route_for_connection(
3978                &client_ctx,
3979                route_frame(
3980                    FrameType::Request,
3981                    pending.client_channel,
3982                    pending.client_epoch + 2,
3983                    4,
3984                ),
3985            )
3986            .await
3987            .unwrap();
3988        let logs = test_log::captured_logs(&output);
3989        assert_eq!(
3990            logs.matches("dropping data-plane frame").count(),
3991            2,
3992            "{logs}"
3993        );
3994        assert!(logs.contains("suppressed=2"), "{logs}");
3995    }
3996
3997    #[tokio::test]
3998    async fn module_request_on_reserved_or_absent_route_receives_unknown_channel() {
3999        let (
4000            reserved_router,
4001            _forwarding,
4002            _client_ctx,
4003            _client_rx,
4004            reserved_module_ctx,
4005            mut reserved_module_egress_rx,
4006            _module_rx,
4007            reserved,
4008        ) = dynamic_route_fixture(false);
4009        reserved_router
4010            .route_for_connection(
4011                &reserved_module_ctx,
4012                route_frame(
4013                    FrameType::Request,
4014                    reserved.module_channel,
4015                    reserved.module_epoch,
4016                    1_001,
4017                ),
4018            )
4019            .await
4020            .unwrap();
4021        let reserved_error = reserved_module_egress_rx.try_recv().unwrap();
4022        let reserved_body: ErrorBody = serde_json::from_slice(&reserved_error.body).unwrap();
4023        assert_eq!(reserved_error.header.ty, FrameType::Error);
4024        assert_eq!(reserved_error.header.channel, reserved.module_channel);
4025        assert_eq!(reserved_error.header.epoch, reserved.module_epoch);
4026        assert_eq!(reserved_error.header.corr, 1_001);
4027        assert_eq!(reserved_body.code, "unknown_channel");
4028        assert_eq!(
4029            reserved_router.counters.snapshot()["module_requests_dropped_stale_route"],
4030            1
4031        );
4032
4033        let (
4034            absent_router,
4035            _forwarding,
4036            _client_ctx,
4037            _client_rx,
4038            absent_module_ctx,
4039            mut absent_module_egress_rx,
4040            _module_rx,
4041            absent,
4042        ) = dynamic_route_fixture(false);
4043        absent_router
4044            .route_for_connection(
4045                &absent_module_ctx,
4046                route_frame(
4047                    FrameType::Request,
4048                    absent.module_channel + 1,
4049                    absent.module_epoch,
4050                    1_002,
4051                ),
4052            )
4053            .await
4054            .unwrap();
4055        let absent_error = absent_module_egress_rx.try_recv().unwrap();
4056        let absent_body: ErrorBody = serde_json::from_slice(&absent_error.body).unwrap();
4057        assert_eq!(absent_error.header.ty, FrameType::Error);
4058        assert_eq!(absent_error.header.channel, absent.module_channel + 1);
4059        assert_eq!(absent_error.header.epoch, absent.module_epoch);
4060        assert_eq!(absent_error.header.corr, 1_002);
4061        assert_eq!(absent_body.code, "unknown_channel");
4062        assert_eq!(
4063            absent_router.counters.snapshot()["module_requests_dropped_stale_route"],
4064            1
4065        );
4066    }
4067
4068    #[tokio::test]
4069    async fn non_request_module_frame_on_dead_route_is_counted_without_error() {
4070        let (
4071            router,
4072            forwarding,
4073            client_ctx,
4074            mut client_rx,
4075            module_ctx,
4076            mut module_egress_rx,
4077            mut module_rx,
4078            pending,
4079        ) = dynamic_route_fixture(true);
4080        let (other_module_tx, _other_module_rx) = mpsc::channel(8);
4081        forwarding
4082            .register_module_connection(
4083                ConnectionId::new(502),
4084                "other-module".into(),
4085                2,
4086                Concurrency::ModuleManaged,
4087                FrameSink::new(other_module_tx),
4088            )
4089            .unwrap();
4090        let _ = client_rx.recv().await.unwrap();
4091
4092        router
4093            .route_for_connection(
4094                &client_ctx,
4095                route_frame(
4096                    FrameType::Goodbye,
4097                    pending.client_channel,
4098                    pending.client_epoch,
4099                    1_003,
4100                ),
4101            )
4102            .await
4103            .unwrap();
4104        let _ = module_rx.recv().await.unwrap();
4105
4106        router
4107            .route_for_connection(
4108                &module_ctx,
4109                route_frame(
4110                    FrameType::StreamData,
4111                    pending.module_channel,
4112                    pending.module_epoch,
4113                    1_004,
4114                ),
4115            )
4116            .await
4117            .unwrap();
4118
4119        // No ERROR goes back for a non-request frame. The one reply is the
4120        // route GOODBYE telling the module to let go of the released route.
4121        let reply = module_egress_rx.try_recv().unwrap();
4122        assert_eq!(reply.header.ty, FrameType::Goodbye);
4123        assert!(module_egress_rx.try_recv().is_err());
4124        let counters = router.counters.snapshot();
4125        assert_eq!(counters["module_frames_dropped_no_route"], 1);
4126        assert_eq!(
4127            counters["module_frames_dropped_no_route_by_module"],
4128            serde_json::json!({ "epoch-router": 1 })
4129        );
4130        assert_eq!(counters["module_requests_dropped_stale_route"], 0);
4131    }
4132
4133    /// Drain whatever the module connection's queue holds right now, the way a
4134    /// module's reader would before it stalls.
4135    fn drain_now(rx: &mut mpsc::Receiver<OutboundFrame>) {
4136        while rx.try_recv().is_ok() {}
4137    }
4138
4139    /// A module that stops reading for a moment when a client closes one of
4140    /// its routes still learns the route is gone: the GOODBYE its full egress
4141    /// queue refused is delivered as soon as it reads again, rather than
4142    /// dropped, which would leave the module holding the route for the rest of
4143    /// its connection.
4144    #[tokio::test]
4145    async fn route_goodbye_refused_by_stalled_module_is_delivered_when_it_resumes_reading() {
4146        const BUDGET: usize = 4_096;
4147        let forwarding = Arc::new(ForwardingTable::default());
4148        let control = Arc::new(ControlHandler::with_forwarding(
4149            Arc::new(Registry::default()),
4150            Arc::clone(&forwarding),
4151        ));
4152        let router = Router::with_control_handler(control);
4153        let module_connection = ConnectionId::new(80);
4154        let client_connection = ConnectionId::new(81);
4155        let (module_tx, mut module_rx) = mpsc::channel(64);
4156        let module_sink = FrameSink::with_byte_budget(module_tx, BUDGET);
4157        forwarding
4158            .register_module_connection(
4159                module_connection,
4160                "stalled-provider".into(),
4161                2,
4162                Concurrency::ModuleManaged,
4163                module_sink.clone(),
4164            )
4165            .unwrap();
4166        let (client_tx, mut client_rx) = mpsc::channel(8);
4167        let client_sink = FrameSink::new(client_tx);
4168        let pending = forwarding
4169            .begin_route_bind_relay_for_test(
4170                client_connection,
4171                client_sink.clone(),
4172                1_100,
4173                "stalled-provider",
4174            )
4175            .unwrap();
4176        forwarding
4177            .complete_pending_relay(
4178                module_connection,
4179                pending.corr,
4180                RouteBindRelayOutcome::Accepted,
4181            )
4182            .unwrap();
4183        let _ = client_rx.recv().await.unwrap();
4184        drain_now(&mut module_rx);
4185
4186        // The module stalls: its queue holds more than the byte budget, so the
4187        // queue refuses anything further.
4188        module_sink
4189            .try_send(stream_frame(9, 1, 0, vec![b'f'; BUDGET]))
4190            .unwrap();
4191        assert!(module_sink
4192            .try_send(stream_frame(9, 1, 1, Vec::new()))
4193            .is_err());
4194
4195        let client_ctx = RouteCtx {
4196            connection_id: client_connection,
4197            egress: client_sink,
4198        };
4199        router
4200            .route_for_connection(
4201                &client_ctx,
4202                route_frame(
4203                    FrameType::Goodbye,
4204                    pending.client_channel,
4205                    pending.client_epoch,
4206                    1_101,
4207                ),
4208            )
4209            .await
4210            .unwrap();
4211        tokio::task::yield_now().await;
4212        assert_eq!(
4213            router.counters.snapshot()["goodbye_relay_module_dropped"],
4214            0,
4215            "a GOODBYE refused by a momentarily full module queue must not be dropped"
4216        );
4217
4218        // The module reads again: the filler comes off, then the GOODBYE.
4219        let filler = module_rx.recv().await.unwrap();
4220        assert_eq!(filler.header.ty, FrameType::StreamData);
4221        drop(filler);
4222        let goodbye = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
4223            .await
4224            .expect("the refused GOODBYE must be delivered once the module frees room")
4225            .unwrap();
4226        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
4227        assert_eq!(goodbye.header.channel, pending.module_channel);
4228        assert_eq!(goodbye.header.epoch, pending.module_epoch);
4229        assert_eq!(
4230            router.counters.snapshot()["goodbye_relay_module_dropped"],
4231            0
4232        );
4233    }
4234
4235    /// A module still sending on a route the daemon released is told, with a
4236    /// route GOODBYE for exactly the (channel, epoch) it sent on. The same
4237    /// happens for a stale epoch on a channel whose route moved on and for a
4238    /// channel that never had a route; only the first is counted as traffic on
4239    /// a released route.
4240    #[tokio::test]
4241    async fn module_frame_on_route_the_daemon_does_not_hold_is_answered_with_goodbye() {
4242        let (
4243            router,
4244            _forwarding,
4245            client_ctx,
4246            mut client_rx,
4247            module_ctx,
4248            mut module_egress_rx,
4249            mut module_rx,
4250            pending,
4251        ) = dynamic_route_fixture(true);
4252        let _ = client_rx.recv().await.unwrap();
4253        router
4254            .route_for_connection(
4255                &client_ctx,
4256                route_frame(
4257                    FrameType::Goodbye,
4258                    pending.client_channel,
4259                    pending.client_epoch,
4260                    1_200,
4261                ),
4262            )
4263            .await
4264            .unwrap();
4265        drain_now(&mut module_rx);
4266
4267        let cases = [
4268            // Released route: the daemon allocated this (channel, epoch).
4269            (pending.module_channel, pending.module_epoch),
4270            // A channel the daemon never allocated on this connection.
4271            (pending.module_channel + 1, 1),
4272        ];
4273        for (channel, epoch) in cases {
4274            router
4275                .route_for_connection(
4276                    &module_ctx,
4277                    route_frame(FrameType::StreamData, channel, epoch, 1_201),
4278                )
4279                .await
4280                .unwrap();
4281            let reply = module_egress_rx
4282                .try_recv()
4283                .expect("a frame on a route the daemon does not hold is answered");
4284            assert_eq!(reply.header.ty, FrameType::Goodbye);
4285            assert_eq!(reply.header.channel, channel);
4286            assert_eq!(reply.header.epoch, epoch);
4287            assert_eq!(reply.header.corr, 0);
4288            assert!(module_egress_rx.try_recv().is_err());
4289        }
4290
4291        // A GOODBYE from the module for a route the daemon already released
4292        // is not answered: the module is letting go already.
4293        router
4294            .route_for_connection(
4295                &module_ctx,
4296                route_frame(FrameType::Goodbye, pending.module_channel + 2, 1, 0),
4297            )
4298            .await
4299            .unwrap();
4300        assert!(module_egress_rx.try_recv().is_err());
4301
4302        let counters = router.counters.snapshot();
4303        assert_eq!(counters["module_frames_dropped_no_route"], 3);
4304        assert_eq!(counters["module_frames_dropped_released_route"], 1);
4305        assert_eq!(
4306            counters["module_frames_dropped_released_route_by_module"],
4307            serde_json::json!({ "epoch-router": 1 })
4308        );
4309        assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4310    }
4311
4312    /// A chatty orphan (a streaming module still producing on a route it
4313    /// missed the GOODBYE for) is answered once per interval, not once per
4314    /// frame, and answered again once the interval has passed.
4315    #[tokio::test(start_paused = true)]
4316    async fn burst_of_orphan_module_frames_is_answered_once_per_interval() {
4317        let (
4318            router,
4319            _forwarding,
4320            client_ctx,
4321            mut client_rx,
4322            module_ctx,
4323            mut module_egress_rx,
4324            mut module_rx,
4325            pending,
4326        ) = dynamic_route_fixture(true);
4327        let _ = client_rx.recv().await.unwrap();
4328        router
4329            .route_for_connection(
4330                &client_ctx,
4331                route_frame(
4332                    FrameType::Goodbye,
4333                    pending.client_channel,
4334                    pending.client_epoch,
4335                    1_300,
4336                ),
4337            )
4338            .await
4339            .unwrap();
4340        drain_now(&mut module_rx);
4341
4342        let send_burst = |corr: u64| {
4343            route_frame(
4344                FrameType::StreamData,
4345                pending.module_channel,
4346                pending.module_epoch,
4347                corr,
4348            )
4349        };
4350        for corr in 0..20 {
4351            router
4352                .route_for_connection(&module_ctx, send_burst(corr))
4353                .await
4354                .unwrap();
4355        }
4356        let mut replies = 0;
4357        while let Ok(reply) = module_egress_rx.try_recv() {
4358            assert_eq!(reply.header.ty, FrameType::Goodbye);
4359            replies += 1;
4360        }
4361        assert_eq!(replies, 1, "a burst within the interval gets one GOODBYE");
4362
4363        tokio::time::advance(ORPHAN_ROUTE_GOODBYE_INTERVAL).await;
4364        router
4365            .route_for_connection(&module_ctx, send_burst(20))
4366            .await
4367            .unwrap();
4368        let retry = module_egress_rx
4369            .try_recv()
4370            .expect("the first orphan frame after the interval is answered again");
4371        assert_eq!(retry.header.channel, pending.module_channel);
4372        assert_eq!(retry.header.epoch, pending.module_epoch);
4373
4374        let counters = router.counters.snapshot();
4375        assert_eq!(counters["module_frames_dropped_no_route"], 21);
4376        assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4377    }
4378
4379    /// The rate-limit memory is per connection and goes away with it.
4380    #[test]
4381    fn orphan_goodbye_rate_limit_state_is_released_with_the_connection() {
4382        let router = Router::with_default_self_handler();
4383        let connection = router.begin_connection();
4384        let id = connection.id();
4385        assert!(router.orphan_goodbyes.claim(id, 7));
4386        assert!(!router.orphan_goodbyes.claim(id, 7));
4387        assert!(router.orphan_goodbyes.claim(id, 8));
4388        drop(connection);
4389        assert!(router
4390            .orphan_goodbyes
4391            .last_sent
4392            .lock()
4393            .unwrap()
4394            .get(&id)
4395            .is_none());
4396    }
4397
4398    #[test]
4399    fn channel_zero_cannot_be_registered_as_backend() {
4400        let mut router = Router::with_default_self_handler();
4401
4402        let err = router.register_backend(0, EchoBackend).unwrap_err();
4403
4404        assert_eq!(err, RouterError::ReservedChannelZero);
4405    }
4406}