Skip to main content

subc_daemon/
router.rs

1use std::{
2    collections::HashMap,
3    error::Error,
4    fmt,
5    sync::{
6        atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
7        Arc, Mutex,
8    },
9    time::{Duration, Instant},
10};
11
12use subc_protocol::{error_codes, ErrorBody, Flags, FrameType, Priority};
13use tokio::sync::{mpsc, Notify};
14use tracing::{debug, warn};
15
16use crate::{
17    control::ControlHandler,
18    forwarding::{
19        principal_label, CloseReason, ConnectionCloseReceiver, DataRoute, DataRouteState,
20        ForwardingError, ForwardingTable, RouteBinding, RouteRelease, UndeliveredFrame,
21    },
22    registry::ConnectionId,
23    DaemonCounters, Frame, FrameBuildError,
24};
25
26/// One queued outbound frame plus the instant it entered the writer queue.
27///
28/// The stamp exists for the reply-path half of slow-control diagnosis: a
29/// handler can finish in microseconds while the reply sits in this queue
30/// waiting for the writer task to be scheduled, and without a per-item stamp
31/// that wait is invisible to every other timing point (the client's round
32/// trip is the only witness, and it cannot say which side ate the time).
33/// Constructed exclusively inside [`FrameSink`] so no caller can forget it.
34#[derive(Debug)]
35pub struct OutboundFrame {
36    pub frame: Frame,
37    pub enqueued_at: std::time::Instant,
38    pub(crate) flushed: Option<tokio::sync::oneshot::Sender<()>>,
39    /// This frame's share of the connection's queued-byte count. Dropping the
40    /// frame (after the writer has written it, or when the queue itself is
41    /// dropped with the frame still in it) gives the bytes back. `None` only for
42    /// frames built outside a [`FrameSink`], which exist in tests alone.
43    pub(crate) charge: Option<EgressCharge>,
44}
45
46impl OutboundFrame {
47    fn charged(frame: Frame, charge: EgressCharge) -> Self {
48        Self {
49            frame,
50            enqueued_at: charge.enqueued_at,
51            flushed: None,
52            charge: Some(charge),
53        }
54    }
55}
56
57/// Bytes a frame occupies in the connection's egress queue for budget
58/// purposes: the fixed envelope header plus the body.
59fn queued_frame_bytes(frame: &Frame) -> usize {
60    subc_protocol::HEADER_LEN + frame.body.len()
61}
62
63/// A point-in-time view of one connection's egress queue, for diagnosing why a
64/// frame did not fit.
65#[derive(Debug, Clone, Copy, PartialEq, Eq)]
66pub(crate) struct EgressBacklog {
67    pub queued_bytes: usize,
68    pub queued_frames: usize,
69    /// Now minus the enqueue time of the frame the writer most recently took
70    /// off the queue (the frame it is writing, or has just written), or, if
71    /// the writer has taken nothing since the queue was last empty, of the
72    /// frame that made the queue non-empty. While the writer is blocked on a
73    /// frame this is exactly the oldest frame's age; between frames it can be
74    /// one frame older than the true head. `None` when nothing is queued.
75    pub oldest_age: Option<Duration>,
76}
77
78/// Queued-byte accounting shared by every clone of one connection's
79/// [`FrameSink`] and by every frame that sink has admitted. Its per-frame
80/// accounting uses atomics: this is on the path of every frame a module sends
81/// to a client, so it takes no lock per frame.
82#[derive(Debug)]
83struct EgressAccounting {
84    byte_budget: usize,
85    queued_bytes: AtomicUsize,
86    queued_frames: AtomicUsize,
87    /// Origin for the nanosecond timestamps in `oldest_enqueued_nanos`.
88    time_base: Instant,
89    /// Enqueue time, as nanoseconds after `time_base` plus one, of the frame
90    /// described by [`EgressBacklog::oldest_age`]; 0 means none.
91    oldest_enqueued_nanos: AtomicU64,
92    /// Awaited senders currently parked on `freed`. A release only pays for a
93    /// notification when this is non-zero.
94    waiters: AtomicUsize,
95    /// Woken when a charged frame releases its bytes while a sender waits, so
96    /// the awaited send can re-check for room.
97    freed: Notify,
98    client_delivery_closed: AtomicBool,
99    client_delivery_stopped: Notify,
100}
101
102impl EgressAccounting {
103    fn new(byte_budget: usize) -> Self {
104        Self {
105            byte_budget,
106            queued_bytes: AtomicUsize::new(0),
107            queued_frames: AtomicUsize::new(0),
108            time_base: tokio::time::Instant::now().into_std(),
109            oldest_enqueued_nanos: AtomicU64::new(0),
110            waiters: AtomicUsize::new(0),
111            freed: Notify::new(),
112            client_delivery_closed: AtomicBool::new(false),
113            client_delivery_stopped: Notify::new(),
114        }
115    }
116
117    fn stamp(&self, at: Instant) -> u64 {
118        (at.saturating_duration_since(self.time_base).as_nanos() as u64).saturating_add(1)
119    }
120
121    /// Charge `bytes` if they fit in the budget. A frame larger than the whole
122    /// budget (bodies may be up to 64 MiB) is still admitted into an EMPTY
123    /// queue, since it could otherwise never be sent at all; it simply has the
124    /// queue to itself until it is written.
125    fn try_charge(self: &Arc<Self>, bytes: usize) -> Option<EgressCharge> {
126        // SeqCst pairs with `release`: either this load sees bytes a release
127        // just freed, or that release sees this sender counted in `waiters`.
128        let mut current = self.queued_bytes.load(Ordering::SeqCst);
129        loop {
130            if current != 0 && current.saturating_add(bytes) > self.byte_budget {
131                return None;
132            }
133            match self.queued_bytes.compare_exchange_weak(
134                current,
135                current + bytes,
136                Ordering::SeqCst,
137                Ordering::SeqCst,
138            ) {
139                Ok(_) => return Some(self.record(bytes)),
140                Err(actual) => current = actual,
141            }
142        }
143    }
144
145    /// Charge `bytes` regardless of the budget. Used only for a frame whose
146    /// queue slot was reserved in advance, which must be sendable.
147    fn charge_unconditionally(self: &Arc<Self>, bytes: usize) -> EgressCharge {
148        self.queued_bytes.fetch_add(bytes, Ordering::SeqCst);
149        self.record(bytes)
150    }
151
152    fn record(self: &Arc<Self>, bytes: usize) -> EgressCharge {
153        let enqueued_at = tokio::time::Instant::now().into_std();
154        let stamp = self.stamp(enqueued_at);
155        if self.queued_frames.fetch_add(1, Ordering::AcqRel) == 0 {
156            // This frame made the queue non-empty, so it is the head until the
157            // writer takes something.
158            self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
159        }
160        EgressCharge {
161            accounting: Arc::clone(self),
162            bytes,
163            stamp,
164            enqueued_at,
165        }
166    }
167
168    /// The writer has taken the frame stamped `stamp` off the queue.
169    fn taken_by_writer(&self, stamp: u64) {
170        self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
171    }
172
173    fn release(&self, bytes: usize, stamp: u64) {
174        self.queued_bytes.fetch_sub(bytes, Ordering::SeqCst);
175        if self.queued_frames.fetch_sub(1, Ordering::AcqRel) == 1 {
176            // The queue drained. Clear the marker only if it still names this
177            // frame, so a frame admitted in the meantime keeps its stamp.
178            let _ = self.oldest_enqueued_nanos.compare_exchange(
179                stamp,
180                0,
181                Ordering::AcqRel,
182                Ordering::Acquire,
183            );
184        }
185        if self.waiters.load(Ordering::SeqCst) != 0 {
186            self.freed.notify_waiters();
187        }
188    }
189
190    fn backlog(&self) -> EgressBacklog {
191        let queued_frames = self.queued_frames.load(Ordering::Acquire);
192        let stamp = self.oldest_enqueued_nanos.load(Ordering::Acquire);
193        let oldest_age = (queued_frames != 0 && stamp != 0).then(|| {
194            let enqueued = self.time_base + Duration::from_nanos(stamp - 1);
195            tokio::time::Instant::from_std(enqueued).elapsed()
196        });
197        EgressBacklog {
198            queued_bytes: self.queued_bytes.load(Ordering::Acquire),
199            queued_frames,
200            oldest_age,
201        }
202    }
203}
204
205/// One admitted frame's claim on its connection's egress byte budget,
206/// returned when the frame is dropped.
207#[derive(Debug)]
208pub(crate) struct EgressCharge {
209    accounting: Arc<EgressAccounting>,
210    bytes: usize,
211    stamp: u64,
212    enqueued_at: Instant,
213}
214
215impl EgressCharge {
216    /// Called by the connection writer when it takes this frame off the queue,
217    /// so the backlog's oldest-age figure follows the writer.
218    pub(crate) fn taken_by_writer(&self) {
219        self.accounting.taken_by_writer(self.stamp);
220    }
221}
222
223impl Drop for EgressCharge {
224    fn drop(&mut self) {
225        self.accounting.release(self.bytes, self.stamp);
226    }
227}
228
229/// A queue slot reserved ahead of time (a pending `route.open` holds one until
230/// its module answers). Sending through it never waits and never fails for
231/// lack of room: the slot is already held, and its frame is charged to the
232/// byte count outside the budget check. At most
233/// `MAX_PENDING_ROUTE_OPENS_PER_CONNECTION` such frames exist per connection.
234#[derive(Debug)]
235pub(crate) struct EgressPermit {
236    permit: mpsc::OwnedPermit<OutboundFrame>,
237    accounting: Arc<EgressAccounting>,
238}
239
240impl EgressPermit {
241    /// Enqueue `frame` in the reserved slot. Returns true when the connection's
242    /// writer had already gone away, meaning the frame will never be written.
243    pub(crate) fn send(self, frame: Frame) -> bool {
244        let charge = self
245            .accounting
246            .charge_unconditionally(queued_frame_bytes(&frame));
247        let sender = self.permit.send(OutboundFrame::charged(frame, charge));
248        sender.is_closed()
249    }
250}
251
252/// An `OutboundFrame` is a stamped `Frame`; deref keeps every existing frame
253/// read (headers, bodies, assertions) working on queued items unchanged.
254impl std::ops::Deref for OutboundFrame {
255    type Target = Frame;
256
257    fn deref(&self) -> &Frame {
258        &self.frame
259    }
260}
261
262/// Shared tracing-capture helpers for timing-observability tests across
263/// modules (router dispatch, server reply path). Test-only.
264#[cfg(test)]
265pub(crate) mod test_log {
266    use std::{
267        io::Write,
268        sync::{Arc, Mutex, OnceLock},
269    };
270
271    #[derive(Clone)]
272    struct TestLogWriter(Arc<Mutex<Vec<u8>>>);
273
274    impl Write for TestLogWriter {
275        fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
276            self.0
277                .lock()
278                .expect("test log capture is not poisoned")
279                .extend(buffer);
280            Ok(buffer.len())
281        }
282
283        fn flush(&mut self) -> std::io::Result<()> {
284            Ok(())
285        }
286    }
287
288    pub(crate) fn log_capture(
289        level: tracing::Level,
290    ) -> (Arc<Mutex<Vec<u8>>>, tracing::dispatcher::DefaultGuard) {
291        // tracing-core's single-dispatch fast path registers callsites using
292        // the emitting thread's default subscriber. A parallel test with no
293        // subscriber can therefore cache `never` for a callsite this capture
294        // needs. Keep a second dispatch alive so registration considers all
295        // live subscribers instead. This registry is never installed as a
296        // default: each test still records only into its own thread's writer.
297        static INTEREST_ANCHOR: OnceLock<tracing::Dispatch> = OnceLock::new();
298        INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
299        let output = Arc::new(Mutex::new(Vec::new()));
300        let writer = Arc::clone(&output);
301        let subscriber = tracing_subscriber::fmt()
302            .with_max_level(level)
303            .with_ansi(false)
304            .without_time()
305            .with_target(false)
306            .with_writer(move || TestLogWriter(Arc::clone(&writer)))
307            .finish();
308        let guard = tracing::subscriber::set_default(subscriber);
309        (output, guard)
310    }
311
312    pub(crate) fn captured_logs(output: &Arc<Mutex<Vec<u8>>>) -> String {
313        String::from_utf8(
314            output
315                .lock()
316                .expect("test log capture is not poisoned")
317                .clone(),
318        )
319        .expect("tracing output is UTF-8")
320    }
321
322    #[cfg(test)]
323    mod tests {
324        use super::*;
325
326        fn emit_capture_probe(source: &str) {
327            tracing::error!(source, "parallel capture probe");
328        }
329
330        #[test]
331        fn parallel_callsite_registration_keeps_the_test_capture_enabled() {
332            let (output, _guard) = log_capture(tracing::Level::ERROR);
333            // First register the very same callsite on a thread without a
334            // default subscriber, while the test's capture remains live.
335            std::thread::spawn(|| emit_capture_probe("other-thread"))
336                .join()
337                .expect("unsubscribed thread exits");
338            emit_capture_probe("captured-thread");
339
340            let logs = captured_logs(&output);
341            assert!(
342                logs.contains("captured-thread"),
343                "capture was disabled: {logs}"
344            );
345            assert!(!logs.contains("other-thread"), "capture leaked: {logs}");
346        }
347    }
348}
349
350/// Cheaply cloneable handle to one connection's bounded outbound frame queue.
351///
352/// Backends emit responses, streaming frames, and future PUSH frames through this
353/// single path. The queue is bounded twice: by queued BYTES (the budget that
354/// matters for memory and for how long a slow reader may pause), and by the
355/// `mpsc` channel's frame count (a backstop). The socket layer owns the sole
356/// receiver/writer.
357#[derive(Debug, Clone)]
358pub struct FrameSink {
359    tx: mpsc::Sender<OutboundFrame>,
360    accounting: Arc<EgressAccounting>,
361}
362
363impl FrameSink {
364    /// A sink with the standard per-connection byte budget
365    /// ([`crate::server::CONNECTION_EGRESS_BYTE_BUDGET`]); the frame-count bound
366    /// is whatever capacity `tx`'s channel was created with.
367    pub fn new(tx: mpsc::Sender<OutboundFrame>) -> Self {
368        Self::with_byte_budget(tx, crate::server::CONNECTION_EGRESS_BYTE_BUDGET)
369    }
370
371    pub(crate) fn with_byte_budget(tx: mpsc::Sender<OutboundFrame>, byte_budget: usize) -> Self {
372        Self {
373            tx,
374            accounting: Arc::new(EgressAccounting::new(byte_budget)),
375        }
376    }
377
378    /// Wait until the frame's bytes fit the budget, then charge them. Awaited
379    /// senders (control replies, error replies, shutdown notices) are never
380    /// refused by the byte budget: they wait for the writer to free bytes, just
381    /// as they wait for a free slot, which is the same backpressure they had
382    /// when the queue was bounded by frame count alone. Returns `None` if the
383    /// writer goes away while waiting.
384    async fn charge_waiting(&self, bytes: usize) -> Option<EgressCharge> {
385        if let Some(charge) = self.accounting.try_charge(bytes) {
386            return Some(charge);
387        }
388        // Counted as a waiter for as long as this future is parked, including
389        // when it is cancelled, so releases notify only while someone waits.
390        struct Waiting<'a>(&'a AtomicUsize);
391        impl Drop for Waiting<'_> {
392            fn drop(&mut self) {
393                self.0.fetch_sub(1, Ordering::SeqCst);
394            }
395        }
396        self.accounting.waiters.fetch_add(1, Ordering::SeqCst);
397        let _waiting = Waiting(&self.accounting.waiters);
398        loop {
399            let freed = self.accounting.freed.notified();
400            tokio::pin!(freed);
401            // Register interest before checking, so a release that happens
402            // between the check and the await still wakes this sender.
403            freed.as_mut().enable();
404            if let Some(charge) = self.accounting.try_charge(bytes) {
405                return Some(charge);
406            }
407            tokio::select! {
408                _ = &mut freed => {}
409                _ = self.tx.closed() => return None,
410            }
411        }
412    }
413
414    pub async fn send(&self, frame: Frame) -> Result<(), RouterError> {
415        let channel = frame.header.channel;
416        let epoch = frame.header.epoch;
417        let corr = frame.header.corr;
418        let closed =
419            || RouterError::backend_with_epoch(channel, epoch, corr, "connection writer closed");
420        let charge = self
421            .charge_waiting(queued_frame_bytes(&frame))
422            .await
423            .ok_or_else(closed)?;
424        self.tx
425            .send(OutboundFrame::charged(frame, charge))
426            .await
427            .map_err(|_| closed())
428    }
429
430    /// Shutdown notices must leave the socket writer before an idle daemon exits.
431    /// Queue admission alone does not prove this; the writer acknowledges flush.
432    ///
433    /// The local endpoint uses it the same way for its clients-only refusal: the
434    /// connection closes right after the refusal is sent, so the writer must
435    /// flush it first.
436    #[cfg(unix)]
437    pub async fn send_flushed(&self, frame: Frame) -> Result<(), RouterError> {
438        let (tx, rx) = tokio::sync::oneshot::channel();
439        let charge = self
440            .charge_waiting(queued_frame_bytes(&frame))
441            .await
442            .ok_or_else(|| RouterError::backend(0, 0, "connection writer closed"))?;
443        let mut outbound = OutboundFrame::charged(frame, charge);
444        outbound.flushed = Some(tx);
445        self.tx
446            .send(outbound)
447            .await
448            .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
449        rx.await
450            .map_err(|_| RouterError::backend(0, 0, "connection flush failed"))
451    }
452
453    pub(crate) async fn reserve_owned(&self) -> Result<EgressPermit, RouterError> {
454        let permit = self
455            .tx
456            .clone()
457            .reserve_owned()
458            .await
459            .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
460        Ok(EgressPermit {
461            permit,
462            accounting: Arc::clone(&self.accounting),
463        })
464    }
465
466    #[cfg(test)]
467    pub(crate) fn try_reserve_owned(&self) -> Result<EgressPermit, RouterError> {
468        let permit = self
469            .tx
470            .clone()
471            .try_reserve_owned()
472            .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
473        Ok(EgressPermit {
474            permit,
475            accounting: Arc::clone(&self.accounting),
476        })
477    }
478
479    pub(crate) fn is_closed(&self) -> bool {
480        self.tx.is_closed()
481    }
482
483    /// What the connection's egress queue holds right now.
484    pub(crate) fn backlog(&self) -> EgressBacklog {
485        self.accounting.backlog()
486    }
487
488    /// Stop module deliveries as soon as the client reader exits, without
489    /// waiting for the socket writer's separate teardown drain grace.
490    pub(crate) fn close_client_delivery(&self) {
491        self.accounting
492            .client_delivery_closed
493            .store(true, Ordering::Release);
494        self.accounting.client_delivery_stopped.notify_waiters();
495    }
496
497    /// A full queue can be a short fanout burst rather than a stopped reader.
498    /// Spend only the grace the atomic backlog age has not already used.
499    async fn send_to_client(&self, frame: Frame) -> Result<(), ClientDeliveryFailure> {
500        let frame_type = frame.header.ty;
501        let flags = frame.header.flags;
502        let stopped = self.accounting.client_delivery_stopped.notified();
503        tokio::pin!(stopped);
504        stopped.as_mut().enable();
505        let failure = |cause, error| ClientDeliveryFailure {
506            diagnosis: ClientDeliveryDiagnosis {
507                frame_type,
508                flags,
509                cause,
510            },
511            error,
512        };
513        let closed = || {
514            RouterError::backend_with_epoch(
515                frame.header.channel,
516                frame.header.epoch,
517                frame.header.corr,
518                "client connection closed",
519            )
520        };
521        if self
522            .accounting
523            .client_delivery_closed
524            .load(Ordering::Acquire)
525        {
526            return Err(failure("client_closed", closed()));
527        }
528        let (frame, error) = match self.try_send_owned(frame) {
529            Ok(()) => return Ok(()),
530            Err(refused) => refused,
531        };
532        if self.is_closed() {
533            return Err(failure("client_closed", error));
534        }
535        let grace = crate::server::CONNECTION_EGRESS_STALL_GRACE;
536        let age = self.backlog().oldest_age.unwrap_or_default();
537        if age > grace {
538            return Err(failure("oldest_frame_older_than_250_ms", error));
539        }
540        let result = tokio::select! {
541            biased;
542            _ = &mut stopped => return Err(failure("client_closed", error)),
543            result = tokio::time::timeout(grace.saturating_sub(age), self.send(frame)) => result,
544        };
545        match result {
546            Ok(Ok(())) => Ok(()),
547            Ok(Err(error)) => Err(failure("client_closed", error)),
548            Err(_) => Err(failure("waited_250_ms_without_room", error)),
549        }
550    }
551
552    /// Enqueue without waiting. Fails when the frame's bytes would take the
553    /// queue past its byte budget, when the channel's frame-count backstop is
554    /// full, or when the writer is gone.
555    pub(crate) fn try_send(&self, frame: Frame) -> Result<(), RouterError> {
556        self.try_send_owned(frame).map_err(|(_, error)| error)
557    }
558
559    fn try_send_owned(&self, frame: Frame) -> Result<(), (Frame, RouterError)> {
560        let channel = frame.header.channel;
561        let epoch = frame.header.epoch;
562        let corr = frame.header.corr;
563        let unavailable = |why: String| {
564            RouterError::backend_with_epoch(
565                channel,
566                epoch,
567                corr,
568                format!("connection writer unavailable: {why}"),
569            )
570        };
571        let bytes = queued_frame_bytes(&frame);
572        let Some(charge) = self.accounting.try_charge(bytes) else {
573            if self.tx.is_closed() {
574                return Err((frame, unavailable("channel closed".to_string())));
575            }
576            return Err((
577                frame,
578                unavailable(format!(
579                    "egress byte budget exhausted ({} queued bytes, frame of {bytes} bytes, budget {})",
580                    self.accounting.queued_bytes.load(Ordering::Acquire),
581                    self.accounting.byte_budget
582                )),
583            ));
584        };
585        // Return the refused frame for a bounded retry; dropping the rest of
586        // the outbound wrapper returns its charge.
587        self.tx
588            .try_send(OutboundFrame::charged(frame, charge))
589            .map_err(|err| {
590                let error = unavailable(err.to_string());
591                (err.into_inner().frame, error)
592            })
593    }
594}
595
596#[derive(Debug)]
597struct ClientDeliveryFailure {
598    diagnosis: ClientDeliveryDiagnosis,
599    error: RouterError,
600}
601
602/// Header facts only: payloads can contain secrets and must never enter logs.
603#[derive(Debug)]
604pub(crate) struct ClientDeliveryDiagnosis {
605    pub frame_type: FrameType,
606    pub flags: Flags,
607    pub cause: &'static str,
608}
609
610impl fmt::Display for ClientDeliveryFailure {
611    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
612        self.error.fmt(f)
613    }
614}
615
616/// Minimum time between two route GOODBYEs the daemon sends one module
617/// connection for one channel in answer to the module's frames on a (channel,
618/// epoch) the daemon holds no route for.
619///
620/// A module that missed a GOODBYE keeps sending on that route (a streaming
621/// module, many frames a second), and every one of those frames lands here. One
622/// answer is enough when it arrives, so answering each frame would only flood
623/// the module's egress queue at the moment it is already behind. Five seconds
624/// is long enough for an answer queued behind a deep backlog to reach the module
625/// and take effect before a second is sent, and short enough that an answer
626/// the queue refused is retried by the next orphan frame well within a minute.
627const ORPHAN_ROUTE_GOODBYE_INTERVAL: Duration = Duration::from_secs(5);
628
629/// Once one connection has this many remembered channels, entries older than
630/// [`ORPHAN_ROUTE_GOODBYE_INTERVAL`] are pruned before another is added. An
631/// expired entry has no effect on rate limiting, so pruning never changes a
632/// decision; it keeps a module that orphaned many channels long ago from
633/// pinning memory for all of them. The map is bounded in any case by the
634/// 16-bit channel space and is dropped whole when the connection ends.
635const ORPHAN_ROUTE_GOODBYE_PRUNE_AT: usize = 256;
636
637/// When each module connection was last sent an orphan-route GOODBYE, per
638/// channel. Shared by the [`Router`] (which consults it) and every
639/// [`RouterConnection`] (which removes its own entry when the connection ends).
640#[derive(Debug, Default)]
641struct OrphanGoodbyeLimiter {
642    last_sent: Mutex<HashMap<ConnectionId, HashMap<u16, tokio::time::Instant>>>,
643}
644
645impl OrphanGoodbyeLimiter {
646    /// True, and the send recorded, when `channel` on `connection_id` has not
647    /// been answered within the interval.
648    fn claim(&self, connection_id: ConnectionId, channel: u16) -> bool {
649        let now = tokio::time::Instant::now();
650        let mut last_sent = self
651            .last_sent
652            .lock()
653            .unwrap_or_else(std::sync::PoisonError::into_inner);
654        let channels = last_sent.entry(connection_id).or_default();
655        if channels.get(&channel).is_some_and(|sent| {
656            now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
657        }) {
658            return false;
659        }
660        if channels.len() >= ORPHAN_ROUTE_GOODBYE_PRUNE_AT {
661            channels.retain(|_, sent| {
662                now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
663            });
664        }
665        channels.insert(channel, now);
666        true
667    }
668
669    fn forget_connection(&self, connection_id: ConnectionId) {
670        self.last_sent
671            .lock()
672            .unwrap_or_else(std::sync::PoisonError::into_inner)
673            .remove(&connection_id);
674    }
675}
676
677/// Per-route context shared with backends besides the frame itself.
678#[derive(Debug, Clone)]
679pub struct RouteCtx {
680    pub connection_id: ConnectionId,
681    pub egress: FrameSink,
682}
683
684/// Closed set of data-plane backends for non-zero channels.
685///
686/// Channel 0 is structurally special and remains [`Router::control`], not an
687/// enum variant. Static/test backends live in [`Router::backends`]; the forwarding
688/// backend is selected dynamically only when a per-connection forwarding binding exists.
689#[derive(Debug, Clone)]
690pub enum Backend {
691    Echo(EchoBackend),
692    Forward(ForwardBackend),
693}
694
695impl From<EchoBackend> for Backend {
696    fn from(backend: EchoBackend) -> Self {
697        Self::Echo(backend)
698    }
699}
700
701impl From<ForwardBackend> for Backend {
702    fn from(backend: ForwardBackend) -> Self {
703        Self::Forward(backend)
704    }
705}
706
707impl Backend {
708    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
709        match self {
710            Self::Echo(backend) => backend.handle(ctx, frame).await,
711            Self::Forward(backend) => backend.handle(ctx, frame).await,
712        }
713    }
714}
715
716/// I/O-agnostic splice router keyed by envelope `channel`.
717///
718/// Channel 0 is reserved for subc itself and is always dispatched to the
719/// dedicated control handler. Other channels must be explicitly registered.
720/// Unknown client-originated non-zero channels are translated to canonical JSON `ERROR` frames on
721/// the connection sink so the peer can continue using the same socket. Module-originated frames for
722/// released route channels are logged and dropped as the channel-gone race backstop.
723///
724/// DATA-PLANE BODIES ARE NEVER DECODED HERE, and the consequence is worth stating
725/// because it looks like a guarantee and is not. Additive fields in a request or
726/// response body reach the far side untouched — not because anything permits them,
727/// but because routing reads only the 21-byte header and treats the body as opaque
728/// bytes. That is a performance property, so **nothing prevents it from changing**:
729/// a future reason to inspect a body would convert a wire-transparent path into a
730/// filtering one, and nobody would think of it as a contract change.
731///
732/// So when a peer asks whether subc sees an additive field, the answer is per-path
733/// and this path's zero means WE NEVER LOOK rather than WE LOOK AT EVERYTHING. The
734/// control plane (typed enums at the frame boundary) and the MCP gateway (envelope
735/// unwrap plus a named struct) both narrow; only this one does not.
736pub struct Router {
737    backends: HashMap<u16, Backend>,
738    control: Arc<ControlHandler>,
739    forwarding: Arc<ForwardingTable>,
740    forward_backend: ForwardBackend,
741    counters: DaemonCounters,
742    next_connection_id: AtomicU64,
743    orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
744}
745
746impl Router {
747    pub fn with_control_handler(control: Arc<ControlHandler>) -> Self {
748        // The handler the router serves is the one a swap must tell when it
749        // promotes a candidate; this is where it first sits behind an `Arc`.
750        control.install_swap_promotion_observer();
751        let forwarding = control.forwarding();
752        let counters = control.counters();
753        Self {
754            backends: HashMap::new(),
755            control,
756            forwarding: Arc::clone(&forwarding),
757            forward_backend: ForwardBackend::new(forwarding),
758            counters,
759            // ConnectionId::LOCAL is 0; real socket ids start at 1 and never collide.
760            next_connection_id: AtomicU64::new(1),
761            orphan_goodbyes: Arc::default(),
762        }
763    }
764
765    pub fn with_default_self_handler() -> Self {
766        Self::with_control_handler(Arc::new(ControlHandler::default()))
767    }
768
769    pub fn forwarding(&self) -> Arc<ForwardingTable> {
770        Arc::clone(&self.forwarding)
771    }
772
773    pub fn register_backend(
774        &mut self,
775        channel: u16,
776        backend: impl Into<Backend>,
777    ) -> Result<(), RouterError> {
778        self.register_backend_arc(channel, Arc::new(backend.into()))
779    }
780
781    pub(crate) fn register_backend_arc(
782        &mut self,
783        channel: u16,
784        backend: Arc<Backend>,
785    ) -> Result<(), RouterError> {
786        if channel == 0 {
787            return Err(RouterError::ReservedChannelZero);
788        }
789        if self.backends.contains_key(&channel) {
790            return Err(RouterError::DuplicateChannel { channel });
791        }
792        self.backends.insert(channel, backend.as_ref().clone());
793        Ok(())
794    }
795
796    /// Start a connection-scoped routing context. Dropping the guard releases
797    /// any control-plane registrations owned by the connection.
798    fn record_module_frame_drop(&self, connection_id: ConnectionId) -> Result<(), RouterError> {
799        let module_id = self
800            .forwarding
801            .module_id_for_connection(connection_id)
802            .map_err(RouterError::Forwarding)?;
803        self.counters
804            .increment_module_frames_dropped_no_route(module_id.as_deref());
805        Ok(())
806    }
807
808    /// A module sent a non-request frame on a (channel, epoch) the daemon holds
809    /// no route for: most often a route the daemon released whose GOODBYE the
810    /// module never received, so the module still believes it is open and keeps
811    /// sending. Count the drop, and tell the module to let go of exactly that
812    /// (channel, epoch) with a route GOODBYE, at most once per
813    /// [`ORPHAN_ROUTE_GOODBYE_INTERVAL`] per connection and channel.
814    ///
815    /// The answer uses `try_send`: it is a best-effort nudge, and if the queue
816    /// refuses it, the module's next frame on that route after the interval
817    /// asks again. A GOODBYE from the module is not answered, since the module
818    /// is already letting go of the route.
819    fn handle_orphan_module_frame(&self, ctx: &RouteCtx, frame: &Frame) -> Result<(), RouterError> {
820        let channel = frame.header.channel;
821        let epoch = frame.header.epoch;
822        let module_id = self
823            .forwarding
824            .module_id_for_connection(ctx.connection_id)
825            .map_err(RouterError::Forwarding)?;
826        self.counters
827            .increment_module_frames_dropped_no_route(module_id.as_deref());
828        if self
829            .forwarding
830            .module_route_epoch_was_allocated(ctx.connection_id, channel, epoch)
831            .map_err(RouterError::Forwarding)?
832        {
833            self.counters
834                .increment_module_frames_dropped_released_route(module_id.as_deref());
835        }
836        if frame.header.ty == FrameType::Goodbye
837            || !self.orphan_goodbyes.claim(ctx.connection_id, channel)
838        {
839            return Ok(());
840        }
841        let goodbye = Frame::build_with_version(
842            frame.header.ver,
843            FrameType::Goodbye,
844            Flags::new(false, Priority::Passive, false),
845            channel,
846            epoch,
847            0,
848            Vec::new(),
849        )
850        .map_err(RouterError::FrameBuild)?;
851        match ctx.egress.try_send(goodbye) {
852            Ok(()) => {
853                self.counters.increment_module_orphan_route_goodbyes_sent();
854                debug!(
855                    connection_id = ctx.connection_id.get(),
856                    module_id = module_id.as_deref().unwrap_or("unknown"),
857                    channel,
858                    epoch,
859                    "answered module frame on a route the daemon does not hold with a route GOODBYE"
860                );
861            }
862            Err(err) => {
863                self.forwarding.warn_frame_drop(
864                    ctx.connection_id,
865                    channel,
866                    epoch,
867                    "daemon_to_module",
868                    if ctx.egress.is_closed() {
869                        "endpoint_gone"
870                    } else {
871                        "egress_budget"
872                    },
873                );
874                debug!(
875                connection_id = ctx.connection_id.get(),
876                module_id = module_id.as_deref().unwrap_or("unknown"),
877                channel,
878                epoch,
879                error = %err,
880                "could not enqueue route GOODBYE for module frame on a route the daemon does not hold; the next such frame retries"
881                );
882            }
883        }
884        Ok(())
885    }
886
887    pub fn begin_connection(&self) -> RouterConnection {
888        let raw = self.next_connection_id.fetch_add(1, Ordering::Relaxed);
889        let id = ConnectionId::new(raw);
890        let close_receiver = self.forwarding.register_connection_close(id);
891        RouterConnection {
892            id,
893            control_handler: Arc::clone(&self.control),
894            forwarding: Arc::clone(&self.forwarding),
895            close_receiver: Some(close_receiver),
896            orphan_goodbyes: Arc::clone(&self.orphan_goodbyes),
897        }
898    }
899
900    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
901        self.control.route_open_target(frame)
902    }
903
904    pub(crate) fn record_captured_peer(
905        &self,
906        connection: ConnectionId,
907        peer: subc_os::local_peer::CapturedPeer,
908    ) {
909        self.control.record_captured_peer(connection, peer);
910    }
911
912    pub(crate) fn record_caller_endpoints(
913        &self,
914        connection: ConnectionId,
915        endpoints: Option<(std::net::SocketAddr, std::net::SocketAddr)>,
916    ) {
917        self.control.record_caller_endpoints(connection, endpoints);
918    }
919
920    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
921        self.control.defers_privileged_control(ctx, frame)
922    }
923
924    pub(crate) fn route_open_capacity_refusal(
925        &self,
926        ctx: &RouteCtx,
927        frame: &Frame,
928        target_module_id: &str,
929        in_flight: usize,
930        limit: usize,
931    ) -> Result<Frame, RouterError> {
932        self.control
933            .route_open_capacity_refusal(ctx, frame, target_module_id, in_flight, limit)
934    }
935
936    pub async fn route_for_connection(
937        &self,
938        ctx: &RouteCtx,
939        frame: Frame,
940    ) -> Result<(), RouterError> {
941        self.route_for_connection_started(ctx, frame, None).await
942    }
943
944    pub(crate) async fn route_for_connection_started(
945        &self,
946        ctx: &RouteCtx,
947        frame: Frame,
948        dispatch_started_at: Option<Instant>,
949    ) -> Result<(), RouterError> {
950        let channel = frame.header.channel;
951        let epoch = frame.header.epoch;
952        let corr = frame.header.corr;
953        if channel == 0 {
954            debug!(
955                connection_id = ctx.connection_id.get(),
956                corr,
957                frame_type = ?frame.header.ty,
958                "routing control frame"
959            );
960            // The connection loop dispatches every control operation directly
961            // except `route.open`, activity writes and privileged requests. Each
962            // task passes a timestamp captured before
963            // spawn, so slow-dispatch timing includes scheduler delay but no
964            // wait in an application-owned queue.
965            let dispatch_started_at = (frame.header.ty == FrameType::Request)
966                .then(|| dispatch_started_at.unwrap_or_else(Instant::now));
967            let responses = self
968                .control
969                .handle_control_frame_timed(ctx, frame, dispatch_started_at)
970                .await?;
971            for response in responses {
972                ctx.egress.send(response).await?;
973            }
974            return Ok(());
975        }
976
977        let data_route = self
978            .forwarding
979            .lookup_data_route(ctx.connection_id, channel, epoch)
980            .map_err(RouterError::Forwarding)?;
981
982        match data_route {
983            DataRoute::Module(DataRouteState::EpochMismatch) => {
984                self.forwarding.warn_frame_drop(
985                    ctx.connection_id,
986                    channel,
987                    epoch,
988                    "module_to_client",
989                    "stale_epoch",
990                );
991                if frame.header.ty == FrameType::Request {
992                    self.counters
993                        .increment_module_requests_dropped_stale_route();
994                    let err = RouterError::StaleRouteEpoch {
995                        channel,
996                        epoch,
997                        corr,
998                    };
999                    if let Some(error_frame) = err.to_error_frame() {
1000                        ctx.egress.send(error_frame).await?;
1001                    }
1002                } else {
1003                    self.handle_orphan_module_frame(ctx, &frame)?;
1004                }
1005                debug!(
1006                    connection_id = ctx.connection_id.get(),
1007                    channel, epoch, corr, "dropping module frame for stale route epoch"
1008                );
1009                return Ok(());
1010            }
1011            DataRoute::Module(DataRouteState::Reserved) => {
1012                self.forwarding.warn_frame_drop(
1013                    ctx.connection_id,
1014                    channel,
1015                    epoch,
1016                    "module_to_client",
1017                    "missing_binding",
1018                );
1019                if frame.header.ty == FrameType::Request {
1020                    self.counters
1021                        .increment_module_requests_dropped_stale_route();
1022                    let err = RouterError::UnknownChannel {
1023                        channel,
1024                        epoch,
1025                        corr,
1026                    };
1027                    if let Some(error_frame) = err.to_error_frame() {
1028                        ctx.egress.send(error_frame).await?;
1029                    }
1030                } else {
1031                    self.record_module_frame_drop(ctx.connection_id)?;
1032                }
1033                debug!(
1034                    connection_id = ctx.connection_id.get(),
1035                    channel, epoch, corr, "dropping module frame for reserved route handle"
1036                );
1037                return Ok(());
1038            }
1039            DataRoute::Module(DataRouteState::Absent) => {
1040                self.forwarding.warn_frame_drop(
1041                    ctx.connection_id,
1042                    channel,
1043                    epoch,
1044                    "module_to_client",
1045                    "unknown_channel",
1046                );
1047                if frame.header.ty == FrameType::Request {
1048                    self.counters
1049                        .increment_module_requests_dropped_stale_route();
1050                    let err = RouterError::UnknownChannel {
1051                        channel,
1052                        epoch,
1053                        corr,
1054                    };
1055                    if let Some(error_frame) = err.to_error_frame() {
1056                        ctx.egress.send(error_frame).await?;
1057                    }
1058                } else {
1059                    self.handle_orphan_module_frame(ctx, &frame)?;
1060                }
1061                debug!(
1062                    connection_id = ctx.connection_id.get(),
1063                    channel, epoch, corr, "dropping module frame for absent route handle"
1064                );
1065                return Ok(());
1066            }
1067            DataRoute::Module(DataRouteState::Bound(route)) => {
1068                if frame.header.ty == FrameType::Goodbye {
1069                    if let RouteRelease::Removed(target) = self
1070                        .forwarding
1071                        .release_module_route(ctx.connection_id, channel, epoch)
1072                        .map_err(RouterError::Forwarding)?
1073                    {
1074                        let mut goodbye = frame;
1075                        goodbye.header.channel = target.channel;
1076                        goodbye.header.epoch = target.epoch;
1077                        // A module must never be able to make a frame look
1078                        // daemon-authored, so the origin bit is cleared on
1079                        // every module frame relayed to a client.
1080                        goodbye.header.flags = goodbye.header.flags.without_daemon_origin();
1081                        if let Err(err) = target.sink.send_to_client(goodbye).await {
1082                            if !target.close_on_delivery_failure() {
1083                                self.forwarding.warn_frame_drop(
1084                                    target.connection_id,
1085                                    target.channel,
1086                                    target.epoch,
1087                                    "module_to_client",
1088                                    err.diagnosis.cause,
1089                                );
1090                            }
1091                            if target.close_on_delivery_failure()
1092                                && self
1093                                    .forwarding
1094                                    .escalate_client_delivery_failure_diagnosed(
1095                                        target.connection_id,
1096                                        target.channel,
1097                                        target.epoch,
1098                                        CloseReason::new(
1099                                            "route_goodbye_delivery_failed",
1100                                            format!(
1101                                                "failed to enqueue route GOODBYE for client channel {}: {err}",
1102                                                target.channel
1103                                            ),
1104                                        ),
1105                                        UndeliveredFrame {
1106                                            module_id: Some(&route.module_id),
1107                                            sink: &target.sink,
1108                                        },
1109                                        Some(&err.diagnosis),
1110                                    )
1111                                    .map_err(RouterError::Forwarding)?
1112                            {
1113                                self.counters.increment_goodbye_relay_client_failed();
1114                            }
1115                        }
1116                    } else {
1117                        self.forwarding.warn_frame_drop(
1118                            ctx.connection_id,
1119                            channel,
1120                            epoch,
1121                            "module_to_client",
1122                            "missing_binding",
1123                        );
1124                    }
1125                    return Ok(());
1126                }
1127
1128                // Accounting allows an oversized frame to occupy an empty queue
1129                // on other send paths. Refuse it at this splice before admission,
1130                // so the next ordinary reply cannot close a multiplexed client.
1131                let bytes = queued_frame_bytes(&frame);
1132                let limit = route.client_sink.accounting.byte_budget;
1133                if bytes > limit {
1134                    let ty = frame.header.ty;
1135                    let flags = frame.header.flags;
1136                    drop(frame);
1137                    return self
1138                        .refuse_oversized_module_frame(ctx, &route, (ty, flags), corr, bytes)
1139                        .await;
1140                }
1141
1142                // A terminal frame ends the request at the module whether or not
1143                // the client can still take it, so its credit is released before
1144                // delivery is attempted. Releasing only after a successful
1145                // enqueue would leave a drain counting a finished request until
1146                // the client connection's cleanup removes the route.
1147                let releases_credit = is_terminal_frame(frame.header.ty);
1148                if releases_credit {
1149                    route.flow.release_corr(corr);
1150                }
1151                let mut frame = frame;
1152                frame.header.channel = route.client_channel;
1153                frame.header.epoch = route.client_epoch;
1154                // Responses, errors, stream data and ends, and reverse
1155                // requests all pass through here. A module must never be able
1156                // to make a frame look daemon-authored, so the origin bit a
1157                // module may have set is cleared; every other flag bit and
1158                // the body travel unchanged.
1159                frame.header.flags = frame.header.flags.without_daemon_origin();
1160                if let Err(err) = route.client_sink.send_to_client(frame).await {
1161                    if self
1162                        .forwarding
1163                        .escalate_client_delivery_failure_diagnosed(
1164                            route.client_connection_id,
1165                            route.client_channel,
1166                            route.client_epoch,
1167                            CloseReason::new(
1168                                "module_to_client_delivery_failed",
1169                                format!(
1170                                    "failed to enqueue module frame for client channel {} corr {corr}: {err}",
1171                                    route.client_channel
1172                                ),
1173                            ),
1174                            UndeliveredFrame {
1175                                module_id: Some(&route.module_id),
1176                                sink: &route.client_sink,
1177                            },
1178                            Some(&err.diagnosis),
1179                        )
1180                        .map_err(RouterError::Forwarding)?
1181                    {
1182                        self.counters
1183                            .increment_client_egress_close_delivery_failed();
1184                    }
1185                    return Ok(());
1186                }
1187                return Ok(());
1188            }
1189            DataRoute::Client(DataRouteState::EpochMismatch) => {
1190                self.forwarding.warn_frame_drop(
1191                    ctx.connection_id,
1192                    channel,
1193                    epoch,
1194                    "client_to_module",
1195                    "stale_epoch",
1196                );
1197                if frame.header.ty == FrameType::Request {
1198                    self.counters.increment_client_frames_dropped_stale_route();
1199                    // Dropped before forwarding; a re-bind retry cannot double-execute this request.
1200                    let err = RouterError::StaleRouteEpoch {
1201                        channel,
1202                        epoch,
1203                        corr,
1204                    };
1205                    if let Some(error_frame) = err.to_error_frame() {
1206                        ctx.egress.send(error_frame).await?;
1207                    }
1208                }
1209                debug!(
1210                    connection_id = ctx.connection_id.get(),
1211                    channel, epoch, corr, "dropping client frame for stale route epoch"
1212                );
1213                return Ok(());
1214            }
1215            DataRoute::Client(DataRouteState::Reserved) => {
1216                self.forwarding.warn_frame_drop(
1217                    ctx.connection_id,
1218                    channel,
1219                    epoch,
1220                    "client_to_module",
1221                    "missing_binding",
1222                );
1223                if frame.header.ty == FrameType::Request {
1224                    let err = RouterError::UnknownChannel {
1225                        channel,
1226                        epoch,
1227                        corr,
1228                    };
1229                    if let Some(error_frame) = err.to_error_frame() {
1230                        ctx.egress.send(error_frame).await?;
1231                    }
1232                }
1233                return Ok(());
1234            }
1235            DataRoute::Client(DataRouteState::Bound(route)) => {
1236                if frame.header.ty == FrameType::Goodbye {
1237                    let _ = self
1238                        .control
1239                        .handle_route_goodbye(ctx.connection_id, channel, epoch)?;
1240                    return Ok(());
1241                }
1242                return self.forward_backend.handle_bound(frame, route).await;
1243            }
1244            DataRoute::Client(DataRouteState::Absent) => {}
1245        }
1246
1247        if let Some(backend) = self.backends.get(&channel) {
1248            return backend.handle(ctx.clone(), frame).await.inspect_err(|_| {
1249                self.forwarding.warn_frame_drop(
1250                    ctx.connection_id,
1251                    channel,
1252                    epoch,
1253                    "client_to_backend",
1254                    "endpoint_gone",
1255                );
1256            });
1257        }
1258        self.forwarding.warn_frame_drop(
1259            ctx.connection_id,
1260            channel,
1261            epoch,
1262            "client_to_module",
1263            "unknown_channel",
1264        );
1265        if frame.header.ty == FrameType::Request {
1266            let err = RouterError::UnknownChannel {
1267                channel,
1268                epoch,
1269                corr,
1270            };
1271            if let Some(error_frame) = err.to_error_frame() {
1272                ctx.egress.send(error_frame).await?;
1273            }
1274        }
1275        Ok(())
1276    }
1277
1278    async fn refuse_oversized_module_frame(
1279        &self,
1280        ctx: &RouteCtx,
1281        route: &RouteBinding,
1282        header: (FrameType, Flags),
1283        corr: u64,
1284        bytes: usize,
1285    ) -> Result<(), RouterError> {
1286        let (ty, flags) = header;
1287        let limit = route.client_sink.accounting.byte_budget;
1288        if ty == FrameType::Push {
1289            // Deployed Rust clients cannot decode a new route.closed reason.
1290            // Until that vocabulary can be extended compatibly, an oversized
1291            // unsolicited push still closes the connection, with its cause named.
1292            let diagnosis = ClientDeliveryDiagnosis {
1293                frame_type: ty,
1294                flags,
1295                cause: "frame_too_large",
1296            };
1297            if self
1298                .forwarding
1299                .escalate_client_delivery_failure_diagnosed(
1300                    route.client_connection_id,
1301                    route.client_channel,
1302                    route.client_epoch,
1303                    CloseReason::new(
1304                        error_codes::FRAME_TOO_LARGE,
1305                        format!(
1306                            "oversized push: size={bytes} limit={limit} (header plus body bytes)"
1307                        ),
1308                    ),
1309                    UndeliveredFrame {
1310                        module_id: Some(&route.module_id),
1311                        sink: &route.client_sink,
1312                    },
1313                    Some(&diagnosis),
1314                )
1315                .map_err(RouterError::Forwarding)?
1316            {
1317                self.counters
1318                    .increment_client_egress_close_delivery_failed();
1319            }
1320            return Ok(());
1321        }
1322
1323        warn!(
1324            connection_id = route.client_connection_id.get(),
1325            module_id = %route.module_id,
1326            client_principal = %principal_label(&route.principal),
1327            client_channel = route.client_channel,
1328            frame_type = ?ty,
1329            frame_bytes = bytes,
1330            byte_limit = limit,
1331            "refusing oversized module-to-client frame with ERROR frame_too_large"
1332        );
1333        if ty == FrameType::Request {
1334            // A reverse request belongs to the module, not to a waiting client.
1335            // Answer its sender without forwarding any part of it to the client.
1336            ctx.egress
1337                .send(frame_too_large_error(
1338                    route.module_channel,
1339                    route.module_epoch,
1340                    corr,
1341                    bytes,
1342                    limit,
1343                )?)
1344                .await?;
1345        } else {
1346            // A refused stream chunk also ends the waiting request. Releasing
1347            // its credit prevents a finished request from keeping a drain alive.
1348            route.flow.release_corr(corr);
1349            let error = frame_too_large_error(
1350                route.client_channel,
1351                route.client_epoch,
1352                corr,
1353                bytes,
1354                limit,
1355            )?;
1356            // The replacement shares the reply's bounded burst grace, so a
1357            // stopped client cannot indefinitely stall the module frame reader.
1358            if let Err(err) = route.client_sink.send_to_client(error).await {
1359                if self
1360                    .forwarding
1361                    .escalate_client_delivery_failure_diagnosed(
1362                        route.client_connection_id,
1363                        route.client_channel,
1364                        route.client_epoch,
1365                        CloseReason::new(
1366                            "module_to_client_delivery_failed",
1367                            format!(
1368                                "failed to enqueue frame_too_large for client channel {} corr {corr}: {err}",
1369                                route.client_channel
1370                            ),
1371                        ),
1372                        UndeliveredFrame {
1373                            module_id: Some(&route.module_id),
1374                            sink: &route.client_sink,
1375                        },
1376                        Some(&err.diagnosis),
1377                    )
1378                    .map_err(RouterError::Forwarding)?
1379                {
1380                    self.counters
1381                        .increment_client_egress_close_delivery_failed();
1382                }
1383            }
1384        }
1385        Ok(())
1386    }
1387}
1388
1389impl Default for Router {
1390    fn default() -> Self {
1391        Self::with_default_self_handler()
1392    }
1393}
1394
1395/// Connection-scoped cleanup guard returned by [`Router::begin_connection`].
1396#[must_use]
1397pub struct RouterConnection {
1398    id: ConnectionId,
1399    control_handler: Arc<ControlHandler>,
1400    forwarding: Arc<ForwardingTable>,
1401    close_receiver: Option<ConnectionCloseReceiver>,
1402    orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
1403}
1404
1405impl RouterConnection {
1406    pub fn id(&self) -> ConnectionId {
1407        self.id
1408    }
1409
1410    pub(crate) fn take_close_receiver(&mut self) -> ConnectionCloseReceiver {
1411        self.close_receiver
1412            .take()
1413            .expect("connection close receiver can only be taken once")
1414    }
1415}
1416
1417impl Drop for RouterConnection {
1418    fn drop(&mut self) {
1419        self.forwarding.unregister_connection_close(self.id);
1420        self.orphan_goodbyes.forget_connection(self.id);
1421        // GOODBYE (explicit) and connection-drop cleanup both call the same
1422        // idempotent deregistration path.
1423        let _ = self.control_handler.cleanup_connection(self.id);
1424    }
1425}
1426
1427/// Minimal in-memory backend used by tests and early wiring: it emits a
1428/// `RESPONSE` on the same channel/correlation id with the exact same body bytes.
1429#[derive(Debug, Default, Clone, Copy)]
1430pub struct EchoBackend;
1431
1432impl EchoBackend {
1433    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1434        let response = Frame::build_with_version(
1435            frame.header.ver,
1436            FrameType::Response,
1437            frame.header.flags,
1438            frame.header.channel,
1439            frame.header.epoch,
1440            frame.header.corr,
1441            frame.body,
1442        )
1443        .map_err(RouterError::FrameBuild)?;
1444        ctx.egress.send(response).await
1445    }
1446}
1447
1448/// Data-plane backend that splices client frames to the module connection bound at attach time.
1449#[derive(Debug, Clone)]
1450pub struct ForwardBackend {
1451    forwarding: Arc<ForwardingTable>,
1452}
1453
1454impl ForwardBackend {
1455    pub fn new(forwarding: Arc<ForwardingTable>) -> Self {
1456        Self { forwarding }
1457    }
1458
1459    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1460        let channel = frame.header.channel;
1461        let corr = frame.header.corr;
1462        let route = match self
1463            .forwarding
1464            .lookup_data_route(ctx.connection_id, channel, frame.header.epoch)
1465            .map_err(RouterError::Forwarding)?
1466        {
1467            DataRoute::Client(DataRouteState::Bound(route)) => route,
1468            DataRoute::Client(_) | DataRoute::Module(_) => {
1469                self.forwarding.warn_frame_drop(
1470                    ctx.connection_id,
1471                    channel,
1472                    frame.header.epoch,
1473                    "client_to_module",
1474                    "missing_binding",
1475                );
1476                return Err(RouterError::UnknownChannel {
1477                    channel,
1478                    epoch: frame.header.epoch,
1479                    corr,
1480                });
1481            }
1482        };
1483        self.handle_bound(frame, route).await
1484    }
1485
1486    pub(crate) async fn handle_bound(
1487        &self,
1488        frame: Frame,
1489        route: Arc<RouteBinding>,
1490    ) -> Result<(), RouterError> {
1491        let channel = frame.header.channel;
1492        let corr = frame.header.corr;
1493        let frame_type = frame.header.ty;
1494
1495        // CANCEL and other non-REQUEST frames bypass the request-credit window;
1496        // the original request's credit is freed only by the module's terminal frame.
1497        let acquired_credit = frame_type == FrameType::Request;
1498        if acquired_credit {
1499            if let Err(err) = route
1500                .flow
1501                .acquire_tagged(corr, frame.header.flags.is_subscription())
1502                .await
1503            {
1504                self.forwarding.warn_frame_drop(
1505                    route.client_connection_id,
1506                    channel,
1507                    frame.header.epoch,
1508                    "client_to_module",
1509                    "endpoint_gone",
1510                );
1511                // `module_reloading` here is answered BEFORE the frame is
1512                // forwarded, so the module never sees the request and callers
1513                // may re-dispatch it after reopening the route. That is a wire
1514                // guarantee documented on `error_codes::MODULE_RELOADING`; do
1515                // not emit this code for a request that already reached
1516                // `module_sink.send` below.
1517                if self
1518                    .forwarding
1519                    .endpoint_is_draining(route.module_endpoint)
1520                    .map_err(RouterError::Forwarding)?
1521                {
1522                    return Err(RouterError::route_error_with_epoch(
1523                        channel,
1524                        frame.header.epoch,
1525                        corr,
1526                        "module_reloading",
1527                        format!("module endpoint for route channel {channel} is reloading"),
1528                    ));
1529                }
1530                return Err(RouterError::backend_with_epoch(
1531                    channel,
1532                    frame.header.epoch,
1533                    corr,
1534                    format!("{err} for route channel {channel}"),
1535                ));
1536            }
1537        }
1538
1539        let mut frame = frame;
1540        frame.header.channel = route.module_channel;
1541        frame.header.epoch = route.module_epoch;
1542        // The origin bit is the daemon's own signal to clients. A client
1543        // frame carrying it must not reach a module appearing daemon-authored,
1544        // so it is cleared here; every other flag bit travels unchanged.
1545        frame.header.flags = frame.header.flags.without_daemon_origin();
1546        let result = route.module_sink.send(frame).await.map_err(|err| {
1547            self.forwarding.warn_frame_drop(
1548                route.client_connection_id,
1549                channel,
1550                route.client_epoch,
1551                "client_to_module",
1552                "endpoint_gone",
1553            );
1554            RouterError::backend_with_epoch(channel, route.client_epoch, corr, err.to_string())
1555        });
1556        if acquired_credit && result.is_err() {
1557            route.flow.release_corr(corr);
1558        }
1559        result
1560    }
1561}
1562
1563fn is_terminal_frame(frame_type: FrameType) -> bool {
1564    matches!(
1565        frame_type,
1566        FrameType::Response | FrameType::Error | FrameType::StreamEnd
1567    )
1568}
1569
1570/// Typed router errors. Routable failures can be translated to canonical JSON
1571/// `ERROR` frames with [`RouterError::to_error_frame`].
1572#[derive(Debug, Clone, PartialEq, Eq)]
1573pub enum RouterError {
1574    ReservedChannelZero,
1575    DuplicateChannel {
1576        channel: u16,
1577    },
1578    UnknownChannel {
1579        channel: u16,
1580        epoch: u32,
1581        corr: u64,
1582    },
1583    StaleRouteEpoch {
1584        channel: u16,
1585        epoch: u32,
1586        corr: u64,
1587    },
1588    Backend {
1589        channel: u16,
1590        epoch: u32,
1591        corr: u64,
1592        message: String,
1593    },
1594    RouteError {
1595        channel: u16,
1596        epoch: u32,
1597        corr: u64,
1598        code: String,
1599        message: String,
1600    },
1601    FrameBuild(FrameBuildError),
1602    Forwarding(ForwardingError),
1603}
1604
1605impl RouterError {
1606    pub fn backend(channel: u16, corr: u64, message: impl Into<String>) -> Self {
1607        Self::backend_with_epoch(channel, 0, corr, message)
1608    }
1609
1610    pub fn backend_with_epoch(
1611        channel: u16,
1612        epoch: u32,
1613        corr: u64,
1614        message: impl Into<String>,
1615    ) -> Self {
1616        Self::Backend {
1617            channel,
1618            epoch,
1619            corr,
1620            message: message.into(),
1621        }
1622    }
1623
1624    pub fn route_error(
1625        channel: u16,
1626        corr: u64,
1627        code: impl Into<String>,
1628        message: impl Into<String>,
1629    ) -> Self {
1630        Self::route_error_with_epoch(channel, 0, corr, code, message)
1631    }
1632
1633    pub fn route_error_with_epoch(
1634        channel: u16,
1635        epoch: u32,
1636        corr: u64,
1637        code: impl Into<String>,
1638        message: impl Into<String>,
1639    ) -> Self {
1640        Self::RouteError {
1641            channel,
1642            epoch,
1643            corr,
1644            code: code.into(),
1645            message: message.into(),
1646        }
1647    }
1648
1649    /// Translate route failures that belong on the wire into an `ERROR` frame.
1650    pub fn to_error_frame(&self) -> Option<Frame> {
1651        match self {
1652            Self::UnknownChannel {
1653                channel,
1654                epoch,
1655                corr,
1656            } => error_frame(
1657                *channel,
1658                *epoch,
1659                *corr,
1660                error_codes::UNKNOWN_CHANNEL,
1661                format!("unknown channel {channel}"),
1662            ),
1663            Self::StaleRouteEpoch {
1664                channel,
1665                epoch,
1666                corr,
1667            } => error_frame(
1668                *channel,
1669                *epoch,
1670                *corr,
1671                error_codes::STALE_ROUTE_EPOCH,
1672                format!("stale route epoch for channel {channel}"),
1673            ),
1674            Self::Backend {
1675                channel,
1676                epoch,
1677                corr,
1678                message,
1679            } => error_frame(*channel, *epoch, *corr, "backend_error", message.clone()),
1680            Self::RouteError {
1681                channel,
1682                epoch,
1683                corr,
1684                code,
1685                message,
1686            } => error_frame(*channel, *epoch, *corr, code, message.clone()),
1687            Self::ReservedChannelZero
1688            | Self::DuplicateChannel { .. }
1689            | Self::FrameBuild(_)
1690            | Self::Forwarding(_) => None,
1691        }
1692    }
1693}
1694
1695/// Flags for an ERROR frame the daemon authors itself.
1696///
1697/// Bit 6 (`FLAG_DAEMON_ORIGIN`) tells the client the daemon wrote this
1698/// error, never a module. What a code then proves is per code: on a bound
1699/// route the daemon writes `module_reloading`, `unknown_channel` and
1700/// `stale_route_epoch` only before forwarding, so with the bit they mean the
1701/// request was not delivered, while `frame_too_large` replaces a reply the
1702/// module already produced. Every daemon-authored ERROR goes through here;
1703/// frames a module sent keep their own flags with the bit cleared at the
1704/// splice.
1705/// Nothing else about the flags differs: passive priority, not binary, not last.
1706pub(crate) fn daemon_error_flags() -> Flags {
1707    Flags::new(false, Priority::Passive, false).with_daemon_origin()
1708}
1709
1710fn error_frame(channel: u16, epoch: u32, corr: u64, code: &str, message: String) -> Option<Frame> {
1711    let body = serde_json::to_vec(&ErrorBody {
1712        code: code.to_string(),
1713        message,
1714        detail: None,
1715    })
1716    .ok()?;
1717
1718    Frame::build(
1719        FrameType::Error,
1720        daemon_error_flags(),
1721        channel,
1722        epoch,
1723        corr,
1724        body,
1725    )
1726    .ok()
1727}
1728
1729fn frame_too_large_error(
1730    channel: u16,
1731    epoch: u32,
1732    corr: u64,
1733    bytes: usize,
1734    limit: usize,
1735) -> Result<Frame, RouterError> {
1736    let body = serde_json::to_vec(&ErrorBody {
1737        code: error_codes::FRAME_TOO_LARGE.to_string(),
1738        message: format!(
1739            "module frame of {bytes} bytes exceeds client egress limit of {limit} bytes"
1740        ),
1741        detail: Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit })),
1742    })
1743    .expect("serializing a frame size error");
1744    Frame::build(
1745        FrameType::Error,
1746        daemon_error_flags(),
1747        channel,
1748        epoch,
1749        corr,
1750        body,
1751    )
1752    .map_err(RouterError::FrameBuild)
1753}
1754
1755impl fmt::Display for RouterError {
1756    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1757        match self {
1758            Self::ReservedChannelZero => write!(f, "channel 0 is reserved for subc"),
1759            Self::DuplicateChannel { channel } => {
1760                write!(f, "backend already registered for channel {channel}")
1761            }
1762            Self::UnknownChannel { channel, corr, .. } => {
1763                write!(f, "unknown channel {channel} for corr {corr}")
1764            }
1765            Self::StaleRouteEpoch { channel, corr, .. } => {
1766                write!(f, "stale route epoch for channel {channel} corr {corr}")
1767            }
1768            Self::Backend {
1769                channel,
1770                corr,
1771                message,
1772                ..
1773            } => write!(
1774                f,
1775                "backend error on channel {channel} corr {corr}: {message}"
1776            ),
1777            Self::RouteError {
1778                channel,
1779                corr,
1780                code,
1781                message,
1782                ..
1783            } => write!(
1784                f,
1785                "route error {code} on channel {channel} corr {corr}: {message}"
1786            ),
1787            Self::FrameBuild(err) => write!(f, "failed to build routed frame: {err}"),
1788            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
1789        }
1790    }
1791}
1792
1793impl Error for RouterError {
1794    fn source(&self) -> Option<&(dyn Error + 'static)> {
1795        match self {
1796            Self::FrameBuild(err) => Some(err),
1797            Self::Forwarding(err) => Some(err),
1798            Self::ReservedChannelZero
1799            | Self::DuplicateChannel { .. }
1800            | Self::UnknownChannel { .. }
1801            | Self::StaleRouteEpoch { .. }
1802            | Self::Backend { .. }
1803            | Self::RouteError { .. } => None,
1804        }
1805    }
1806}
1807
1808#[cfg(test)]
1809mod tests {
1810    use super::*;
1811    use crate::{
1812        forwarding::RouteBindRelayOutcome,
1813        supervise::{ModuleSpec, RestartPolicy, Supervisor, SupervisorHandle},
1814        ControlHandler, Registry,
1815    };
1816    use std::{
1817        sync::{mpsc as std_mpsc, Arc},
1818        time::Duration,
1819    };
1820    use subc_control::ModuleProtocol;
1821    use subc_protocol::{manifest::Concurrency, ErrorBody, Flags, FrameType, Priority};
1822    use tokio::sync::mpsc;
1823
1824    pub(crate) use crate::router::test_log::{captured_logs, log_capture};
1825
1826    fn logged_millis(logs: &str, field: &str) -> u64 {
1827        logs.split_whitespace()
1828            .find_map(|part| part.strip_prefix(field))
1829            .and_then(|value| value.parse().ok())
1830            .unwrap_or_else(|| panic!("missing numeric {field} in logs: {logs}"))
1831    }
1832
1833    fn request(channel: u16, corr: u64, body: &[u8]) -> Frame {
1834        Frame::build(
1835            FrameType::Request,
1836            Flags::new(true, Priority::Interactive, false),
1837            channel,
1838            0,
1839            corr,
1840            body.to_vec(),
1841        )
1842        .unwrap()
1843    }
1844
1845    fn ping(corr: u64) -> Frame {
1846        Frame::build(
1847            FrameType::Ping,
1848            Flags::new(false, Priority::Passive, false),
1849            0,
1850            0,
1851            corr,
1852            Vec::new(),
1853        )
1854        .unwrap()
1855    }
1856
1857    fn route_ctx() -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
1858        let (tx, rx) = mpsc::channel(8);
1859        (
1860            RouteCtx {
1861                connection_id: ConnectionId::LOCAL,
1862                egress: FrameSink::new(tx),
1863            },
1864            rx,
1865        )
1866    }
1867
1868    #[tokio::test]
1869    async fn echo_backend_returns_response_with_byte_identical_body() {
1870        let mut router = Router::with_default_self_handler();
1871        router.register_backend(7, EchoBackend).unwrap();
1872        let (ctx, mut rx) = route_ctx();
1873        let body = b"{not parsed}\0\xff";
1874
1875        router
1876            .route_for_connection(&ctx, request(7, 123, body))
1877            .await
1878            .unwrap();
1879        let response = rx.recv().await.unwrap();
1880
1881        assert_eq!(response.header.ty, FrameType::Response);
1882        assert_eq!(response.header.channel, 7);
1883        assert_eq!(response.header.corr, 123);
1884        assert_eq!(response.body, body);
1885        assert!(rx.try_recv().is_err());
1886    }
1887
1888    #[tokio::test]
1889    async fn unknown_channel_emits_canonical_error_frame() {
1890        let router = Router::with_default_self_handler();
1891        let (ctx, mut rx) = route_ctx();
1892
1893        router
1894            .route_for_connection(&ctx, request(99, 5, b"payload"))
1895            .await
1896            .unwrap();
1897        let error_frame = rx.recv().await.unwrap();
1898
1899        assert_eq!(error_frame.header.ty, FrameType::Error);
1900        assert_eq!(error_frame.header.channel, 99);
1901        assert_eq!(error_frame.header.corr, 5);
1902        let body: ErrorBody = serde_json::from_slice(&error_frame.body).unwrap();
1903        assert_eq!(body.code, "unknown_channel");
1904        assert_eq!(body.message, "unknown channel 99");
1905    }
1906
1907    /// Every ERROR the daemon authors carries the daemon-origin bit, so a
1908    /// client can tell a daemon refusal from a module's reply. Both the
1909    /// unknown-channel and the stale-epoch errors below are received on the
1910    /// client's own egress, not merely built.
1911    #[tokio::test]
1912    async fn daemon_authored_data_plane_errors_carry_daemon_origin() {
1913        let router = Router::with_default_self_handler();
1914        let (ctx, mut rx) = route_ctx();
1915        router
1916            .route_for_connection(&ctx, request(99, 5, b"payload"))
1917            .await
1918            .unwrap();
1919        let unknown = rx.recv().await.unwrap();
1920        assert_eq!(unknown.header.ty, FrameType::Error);
1921        let body: ErrorBody = serde_json::from_slice(&unknown.body).unwrap();
1922        assert_eq!(body.code, "unknown_channel");
1923        assert!(
1924            unknown.header.flags.is_daemon_origin(),
1925            "unknown_channel must carry daemon origin: {:?}",
1926            unknown.header.flags
1927        );
1928
1929        let (
1930            router,
1931            _forwarding,
1932            client_ctx,
1933            mut client_rx,
1934            _module_ctx,
1935            _module_egress_rx,
1936            _module_rx,
1937            pending,
1938        ) = dynamic_route_fixture(true);
1939        let _ = client_rx.recv().await;
1940        router
1941            .route_for_connection(
1942                &client_ctx,
1943                route_frame(
1944                    FrameType::Request,
1945                    pending.client_channel,
1946                    pending.client_epoch + 1,
1947                    702,
1948                ),
1949            )
1950            .await
1951            .unwrap();
1952        let stale = client_rx.recv().await.unwrap();
1953        assert_eq!(stale.header.ty, FrameType::Error);
1954        let body: ErrorBody = serde_json::from_slice(&stale.body).unwrap();
1955        assert_eq!(body.code, "stale_route_epoch");
1956        assert!(
1957            stale.header.flags.is_daemon_origin(),
1958            "stale_route_epoch must carry daemon origin: {:?}",
1959            stale.header.flags
1960        );
1961    }
1962
1963    /// A module ERROR that spoofs a daemon code with the daemon-origin bit
1964    /// set must reach the client with the bit cleared and the body
1965    /// byte-identical: the bit means the daemon wrote the frame, and a module
1966    /// must never be able to claim that.
1967    #[tokio::test]
1968    async fn module_error_with_daemon_bit_is_stripped_before_client_delivery() {
1969        let (
1970            router,
1971            _forwarding,
1972            client_ctx,
1973            mut client_rx,
1974            module_ctx,
1975            _module_egress_rx,
1976            mut module_rx,
1977            pending,
1978        ) = dynamic_route_fixture(true);
1979        let _ = client_rx.recv().await;
1980        // Open flow credit the way a real request would, so the terminal
1981        // error exercises the credit-release path too.
1982        router
1983            .route_for_connection(
1984                &client_ctx,
1985                route_frame(
1986                    FrameType::Request,
1987                    pending.client_channel,
1988                    pending.client_epoch,
1989                    701,
1990                ),
1991            )
1992            .await
1993            .unwrap();
1994        let _ = module_rx.recv().await;
1995
1996        let body = serde_json::to_vec(&ErrorBody::new("module_reloading", "try again")).unwrap();
1997        let spoof = Frame::build(
1998            FrameType::Error,
1999            Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2000            pending.module_channel,
2001            pending.module_epoch,
2002            701,
2003            body.clone(),
2004        )
2005        .unwrap();
2006        router
2007            .route_for_connection(&module_ctx, spoof)
2008            .await
2009            .unwrap();
2010        let delivered = client_rx.recv().await.unwrap();
2011        assert_eq!(delivered.header.ty, FrameType::Error);
2012        assert_eq!(delivered.header.channel, pending.client_channel);
2013        assert_eq!(delivered.header.epoch, pending.client_epoch);
2014        assert_eq!(delivered.header.corr, 701);
2015        assert!(
2016            !delivered.header.flags.is_daemon_origin(),
2017            "module-set daemon origin must be cleared: {:?}",
2018            delivered.header.flags
2019        );
2020        assert_eq!(delivered.body, body, "the body travels byte-identical");
2021    }
2022
2023    /// Clearing the origin bit strips exactly that bit: priority, LAST and
2024    /// subscription survive the splice unchanged.
2025    #[tokio::test]
2026    async fn module_response_with_daemon_bit_is_stripped_preserving_other_flags() {
2027        use subc_protocol::FLAG_SUBSCRIPTION;
2028        let (
2029            router,
2030            _forwarding,
2031            client_ctx,
2032            mut client_rx,
2033            module_ctx,
2034            _module_egress_rx,
2035            mut module_rx,
2036            pending,
2037        ) = dynamic_route_fixture(true);
2038        let _ = client_rx.recv().await;
2039        router
2040            .route_for_connection(
2041                &client_ctx,
2042                route_frame(
2043                    FrameType::Request,
2044                    pending.client_channel,
2045                    pending.client_epoch,
2046                    701,
2047                ),
2048            )
2049            .await
2050            .unwrap();
2051        let _ = module_rx.recv().await;
2052
2053        let sent_flags = Flags(
2054            Flags::new(true, Priority::Background, true).0
2055                | FLAG_SUBSCRIPTION
2056                | subc_protocol::FLAG_DAEMON_ORIGIN,
2057        );
2058        assert!(sent_flags.is_daemon_origin());
2059        let response = Frame::build(
2060            FrameType::Response,
2061            sent_flags,
2062            pending.module_channel,
2063            pending.module_epoch,
2064            701,
2065            b"route-body".to_vec(),
2066        )
2067        .unwrap();
2068        router
2069            .route_for_connection(&module_ctx, response)
2070            .await
2071            .unwrap();
2072        let delivered = client_rx.recv().await.unwrap();
2073        assert_eq!(delivered.header.ty, FrameType::Response);
2074        assert!(
2075            !delivered.header.flags.is_daemon_origin(),
2076            "module-set daemon origin must be cleared: {:?}",
2077            delivered.header.flags
2078        );
2079        assert!(
2080            delivered.header.flags.is_binary(),
2081            "binary flag must survive: {:?}",
2082            delivered.header.flags
2083        );
2084        assert!(
2085            delivered.header.flags.is_last(),
2086            "LAST flag must survive: {:?}",
2087            delivered.header.flags
2088        );
2089        assert!(
2090            delivered.header.flags.is_subscription(),
2091            "subscription flag must survive: {:?}",
2092            delivered.header.flags
2093        );
2094        assert_eq!(
2095            delivered.header.flags.priority(),
2096            Some(Priority::Background),
2097            "priority must survive"
2098        );
2099    }
2100
2101    /// A client frame carrying the origin bit must not reach the module
2102    /// looking daemon-authored either: the splice clears it symmetrically.
2103    #[tokio::test]
2104    async fn client_frame_with_daemon_bit_is_stripped_before_module_delivery() {
2105        let (
2106            router,
2107            _forwarding,
2108            client_ctx,
2109            mut client_rx,
2110            _module_ctx,
2111            _module_egress_rx,
2112            mut module_rx,
2113            pending,
2114        ) = dynamic_route_fixture(true);
2115        let _ = client_rx.recv().await;
2116        let spoof = Frame::build(
2117            FrameType::Request,
2118            Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2119            pending.client_channel,
2120            pending.client_epoch,
2121            701,
2122            b"route-body".to_vec(),
2123        )
2124        .unwrap();
2125        router
2126            .route_for_connection(&client_ctx, spoof)
2127            .await
2128            .unwrap();
2129        let delivered = module_rx.recv().await.unwrap();
2130        assert_eq!(delivered.header.ty, FrameType::Request);
2131        assert!(
2132            !delivered.header.flags.is_daemon_origin(),
2133            "client-set daemon origin must be cleared: {:?}",
2134            delivered.header.flags
2135        );
2136    }
2137
2138    #[tokio::test]
2139    async fn channel_zero_uses_control_handler_not_backend_registry() {
2140        let mut router = Router::with_default_self_handler();
2141        router.register_backend(1, EchoBackend).unwrap();
2142        let (ctx, mut rx) = route_ctx();
2143
2144        router.route_for_connection(&ctx, ping(77)).await.unwrap();
2145        let response = rx.recv().await.unwrap();
2146
2147        assert_eq!(response.header.ty, FrameType::Pong);
2148        assert_eq!(response.header.channel, 0);
2149        assert_eq!(response.header.corr, 77);
2150        assert!(response.body.is_empty());
2151    }
2152
2153    #[tokio::test]
2154    async fn slow_control_dispatch_logs_decoded_op_and_elapsed_time() {
2155        let control = Arc::new(
2156            ControlHandler::new(Arc::new(Registry::default()))
2157                .with_control_dispatch_delay(Duration::from_millis(1050)),
2158        );
2159        let router = Router::with_control_handler(control);
2160        let (ctx, mut rx) = route_ctx();
2161        let (output, guard) = log_capture(tracing::Level::WARN);
2162
2163        router
2164            .route_for_connection(&ctx, request(0, 41, br#"{"op":"server.describe"}"#))
2165            .await
2166            .expect("slow request routes");
2167        assert!(rx.recv().await.is_some(), "request receives a response");
2168        drop(guard);
2169
2170        let logs = captured_logs(&output);
2171        assert!(logs.contains("slow control dispatch"));
2172        assert!(logs.contains("op=server.describe"));
2173        assert!(logs.contains("connection_id=0"));
2174        assert!(logs.contains("corr=41"));
2175        assert!(
2176            logged_millis(&logs, "elapsed_ms=") >= 1050,
2177            "elapsed must include the injected handler delay: {logs}"
2178        );
2179    }
2180
2181    #[tokio::test]
2182    async fn fast_control_dispatch_emits_arrival_without_slow_warning() {
2183        let router = Router::with_default_self_handler();
2184        let (ctx, mut rx) = route_ctx();
2185        let (output, guard) = log_capture(tracing::Level::DEBUG);
2186
2187        router
2188            .route_for_connection(&ctx, request(0, 42, br#"{"op":"server.describe"}"#))
2189            .await
2190            .expect("fast request routes");
2191        assert!(rx.recv().await.is_some(), "request receives a response");
2192        drop(guard);
2193
2194        let logs = captured_logs(&output);
2195        assert!(logs.contains("control dispatch op=server.describe connection_id=0 corr=42"));
2196        assert!(!logs.contains("slow control dispatch"));
2197    }
2198
2199    #[tokio::test]
2200    async fn control_dispatch_arrival_is_hidden_at_info() {
2201        let router = Router::with_default_self_handler();
2202        let (ctx, mut rx) = route_ctx();
2203        let (output, guard) = log_capture(tracing::Level::INFO);
2204
2205        router
2206            .route_for_connection(&ctx, request(0, 43, br#"{"op":"server.describe"}"#))
2207            .await
2208            .expect("fast request routes");
2209        assert!(rx.recv().await.is_some(), "request receives a response");
2210        drop(guard);
2211
2212        assert!(
2213            !captured_logs(&output).contains("control dispatch"),
2214            "arrival logging must stay hidden at INFO"
2215        );
2216    }
2217
2218    #[tokio::test]
2219    async fn supervisor_list_logs_contended_snapshot_lock_only() {
2220        let registry = Arc::new(Registry::default());
2221        let handle = SupervisorHandle::new();
2222        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
2223            .with_handle(handle.clone());
2224        let module = supervisor
2225            .supervise_configured(
2226                ModuleSpec {
2227                    module_id: "held-module".to_string(),
2228                    program: "test-module".into(),
2229                    args: Vec::new(),
2230                    env: Vec::new(),
2231                    reserved: false,
2232                    reserved_prefixes: Vec::new(),
2233                    protocol: ModuleProtocol::Subc,
2234                    overlap: Default::default(),
2235                },
2236                false,
2237            )
2238            .expect("disabled test module is supervised");
2239        let router = Router::with_control_handler(Arc::new(
2240            ControlHandler::new(Arc::clone(&registry)).with_supervisor(handle),
2241        ));
2242        let (ctx, mut rx) = route_ctx();
2243        let (acquired, ready) = std_mpsc::channel();
2244        let holder = module.hold_snapshot_for_test(acquired, Duration::from_millis(400));
2245        ready.recv().expect("holder acquired snapshot lock");
2246        let (output, guard) = log_capture(tracing::Level::WARN);
2247
2248        router
2249            .route_for_connection(&ctx, request(0, 44, br#"{"op":"supervisor.list"}"#))
2250            .await
2251            .expect("list request routes after the lock releases");
2252        assert!(
2253            rx.recv().await.is_some(),
2254            "list request receives a response"
2255        );
2256        holder.join().expect("snapshot holder exits cleanly");
2257        drop(guard);
2258
2259        let logs = captured_logs(&output);
2260        assert!(logs.contains("slow snapshot lock"));
2261        assert!(logs.contains("module_id=held-module"));
2262        assert!(logs.contains("caller=list"));
2263        assert!(
2264            logged_millis(&logs, "waited_ms=") >= 250,
2265            "wait must exceed the slow-lock threshold: {logs}"
2266        );
2267
2268        let (output, guard) = log_capture(tracing::Level::WARN);
2269        router
2270            .route_for_connection(&ctx, request(0, 45, br#"{"op":"supervisor.list"}"#))
2271            .await
2272            .expect("uncontended list request routes");
2273        assert!(
2274            rx.recv().await.is_some(),
2275            "uncontended list receives a response"
2276        );
2277        drop(guard);
2278        assert!(
2279            !captured_logs(&output).contains("slow snapshot lock"),
2280            "uncontended list acquisition must not warn"
2281        );
2282    }
2283
2284    #[tokio::test]
2285    async fn full_module_to_client_sink_requests_client_close_without_erroring_module() {
2286        let forwarding = Arc::new(ForwardingTable::default());
2287        let control = Arc::new(ControlHandler::with_forwarding(
2288            Arc::new(crate::Registry::default()),
2289            Arc::clone(&forwarding),
2290        ));
2291        let router = Router::with_control_handler(control);
2292        let module_connection = ConnectionId::new(10);
2293        let client_connection = ConnectionId::new(20);
2294        let mut close_receiver = forwarding.register_connection_close(client_connection);
2295        let (module_tx, _module_rx) = mpsc::channel(1);
2296        forwarding
2297            .register_module_connection(
2298                module_connection,
2299                "full-sink-provider".to_string(),
2300                1,
2301                Concurrency::ModuleManaged,
2302                FrameSink::new(module_tx),
2303            )
2304            .unwrap();
2305        let (client_tx, mut client_rx) = mpsc::channel(1);
2306        let pending = forwarding
2307            .begin_route_bind_relay_for_test(
2308                client_connection,
2309                FrameSink::new(client_tx),
2310                700,
2311                "full-sink-provider",
2312            )
2313            .unwrap();
2314        forwarding
2315            .complete_pending_relay(
2316                module_connection,
2317                pending.corr,
2318                RouteBindRelayOutcome::Accepted,
2319            )
2320            .unwrap();
2321
2322        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2323        let module_ctx = RouteCtx {
2324            connection_id: module_connection,
2325            egress: FrameSink::new(module_egress_tx),
2326        };
2327        let terminal = Frame::build(
2328            FrameType::Response,
2329            Flags::new(false, Priority::Interactive, true),
2330            pending.module_channel,
2331            pending.module_epoch,
2332            701,
2333            b"terminal".to_vec(),
2334        )
2335        .unwrap();
2336
2337        router
2338            .route_for_connection(&module_ctx, terminal)
2339            .await
2340            .unwrap();
2341        let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2342            .await
2343            .expect("close request should be sent for the full client sink")
2344            .expect("close sender should include a reason");
2345        assert!(
2346            reason
2347                .to_string()
2348                .contains("module_to_client_delivery_failed"),
2349            "unexpected close reason: {reason}"
2350        );
2351        assert_eq!(client_rx.try_recv().unwrap().header.corr, 700);
2352        assert!(client_rx.try_recv().is_err());
2353        assert_eq!(
2354            router.counters.snapshot()["client_egress_close_delivery_failed"],
2355            1
2356        );
2357    }
2358
2359    /// A terminal frame ends the request at the module even when the client
2360    /// cannot take it, so the drain must stop counting it at once rather than
2361    /// when the client connection's cleanup later removes the route.
2362    #[tokio::test]
2363    async fn terminal_frame_releases_its_credit_even_when_client_delivery_fails() {
2364        let forwarding = Arc::new(ForwardingTable::default());
2365        let control = Arc::new(ControlHandler::with_forwarding(
2366            Arc::new(crate::Registry::default()),
2367            Arc::clone(&forwarding),
2368        ));
2369        let router = Router::with_control_handler(control);
2370        let module_connection = ConnectionId::new(11);
2371        let client_connection = ConnectionId::new(21);
2372        let _close_receiver = forwarding.register_connection_close(client_connection);
2373        let (module_tx, _module_rx) = mpsc::channel(1);
2374        forwarding
2375            .register_module_connection(
2376                module_connection,
2377                "credit-provider".to_string(),
2378                1,
2379                Concurrency::ModuleManaged,
2380                FrameSink::new(module_tx),
2381            )
2382            .unwrap();
2383        // Capacity one, filled by the route.open response, so the terminal
2384        // frame below cannot be enqueued for the client.
2385        let (client_tx, _client_rx) = mpsc::channel(1);
2386        let pending = forwarding
2387            .begin_route_bind_relay_for_test(
2388                client_connection,
2389                FrameSink::new(client_tx),
2390                800,
2391                "credit-provider",
2392            )
2393            .unwrap();
2394        forwarding
2395            .complete_pending_relay(
2396                module_connection,
2397                pending.corr,
2398                RouteBindRelayOutcome::Accepted,
2399            )
2400            .unwrap();
2401        let DataRoute::Client(DataRouteState::Bound(route)) = forwarding
2402            .lookup_data_route(
2403                client_connection,
2404                pending.client_channel,
2405                pending.client_epoch,
2406            )
2407            .unwrap()
2408        else {
2409            panic!("expected a bound client route");
2410        };
2411        route.flow.acquire_tagged(801, false).await.unwrap();
2412        assert_eq!(route.flow.drain_in_flight(), 1);
2413
2414        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2415        let module_ctx = RouteCtx {
2416            connection_id: module_connection,
2417            egress: FrameSink::new(module_egress_tx),
2418        };
2419        let terminal = Frame::build(
2420            FrameType::Response,
2421            Flags::new(false, Priority::Interactive, true),
2422            pending.module_channel,
2423            pending.module_epoch,
2424            801,
2425            b"terminal".to_vec(),
2426        )
2427        .unwrap();
2428        router
2429            .route_for_connection(&module_ctx, terminal)
2430            .await
2431            .unwrap();
2432
2433        assert_eq!(
2434            router.counters.snapshot()["client_egress_close_delivery_failed"],
2435            1,
2436            "the client delivery must have failed for this test to mean anything"
2437        );
2438        assert_eq!(
2439            route.flow.drain_in_flight(),
2440            0,
2441            "the module's terminal frame must release its credit even though the client could not take it"
2442        );
2443    }
2444
2445    #[tokio::test]
2446    async fn full_route_goodbye_sink_requests_target_close_without_erroring_module() {
2447        let forwarding = Arc::new(ForwardingTable::default());
2448        let control = Arc::new(ControlHandler::with_forwarding(
2449            Arc::new(crate::Registry::default()),
2450            Arc::clone(&forwarding),
2451        ));
2452        let router = Router::with_control_handler(control);
2453        let module_connection = ConnectionId::new(30);
2454        let client_connection = ConnectionId::new(40);
2455        let mut close_receiver = forwarding.register_connection_close(client_connection);
2456        let (module_tx, _module_rx) = mpsc::channel(1);
2457        forwarding
2458            .register_module_connection(
2459                module_connection,
2460                "goodbye-full-provider".to_string(),
2461                1,
2462                Concurrency::ModuleManaged,
2463                FrameSink::new(module_tx),
2464            )
2465            .unwrap();
2466        let (client_tx, mut client_rx) = mpsc::channel(1);
2467        let pending = forwarding
2468            .begin_route_bind_relay_for_test(
2469                client_connection,
2470                FrameSink::new(client_tx),
2471                800,
2472                "goodbye-full-provider",
2473            )
2474            .unwrap();
2475        forwarding
2476            .complete_pending_relay(
2477                module_connection,
2478                pending.corr,
2479                RouteBindRelayOutcome::Accepted,
2480            )
2481            .unwrap();
2482
2483        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2484        let module_ctx = RouteCtx {
2485            connection_id: module_connection,
2486            egress: FrameSink::new(module_egress_tx),
2487        };
2488        let goodbye = Frame::build(
2489            FrameType::Goodbye,
2490            Flags::new(false, Priority::Passive, true),
2491            pending.module_channel,
2492            pending.module_epoch,
2493            801,
2494            Vec::new(),
2495        )
2496        .unwrap();
2497
2498        router
2499            .route_for_connection(&module_ctx, goodbye)
2500            .await
2501            .unwrap();
2502        let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2503            .await
2504            .expect("close request should be sent for the full GOODBYE sink")
2505            .expect("close sender should include a reason");
2506        assert!(
2507            reason.to_string().contains("route_goodbye_delivery_failed"),
2508            "unexpected close reason: {reason}"
2509        );
2510        assert_eq!(client_rx.try_recv().unwrap().header.corr, 800);
2511        assert!(client_rx.try_recv().is_err());
2512        assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 1);
2513        assert_eq!(router.counters.snapshot()["route_released_epoch_fenced"], 1);
2514    }
2515
2516    /// One module and one client connection with `routes` routes bound between
2517    /// them. The client uses a real connection egress queue (the same byte
2518    /// budget and frame-count backstop as a live connection), and its route.open
2519    /// responses are drained so the queue starts empty.
2520    async fn multi_route_client(
2521        module_id: &str,
2522        module_connection: ConnectionId,
2523        client_connection: ConnectionId,
2524        routes: usize,
2525    ) -> (
2526        Router,
2527        FrameSink,
2528        mpsc::Receiver<OutboundFrame>,
2529        RouteCtx,
2530        Vec<crate::forwarding::PendingRouteBindRelay>,
2531        ConnectionCloseReceiver,
2532    ) {
2533        let forwarding = Arc::new(ForwardingTable::default());
2534        let control = Arc::new(ControlHandler::with_forwarding(
2535            Arc::new(crate::Registry::default()),
2536            Arc::clone(&forwarding),
2537        ));
2538        let router = Router::with_control_handler(control);
2539        let close_receiver = forwarding.register_connection_close(client_connection);
2540        let (module_tx, _module_rx) = mpsc::channel(8);
2541        forwarding
2542            .register_module_connection(
2543                module_connection,
2544                module_id.to_string(),
2545                1,
2546                Concurrency::ModuleManaged,
2547                FrameSink::new(module_tx),
2548            )
2549            .unwrap();
2550        let (client_sink, mut client_rx) = crate::server::connection_egress();
2551        let mut bound = Vec::with_capacity(routes);
2552        for index in 0..routes {
2553            let pending = forwarding
2554                .begin_route_bind_relay_for_test(
2555                    client_connection,
2556                    client_sink.clone(),
2557                    900 + index as u64,
2558                    module_id,
2559                )
2560                .unwrap();
2561            forwarding
2562                .complete_pending_relay(
2563                    module_connection,
2564                    pending.corr,
2565                    RouteBindRelayOutcome::Accepted,
2566                )
2567                .unwrap();
2568            assert_eq!(
2569                client_rx.recv().await.unwrap().header.corr,
2570                900 + index as u64
2571            );
2572            bound.push(pending);
2573        }
2574        let (module_egress_tx, _module_egress_rx) = mpsc::channel(8);
2575        let module_ctx = RouteCtx {
2576            connection_id: module_connection,
2577            egress: FrameSink::new(module_egress_tx),
2578        };
2579        (
2580            router,
2581            client_sink,
2582            client_rx,
2583            module_ctx,
2584            bound,
2585            close_receiver,
2586        )
2587    }
2588
2589    /// Per-frame cost of the egress sink's admission and release accounting:
2590    /// one million 200-byte frames enqueued with `try_send` and taken off the
2591    /// queue the way the connection writer does, in batches of 1,000 so the
2592    /// queue stays well inside its budget. Prints nanoseconds per frame; it is
2593    /// a measurement, not a gate. Run with
2594    /// `cargo test --release -p subc-daemon --lib egress_sink_per_frame_cost -- --ignored --nocapture`.
2595    #[test]
2596    #[ignore = "timing measurement, run on demand"]
2597    fn egress_sink_per_frame_cost() {
2598        const FRAMES: usize = 1_000_000;
2599        const BATCH: usize = 1_000;
2600        let (sink, mut rx) = crate::server::connection_egress();
2601        let template = stream_frame(9, 1, 0, vec![b't'; 200]);
2602        let started = Instant::now();
2603        for _ in 0..FRAMES / BATCH {
2604            for _ in 0..BATCH {
2605                sink.try_send(template.clone()).unwrap();
2606            }
2607            for _ in 0..BATCH {
2608                let outbound = rx.try_recv().unwrap();
2609                if let Some(charge) = &outbound.charge {
2610                    charge.taken_by_writer();
2611                }
2612                drop(std::hint::black_box(outbound));
2613            }
2614        }
2615        let elapsed = started.elapsed();
2616        assert_eq!(sink.backlog().queued_bytes, 0);
2617        println!(
2618            "egress sink: {FRAMES} frames in {elapsed:?}, {:.1} ns/frame",
2619            elapsed.as_nanos() as f64 / FRAMES as f64
2620        );
2621    }
2622
2623    fn stream_frame(channel: u16, epoch: u32, corr: u64, body: Vec<u8>) -> Frame {
2624        Frame::build(
2625            FrameType::StreamData,
2626            Flags::new(false, Priority::Interactive, false),
2627            channel,
2628            epoch,
2629            corr,
2630            body,
2631        )
2632        .unwrap()
2633    }
2634
2635    fn module_route_for_test(
2636        router: &Router,
2637        ctx: &RouteCtx,
2638        route: &crate::forwarding::PendingRouteBindRelay,
2639    ) -> Arc<RouteBinding> {
2640        match router
2641            .forwarding
2642            .lookup_data_route(ctx.connection_id, route.module_channel, route.module_epoch)
2643            .unwrap()
2644        {
2645            DataRoute::Module(DataRouteState::Bound(binding)) => binding,
2646            other => panic!("expected a bound module route, got {other:?}"),
2647        }
2648    }
2649
2650    fn assert_frame_size_error(frame: &Frame, channel: u16, epoch: u32, corr: u64, bytes: usize) {
2651        assert_eq!(frame.header.ty, FrameType::Error);
2652        assert_eq!(frame.header.channel, channel);
2653        assert_eq!(frame.header.epoch, epoch);
2654        assert_eq!(frame.header.corr, corr);
2655        assert!(frame.header.flags.is_daemon_origin());
2656        let body: ErrorBody = serde_json::from_slice(&frame.body).unwrap();
2657        assert_eq!(body.code, error_codes::FRAME_TOO_LARGE);
2658        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2659        assert!(body.message.contains(&bytes.to_string()));
2660        assert!(body.message.contains(&limit.to_string()));
2661        assert_eq!(
2662            body.detail,
2663            Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit }))
2664        );
2665    }
2666
2667    #[tokio::test]
2668    async fn module_egress_response_oversized_reply_keeps_connection_and_other_route() {
2669        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
2670        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2671            "oversize-provider",
2672            ConnectionId::new(80),
2673            ConnectionId::new(81),
2674            2,
2675        )
2676        .await;
2677        let binding = module_route_for_test(&router, &ctx, &routes[0]);
2678        binding.flow.acquire_tagged(101, false).await.unwrap();
2679        module_route_for_test(&router, &ctx, &routes[1])
2680            .flow
2681            .acquire_tagged(102, false)
2682            .await
2683            .unwrap();
2684        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2685        let oversized = Frame::build(
2686            FrameType::Response,
2687            Flags::new(false, Priority::Interactive, true),
2688            routes[0].module_channel,
2689            routes[0].module_epoch,
2690            101,
2691            vec![b'x'; limit - subc_protocol::HEADER_LEN + 1],
2692        )
2693        .unwrap();
2694        assert_eq!(sink.backlog().queued_bytes, 0);
2695        router.route_for_connection(&ctx, oversized).await.unwrap();
2696
2697        // Reproduce a large reply followed by ordinary traffic on another route
2698        // before the client reads anything. The large reply must not monopolize
2699        // the queue and make that second delivery close the shared connection.
2700        router
2701            .route_for_connection(
2702                &ctx,
2703                route_frame(
2704                    FrameType::Response,
2705                    routes[1].module_channel,
2706                    routes[1].module_epoch,
2707                    102,
2708                ),
2709            )
2710            .await
2711            .unwrap();
2712        assert!(
2713            close.try_recv().is_err(),
2714            "an oversized reply must not close the multiplexed client"
2715        );
2716        let error = rx.try_recv().expect("the waiting client gets an ERROR");
2717        assert_frame_size_error(
2718            &error,
2719            routes[0].client_channel,
2720            routes[0].client_epoch,
2721            101,
2722            limit + 1,
2723        );
2724        let second = rx.try_recv().expect("the other route's reply arrives");
2725        assert_eq!(second.header.ty, FrameType::Response);
2726        assert_eq!(second.header.channel, routes[1].client_channel);
2727        assert_eq!(second.header.epoch, routes[1].client_epoch);
2728        assert_eq!(second.header.corr, 102);
2729        assert_eq!(second.body, b"route-body");
2730        assert_eq!(binding.flow.drain_in_flight(), 0);
2731        for route in &routes {
2732            module_route_for_test(&router, &ctx, route);
2733        }
2734        assert!(rx.try_recv().is_err());
2735        drop((error, second));
2736        assert_eq!(sink.backlog().queued_bytes, 0);
2737        assert_eq!(
2738            router.counters.snapshot()["client_egress_close_delivery_failed"],
2739            0
2740        );
2741
2742        let captured = test_log::captured_logs(&logs);
2743        let warnings: Vec<_> = captured
2744            .lines()
2745            .filter(|line| line.contains("WARN"))
2746            .collect();
2747        assert_eq!(warnings.len(), 1, "{captured}");
2748        let warning = warnings[0];
2749        for expected in [
2750            "module_id=oversize-provider".to_string(),
2751            "client_principal=direct".to_string(),
2752            format!("client_channel={}", routes[0].client_channel),
2753            format!("frame_bytes={}", limit + 1),
2754            format!("byte_limit={limit}"),
2755        ] {
2756            assert!(warning.contains(&expected), "missing {expected}: {warning}");
2757        }
2758    }
2759
2760    async fn poll_once<F: std::future::Future>(
2761        mut future: std::pin::Pin<&mut F>,
2762    ) -> std::task::Poll<F::Output> {
2763        std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
2764    }
2765
2766    fn fill_client_budget(sink: &FrameSink, route: &crate::forwarding::PendingRouteBindRelay) {
2767        sink.try_send(
2768            Frame::build(
2769                FrameType::Response,
2770                Flags::new(false, Priority::Interactive, true),
2771                route.client_channel,
2772                route.client_epoch,
2773                100,
2774                vec![
2775                    b'x';
2776                    crate::server::CONNECTION_EGRESS_BYTE_BUDGET - subc_protocol::HEADER_LEN
2777                ],
2778            )
2779            .unwrap(),
2780        )
2781        .unwrap();
2782        assert_eq!(
2783            sink.backlog().queued_bytes,
2784            crate::server::CONNECTION_EGRESS_BYTE_BUDGET
2785        );
2786    }
2787
2788    async fn assert_full_client_burst_survives(ty: FrameType, oversized: bool) {
2789        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2790            "draining-burst",
2791            ConnectionId::new(92),
2792            ConnectionId::new(93),
2793            2,
2794        )
2795        .await;
2796        fill_client_budget(&sink, &routes[0]);
2797        let frame = if oversized {
2798            Frame::build(
2799                ty,
2800                Flags::new(false, Priority::Interactive, false),
2801                routes[1].module_channel,
2802                routes[1].module_epoch,
2803                101,
2804                vec![b's'; crate::server::CONNECTION_EGRESS_BYTE_BUDGET],
2805            )
2806            .unwrap()
2807        } else {
2808            route_frame(ty, routes[1].module_channel, routes[1].module_epoch, 101)
2809        };
2810        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2811        assert!(poll_once(delivery.as_mut()).await.is_pending());
2812        assert!(
2813            close.try_recv().is_err(),
2814            "fresh bursts must not close: {ty:?}"
2815        );
2816        tokio::time::advance(Duration::from_millis(50)).await;
2817        let queued = rx.try_recv().unwrap();
2818        queued.charge.as_ref().unwrap().taken_by_writer();
2819        drop(queued);
2820        assert!(matches!(
2821            poll_once(delivery.as_mut()).await,
2822            std::task::Poll::Ready(Ok(()))
2823        ));
2824        let delivered = rx
2825            .try_recv()
2826            .expect("the burst frame must be delivered after draining");
2827        assert_eq!(delivered.header.channel, routes[1].client_channel);
2828        assert_eq!(delivered.header.epoch, routes[1].client_epoch);
2829        assert_eq!(delivered.header.corr, 101);
2830        if oversized {
2831            assert_frame_size_error(
2832                &delivered,
2833                routes[1].client_channel,
2834                routes[1].client_epoch,
2835                101,
2836                crate::server::CONNECTION_EGRESS_BYTE_BUDGET + subc_protocol::HEADER_LEN,
2837            );
2838        } else {
2839            assert_eq!(delivered.header.ty, ty);
2840        }
2841        assert!(rx.try_recv().is_err());
2842        assert!(close.try_recv().is_err());
2843        assert_eq!(
2844            router.counters.snapshot()["client_egress_close_delivery_failed"],
2845            0
2846        );
2847        assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 0);
2848    }
2849
2850    #[tokio::test(start_paused = true)]
2851    async fn module_egress_fresh_full_queue_burst_survives() {
2852        for ty in [
2853            FrameType::Response,
2854            FrameType::StreamData,
2855            FrameType::Request,
2856            FrameType::Push,
2857        ] {
2858            assert_full_client_burst_survives(ty, false).await;
2859        }
2860    }
2861
2862    #[tokio::test(start_paused = true)]
2863    async fn module_goodbye_fresh_full_queue_burst_survives() {
2864        assert_full_client_burst_survives(FrameType::Goodbye, false).await;
2865    }
2866
2867    #[tokio::test(start_paused = true)]
2868    async fn frame_count_backstop_also_allows_a_draining_burst() {
2869        let (tx, mut rx) = mpsc::channel(1);
2870        let sink = FrameSink::with_byte_budget(tx, 2_000);
2871        sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
2872            .unwrap();
2873        let mut delivery = Box::pin(sink.send_to_client(stream_frame(1, 1, 2, vec![b'y'; 100])));
2874        assert!(poll_once(delivery.as_mut()).await.is_pending());
2875        assert!(
2876            sink.backlog().queued_bytes < 2_000,
2877            "only frame capacity is exhausted"
2878        );
2879        tokio::time::advance(Duration::from_millis(50)).await;
2880        drop(rx.try_recv().unwrap());
2881        assert!(matches!(
2882            poll_once(delivery.as_mut()).await,
2883            std::task::Poll::Ready(Ok(()))
2884        ));
2885        assert_eq!(rx.try_recv().unwrap().header.corr, 2);
2886    }
2887
2888    #[tokio::test(start_paused = true)]
2889    async fn oversized_reply_replacement_fresh_full_queue_burst_survives() {
2890        for ty in [FrameType::Response, FrameType::StreamData] {
2891            assert_full_client_burst_survives(ty, true).await;
2892        }
2893    }
2894
2895    #[tokio::test(start_paused = true)]
2896    async fn module_egress_stopped_reader_closes_without_waiting() {
2897        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
2898        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2899            "oldest-stopped-reader",
2900            ConnectionId::new(94),
2901            ConnectionId::new(95),
2902            2,
2903        )
2904        .await;
2905        fill_client_budget(&sink, &routes[0]);
2906        // The writer has taken the head but cannot finish writing it. Its
2907        // bytes stay charged, and the atomic age still diagnoses that stall.
2908        let writing = rx.try_recv().unwrap();
2909        writing.charge.as_ref().unwrap().taken_by_writer();
2910        tokio::time::advance(Duration::from_millis(251)).await;
2911        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(251)));
2912        assert_eq!(sink.backlog().queued_bytes, queued_frame_bytes(&writing));
2913        let started = tokio::time::Instant::now();
2914        let frame = stream_frame(
2915            routes[0].module_channel,
2916            routes[0].module_epoch,
2917            101,
2918            b"secret-body-never-log".to_vec(),
2919        );
2920        let flags = frame.header.flags.0;
2921        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2922        assert!(
2923            matches!(
2924                poll_once(delivery.as_mut()).await,
2925                std::task::Poll::Ready(Ok(()))
2926            ),
2927            "an oldest frame older than 250 ms must close immediately, without a new wait"
2928        );
2929        assert_eq!(started.elapsed(), Duration::ZERO);
2930        assert!(close
2931            .try_recv()
2932            .unwrap()
2933            .to_string()
2934            .contains("module_to_client_delivery_failed"));
2935        let captured = test_log::captured_logs(&logs);
2936        for expected in [
2937            "frame_type=Some(StreamData)".to_string(),
2938            format!("frame_flags=Some({flags})"),
2939            "oldest_queued_ms=251".to_string(),
2940            "close_cause=\"oldest_frame_older_than_250_ms\"".to_string(),
2941        ] {
2942            assert!(
2943                captured.contains(&expected),
2944                "missing {expected}: {captured}"
2945            );
2946        }
2947        assert!(!captured.contains("secret-body-never-log"));
2948    }
2949
2950    #[tokio::test(start_paused = true)]
2951    async fn client_gone_mid_wait_resumes_module_reader_immediately() {
2952        for writer_gone in [false, true] {
2953            let (router, sink, mut rx, ctx, routes, _close) = multi_route_client(
2954                "gone-during-burst",
2955                ConnectionId::new(96),
2956                ConnectionId::new(97),
2957                1,
2958            )
2959            .await;
2960            fill_client_budget(&sink, &routes[0]);
2961            let frame = route_frame(
2962                FrameType::Response,
2963                routes[0].module_channel,
2964                routes[0].module_epoch,
2965                101,
2966            );
2967            let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2968            assert!(poll_once(delivery.as_mut()).await.is_pending());
2969            tokio::time::advance(Duration::from_millis(50)).await;
2970            let started = tokio::time::Instant::now();
2971            if writer_gone {
2972                rx.close();
2973            } else {
2974                sink.close_client_delivery();
2975            }
2976            assert!(
2977                matches!(
2978                    poll_once(delivery.as_mut()).await,
2979                    std::task::Poll::Ready(Ok(()))
2980                ),
2981                "client teardown must wake the module reader before the writer drains"
2982            );
2983            assert_eq!(started.elapsed(), Duration::ZERO);
2984            assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
2985            assert_eq!(rx.try_recv().unwrap().header.corr, 100);
2986            assert!(rx.try_recv().is_err());
2987        }
2988    }
2989
2990    #[tokio::test(start_paused = true)]
2991    async fn cancelled_module_delivery_releases_waiter_without_closing_client() {
2992        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2993            "cancelled-burst",
2994            ConnectionId::new(98),
2995            ConnectionId::new(99),
2996            1,
2997        )
2998        .await;
2999        fill_client_budget(&sink, &routes[0]);
3000        let frame = route_frame(
3001            FrameType::Response,
3002            routes[0].module_channel,
3003            routes[0].module_epoch,
3004            101,
3005        );
3006        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3007        assert!(poll_once(delivery.as_mut()).await.is_pending());
3008        assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 1);
3009        drop(delivery);
3010        assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
3011        tokio::time::advance(Duration::from_secs(1)).await;
3012        assert!(close.try_recv().is_err());
3013        drop(rx.try_recv().unwrap());
3014        assert_eq!(sink.backlog().queued_bytes, 0);
3015        assert!(sink.backlog().oldest_age.is_none());
3016    }
3017
3018    #[tokio::test(start_paused = true)]
3019    async fn egress_atomic_age_follows_writer_and_clears_when_drained() {
3020        let (sink, mut rx) = crate::server::connection_egress();
3021        sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
3022            .unwrap();
3023        tokio::time::advance(Duration::from_millis(200)).await;
3024        sink.try_send(stream_frame(2, 1, 2, vec![b'y'; 100]))
3025            .unwrap();
3026        let writing = rx.try_recv().unwrap();
3027        writing.charge.as_ref().unwrap().taken_by_writer();
3028        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3029        drop(writing);
3030        // Between frames the lock-free marker can still name the previous
3031        // head. The next writer take advances it without scanning the queue.
3032        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3033        tokio::time::advance(Duration::from_millis(50)).await;
3034        let next = rx.try_recv().unwrap();
3035        next.charge.as_ref().unwrap().taken_by_writer();
3036        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(50)));
3037        assert!(
3038            sink.backlog().queued_bytes > 0,
3039            "in-flight bytes remain charged until written"
3040        );
3041        drop(next);
3042        assert_eq!(sink.backlog().queued_bytes, 0);
3043        assert_eq!(sink.backlog().oldest_age, None);
3044    }
3045
3046    #[tokio::test(start_paused = true)]
3047    async fn module_egress_response_full_client_waits_only_remaining_grace() {
3048        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3049            "backpressure-provider",
3050            ConnectionId::new(82),
3051            ConnectionId::new(83),
3052            2,
3053        )
3054        .await;
3055        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3056        // Exactly the limit, including its header, must be admitted to an empty
3057        // queue. Leave it unread to deterministically exhaust only byte capacity.
3058        router
3059            .route_for_connection(
3060                &ctx,
3061                Frame::build(
3062                    FrameType::Response,
3063                    Flags::new(false, Priority::Interactive, true),
3064                    routes[0].module_channel,
3065                    routes[0].module_epoch,
3066                    201,
3067                    vec![b'x'; limit - subc_protocol::HEADER_LEN],
3068                )
3069                .unwrap(),
3070            )
3071            .await
3072            .unwrap();
3073        assert_eq!(sink.backlog().queued_bytes, limit);
3074        assert_eq!(sink.backlog().queued_frames, 1);
3075        let ordinary = route_frame(
3076            FrameType::Response,
3077            routes[1].module_channel,
3078            routes[1].module_epoch,
3079            202,
3080        );
3081        assert!(queued_frame_bytes(&ordinary) < limit);
3082        // A fitting reply gets only the unused portion of the oldest frame's
3083        // 250 ms grace, not an immediate close or a fresh 250 ms wait.
3084        tokio::time::advance(Duration::from_millis(100)).await;
3085        let mut delivery = Box::pin(router.route_for_connection(&ctx, ordinary));
3086        assert!(poll_once(delivery.as_mut()).await.is_pending());
3087        assert!(close.try_recv().is_err());
3088        tokio::time::advance(Duration::from_millis(149)).await;
3089        assert!(poll_once(delivery.as_mut()).await.is_pending());
3090        assert!(close.try_recv().is_err());
3091        tokio::time::advance(Duration::from_millis(1)).await;
3092        assert!(matches!(
3093            poll_once(delivery.as_mut()).await,
3094            std::task::Poll::Ready(Ok(()))
3095        ));
3096        drop(delivery);
3097        let reason = close.try_recv().expect("the full client must close");
3098        assert!(reason
3099            .to_string()
3100            .contains("module_to_client_delivery_failed"));
3101        assert!(!reason.to_string().contains("frame_too_large"));
3102        assert_eq!(
3103            router.counters.snapshot()["client_egress_close_delivery_failed"],
3104            1
3105        );
3106        let admitted = rx.try_recv().expect("the budget-sized frame was admitted");
3107        assert_eq!(admitted.header.ty, FrameType::Response);
3108        assert_eq!(admitted.header.corr, 201);
3109        assert!(
3110            rx.try_recv().is_err(),
3111            "no oversized-frame ERROR is substituted for backpressure"
3112        );
3113    }
3114
3115    #[tokio::test(start_paused = true)]
3116    async fn oversized_reply_to_a_full_client_waits_only_remaining_grace() {
3117        let (router, sink, _rx, ctx, routes, mut close) = multi_route_client(
3118            "full-oversize-provider",
3119            ConnectionId::new(90),
3120            ConnectionId::new(91),
3121            2,
3122        )
3123        .await;
3124        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3125        // Fill the client's queue to its budget and leave it unread.
3126        router
3127            .route_for_connection(
3128                &ctx,
3129                Frame::build(
3130                    FrameType::Response,
3131                    Flags::new(false, Priority::Interactive, true),
3132                    routes[0].module_channel,
3133                    routes[0].module_epoch,
3134                    501,
3135                    vec![b'x'; limit - subc_protocol::HEADER_LEN],
3136                )
3137                .unwrap(),
3138            )
3139            .await
3140            .unwrap();
3141        assert_eq!(sink.backlog().queued_bytes, limit);
3142        let oversized = Frame::build(
3143            FrameType::Response,
3144            Flags::new(false, Priority::Interactive, true),
3145            routes[1].module_channel,
3146            routes[1].module_epoch,
3147            502,
3148            vec![b'y'; limit - subc_protocol::HEADER_LEN + 1],
3149        )
3150        .unwrap();
3151        // The replacement ERROR gets the same bounded burst grace as the
3152        // ordinary reply; oversized payloads themselves are never queued.
3153        tokio::time::advance(Duration::from_millis(80)).await;
3154        let mut delivery = Box::pin(router.route_for_connection(&ctx, oversized));
3155        assert!(poll_once(delivery.as_mut()).await.is_pending());
3156        assert!(close.try_recv().is_err());
3157        tokio::time::advance(Duration::from_millis(169)).await;
3158        assert!(poll_once(delivery.as_mut()).await.is_pending());
3159        assert!(close.try_recv().is_err());
3160        tokio::time::advance(Duration::from_millis(1)).await;
3161        assert!(matches!(
3162            poll_once(delivery.as_mut()).await,
3163            std::task::Poll::Ready(Ok(()))
3164        ));
3165        drop(delivery);
3166        let reason = close.try_recv().expect("the full client must close");
3167        assert!(reason
3168            .to_string()
3169            .contains("module_to_client_delivery_failed"));
3170    }
3171
3172    #[tokio::test]
3173    async fn oversized_reverse_request_errors_module_without_reaching_client() {
3174        let (router, _sink, mut rx, mut ctx, routes, mut close) = multi_route_client(
3175            "reverse-oversize-provider",
3176            ConnectionId::new(84),
3177            ConnectionId::new(85),
3178            2,
3179        )
3180        .await;
3181        let (module_tx, mut module_rx) = mpsc::channel(8);
3182        ctx.egress = FrameSink::new(module_tx);
3183        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3184        router
3185            .route_for_connection(
3186                &ctx,
3187                Frame::build(
3188                    FrameType::Request,
3189                    Flags::new(false, Priority::Interactive, false),
3190                    routes[0].module_channel,
3191                    routes[0].module_epoch,
3192                    301,
3193                    vec![b'r'; limit - subc_protocol::HEADER_LEN + 1],
3194                )
3195                .unwrap(),
3196            )
3197            .await
3198            .unwrap();
3199        assert!(
3200            rx.try_recv().is_err(),
3201            "the reverse request never reaches the client"
3202        );
3203        let error = module_rx.try_recv().expect("the module gets the refusal");
3204        assert_frame_size_error(
3205            &error,
3206            routes[0].module_channel,
3207            routes[0].module_epoch,
3208            301,
3209            limit + 1,
3210        );
3211        assert!(module_rx.try_recv().is_err());
3212        for (index, route) in routes.iter().enumerate() {
3213            module_route_for_test(&router, &ctx, route);
3214            router
3215                .route_for_connection(
3216                    &ctx,
3217                    route_frame(
3218                        FrameType::Response,
3219                        route.module_channel,
3220                        route.module_epoch,
3221                        302 + index as u64,
3222                    ),
3223                )
3224                .await
3225                .unwrap();
3226            let reply = rx.try_recv().expect("both routes still deliver");
3227            assert_eq!(reply.header.ty, FrameType::Response);
3228            assert_eq!(reply.header.channel, route.client_channel);
3229        }
3230        assert!(close.try_recv().is_err());
3231        assert_eq!(
3232            router.counters.snapshot()["client_egress_close_delivery_failed"],
3233            0
3234        );
3235    }
3236
3237    #[tokio::test]
3238    async fn oversized_push_closes_client_with_precise_warn() {
3239        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3240        let (router, _sink, mut rx, ctx, routes, mut close) = multi_route_client(
3241            "push-oversize-provider",
3242            ConnectionId::new(86),
3243            ConnectionId::new(87),
3244            2,
3245        )
3246        .await;
3247        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3248        router
3249            .route_for_connection(
3250                &ctx,
3251                Frame::build(
3252                    FrameType::Push,
3253                    Flags::new(false, Priority::Interactive, false),
3254                    routes[0].module_channel,
3255                    routes[0].module_epoch,
3256                    0,
3257                    vec![b'p'; limit - subc_protocol::HEADER_LEN + 1],
3258                )
3259                .unwrap(),
3260            )
3261            .await
3262            .unwrap();
3263        let reason = close
3264            .try_recv()
3265            .expect("unsolicited oversize still closes the client");
3266        assert!(reason
3267            .to_string()
3268            .contains("frame_too_large: oversized push"));
3269        assert!(
3270            rx.try_recv().is_err(),
3271            "no part of the oversized push is forwarded"
3272        );
3273        let captured = test_log::captured_logs(&logs);
3274        let warnings: Vec<_> = captured
3275            .lines()
3276            .filter(|line| line.contains("WARN"))
3277            .collect();
3278        assert_eq!(warnings.len(), 1, "{captured}");
3279        let warning = warnings[0];
3280        for expected in [
3281            "oversized push".to_string(),
3282            "module_id=\"push-oversize-provider\"".to_string(),
3283            "principals=direct".to_string(),
3284            format!("client_channel={}", routes[0].client_channel),
3285            format!("size={}", limit + 1),
3286            format!("limit={limit}"),
3287        ] {
3288            assert!(warning.contains(&expected), "missing {expected}: {warning}");
3289        }
3290        assert_eq!(
3291            router.counters.snapshot()["client_egress_close_delivery_failed"],
3292            1
3293        );
3294    }
3295
3296    #[tokio::test]
3297    async fn oversized_stream_and_error_frames_end_request_even_with_queued_traffic() {
3298        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3299            "stream-oversize-provider",
3300            ConnectionId::new(88),
3301            ConnectionId::new(89),
3302            2,
3303        )
3304        .await;
3305        let binding = module_route_for_test(&router, &ctx, &routes[0]);
3306        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3307        for ty in [
3308            FrameType::StreamData,
3309            FrameType::StreamEnd,
3310            FrameType::Error,
3311        ] {
3312            binding.flow.acquire_tagged(401, false).await.unwrap();
3313            sink.try_send(route_frame(
3314                FrameType::Response,
3315                routes[1].client_channel,
3316                routes[1].client_epoch,
3317                402,
3318            ))
3319            .unwrap();
3320            router
3321                .route_for_connection(
3322                    &ctx,
3323                    Frame::build(
3324                        ty,
3325                        Flags::new(false, Priority::Interactive, false),
3326                        routes[0].module_channel,
3327                        routes[0].module_epoch,
3328                        401,
3329                        vec![b's'; limit - subc_protocol::HEADER_LEN + 1],
3330                    )
3331                    .unwrap(),
3332                )
3333                .await
3334                .unwrap();
3335            assert!(close.try_recv().is_err());
3336            assert_eq!(rx.try_recv().unwrap().header.corr, 402);
3337            let error = rx
3338                .try_recv()
3339                .expect("every correlated oversize frame becomes ERROR");
3340            assert_frame_size_error(
3341                &error,
3342                routes[0].client_channel,
3343                routes[0].client_epoch,
3344                401,
3345                limit + 1,
3346            );
3347            assert_eq!(binding.flow.drain_in_flight(), 0, "{ty:?} ends the request");
3348            assert!(rx.try_recv().is_err());
3349        }
3350    }
3351
3352    /// An awaited send (a control or error reply) is never refused by the byte
3353    /// budget: it parks until the writer frees bytes. If a release could miss a
3354    /// parked sender, that reply would hang for the connection's lifetime, so
3355    /// this pins the wakeup: the send stays parked while the queue is full and
3356    /// completes as soon as one frame leaves it.
3357    #[tokio::test]
3358    async fn awaited_send_parked_behind_a_full_byte_budget_wakes_when_bytes_free() {
3359        let (tx, mut rx) = mpsc::channel(1024);
3360        let sink = FrameSink::with_byte_budget(tx, 2_000);
3361        let mut queued = 0u64;
3362        while sink
3363            .try_send(stream_frame(7, 1, queued, vec![b'x'; 200]))
3364            .is_ok()
3365        {
3366            queued += 1;
3367        }
3368        assert!(queued > 0, "the budget admitted nothing");
3369
3370        let parked = tokio::spawn({
3371            let sink = sink.clone();
3372            async move { sink.send(stream_frame(0, 0, 999, vec![b'r'; 200])).await }
3373        });
3374        tokio::time::sleep(Duration::from_millis(100)).await;
3375        assert!(
3376            !parked.is_finished(),
3377            "the awaited send must wait while the byte budget is full"
3378        );
3379
3380        // The writer takes one frame and drops it, releasing its bytes.
3381        drop(rx.recv().await.expect("a queued frame"));
3382        tokio::time::timeout(Duration::from_secs(2), parked)
3383            .await
3384            .expect("the parked send was never woken after bytes were freed")
3385            .unwrap()
3386            .unwrap();
3387    }
3388
3389    /// A client multiplexing several token streams pauses its reader while the
3390    /// module keeps producing small frames. Far more frames than the old
3391    /// 64-frame queue allowed, but far fewer bytes than the budget, must all be
3392    /// held without closing the connection and delivered in order afterwards.
3393    #[tokio::test]
3394    async fn paused_client_reader_keeps_connection_through_small_frame_burst() {
3395        const ROUTES: usize = 4;
3396        const FRAMES_PER_ROUTE: usize = 250;
3397        let (router, client_sink, mut client_rx, module_ctx, routes, mut close_receiver) =
3398            multi_route_client(
3399                "burst-provider",
3400                ConnectionId::new(60),
3401                ConnectionId::new(61),
3402                ROUTES,
3403            )
3404            .await;
3405
3406        // The reader is paused: nothing is received until every frame is sent.
3407        for seq in 0..FRAMES_PER_ROUTE as u64 {
3408            for (index, route) in routes.iter().enumerate() {
3409                let body = format!("route-{index}-token-{seq:05}-{}", "t".repeat(170));
3410                router
3411                    .route_for_connection(
3412                        &module_ctx,
3413                        stream_frame(
3414                            route.module_channel,
3415                            route.module_epoch,
3416                            seq,
3417                            body.into_bytes(),
3418                        ),
3419                    )
3420                    .await
3421                    .unwrap();
3422            }
3423        }
3424
3425        let backlog = client_sink.backlog();
3426        assert_eq!(backlog.queued_frames, ROUTES * FRAMES_PER_ROUTE);
3427        assert!(backlog.queued_bytes < crate::server::CONNECTION_EGRESS_BYTE_BUDGET);
3428        assert!(
3429            close_receiver.try_recv().is_err(),
3430            "a paused reader under the byte budget must not be closed"
3431        );
3432        assert_eq!(
3433            router.counters.snapshot()["client_egress_close_delivery_failed"],
3434            0
3435        );
3436
3437        // The reader resumes: every frame arrives, in order within each route.
3438        let mut next_seq = vec![0u64; ROUTES];
3439        for _ in 0..ROUTES * FRAMES_PER_ROUTE {
3440            let frame = client_rx.try_recv().expect("every queued frame arrives");
3441            let index = routes
3442                .iter()
3443                .position(|route| route.client_channel == frame.header.channel)
3444                .expect("frame arrives on one of the bound client channels");
3445            assert_eq!(frame.header.corr, next_seq[index], "route {index} order");
3446            let expected_prefix = format!("route-{index}-token-{:05}-", next_seq[index]);
3447            assert!(frame.body.starts_with(expected_prefix.as_bytes()));
3448            next_seq[index] += 1;
3449        }
3450        assert!(client_rx.try_recv().is_err());
3451        assert_eq!(next_seq, vec![FRAMES_PER_ROUTE as u64; ROUTES]);
3452        assert_eq!(client_sink.backlog().queued_bytes, 0);
3453    }
3454
3455    /// A client that never reads exhausts its byte budget and the oldest
3456    /// frame's 250 ms grace, then closes with one header-only diagnosis.
3457    #[tokio::test(start_paused = true)]
3458    async fn never_reading_client_is_closed_at_byte_budget_with_warn_diagnosis() {
3459        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3460        const BODY: usize = 16 * 1024;
3461        let (router, client_sink, _client_rx, module_ctx, routes, mut close_receiver) =
3462            multi_route_client(
3463                "stuck-reader-provider",
3464                ConnectionId::new(70),
3465                ConnectionId::new(71),
3466                2,
3467            )
3468            .await;
3469
3470        let mut admitted = 0usize;
3471        let mut sent = 0u64;
3472        let started = tokio::time::Instant::now();
3473        while router.counters.snapshot()["client_egress_close_delivery_failed"] == 0 {
3474            assert!(sent < 1_000, "the byte budget never refused a frame");
3475            let route = &routes[(sent % 2) as usize];
3476            router
3477                .route_for_connection(
3478                    &module_ctx,
3479                    stream_frame(
3480                        route.module_channel,
3481                        route.module_epoch,
3482                        sent,
3483                        vec![b'z'; BODY],
3484                    ),
3485                )
3486                .await
3487                .unwrap();
3488            sent += 1;
3489            admitted = client_sink.backlog().queued_frames;
3490        }
3491        assert_eq!(
3492            started.elapsed(),
3493            crate::server::CONNECTION_EGRESS_STALL_GRACE
3494        );
3495        // The budget, not the frame-count backstop, did the refusing.
3496        let frame_bytes = subc_protocol::HEADER_LEN + BODY;
3497        assert_eq!(
3498            admitted,
3499            crate::server::CONNECTION_EGRESS_BYTE_BUDGET / frame_bytes
3500        );
3501        let reason = close_receiver
3502            .try_recv()
3503            .expect("the client connection must be asked to close");
3504        assert!(reason
3505            .to_string()
3506            .contains("module_to_client_delivery_failed"));
3507
3508        // A second refused frame for the same connection adds no second WARN.
3509        router
3510            .route_for_connection(
3511                &module_ctx,
3512                stream_frame(
3513                    routes[0].module_channel,
3514                    routes[0].module_epoch,
3515                    sent,
3516                    vec![b'z'; BODY],
3517                ),
3518            )
3519            .await
3520            .unwrap();
3521
3522        let captured = test_log::captured_logs(&logs);
3523        let warn_lines = captured
3524            .lines()
3525            .filter(|line| {
3526                line.contains("closing client connection: its egress queue could not take a frame")
3527            })
3528            .collect::<Vec<_>>();
3529        assert_eq!(warn_lines.len(), 1, "exactly one WARN, got: {captured}");
3530        let line = warn_lines[0];
3531        assert!(line.contains("WARN"), "{line}");
3532        assert!(line.contains("connection_id=71"), "{line}");
3533        assert!(
3534            line.contains("module_id=\"stuck-reader-provider\""),
3535            "{line}"
3536        );
3537        assert!(line.contains("client_channel="), "{line}");
3538        assert!(line.contains("principals=direct"), "{line}");
3539        let queued_bytes: usize = line
3540            .split("queued_bytes=")
3541            .nth(1)
3542            .and_then(|rest| rest.split_whitespace().next())
3543            .and_then(|value| value.parse().ok())
3544            .expect("queued_bytes is logged");
3545        assert_eq!(queued_bytes, admitted * frame_bytes);
3546        assert!(
3547            line.contains(&format!("queued_frames={admitted}")),
3548            "{line}"
3549        );
3550        assert!(line.contains("oldest_queued_ms="), "{line}");
3551        assert!(line.contains("oldest_queued_ms=250"), "{line}");
3552        assert!(line.contains("frame_type=Some(StreamData)"), "{line}");
3553        assert!(
3554            line.contains(&format!(
3555                "frame_flags=Some({})",
3556                Flags::new(false, Priority::Interactive, false).0
3557            )),
3558            "{line}"
3559        );
3560        assert!(
3561            line.contains("close_cause=\"waited_250_ms_without_room\""),
3562            "{line}"
3563        );
3564        assert!(
3565            !line.contains(&"z".repeat(20)),
3566            "body bytes must never be logged: {line}"
3567        );
3568    }
3569
3570    fn route_frame(ty: FrameType, channel: u16, epoch: u32, corr: u64) -> Frame {
3571        Frame::build(
3572            ty,
3573            Flags::new(false, Priority::Interactive, false),
3574            channel,
3575            epoch,
3576            corr,
3577            if ty == FrameType::Request || ty == FrameType::Response {
3578                b"route-body".to_vec()
3579            } else {
3580                Vec::new()
3581            },
3582        )
3583        .unwrap()
3584    }
3585
3586    type DynamicRouteFixture = (
3587        Router,
3588        Arc<ForwardingTable>,
3589        RouteCtx,
3590        mpsc::Receiver<crate::router::OutboundFrame>,
3591        RouteCtx,
3592        mpsc::Receiver<crate::router::OutboundFrame>,
3593        mpsc::Receiver<crate::router::OutboundFrame>,
3594        crate::forwarding::PendingRouteBindRelay,
3595    );
3596
3597    fn dynamic_route_fixture(commit: bool) -> DynamicRouteFixture {
3598        let forwarding = Arc::new(ForwardingTable::default());
3599        let control = Arc::new(crate::ControlHandler::with_forwarding(
3600            Arc::new(crate::Registry::default()),
3601            Arc::clone(&forwarding),
3602        ));
3603        let router = Router::with_control_handler(control);
3604        let module_connection = ConnectionId::new(500);
3605        let client_connection = ConnectionId::new(501);
3606        let (module_tx, module_rx) = mpsc::channel(8);
3607        forwarding
3608            .register_module_connection(
3609                module_connection,
3610                "epoch-router".into(),
3611                2,
3612                Concurrency::ModuleManaged,
3613                FrameSink::new(module_tx),
3614            )
3615            .unwrap();
3616        let (client_tx, client_rx) = mpsc::channel(8);
3617        let client_sink = FrameSink::new(client_tx);
3618        let pending = forwarding
3619            .begin_route_bind_relay_for_test(
3620                client_connection,
3621                client_sink.clone(),
3622                700,
3623                "epoch-router",
3624            )
3625            .unwrap();
3626        if commit {
3627            forwarding
3628                .complete_pending_relay(
3629                    module_connection,
3630                    pending.corr,
3631                    RouteBindRelayOutcome::Accepted,
3632                )
3633                .unwrap();
3634        }
3635        let (module_egress_tx, module_egress_rx) = mpsc::channel(8);
3636        (
3637            router,
3638            forwarding,
3639            RouteCtx {
3640                connection_id: client_connection,
3641                egress: client_sink,
3642            },
3643            client_rx,
3644            RouteCtx {
3645                connection_id: module_connection,
3646                egress: FrameSink::new(module_egress_tx),
3647            },
3648            module_egress_rx,
3649            module_rx,
3650            pending,
3651        )
3652    }
3653
3654    #[tokio::test]
3655    async fn route_epochs_validate_both_directions_and_rewrite_to_peer_handle() {
3656        let (
3657            router,
3658            _forwarding,
3659            client_ctx,
3660            mut client_rx,
3661            module_ctx,
3662            _module_egress_rx,
3663            mut module_rx,
3664            pending,
3665        ) = dynamic_route_fixture(true);
3666        let route_open = client_rx.recv().await.unwrap();
3667        assert_eq!(route_open.header.corr, 700);
3668
3669        router
3670            .route_for_connection(
3671                &client_ctx,
3672                route_frame(
3673                    FrameType::Request,
3674                    pending.client_channel,
3675                    pending.client_epoch,
3676                    701,
3677                ),
3678            )
3679            .await
3680            .unwrap();
3681        let forwarded = module_rx.recv().await.unwrap();
3682        assert_eq!(forwarded.header.channel, pending.module_channel);
3683        assert_eq!(forwarded.header.epoch, pending.module_epoch);
3684
3685        router
3686            .route_for_connection(
3687                &module_ctx,
3688                route_frame(
3689                    FrameType::Response,
3690                    pending.module_channel,
3691                    pending.module_epoch,
3692                    701,
3693                ),
3694            )
3695            .await
3696            .unwrap();
3697        let delivered = client_rx.recv().await.unwrap();
3698        assert_eq!(delivered.header.channel, pending.client_channel);
3699        assert_eq!(delivered.header.epoch, pending.client_epoch);
3700
3701        router
3702            .route_for_connection(
3703                &client_ctx,
3704                route_frame(
3705                    FrameType::Request,
3706                    pending.client_channel,
3707                    pending.client_epoch + 1,
3708                    702,
3709                ),
3710            )
3711            .await
3712            .unwrap();
3713        router
3714            .route_for_connection(
3715                &module_ctx,
3716                route_frame(
3717                    FrameType::Response,
3718                    pending.module_channel,
3719                    pending.module_epoch + 1,
3720                    703,
3721                ),
3722            )
3723            .await
3724            .unwrap();
3725        let stale_error = client_rx.recv().await.unwrap();
3726        assert_eq!(stale_error.header.ty, FrameType::Error);
3727        assert_eq!(stale_error.header.channel, pending.client_channel);
3728        assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
3729        assert_eq!(stale_error.header.corr, 702);
3730        let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
3731        assert_eq!(body.code, "stale_route_epoch");
3732        assert!(module_rx.try_recv().is_err());
3733        assert!(client_rx.try_recv().is_err());
3734        let counters = router.counters.snapshot();
3735        assert_eq!(counters["client_frames_dropped_stale_route"], 1);
3736        assert_eq!(counters["module_frames_dropped_no_route"], 1);
3737    }
3738
3739    #[tokio::test]
3740    async fn accepted_route_publishes_route_open_before_immediate_reverse_request() {
3741        let (
3742            router,
3743            _,
3744            _client_ctx,
3745            mut client_rx,
3746            module_ctx,
3747            _module_egress_rx,
3748            _module_rx,
3749            pending,
3750        ) = dynamic_route_fixture(true);
3751        router
3752            .route_for_connection(
3753                &module_ctx,
3754                route_frame(
3755                    FrameType::Request,
3756                    pending.module_channel,
3757                    pending.module_epoch,
3758                    800,
3759                ),
3760            )
3761            .await
3762            .unwrap();
3763
3764        let first = client_rx.recv().await.unwrap();
3765        let second = client_rx.recv().await.unwrap();
3766        assert_eq!(first.header.channel, 0);
3767        assert_eq!(first.header.corr, 700);
3768        assert_eq!(second.header.channel, pending.client_channel);
3769        assert_eq!(second.header.epoch, pending.client_epoch);
3770        assert_eq!(second.header.corr, 800);
3771    }
3772
3773    #[tokio::test]
3774    async fn reserved_slot_ingress_errors_only_matching_client_requests() {
3775        let (
3776            router,
3777            _forwarding,
3778            client_ctx,
3779            mut client_rx,
3780            _module_ctx,
3781            _module_egress_rx,
3782            mut module_rx,
3783            pending,
3784        ) = dynamic_route_fixture(false);
3785        router
3786            .route_for_connection(
3787                &client_ctx,
3788                route_frame(
3789                    FrameType::Request,
3790                    pending.client_channel,
3791                    pending.client_epoch,
3792                    900,
3793                ),
3794            )
3795            .await
3796            .unwrap();
3797        let error = client_rx.recv().await.unwrap();
3798        assert_eq!(error.header.ty, FrameType::Error);
3799        assert_eq!(error.header.channel, pending.client_channel);
3800        assert_eq!(error.header.epoch, pending.client_epoch);
3801        assert_eq!(error.header.corr, 900);
3802
3803        router
3804            .route_for_connection(
3805                &client_ctx,
3806                route_frame(
3807                    FrameType::Response,
3808                    pending.client_channel,
3809                    pending.client_epoch,
3810                    901,
3811                ),
3812            )
3813            .await
3814            .unwrap();
3815        router
3816            .route_for_connection(
3817                &client_ctx,
3818                route_frame(
3819                    FrameType::Request,
3820                    pending.client_channel,
3821                    pending.client_epoch + 1,
3822                    902,
3823                ),
3824            )
3825            .await
3826            .unwrap();
3827        let stale_error = client_rx.recv().await.unwrap();
3828        assert_eq!(stale_error.header.ty, FrameType::Error);
3829        assert_eq!(stale_error.header.channel, pending.client_channel);
3830        assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
3831        assert_eq!(stale_error.header.corr, 902);
3832        let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
3833        assert_eq!(body.code, "stale_route_epoch");
3834        assert!(module_rx.try_recv().is_err());
3835        let counters = router.counters.snapshot();
3836        assert_eq!(counters["client_frames_dropped_stale_route"], 1);
3837        assert_eq!(counters["module_frames_dropped_no_route"], 0);
3838    }
3839
3840    #[tokio::test]
3841    async fn dropped_module_route_goodbye_increments_counter() {
3842        let (
3843            router,
3844            _forwarding,
3845            client_ctx,
3846            mut client_rx,
3847            _module_ctx,
3848            _module_egress_rx,
3849            mut module_rx,
3850            pending,
3851        ) = dynamic_route_fixture(true);
3852        let _ = client_rx.recv().await;
3853        module_rx.close();
3854
3855        router
3856            .route_for_connection(
3857                &client_ctx,
3858                route_frame(
3859                    FrameType::Goodbye,
3860                    pending.client_channel,
3861                    pending.client_epoch,
3862                    999,
3863                ),
3864            )
3865            .await
3866            .unwrap();
3867
3868        let counters = router.counters.snapshot();
3869        assert_eq!(counters["goodbye_relay_module_dropped"], 1);
3870        assert_eq!(
3871            counters["goodbye_relay_module_dropped_by_module"],
3872            serde_json::json!({ "epoch-router": 1 })
3873        );
3874        assert_eq!(counters["route_released_epoch_fenced"], 1);
3875    }
3876
3877    #[tokio::test]
3878    async fn module_request_on_stale_epoch_receives_stale_route_epoch() {
3879        let (
3880            router,
3881            _forwarding,
3882            _client_ctx,
3883            _client_rx,
3884            module_ctx,
3885            mut module_egress_rx,
3886            mut module_rx,
3887            pending,
3888        ) = dynamic_route_fixture(true);
3889
3890        router
3891            .route_for_connection(
3892                &module_ctx,
3893                route_frame(
3894                    FrameType::Request,
3895                    pending.module_channel,
3896                    pending.module_epoch + 1,
3897                    1_000,
3898                ),
3899            )
3900            .await
3901            .unwrap();
3902
3903        let error = module_egress_rx.try_recv().unwrap();
3904        assert_eq!(error.header.ty, FrameType::Error);
3905        assert_eq!(error.header.channel, pending.module_channel);
3906        assert_eq!(error.header.epoch, pending.module_epoch + 1);
3907        assert_eq!(error.header.corr, 1_000);
3908        let body: ErrorBody = serde_json::from_slice(&error.body).unwrap();
3909        assert_eq!(body.code, "stale_route_epoch");
3910        assert!(module_rx.try_recv().is_err());
3911        let counters = router.counters.snapshot();
3912        assert_eq!(counters["module_requests_dropped_stale_route"], 1);
3913        assert_eq!(counters["module_frames_dropped_no_route"], 0);
3914    }
3915
3916    #[tokio::test(start_paused = true)]
3917    async fn stale_epoch_drop_warns_once_per_minute_with_suppressed_count() {
3918        let (output, _guard) = test_log::log_capture(tracing::Level::WARN);
3919        let (
3920            router,
3921            _forwarding,
3922            client_ctx,
3923            mut client_rx,
3924            _module_ctx,
3925            _module_egress_rx,
3926            mut module_rx,
3927            pending,
3928        ) = dynamic_route_fixture(true);
3929        let _ = client_rx.recv().await;
3930        for corr in 1..=3 {
3931            router
3932                .route_for_connection(
3933                    &client_ctx,
3934                    route_frame(
3935                        FrameType::Request,
3936                        pending.client_channel,
3937                        pending.client_epoch + 1,
3938                        corr,
3939                    ),
3940                )
3941                .await
3942                .unwrap();
3943            assert_eq!(client_rx.recv().await.unwrap().header.ty, FrameType::Error);
3944        }
3945        assert!(module_rx.try_recv().is_err());
3946        let logs = test_log::captured_logs(&output);
3947        assert_eq!(
3948            logs.matches("dropping data-plane frame").count(),
3949            1,
3950            "{logs}"
3951        );
3952        assert!(logs.contains("WARN"), "{logs}");
3953        assert!(
3954            logs.contains(&format!("connection_id={}", client_ctx.connection_id.get())),
3955            "{logs}"
3956        );
3957        assert!(
3958            logs.contains(&format!("channel={}", pending.client_channel)),
3959            "{logs}"
3960        );
3961        assert!(
3962            logs.contains(&format!("epoch={}", pending.client_epoch + 1)),
3963            "{logs}"
3964        );
3965        assert!(
3966            logs.contains("client_to_module") && logs.contains("stale_epoch"),
3967            "{logs}"
3968        );
3969        assert!(logs.contains("suppressed=0"), "{logs}");
3970
3971        tokio::time::advance(Duration::from_secs(60)).await;
3972        router
3973            .route_for_connection(
3974                &client_ctx,
3975                route_frame(
3976                    FrameType::Request,
3977                    pending.client_channel,
3978                    pending.client_epoch + 2,
3979                    4,
3980                ),
3981            )
3982            .await
3983            .unwrap();
3984        let logs = test_log::captured_logs(&output);
3985        assert_eq!(
3986            logs.matches("dropping data-plane frame").count(),
3987            2,
3988            "{logs}"
3989        );
3990        assert!(logs.contains("suppressed=2"), "{logs}");
3991    }
3992
3993    #[tokio::test]
3994    async fn module_request_on_reserved_or_absent_route_receives_unknown_channel() {
3995        let (
3996            reserved_router,
3997            _forwarding,
3998            _client_ctx,
3999            _client_rx,
4000            reserved_module_ctx,
4001            mut reserved_module_egress_rx,
4002            _module_rx,
4003            reserved,
4004        ) = dynamic_route_fixture(false);
4005        reserved_router
4006            .route_for_connection(
4007                &reserved_module_ctx,
4008                route_frame(
4009                    FrameType::Request,
4010                    reserved.module_channel,
4011                    reserved.module_epoch,
4012                    1_001,
4013                ),
4014            )
4015            .await
4016            .unwrap();
4017        let reserved_error = reserved_module_egress_rx.try_recv().unwrap();
4018        let reserved_body: ErrorBody = serde_json::from_slice(&reserved_error.body).unwrap();
4019        assert_eq!(reserved_error.header.ty, FrameType::Error);
4020        assert_eq!(reserved_error.header.channel, reserved.module_channel);
4021        assert_eq!(reserved_error.header.epoch, reserved.module_epoch);
4022        assert_eq!(reserved_error.header.corr, 1_001);
4023        assert_eq!(reserved_body.code, "unknown_channel");
4024        assert_eq!(
4025            reserved_router.counters.snapshot()["module_requests_dropped_stale_route"],
4026            1
4027        );
4028
4029        let (
4030            absent_router,
4031            _forwarding,
4032            _client_ctx,
4033            _client_rx,
4034            absent_module_ctx,
4035            mut absent_module_egress_rx,
4036            _module_rx,
4037            absent,
4038        ) = dynamic_route_fixture(false);
4039        absent_router
4040            .route_for_connection(
4041                &absent_module_ctx,
4042                route_frame(
4043                    FrameType::Request,
4044                    absent.module_channel + 1,
4045                    absent.module_epoch,
4046                    1_002,
4047                ),
4048            )
4049            .await
4050            .unwrap();
4051        let absent_error = absent_module_egress_rx.try_recv().unwrap();
4052        let absent_body: ErrorBody = serde_json::from_slice(&absent_error.body).unwrap();
4053        assert_eq!(absent_error.header.ty, FrameType::Error);
4054        assert_eq!(absent_error.header.channel, absent.module_channel + 1);
4055        assert_eq!(absent_error.header.epoch, absent.module_epoch);
4056        assert_eq!(absent_error.header.corr, 1_002);
4057        assert_eq!(absent_body.code, "unknown_channel");
4058        assert_eq!(
4059            absent_router.counters.snapshot()["module_requests_dropped_stale_route"],
4060            1
4061        );
4062    }
4063
4064    #[tokio::test]
4065    async fn non_request_module_frame_on_dead_route_is_counted_without_error() {
4066        let (
4067            router,
4068            forwarding,
4069            client_ctx,
4070            mut client_rx,
4071            module_ctx,
4072            mut module_egress_rx,
4073            mut module_rx,
4074            pending,
4075        ) = dynamic_route_fixture(true);
4076        let (other_module_tx, _other_module_rx) = mpsc::channel(8);
4077        forwarding
4078            .register_module_connection(
4079                ConnectionId::new(502),
4080                "other-module".into(),
4081                2,
4082                Concurrency::ModuleManaged,
4083                FrameSink::new(other_module_tx),
4084            )
4085            .unwrap();
4086        let _ = client_rx.recv().await.unwrap();
4087
4088        router
4089            .route_for_connection(
4090                &client_ctx,
4091                route_frame(
4092                    FrameType::Goodbye,
4093                    pending.client_channel,
4094                    pending.client_epoch,
4095                    1_003,
4096                ),
4097            )
4098            .await
4099            .unwrap();
4100        let _ = module_rx.recv().await.unwrap();
4101
4102        router
4103            .route_for_connection(
4104                &module_ctx,
4105                route_frame(
4106                    FrameType::StreamData,
4107                    pending.module_channel,
4108                    pending.module_epoch,
4109                    1_004,
4110                ),
4111            )
4112            .await
4113            .unwrap();
4114
4115        // No ERROR goes back for a non-request frame. The one reply is the
4116        // route GOODBYE telling the module to let go of the released route.
4117        let reply = module_egress_rx.try_recv().unwrap();
4118        assert_eq!(reply.header.ty, FrameType::Goodbye);
4119        assert!(module_egress_rx.try_recv().is_err());
4120        let counters = router.counters.snapshot();
4121        assert_eq!(counters["module_frames_dropped_no_route"], 1);
4122        assert_eq!(
4123            counters["module_frames_dropped_no_route_by_module"],
4124            serde_json::json!({ "epoch-router": 1 })
4125        );
4126        assert_eq!(counters["module_requests_dropped_stale_route"], 0);
4127    }
4128
4129    /// Drain whatever the module connection's queue holds right now, the way a
4130    /// module's reader would before it stalls.
4131    fn drain_now(rx: &mut mpsc::Receiver<OutboundFrame>) {
4132        while rx.try_recv().is_ok() {}
4133    }
4134
4135    /// A module that stops reading for a moment when a client closes one of
4136    /// its routes still learns the route is gone: the GOODBYE its full egress
4137    /// queue refused is delivered as soon as it reads again, rather than
4138    /// dropped, which would leave the module holding the route for the rest of
4139    /// its connection.
4140    #[tokio::test]
4141    async fn route_goodbye_refused_by_stalled_module_is_delivered_when_it_resumes_reading() {
4142        const BUDGET: usize = 4_096;
4143        let forwarding = Arc::new(ForwardingTable::default());
4144        let control = Arc::new(ControlHandler::with_forwarding(
4145            Arc::new(Registry::default()),
4146            Arc::clone(&forwarding),
4147        ));
4148        let router = Router::with_control_handler(control);
4149        let module_connection = ConnectionId::new(80);
4150        let client_connection = ConnectionId::new(81);
4151        let (module_tx, mut module_rx) = mpsc::channel(64);
4152        let module_sink = FrameSink::with_byte_budget(module_tx, BUDGET);
4153        forwarding
4154            .register_module_connection(
4155                module_connection,
4156                "stalled-provider".into(),
4157                2,
4158                Concurrency::ModuleManaged,
4159                module_sink.clone(),
4160            )
4161            .unwrap();
4162        let (client_tx, mut client_rx) = mpsc::channel(8);
4163        let client_sink = FrameSink::new(client_tx);
4164        let pending = forwarding
4165            .begin_route_bind_relay_for_test(
4166                client_connection,
4167                client_sink.clone(),
4168                1_100,
4169                "stalled-provider",
4170            )
4171            .unwrap();
4172        forwarding
4173            .complete_pending_relay(
4174                module_connection,
4175                pending.corr,
4176                RouteBindRelayOutcome::Accepted,
4177            )
4178            .unwrap();
4179        let _ = client_rx.recv().await.unwrap();
4180        drain_now(&mut module_rx);
4181
4182        // The module stalls: its queue holds more than the byte budget, so the
4183        // queue refuses anything further.
4184        module_sink
4185            .try_send(stream_frame(9, 1, 0, vec![b'f'; BUDGET]))
4186            .unwrap();
4187        assert!(module_sink
4188            .try_send(stream_frame(9, 1, 1, Vec::new()))
4189            .is_err());
4190
4191        let client_ctx = RouteCtx {
4192            connection_id: client_connection,
4193            egress: client_sink,
4194        };
4195        router
4196            .route_for_connection(
4197                &client_ctx,
4198                route_frame(
4199                    FrameType::Goodbye,
4200                    pending.client_channel,
4201                    pending.client_epoch,
4202                    1_101,
4203                ),
4204            )
4205            .await
4206            .unwrap();
4207        tokio::task::yield_now().await;
4208        assert_eq!(
4209            router.counters.snapshot()["goodbye_relay_module_dropped"],
4210            0,
4211            "a GOODBYE refused by a momentarily full module queue must not be dropped"
4212        );
4213
4214        // The module reads again: the filler comes off, then the GOODBYE.
4215        let filler = module_rx.recv().await.unwrap();
4216        assert_eq!(filler.header.ty, FrameType::StreamData);
4217        drop(filler);
4218        let goodbye = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
4219            .await
4220            .expect("the refused GOODBYE must be delivered once the module frees room")
4221            .unwrap();
4222        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
4223        assert_eq!(goodbye.header.channel, pending.module_channel);
4224        assert_eq!(goodbye.header.epoch, pending.module_epoch);
4225        assert_eq!(
4226            router.counters.snapshot()["goodbye_relay_module_dropped"],
4227            0
4228        );
4229    }
4230
4231    /// A module still sending on a route the daemon released is told, with a
4232    /// route GOODBYE for exactly the (channel, epoch) it sent on. The same
4233    /// happens for a stale epoch on a channel whose route moved on and for a
4234    /// channel that never had a route; only the first is counted as traffic on
4235    /// a released route.
4236    #[tokio::test]
4237    async fn module_frame_on_route_the_daemon_does_not_hold_is_answered_with_goodbye() {
4238        let (
4239            router,
4240            _forwarding,
4241            client_ctx,
4242            mut client_rx,
4243            module_ctx,
4244            mut module_egress_rx,
4245            mut module_rx,
4246            pending,
4247        ) = dynamic_route_fixture(true);
4248        let _ = client_rx.recv().await.unwrap();
4249        router
4250            .route_for_connection(
4251                &client_ctx,
4252                route_frame(
4253                    FrameType::Goodbye,
4254                    pending.client_channel,
4255                    pending.client_epoch,
4256                    1_200,
4257                ),
4258            )
4259            .await
4260            .unwrap();
4261        drain_now(&mut module_rx);
4262
4263        let cases = [
4264            // Released route: the daemon allocated this (channel, epoch).
4265            (pending.module_channel, pending.module_epoch),
4266            // A channel the daemon never allocated on this connection.
4267            (pending.module_channel + 1, 1),
4268        ];
4269        for (channel, epoch) in cases {
4270            router
4271                .route_for_connection(
4272                    &module_ctx,
4273                    route_frame(FrameType::StreamData, channel, epoch, 1_201),
4274                )
4275                .await
4276                .unwrap();
4277            let reply = module_egress_rx
4278                .try_recv()
4279                .expect("a frame on a route the daemon does not hold is answered");
4280            assert_eq!(reply.header.ty, FrameType::Goodbye);
4281            assert_eq!(reply.header.channel, channel);
4282            assert_eq!(reply.header.epoch, epoch);
4283            assert_eq!(reply.header.corr, 0);
4284            assert!(module_egress_rx.try_recv().is_err());
4285        }
4286
4287        // A GOODBYE from the module for a route the daemon already released
4288        // is not answered: the module is letting go already.
4289        router
4290            .route_for_connection(
4291                &module_ctx,
4292                route_frame(FrameType::Goodbye, pending.module_channel + 2, 1, 0),
4293            )
4294            .await
4295            .unwrap();
4296        assert!(module_egress_rx.try_recv().is_err());
4297
4298        let counters = router.counters.snapshot();
4299        assert_eq!(counters["module_frames_dropped_no_route"], 3);
4300        assert_eq!(counters["module_frames_dropped_released_route"], 1);
4301        assert_eq!(
4302            counters["module_frames_dropped_released_route_by_module"],
4303            serde_json::json!({ "epoch-router": 1 })
4304        );
4305        assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4306    }
4307
4308    /// A chatty orphan (a streaming module still producing on a route it
4309    /// missed the GOODBYE for) is answered once per interval, not once per
4310    /// frame, and answered again once the interval has passed.
4311    #[tokio::test(start_paused = true)]
4312    async fn burst_of_orphan_module_frames_is_answered_once_per_interval() {
4313        let (
4314            router,
4315            _forwarding,
4316            client_ctx,
4317            mut client_rx,
4318            module_ctx,
4319            mut module_egress_rx,
4320            mut module_rx,
4321            pending,
4322        ) = dynamic_route_fixture(true);
4323        let _ = client_rx.recv().await.unwrap();
4324        router
4325            .route_for_connection(
4326                &client_ctx,
4327                route_frame(
4328                    FrameType::Goodbye,
4329                    pending.client_channel,
4330                    pending.client_epoch,
4331                    1_300,
4332                ),
4333            )
4334            .await
4335            .unwrap();
4336        drain_now(&mut module_rx);
4337
4338        let send_burst = |corr: u64| {
4339            route_frame(
4340                FrameType::StreamData,
4341                pending.module_channel,
4342                pending.module_epoch,
4343                corr,
4344            )
4345        };
4346        for corr in 0..20 {
4347            router
4348                .route_for_connection(&module_ctx, send_burst(corr))
4349                .await
4350                .unwrap();
4351        }
4352        let mut replies = 0;
4353        while let Ok(reply) = module_egress_rx.try_recv() {
4354            assert_eq!(reply.header.ty, FrameType::Goodbye);
4355            replies += 1;
4356        }
4357        assert_eq!(replies, 1, "a burst within the interval gets one GOODBYE");
4358
4359        tokio::time::advance(ORPHAN_ROUTE_GOODBYE_INTERVAL).await;
4360        router
4361            .route_for_connection(&module_ctx, send_burst(20))
4362            .await
4363            .unwrap();
4364        let retry = module_egress_rx
4365            .try_recv()
4366            .expect("the first orphan frame after the interval is answered again");
4367        assert_eq!(retry.header.channel, pending.module_channel);
4368        assert_eq!(retry.header.epoch, pending.module_epoch);
4369
4370        let counters = router.counters.snapshot();
4371        assert_eq!(counters["module_frames_dropped_no_route"], 21);
4372        assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4373    }
4374
4375    /// The rate-limit memory is per connection and goes away with it.
4376    #[test]
4377    fn orphan_goodbye_rate_limit_state_is_released_with_the_connection() {
4378        let router = Router::with_default_self_handler();
4379        let connection = router.begin_connection();
4380        let id = connection.id();
4381        assert!(router.orphan_goodbyes.claim(id, 7));
4382        assert!(!router.orphan_goodbyes.claim(id, 7));
4383        assert!(router.orphan_goodbyes.claim(id, 8));
4384        drop(connection);
4385        assert!(router
4386            .orphan_goodbyes
4387            .last_sent
4388            .lock()
4389            .unwrap()
4390            .get(&id)
4391            .is_none());
4392    }
4393
4394    #[test]
4395    fn channel_zero_cannot_be_registered_as_backend() {
4396        let mut router = Router::with_default_self_handler();
4397
4398        let err = router.register_backend(0, EchoBackend).unwrap_err();
4399
4400        assert_eq!(err, RouterError::ReservedChannelZero);
4401    }
4402}