Skip to main content

subc_daemon/
router.rs

1use std::{
2    collections::HashMap,
3    error::Error,
4    fmt,
5    sync::{
6        atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
7        Arc, Mutex,
8    },
9    time::{Duration, Instant},
10};
11
12use subc_protocol::{error_codes, ErrorBody, Flags, FrameType, Priority};
13use tokio::sync::{mpsc, Notify};
14use tracing::{debug, warn};
15
16use crate::{
17    control::ControlHandler,
18    forwarding::{
19        principal_label, CloseReason, ConnectionCloseReceiver, DataRoute, DataRouteState,
20        ForwardingError, ForwardingTable, RouteBinding, RouteRelease, UndeliveredFrame,
21    },
22    registry::ConnectionId,
23    DaemonCounters, Frame, FrameBuildError,
24};
25
26/// One queued outbound frame plus the instant it entered the writer queue.
27///
28/// The stamp exists for the reply-path half of slow-control diagnosis: a
29/// handler can finish in microseconds while the reply sits in this queue
30/// waiting for the writer task to be scheduled, and without a per-item stamp
31/// that wait is invisible to every other timing point (the client's round
32/// trip is the only witness, and it cannot say which side ate the time).
33/// Constructed exclusively inside [`FrameSink`] so no caller can forget it.
34#[derive(Debug)]
35pub struct OutboundFrame {
36    pub frame: Frame,
37    pub enqueued_at: std::time::Instant,
38    pub(crate) flushed: Option<tokio::sync::oneshot::Sender<()>>,
39    /// This frame's share of the connection's queued-byte count. Dropping the
40    /// frame (after the writer has written it, or when the queue itself is
41    /// dropped with the frame still in it) gives the bytes back. `None` only for
42    /// frames built outside a [`FrameSink`], which exist in tests alone.
43    pub(crate) charge: Option<EgressCharge>,
44}
45
46impl OutboundFrame {
47    fn charged(frame: Frame, charge: EgressCharge) -> Self {
48        Self {
49            frame,
50            enqueued_at: charge.enqueued_at,
51            flushed: None,
52            charge: Some(charge),
53        }
54    }
55}
56
57/// Bytes a frame occupies in the connection's egress queue for budget
58/// purposes: the fixed envelope header plus the body.
59fn queued_frame_bytes(frame: &Frame) -> usize {
60    subc_protocol::HEADER_LEN + frame.body.len()
61}
62
63/// A point-in-time view of one connection's egress queue, for diagnosing why a
64/// frame did not fit.
65#[derive(Debug, Clone, Copy, PartialEq, Eq)]
66pub(crate) struct EgressBacklog {
67    pub queued_bytes: usize,
68    pub queued_frames: usize,
69    /// Now minus the enqueue time of the frame the writer most recently took
70    /// off the queue (the frame it is writing, or has just written), or, if
71    /// the writer has taken nothing since the queue was last empty, of the
72    /// frame that made the queue non-empty. While the writer is blocked on a
73    /// frame this is exactly the oldest frame's age; between frames it can be
74    /// one frame older than the true head. `None` when nothing is queued.
75    pub oldest_age: Option<Duration>,
76}
77
78/// Queued-byte accounting shared by every clone of one connection's
79/// [`FrameSink`] and by every frame that sink has admitted. Its per-frame
80/// accounting uses atomics: this is on the path of every frame a module sends
81/// to a client, so it takes no lock per frame.
82#[derive(Debug)]
83struct EgressAccounting {
84    byte_budget: usize,
85    queued_bytes: AtomicUsize,
86    queued_frames: AtomicUsize,
87    /// Origin for the nanosecond timestamps in `oldest_enqueued_nanos`.
88    time_base: Instant,
89    /// Enqueue time, as nanoseconds after `time_base` plus one, of the frame
90    /// described by [`EgressBacklog::oldest_age`]; 0 means none.
91    oldest_enqueued_nanos: AtomicU64,
92    /// Awaited senders currently parked on `freed`. A release only pays for a
93    /// notification when this is non-zero.
94    waiters: AtomicUsize,
95    /// Woken when a charged frame releases its bytes while a sender waits, so
96    /// the awaited send can re-check for room.
97    freed: Notify,
98    client_delivery_closed: AtomicBool,
99    client_delivery_stopped: Notify,
100}
101
102impl EgressAccounting {
103    fn new(byte_budget: usize) -> Self {
104        Self {
105            byte_budget,
106            queued_bytes: AtomicUsize::new(0),
107            queued_frames: AtomicUsize::new(0),
108            time_base: tokio::time::Instant::now().into_std(),
109            oldest_enqueued_nanos: AtomicU64::new(0),
110            waiters: AtomicUsize::new(0),
111            freed: Notify::new(),
112            client_delivery_closed: AtomicBool::new(false),
113            client_delivery_stopped: Notify::new(),
114        }
115    }
116
117    fn stamp(&self, at: Instant) -> u64 {
118        (at.saturating_duration_since(self.time_base).as_nanos() as u64).saturating_add(1)
119    }
120
121    /// Charge `bytes` if they fit in the budget. A frame larger than the whole
122    /// budget (bodies may be up to 64 MiB) is still admitted into an EMPTY
123    /// queue, since it could otherwise never be sent at all; it simply has the
124    /// queue to itself until it is written.
125    fn try_charge(self: &Arc<Self>, bytes: usize) -> Option<EgressCharge> {
126        // SeqCst pairs with `release`: either this load sees bytes a release
127        // just freed, or that release sees this sender counted in `waiters`.
128        let mut current = self.queued_bytes.load(Ordering::SeqCst);
129        loop {
130            if current != 0 && current.saturating_add(bytes) > self.byte_budget {
131                return None;
132            }
133            match self.queued_bytes.compare_exchange_weak(
134                current,
135                current + bytes,
136                Ordering::SeqCst,
137                Ordering::SeqCst,
138            ) {
139                Ok(_) => return Some(self.record(bytes)),
140                Err(actual) => current = actual,
141            }
142        }
143    }
144
145    /// Charge `bytes` regardless of the budget. Used only for a frame whose
146    /// queue slot was reserved in advance, which must be sendable.
147    fn charge_unconditionally(self: &Arc<Self>, bytes: usize) -> EgressCharge {
148        self.queued_bytes.fetch_add(bytes, Ordering::SeqCst);
149        self.record(bytes)
150    }
151
152    fn record(self: &Arc<Self>, bytes: usize) -> EgressCharge {
153        let enqueued_at = tokio::time::Instant::now().into_std();
154        let stamp = self.stamp(enqueued_at);
155        if self.queued_frames.fetch_add(1, Ordering::AcqRel) == 0 {
156            // This frame made the queue non-empty, so it is the head until the
157            // writer takes something.
158            self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
159        }
160        EgressCharge {
161            accounting: Arc::clone(self),
162            bytes,
163            stamp,
164            enqueued_at,
165        }
166    }
167
168    /// The writer has taken the frame stamped `stamp` off the queue.
169    fn taken_by_writer(&self, stamp: u64) {
170        self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
171    }
172
173    fn release(&self, bytes: usize, stamp: u64) {
174        self.queued_bytes.fetch_sub(bytes, Ordering::SeqCst);
175        if self.queued_frames.fetch_sub(1, Ordering::AcqRel) == 1 {
176            // The queue drained. Clear the marker only if it still names this
177            // frame, so a frame admitted in the meantime keeps its stamp.
178            let _ = self.oldest_enqueued_nanos.compare_exchange(
179                stamp,
180                0,
181                Ordering::AcqRel,
182                Ordering::Acquire,
183            );
184        }
185        if self.waiters.load(Ordering::SeqCst) != 0 {
186            self.freed.notify_waiters();
187        }
188    }
189
190    fn backlog(&self) -> EgressBacklog {
191        let queued_frames = self.queued_frames.load(Ordering::Acquire);
192        let stamp = self.oldest_enqueued_nanos.load(Ordering::Acquire);
193        let oldest_age = (queued_frames != 0 && stamp != 0).then(|| {
194            let enqueued = self.time_base + Duration::from_nanos(stamp - 1);
195            tokio::time::Instant::from_std(enqueued).elapsed()
196        });
197        EgressBacklog {
198            queued_bytes: self.queued_bytes.load(Ordering::Acquire),
199            queued_frames,
200            oldest_age,
201        }
202    }
203}
204
205/// One admitted frame's claim on its connection's egress byte budget,
206/// returned when the frame is dropped.
207#[derive(Debug)]
208pub(crate) struct EgressCharge {
209    accounting: Arc<EgressAccounting>,
210    bytes: usize,
211    stamp: u64,
212    enqueued_at: Instant,
213}
214
215impl EgressCharge {
216    /// Called by the connection writer when it takes this frame off the queue,
217    /// so the backlog's oldest-age figure follows the writer.
218    pub(crate) fn taken_by_writer(&self) {
219        self.accounting.taken_by_writer(self.stamp);
220    }
221}
222
223impl Drop for EgressCharge {
224    fn drop(&mut self) {
225        self.accounting.release(self.bytes, self.stamp);
226    }
227}
228
229/// A queue slot reserved ahead of time (a pending `route.open` holds one until
230/// its module answers). Sending through it never waits and never fails for
231/// lack of room: the slot is already held, and its frame is charged to the
232/// byte count outside the budget check. At most
233/// `MAX_PENDING_ROUTE_OPENS_PER_CONNECTION` such frames exist per connection.
234#[derive(Debug)]
235pub(crate) struct EgressPermit {
236    permit: mpsc::OwnedPermit<OutboundFrame>,
237    accounting: Arc<EgressAccounting>,
238}
239
240impl EgressPermit {
241    /// Enqueue `frame` in the reserved slot. Returns true when the connection's
242    /// writer had already gone away, meaning the frame will never be written.
243    pub(crate) fn send(self, frame: Frame) -> bool {
244        let charge = self
245            .accounting
246            .charge_unconditionally(queued_frame_bytes(&frame));
247        let sender = self.permit.send(OutboundFrame::charged(frame, charge));
248        sender.is_closed()
249    }
250}
251
252/// An `OutboundFrame` is a stamped `Frame`; deref keeps every existing frame
253/// read (headers, bodies, assertions) working on queued items unchanged.
254impl std::ops::Deref for OutboundFrame {
255    type Target = Frame;
256
257    fn deref(&self) -> &Frame {
258        &self.frame
259    }
260}
261
262/// Shared tracing-capture helpers for timing-observability tests across
263/// modules (router dispatch, server reply path). Test-only.
264#[cfg(test)]
265pub(crate) mod test_log {
266    use std::{
267        io::Write,
268        sync::{Arc, Mutex, OnceLock},
269    };
270
271    #[derive(Clone)]
272    struct TestLogWriter(Arc<Mutex<Vec<u8>>>);
273
274    impl Write for TestLogWriter {
275        fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
276            self.0
277                .lock()
278                .expect("test log capture is not poisoned")
279                .extend(buffer);
280            Ok(buffer.len())
281        }
282
283        fn flush(&mut self) -> std::io::Result<()> {
284            Ok(())
285        }
286    }
287
288    pub(crate) fn log_capture(
289        level: tracing::Level,
290    ) -> (Arc<Mutex<Vec<u8>>>, tracing::dispatcher::DefaultGuard) {
291        // tracing-core's single-dispatch fast path registers callsites using
292        // the emitting thread's default subscriber. A parallel test with no
293        // subscriber can therefore cache `never` for a callsite this capture
294        // needs. Keep a second dispatch alive so registration considers all
295        // live subscribers instead. This registry is never installed as a
296        // default: each test still records only into its own thread's writer.
297        static INTEREST_ANCHOR: OnceLock<tracing::Dispatch> = OnceLock::new();
298        INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
299        let output = Arc::new(Mutex::new(Vec::new()));
300        let writer = Arc::clone(&output);
301        let subscriber = tracing_subscriber::fmt()
302            .with_max_level(level)
303            .with_ansi(false)
304            .without_time()
305            .with_target(false)
306            .with_writer(move || TestLogWriter(Arc::clone(&writer)))
307            .finish();
308        let guard = tracing::subscriber::set_default(subscriber);
309        (output, guard)
310    }
311
312    pub(crate) fn captured_logs(output: &Arc<Mutex<Vec<u8>>>) -> String {
313        String::from_utf8(
314            output
315                .lock()
316                .expect("test log capture is not poisoned")
317                .clone(),
318        )
319        .expect("tracing output is UTF-8")
320    }
321
322    #[cfg(test)]
323    mod tests {
324        use super::*;
325
326        fn emit_capture_probe(source: &str) {
327            tracing::error!(source, "parallel capture probe");
328        }
329
330        #[test]
331        fn parallel_callsite_registration_keeps_the_test_capture_enabled() {
332            let (output, _guard) = log_capture(tracing::Level::ERROR);
333            // First register the very same callsite on a thread without a
334            // default subscriber, while the test's capture remains live.
335            std::thread::spawn(|| emit_capture_probe("other-thread"))
336                .join()
337                .expect("unsubscribed thread exits");
338            emit_capture_probe("captured-thread");
339
340            let logs = captured_logs(&output);
341            assert!(
342                logs.contains("captured-thread"),
343                "capture was disabled: {logs}"
344            );
345            assert!(!logs.contains("other-thread"), "capture leaked: {logs}");
346        }
347    }
348}
349
350/// Cheaply cloneable handle to one connection's bounded outbound frame queue.
351///
352/// Backends emit responses, streaming frames, and future PUSH frames through this
353/// single path. The queue is bounded twice: by queued BYTES (the budget that
354/// matters for memory and for how long a slow reader may pause), and by the
355/// `mpsc` channel's frame count (a backstop). The socket layer owns the sole
356/// receiver/writer.
357#[derive(Debug, Clone)]
358pub struct FrameSink {
359    tx: mpsc::Sender<OutboundFrame>,
360    accounting: Arc<EgressAccounting>,
361}
362
363impl FrameSink {
364    /// A sink with the standard per-connection byte budget
365    /// ([`crate::server::CONNECTION_EGRESS_BYTE_BUDGET`]); the frame-count bound
366    /// is whatever capacity `tx`'s channel was created with.
367    pub fn new(tx: mpsc::Sender<OutboundFrame>) -> Self {
368        Self::with_byte_budget(tx, crate::server::CONNECTION_EGRESS_BYTE_BUDGET)
369    }
370
371    pub(crate) fn with_byte_budget(tx: mpsc::Sender<OutboundFrame>, byte_budget: usize) -> Self {
372        Self {
373            tx,
374            accounting: Arc::new(EgressAccounting::new(byte_budget)),
375        }
376    }
377
378    /// Wait until the frame's bytes fit the budget, then charge them. Awaited
379    /// senders (control replies, error replies, shutdown notices) are never
380    /// refused by the byte budget: they wait for the writer to free bytes, just
381    /// as they wait for a free slot, which is the same backpressure they had
382    /// when the queue was bounded by frame count alone. Returns `None` if the
383    /// writer goes away while waiting.
384    async fn charge_waiting(&self, bytes: usize) -> Option<EgressCharge> {
385        if let Some(charge) = self.accounting.try_charge(bytes) {
386            return Some(charge);
387        }
388        // Counted as a waiter for as long as this future is parked, including
389        // when it is cancelled, so releases notify only while someone waits.
390        struct Waiting<'a>(&'a AtomicUsize);
391        impl Drop for Waiting<'_> {
392            fn drop(&mut self) {
393                self.0.fetch_sub(1, Ordering::SeqCst);
394            }
395        }
396        self.accounting.waiters.fetch_add(1, Ordering::SeqCst);
397        let _waiting = Waiting(&self.accounting.waiters);
398        loop {
399            let freed = self.accounting.freed.notified();
400            tokio::pin!(freed);
401            // Register interest before checking, so a release that happens
402            // between the check and the await still wakes this sender.
403            freed.as_mut().enable();
404            if let Some(charge) = self.accounting.try_charge(bytes) {
405                return Some(charge);
406            }
407            tokio::select! {
408                _ = &mut freed => {}
409                _ = self.tx.closed() => return None,
410            }
411        }
412    }
413
414    pub async fn send(&self, frame: Frame) -> Result<(), RouterError> {
415        let channel = frame.header.channel;
416        let epoch = frame.header.epoch;
417        let corr = frame.header.corr;
418        let closed =
419            || RouterError::backend_with_epoch(channel, epoch, corr, "connection writer closed");
420        let charge = self
421            .charge_waiting(queued_frame_bytes(&frame))
422            .await
423            .ok_or_else(closed)?;
424        self.tx
425            .send(OutboundFrame::charged(frame, charge))
426            .await
427            .map_err(|_| closed())
428    }
429
430    /// Shutdown notices must leave the socket writer before an idle daemon exits.
431    /// Queue admission alone does not prove this; the writer acknowledges flush.
432    #[cfg(unix)]
433    pub(crate) async fn send_flushed(&self, frame: Frame) -> Result<(), RouterError> {
434        let (tx, rx) = tokio::sync::oneshot::channel();
435        let charge = self
436            .charge_waiting(queued_frame_bytes(&frame))
437            .await
438            .ok_or_else(|| RouterError::backend(0, 0, "connection writer closed"))?;
439        let mut outbound = OutboundFrame::charged(frame, charge);
440        outbound.flushed = Some(tx);
441        self.tx
442            .send(outbound)
443            .await
444            .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
445        rx.await
446            .map_err(|_| RouterError::backend(0, 0, "connection flush failed"))
447    }
448
449    pub(crate) async fn reserve_owned(&self) -> Result<EgressPermit, RouterError> {
450        let permit = self
451            .tx
452            .clone()
453            .reserve_owned()
454            .await
455            .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
456        Ok(EgressPermit {
457            permit,
458            accounting: Arc::clone(&self.accounting),
459        })
460    }
461
462    #[cfg(test)]
463    pub(crate) fn try_reserve_owned(&self) -> Result<EgressPermit, RouterError> {
464        let permit = self
465            .tx
466            .clone()
467            .try_reserve_owned()
468            .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
469        Ok(EgressPermit {
470            permit,
471            accounting: Arc::clone(&self.accounting),
472        })
473    }
474
475    pub(crate) fn is_closed(&self) -> bool {
476        self.tx.is_closed()
477    }
478
479    /// What the connection's egress queue holds right now.
480    pub(crate) fn backlog(&self) -> EgressBacklog {
481        self.accounting.backlog()
482    }
483
484    /// Stop module deliveries as soon as the client reader exits, without
485    /// waiting for the socket writer's separate teardown drain grace.
486    pub(crate) fn close_client_delivery(&self) {
487        self.accounting
488            .client_delivery_closed
489            .store(true, Ordering::Release);
490        self.accounting.client_delivery_stopped.notify_waiters();
491    }
492
493    /// A full queue can be a short fanout burst rather than a stopped reader.
494    /// Spend only the grace the atomic backlog age has not already used.
495    async fn send_to_client(&self, frame: Frame) -> Result<(), ClientDeliveryFailure> {
496        let frame_type = frame.header.ty;
497        let flags = frame.header.flags;
498        let stopped = self.accounting.client_delivery_stopped.notified();
499        tokio::pin!(stopped);
500        stopped.as_mut().enable();
501        let failure = |cause, error| ClientDeliveryFailure {
502            diagnosis: ClientDeliveryDiagnosis {
503                frame_type,
504                flags,
505                cause,
506            },
507            error,
508        };
509        let closed = || {
510            RouterError::backend_with_epoch(
511                frame.header.channel,
512                frame.header.epoch,
513                frame.header.corr,
514                "client connection closed",
515            )
516        };
517        if self
518            .accounting
519            .client_delivery_closed
520            .load(Ordering::Acquire)
521        {
522            return Err(failure("client_closed", closed()));
523        }
524        let (frame, error) = match self.try_send_owned(frame) {
525            Ok(()) => return Ok(()),
526            Err(refused) => refused,
527        };
528        if self.is_closed() {
529            return Err(failure("client_closed", error));
530        }
531        let grace = crate::server::CONNECTION_EGRESS_STALL_GRACE;
532        let age = self.backlog().oldest_age.unwrap_or_default();
533        if age > grace {
534            return Err(failure("oldest_frame_older_than_250_ms", error));
535        }
536        let result = tokio::select! {
537            biased;
538            _ = &mut stopped => return Err(failure("client_closed", error)),
539            result = tokio::time::timeout(grace.saturating_sub(age), self.send(frame)) => result,
540        };
541        match result {
542            Ok(Ok(())) => Ok(()),
543            Ok(Err(error)) => Err(failure("client_closed", error)),
544            Err(_) => Err(failure("waited_250_ms_without_room", error)),
545        }
546    }
547
548    /// Enqueue without waiting. Fails when the frame's bytes would take the
549    /// queue past its byte budget, when the channel's frame-count backstop is
550    /// full, or when the writer is gone.
551    pub(crate) fn try_send(&self, frame: Frame) -> Result<(), RouterError> {
552        self.try_send_owned(frame).map_err(|(_, error)| error)
553    }
554
555    fn try_send_owned(&self, frame: Frame) -> Result<(), (Frame, RouterError)> {
556        let channel = frame.header.channel;
557        let epoch = frame.header.epoch;
558        let corr = frame.header.corr;
559        let unavailable = |why: String| {
560            RouterError::backend_with_epoch(
561                channel,
562                epoch,
563                corr,
564                format!("connection writer unavailable: {why}"),
565            )
566        };
567        let bytes = queued_frame_bytes(&frame);
568        let Some(charge) = self.accounting.try_charge(bytes) else {
569            if self.tx.is_closed() {
570                return Err((frame, unavailable("channel closed".to_string())));
571            }
572            return Err((
573                frame,
574                unavailable(format!(
575                    "egress byte budget exhausted ({} queued bytes, frame of {bytes} bytes, budget {})",
576                    self.accounting.queued_bytes.load(Ordering::Acquire),
577                    self.accounting.byte_budget
578                )),
579            ));
580        };
581        // Return the refused frame for a bounded retry; dropping the rest of
582        // the outbound wrapper returns its charge.
583        self.tx
584            .try_send(OutboundFrame::charged(frame, charge))
585            .map_err(|err| {
586                let error = unavailable(err.to_string());
587                (err.into_inner().frame, error)
588            })
589    }
590}
591
592#[derive(Debug)]
593struct ClientDeliveryFailure {
594    diagnosis: ClientDeliveryDiagnosis,
595    error: RouterError,
596}
597
598/// Header facts only: payloads can contain secrets and must never enter logs.
599#[derive(Debug)]
600pub(crate) struct ClientDeliveryDiagnosis {
601    pub frame_type: FrameType,
602    pub flags: Flags,
603    pub cause: &'static str,
604}
605
606impl fmt::Display for ClientDeliveryFailure {
607    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
608        self.error.fmt(f)
609    }
610}
611
612/// Minimum time between two route GOODBYEs the daemon sends one module
613/// connection for one channel in answer to the module's frames on a (channel,
614/// epoch) the daemon holds no route for.
615///
616/// A module that missed a GOODBYE keeps sending on that route (a streaming
617/// module, many frames a second), and every one of those frames lands here. One
618/// answer is enough when it arrives, so answering each frame would only flood
619/// the module's egress queue at the moment it is already behind. Five seconds
620/// is long enough for an answer queued behind a deep backlog to reach the module
621/// and take effect before a second is sent, and short enough that an answer
622/// the queue refused is retried by the next orphan frame well within a minute.
623const ORPHAN_ROUTE_GOODBYE_INTERVAL: Duration = Duration::from_secs(5);
624
625/// Once one connection has this many remembered channels, entries older than
626/// [`ORPHAN_ROUTE_GOODBYE_INTERVAL`] are pruned before another is added. An
627/// expired entry has no effect on rate limiting, so pruning never changes a
628/// decision; it keeps a module that orphaned many channels long ago from
629/// pinning memory for all of them. The map is bounded in any case by the
630/// 16-bit channel space and is dropped whole when the connection ends.
631const ORPHAN_ROUTE_GOODBYE_PRUNE_AT: usize = 256;
632
633/// When each module connection was last sent an orphan-route GOODBYE, per
634/// channel. Shared by the [`Router`] (which consults it) and every
635/// [`RouterConnection`] (which removes its own entry when the connection ends).
636#[derive(Debug, Default)]
637struct OrphanGoodbyeLimiter {
638    last_sent: Mutex<HashMap<ConnectionId, HashMap<u16, tokio::time::Instant>>>,
639}
640
641impl OrphanGoodbyeLimiter {
642    /// True, and the send recorded, when `channel` on `connection_id` has not
643    /// been answered within the interval.
644    fn claim(&self, connection_id: ConnectionId, channel: u16) -> bool {
645        let now = tokio::time::Instant::now();
646        let mut last_sent = self
647            .last_sent
648            .lock()
649            .unwrap_or_else(std::sync::PoisonError::into_inner);
650        let channels = last_sent.entry(connection_id).or_default();
651        if channels.get(&channel).is_some_and(|sent| {
652            now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
653        }) {
654            return false;
655        }
656        if channels.len() >= ORPHAN_ROUTE_GOODBYE_PRUNE_AT {
657            channels.retain(|_, sent| {
658                now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
659            });
660        }
661        channels.insert(channel, now);
662        true
663    }
664
665    fn forget_connection(&self, connection_id: ConnectionId) {
666        self.last_sent
667            .lock()
668            .unwrap_or_else(std::sync::PoisonError::into_inner)
669            .remove(&connection_id);
670    }
671}
672
673/// Per-route context shared with backends besides the frame itself.
674#[derive(Debug, Clone)]
675pub struct RouteCtx {
676    pub connection_id: ConnectionId,
677    pub egress: FrameSink,
678}
679
680/// Closed set of data-plane backends for non-zero channels.
681///
682/// Channel 0 is structurally special and remains [`Router::control`], not an
683/// enum variant. Static/test backends live in [`Router::backends`]; the forwarding
684/// backend is selected dynamically only when a per-connection forwarding binding exists.
685#[derive(Debug, Clone)]
686pub enum Backend {
687    Echo(EchoBackend),
688    Forward(ForwardBackend),
689}
690
691impl From<EchoBackend> for Backend {
692    fn from(backend: EchoBackend) -> Self {
693        Self::Echo(backend)
694    }
695}
696
697impl From<ForwardBackend> for Backend {
698    fn from(backend: ForwardBackend) -> Self {
699        Self::Forward(backend)
700    }
701}
702
703impl Backend {
704    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
705        match self {
706            Self::Echo(backend) => backend.handle(ctx, frame).await,
707            Self::Forward(backend) => backend.handle(ctx, frame).await,
708        }
709    }
710}
711
712/// I/O-agnostic splice router keyed by envelope `channel`.
713///
714/// Channel 0 is reserved for subc itself and is always dispatched to the
715/// dedicated control handler. Other channels must be explicitly registered.
716/// Unknown client-originated non-zero channels are translated to canonical JSON `ERROR` frames on
717/// the connection sink so the peer can continue using the same socket. Module-originated frames for
718/// released route channels are logged and dropped as the channel-gone race backstop.
719///
720/// DATA-PLANE BODIES ARE NEVER DECODED HERE, and the consequence is worth stating
721/// because it looks like a guarantee and is not. Additive fields in a request or
722/// response body reach the far side untouched — not because anything permits them,
723/// but because routing reads only the 21-byte header and treats the body as opaque
724/// bytes. That is a performance property, so **nothing prevents it from changing**:
725/// a future reason to inspect a body would convert a wire-transparent path into a
726/// filtering one, and nobody would think of it as a contract change.
727///
728/// So when a peer asks whether subc sees an additive field, the answer is per-path
729/// and this path's zero means WE NEVER LOOK rather than WE LOOK AT EVERYTHING. The
730/// control plane (typed enums at the frame boundary) and the MCP gateway (envelope
731/// unwrap plus a named struct) both narrow; only this one does not.
732pub struct Router {
733    backends: HashMap<u16, Backend>,
734    control: Arc<ControlHandler>,
735    forwarding: Arc<ForwardingTable>,
736    forward_backend: ForwardBackend,
737    counters: DaemonCounters,
738    next_connection_id: AtomicU64,
739    orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
740}
741
742impl Router {
743    pub fn with_control_handler(control: Arc<ControlHandler>) -> Self {
744        // The handler the router serves is the one a swap must tell when it
745        // promotes a candidate; this is where it first sits behind an `Arc`.
746        control.install_swap_promotion_observer();
747        let forwarding = control.forwarding();
748        let counters = control.counters();
749        Self {
750            backends: HashMap::new(),
751            control,
752            forwarding: Arc::clone(&forwarding),
753            forward_backend: ForwardBackend::new(forwarding),
754            counters,
755            // ConnectionId::LOCAL is 0; real socket ids start at 1 and never collide.
756            next_connection_id: AtomicU64::new(1),
757            orphan_goodbyes: Arc::default(),
758        }
759    }
760
761    pub fn with_default_self_handler() -> Self {
762        Self::with_control_handler(Arc::new(ControlHandler::default()))
763    }
764
765    pub fn forwarding(&self) -> Arc<ForwardingTable> {
766        Arc::clone(&self.forwarding)
767    }
768
769    pub fn register_backend(
770        &mut self,
771        channel: u16,
772        backend: impl Into<Backend>,
773    ) -> Result<(), RouterError> {
774        self.register_backend_arc(channel, Arc::new(backend.into()))
775    }
776
777    pub(crate) fn register_backend_arc(
778        &mut self,
779        channel: u16,
780        backend: Arc<Backend>,
781    ) -> Result<(), RouterError> {
782        if channel == 0 {
783            return Err(RouterError::ReservedChannelZero);
784        }
785        if self.backends.contains_key(&channel) {
786            return Err(RouterError::DuplicateChannel { channel });
787        }
788        self.backends.insert(channel, backend.as_ref().clone());
789        Ok(())
790    }
791
792    /// Start a connection-scoped routing context. Dropping the guard releases
793    /// any control-plane registrations owned by the connection.
794    fn record_module_frame_drop(&self, connection_id: ConnectionId) -> Result<(), RouterError> {
795        let module_id = self
796            .forwarding
797            .module_id_for_connection(connection_id)
798            .map_err(RouterError::Forwarding)?;
799        self.counters
800            .increment_module_frames_dropped_no_route(module_id.as_deref());
801        Ok(())
802    }
803
804    /// A module sent a non-request frame on a (channel, epoch) the daemon holds
805    /// no route for: most often a route the daemon released whose GOODBYE the
806    /// module never received, so the module still believes it is open and keeps
807    /// sending. Count the drop, and tell the module to let go of exactly that
808    /// (channel, epoch) with a route GOODBYE, at most once per
809    /// [`ORPHAN_ROUTE_GOODBYE_INTERVAL`] per connection and channel.
810    ///
811    /// The answer uses `try_send`: it is a best-effort nudge, and if the queue
812    /// refuses it, the module's next frame on that route after the interval
813    /// asks again. A GOODBYE from the module is not answered, since the module
814    /// is already letting go of the route.
815    fn handle_orphan_module_frame(&self, ctx: &RouteCtx, frame: &Frame) -> Result<(), RouterError> {
816        let channel = frame.header.channel;
817        let epoch = frame.header.epoch;
818        let module_id = self
819            .forwarding
820            .module_id_for_connection(ctx.connection_id)
821            .map_err(RouterError::Forwarding)?;
822        self.counters
823            .increment_module_frames_dropped_no_route(module_id.as_deref());
824        if self
825            .forwarding
826            .module_route_epoch_was_allocated(ctx.connection_id, channel, epoch)
827            .map_err(RouterError::Forwarding)?
828        {
829            self.counters
830                .increment_module_frames_dropped_released_route(module_id.as_deref());
831        }
832        if frame.header.ty == FrameType::Goodbye
833            || !self.orphan_goodbyes.claim(ctx.connection_id, channel)
834        {
835            return Ok(());
836        }
837        let goodbye = Frame::build_with_version(
838            frame.header.ver,
839            FrameType::Goodbye,
840            Flags::new(false, Priority::Passive, false),
841            channel,
842            epoch,
843            0,
844            Vec::new(),
845        )
846        .map_err(RouterError::FrameBuild)?;
847        match ctx.egress.try_send(goodbye) {
848            Ok(()) => {
849                self.counters.increment_module_orphan_route_goodbyes_sent();
850                debug!(
851                    connection_id = ctx.connection_id.get(),
852                    module_id = module_id.as_deref().unwrap_or("unknown"),
853                    channel,
854                    epoch,
855                    "answered module frame on a route the daemon does not hold with a route GOODBYE"
856                );
857            }
858            Err(err) => debug!(
859                connection_id = ctx.connection_id.get(),
860                module_id = module_id.as_deref().unwrap_or("unknown"),
861                channel,
862                epoch,
863                error = %err,
864                "could not enqueue route GOODBYE for module frame on a route the daemon does not hold; the next such frame retries"
865            ),
866        }
867        Ok(())
868    }
869
870    pub fn begin_connection(&self) -> RouterConnection {
871        let raw = self.next_connection_id.fetch_add(1, Ordering::Relaxed);
872        let id = ConnectionId::new(raw);
873        let close_receiver = self.forwarding.register_connection_close(id);
874        RouterConnection {
875            id,
876            control_handler: Arc::clone(&self.control),
877            forwarding: Arc::clone(&self.forwarding),
878            close_receiver: Some(close_receiver),
879            orphan_goodbyes: Arc::clone(&self.orphan_goodbyes),
880        }
881    }
882
883    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
884        self.control.route_open_target(frame)
885    }
886
887    pub(crate) fn record_caller_endpoints(
888        &self,
889        connection: ConnectionId,
890        endpoints: Option<(std::net::SocketAddr, std::net::SocketAddr)>,
891    ) {
892        self.control.record_caller_endpoints(connection, endpoints);
893    }
894
895    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
896        self.control.defers_privileged_control(ctx, frame)
897    }
898
899    pub(crate) fn route_open_capacity_refusal(
900        &self,
901        ctx: &RouteCtx,
902        frame: &Frame,
903        target_module_id: &str,
904        in_flight: usize,
905        limit: usize,
906    ) -> Result<Frame, RouterError> {
907        self.control
908            .route_open_capacity_refusal(ctx, frame, target_module_id, in_flight, limit)
909    }
910
911    pub async fn route_for_connection(
912        &self,
913        ctx: &RouteCtx,
914        frame: Frame,
915    ) -> Result<(), RouterError> {
916        self.route_for_connection_started(ctx, frame, None).await
917    }
918
919    pub(crate) async fn route_for_connection_started(
920        &self,
921        ctx: &RouteCtx,
922        frame: Frame,
923        dispatch_started_at: Option<Instant>,
924    ) -> Result<(), RouterError> {
925        let channel = frame.header.channel;
926        let epoch = frame.header.epoch;
927        let corr = frame.header.corr;
928        if channel == 0 {
929            debug!(
930                connection_id = ctx.connection_id.get(),
931                corr,
932                frame_type = ?frame.header.ty,
933                "routing control frame"
934            );
935            // The connection loop dispatches every control operation directly
936            // except `route.open`, activity writes and privileged requests. Each
937            // task passes a timestamp captured before
938            // spawn, so slow-dispatch timing includes scheduler delay but no
939            // wait in an application-owned queue.
940            let dispatch_started_at = (frame.header.ty == FrameType::Request)
941                .then(|| dispatch_started_at.unwrap_or_else(Instant::now));
942            let responses = self
943                .control
944                .handle_control_frame_timed(ctx, frame, dispatch_started_at)
945                .await?;
946            for response in responses {
947                ctx.egress.send(response).await?;
948            }
949            return Ok(());
950        }
951
952        let data_route = self
953            .forwarding
954            .lookup_data_route(ctx.connection_id, channel, epoch)
955            .map_err(RouterError::Forwarding)?;
956
957        match data_route {
958            DataRoute::Module(DataRouteState::EpochMismatch) => {
959                if frame.header.ty == FrameType::Request {
960                    self.counters
961                        .increment_module_requests_dropped_stale_route();
962                    let err = RouterError::StaleRouteEpoch {
963                        channel,
964                        epoch,
965                        corr,
966                    };
967                    if let Some(error_frame) = err.to_error_frame() {
968                        ctx.egress.send(error_frame).await?;
969                    }
970                } else {
971                    self.handle_orphan_module_frame(ctx, &frame)?;
972                }
973                debug!(
974                    connection_id = ctx.connection_id.get(),
975                    channel, epoch, corr, "dropping module frame for stale route epoch"
976                );
977                return Ok(());
978            }
979            DataRoute::Module(DataRouteState::Reserved) => {
980                if frame.header.ty == FrameType::Request {
981                    self.counters
982                        .increment_module_requests_dropped_stale_route();
983                    let err = RouterError::UnknownChannel {
984                        channel,
985                        epoch,
986                        corr,
987                    };
988                    if let Some(error_frame) = err.to_error_frame() {
989                        ctx.egress.send(error_frame).await?;
990                    }
991                } else {
992                    self.record_module_frame_drop(ctx.connection_id)?;
993                }
994                debug!(
995                    connection_id = ctx.connection_id.get(),
996                    channel, epoch, corr, "dropping module frame for reserved route handle"
997                );
998                return Ok(());
999            }
1000            DataRoute::Module(DataRouteState::Absent) => {
1001                if frame.header.ty == FrameType::Request {
1002                    self.counters
1003                        .increment_module_requests_dropped_stale_route();
1004                    let err = RouterError::UnknownChannel {
1005                        channel,
1006                        epoch,
1007                        corr,
1008                    };
1009                    if let Some(error_frame) = err.to_error_frame() {
1010                        ctx.egress.send(error_frame).await?;
1011                    }
1012                } else {
1013                    self.handle_orphan_module_frame(ctx, &frame)?;
1014                }
1015                debug!(
1016                    connection_id = ctx.connection_id.get(),
1017                    channel, epoch, corr, "dropping module frame for absent route handle"
1018                );
1019                return Ok(());
1020            }
1021            DataRoute::Module(DataRouteState::Bound(route)) => {
1022                if frame.header.ty == FrameType::Goodbye {
1023                    if let RouteRelease::Removed(target) = self
1024                        .forwarding
1025                        .release_module_route(ctx.connection_id, channel, epoch)
1026                        .map_err(RouterError::Forwarding)?
1027                    {
1028                        let mut goodbye = frame;
1029                        goodbye.header.channel = target.channel;
1030                        goodbye.header.epoch = target.epoch;
1031                        if let Err(err) = target.sink.send_to_client(goodbye).await {
1032                            if target.close_on_delivery_failure()
1033                                && self
1034                                    .forwarding
1035                                    .escalate_client_delivery_failure_diagnosed(
1036                                        target.connection_id,
1037                                        target.channel,
1038                                        target.epoch,
1039                                        CloseReason::new(
1040                                            "route_goodbye_delivery_failed",
1041                                            format!(
1042                                                "failed to enqueue route GOODBYE for client channel {}: {err}",
1043                                                target.channel
1044                                            ),
1045                                        ),
1046                                        UndeliveredFrame {
1047                                            module_id: Some(&route.module_id),
1048                                            sink: &target.sink,
1049                                        },
1050                                        Some(&err.diagnosis),
1051                                    )
1052                                    .map_err(RouterError::Forwarding)?
1053                            {
1054                                self.counters.increment_goodbye_relay_client_failed();
1055                            }
1056                        }
1057                    }
1058                    return Ok(());
1059                }
1060
1061                // Accounting allows an oversized frame to occupy an empty queue
1062                // on other send paths. Refuse it at this splice before admission,
1063                // so the next ordinary reply cannot close a multiplexed client.
1064                let bytes = queued_frame_bytes(&frame);
1065                let limit = route.client_sink.accounting.byte_budget;
1066                if bytes > limit {
1067                    let ty = frame.header.ty;
1068                    let flags = frame.header.flags;
1069                    drop(frame);
1070                    return self
1071                        .refuse_oversized_module_frame(ctx, &route, (ty, flags), corr, bytes)
1072                        .await;
1073                }
1074
1075                // A terminal frame ends the request at the module whether or not
1076                // the client can still take it, so its credit is released before
1077                // delivery is attempted. Releasing only after a successful
1078                // enqueue would leave a drain counting a finished request until
1079                // the client connection's cleanup removes the route.
1080                let releases_credit = is_terminal_frame(frame.header.ty);
1081                if releases_credit {
1082                    route.flow.release_corr(corr);
1083                }
1084                let mut frame = frame;
1085                frame.header.channel = route.client_channel;
1086                frame.header.epoch = route.client_epoch;
1087                if let Err(err) = route.client_sink.send_to_client(frame).await {
1088                    if self
1089                        .forwarding
1090                        .escalate_client_delivery_failure_diagnosed(
1091                            route.client_connection_id,
1092                            route.client_channel,
1093                            route.client_epoch,
1094                            CloseReason::new(
1095                                "module_to_client_delivery_failed",
1096                                format!(
1097                                    "failed to enqueue module frame for client channel {} corr {corr}: {err}",
1098                                    route.client_channel
1099                                ),
1100                            ),
1101                            UndeliveredFrame {
1102                                module_id: Some(&route.module_id),
1103                                sink: &route.client_sink,
1104                            },
1105                            Some(&err.diagnosis),
1106                        )
1107                        .map_err(RouterError::Forwarding)?
1108                    {
1109                        self.counters
1110                            .increment_client_egress_close_delivery_failed();
1111                    }
1112                    return Ok(());
1113                }
1114                return Ok(());
1115            }
1116            DataRoute::Client(DataRouteState::EpochMismatch) => {
1117                if frame.header.ty == FrameType::Request {
1118                    self.counters.increment_client_frames_dropped_stale_route();
1119                    // Dropped before forwarding; a re-bind retry cannot double-execute this request.
1120                    let err = RouterError::StaleRouteEpoch {
1121                        channel,
1122                        epoch,
1123                        corr,
1124                    };
1125                    if let Some(error_frame) = err.to_error_frame() {
1126                        ctx.egress.send(error_frame).await?;
1127                    }
1128                }
1129                debug!(
1130                    connection_id = ctx.connection_id.get(),
1131                    channel, epoch, corr, "dropping client frame for stale route epoch"
1132                );
1133                return Ok(());
1134            }
1135            DataRoute::Client(DataRouteState::Reserved) => {
1136                if frame.header.ty == FrameType::Request {
1137                    let err = RouterError::UnknownChannel {
1138                        channel,
1139                        epoch,
1140                        corr,
1141                    };
1142                    if let Some(error_frame) = err.to_error_frame() {
1143                        ctx.egress.send(error_frame).await?;
1144                    }
1145                }
1146                return Ok(());
1147            }
1148            DataRoute::Client(DataRouteState::Bound(route)) => {
1149                if frame.header.ty == FrameType::Goodbye {
1150                    let _ = self
1151                        .control
1152                        .handle_route_goodbye(ctx.connection_id, channel, epoch)?;
1153                    return Ok(());
1154                }
1155                return self.forward_backend.handle_bound(frame, route).await;
1156            }
1157            DataRoute::Client(DataRouteState::Absent) => {}
1158        }
1159
1160        if let Some(backend) = self.backends.get(&channel) {
1161            return backend.handle(ctx.clone(), frame).await;
1162        }
1163        if frame.header.ty == FrameType::Request {
1164            let err = RouterError::UnknownChannel {
1165                channel,
1166                epoch,
1167                corr,
1168            };
1169            if let Some(error_frame) = err.to_error_frame() {
1170                ctx.egress.send(error_frame).await?;
1171            }
1172        }
1173        Ok(())
1174    }
1175
1176    async fn refuse_oversized_module_frame(
1177        &self,
1178        ctx: &RouteCtx,
1179        route: &RouteBinding,
1180        header: (FrameType, Flags),
1181        corr: u64,
1182        bytes: usize,
1183    ) -> Result<(), RouterError> {
1184        let (ty, flags) = header;
1185        let limit = route.client_sink.accounting.byte_budget;
1186        if ty == FrameType::Push {
1187            // Deployed Rust clients cannot decode a new route.closed reason.
1188            // Until that vocabulary can be extended compatibly, an oversized
1189            // unsolicited push still closes the connection, with its cause named.
1190            let diagnosis = ClientDeliveryDiagnosis {
1191                frame_type: ty,
1192                flags,
1193                cause: "frame_too_large",
1194            };
1195            if self
1196                .forwarding
1197                .escalate_client_delivery_failure_diagnosed(
1198                    route.client_connection_id,
1199                    route.client_channel,
1200                    route.client_epoch,
1201                    CloseReason::new(
1202                        error_codes::FRAME_TOO_LARGE,
1203                        format!(
1204                            "oversized push: size={bytes} limit={limit} (header plus body bytes)"
1205                        ),
1206                    ),
1207                    UndeliveredFrame {
1208                        module_id: Some(&route.module_id),
1209                        sink: &route.client_sink,
1210                    },
1211                    Some(&diagnosis),
1212                )
1213                .map_err(RouterError::Forwarding)?
1214            {
1215                self.counters
1216                    .increment_client_egress_close_delivery_failed();
1217            }
1218            return Ok(());
1219        }
1220
1221        warn!(
1222            connection_id = route.client_connection_id.get(),
1223            module_id = %route.module_id,
1224            client_principal = %principal_label(&route.principal),
1225            client_channel = route.client_channel,
1226            frame_type = ?ty,
1227            frame_bytes = bytes,
1228            byte_limit = limit,
1229            "refusing oversized module-to-client frame with ERROR frame_too_large"
1230        );
1231        if ty == FrameType::Request {
1232            // A reverse request belongs to the module, not to a waiting client.
1233            // Answer its sender without forwarding any part of it to the client.
1234            ctx.egress
1235                .send(frame_too_large_error(
1236                    route.module_channel,
1237                    route.module_epoch,
1238                    corr,
1239                    bytes,
1240                    limit,
1241                )?)
1242                .await?;
1243        } else {
1244            // A refused stream chunk also ends the waiting request. Releasing
1245            // its credit prevents a finished request from keeping a drain alive.
1246            route.flow.release_corr(corr);
1247            let error = frame_too_large_error(
1248                route.client_channel,
1249                route.client_epoch,
1250                corr,
1251                bytes,
1252                limit,
1253            )?;
1254            // The replacement shares the reply's bounded burst grace, so a
1255            // stopped client cannot indefinitely stall the module frame reader.
1256            if let Err(err) = route.client_sink.send_to_client(error).await {
1257                if self
1258                    .forwarding
1259                    .escalate_client_delivery_failure_diagnosed(
1260                        route.client_connection_id,
1261                        route.client_channel,
1262                        route.client_epoch,
1263                        CloseReason::new(
1264                            "module_to_client_delivery_failed",
1265                            format!(
1266                                "failed to enqueue frame_too_large for client channel {} corr {corr}: {err}",
1267                                route.client_channel
1268                            ),
1269                        ),
1270                        UndeliveredFrame {
1271                            module_id: Some(&route.module_id),
1272                            sink: &route.client_sink,
1273                        },
1274                        Some(&err.diagnosis),
1275                    )
1276                    .map_err(RouterError::Forwarding)?
1277                {
1278                    self.counters
1279                        .increment_client_egress_close_delivery_failed();
1280                }
1281            }
1282        }
1283        Ok(())
1284    }
1285}
1286
1287impl Default for Router {
1288    fn default() -> Self {
1289        Self::with_default_self_handler()
1290    }
1291}
1292
1293/// Connection-scoped cleanup guard returned by [`Router::begin_connection`].
1294#[must_use]
1295pub struct RouterConnection {
1296    id: ConnectionId,
1297    control_handler: Arc<ControlHandler>,
1298    forwarding: Arc<ForwardingTable>,
1299    close_receiver: Option<ConnectionCloseReceiver>,
1300    orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
1301}
1302
1303impl RouterConnection {
1304    pub fn id(&self) -> ConnectionId {
1305        self.id
1306    }
1307
1308    pub(crate) fn take_close_receiver(&mut self) -> ConnectionCloseReceiver {
1309        self.close_receiver
1310            .take()
1311            .expect("connection close receiver can only be taken once")
1312    }
1313}
1314
1315impl Drop for RouterConnection {
1316    fn drop(&mut self) {
1317        self.forwarding.unregister_connection_close(self.id);
1318        self.orphan_goodbyes.forget_connection(self.id);
1319        // GOODBYE (explicit) and connection-drop cleanup both call the same
1320        // idempotent deregistration path.
1321        let _ = self.control_handler.cleanup_connection(self.id);
1322    }
1323}
1324
1325/// Minimal in-memory backend used by tests and early wiring: it emits a
1326/// `RESPONSE` on the same channel/correlation id with the exact same body bytes.
1327#[derive(Debug, Default, Clone, Copy)]
1328pub struct EchoBackend;
1329
1330impl EchoBackend {
1331    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1332        let response = Frame::build_with_version(
1333            frame.header.ver,
1334            FrameType::Response,
1335            frame.header.flags,
1336            frame.header.channel,
1337            frame.header.epoch,
1338            frame.header.corr,
1339            frame.body,
1340        )
1341        .map_err(RouterError::FrameBuild)?;
1342        ctx.egress.send(response).await
1343    }
1344}
1345
1346/// Data-plane backend that splices client frames to the module connection bound at attach time.
1347#[derive(Debug, Clone)]
1348pub struct ForwardBackend {
1349    forwarding: Arc<ForwardingTable>,
1350}
1351
1352impl ForwardBackend {
1353    pub fn new(forwarding: Arc<ForwardingTable>) -> Self {
1354        Self { forwarding }
1355    }
1356
1357    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1358        let channel = frame.header.channel;
1359        let corr = frame.header.corr;
1360        let route = match self
1361            .forwarding
1362            .lookup_data_route(ctx.connection_id, channel, frame.header.epoch)
1363            .map_err(RouterError::Forwarding)?
1364        {
1365            DataRoute::Client(DataRouteState::Bound(route)) => route,
1366            DataRoute::Client(_) | DataRoute::Module(_) => {
1367                return Err(RouterError::UnknownChannel {
1368                    channel,
1369                    epoch: frame.header.epoch,
1370                    corr,
1371                });
1372            }
1373        };
1374        self.handle_bound(frame, route).await
1375    }
1376
1377    pub(crate) async fn handle_bound(
1378        &self,
1379        frame: Frame,
1380        route: Arc<RouteBinding>,
1381    ) -> Result<(), RouterError> {
1382        let channel = frame.header.channel;
1383        let corr = frame.header.corr;
1384        let frame_type = frame.header.ty;
1385
1386        // CANCEL and other non-REQUEST frames bypass the request-credit window;
1387        // the original request's credit is freed only by the module's terminal frame.
1388        let acquired_credit = frame_type == FrameType::Request;
1389        if acquired_credit {
1390            if let Err(err) = route
1391                .flow
1392                .acquire_tagged(corr, frame.header.flags.is_subscription())
1393                .await
1394            {
1395                // `module_reloading` here is answered BEFORE the frame is
1396                // forwarded, so the module never sees the request and callers
1397                // may re-dispatch it after reopening the route. That is a wire
1398                // guarantee documented on `error_codes::MODULE_RELOADING`; do
1399                // not emit this code for a request that already reached
1400                // `module_sink.send` below.
1401                if self
1402                    .forwarding
1403                    .endpoint_is_draining(route.module_endpoint)
1404                    .map_err(RouterError::Forwarding)?
1405                {
1406                    return Err(RouterError::route_error_with_epoch(
1407                        channel,
1408                        frame.header.epoch,
1409                        corr,
1410                        "module_reloading",
1411                        format!("module endpoint for route channel {channel} is reloading"),
1412                    ));
1413                }
1414                return Err(RouterError::backend_with_epoch(
1415                    channel,
1416                    frame.header.epoch,
1417                    corr,
1418                    format!("{err} for route channel {channel}"),
1419                ));
1420            }
1421        }
1422
1423        let mut frame = frame;
1424        frame.header.channel = route.module_channel;
1425        frame.header.epoch = route.module_epoch;
1426        let result = route.module_sink.send(frame).await.map_err(|err| {
1427            RouterError::backend_with_epoch(channel, route.client_epoch, corr, err.to_string())
1428        });
1429        if acquired_credit && result.is_err() {
1430            route.flow.release_corr(corr);
1431        }
1432        result
1433    }
1434}
1435
1436fn is_terminal_frame(frame_type: FrameType) -> bool {
1437    matches!(
1438        frame_type,
1439        FrameType::Response | FrameType::Error | FrameType::StreamEnd
1440    )
1441}
1442
1443/// Typed router errors. Routable failures can be translated to canonical JSON
1444/// `ERROR` frames with [`RouterError::to_error_frame`].
1445#[derive(Debug, Clone, PartialEq, Eq)]
1446pub enum RouterError {
1447    ReservedChannelZero,
1448    DuplicateChannel {
1449        channel: u16,
1450    },
1451    UnknownChannel {
1452        channel: u16,
1453        epoch: u32,
1454        corr: u64,
1455    },
1456    StaleRouteEpoch {
1457        channel: u16,
1458        epoch: u32,
1459        corr: u64,
1460    },
1461    Backend {
1462        channel: u16,
1463        epoch: u32,
1464        corr: u64,
1465        message: String,
1466    },
1467    RouteError {
1468        channel: u16,
1469        epoch: u32,
1470        corr: u64,
1471        code: String,
1472        message: String,
1473    },
1474    FrameBuild(FrameBuildError),
1475    Forwarding(ForwardingError),
1476}
1477
1478impl RouterError {
1479    pub fn backend(channel: u16, corr: u64, message: impl Into<String>) -> Self {
1480        Self::backend_with_epoch(channel, 0, corr, message)
1481    }
1482
1483    pub fn backend_with_epoch(
1484        channel: u16,
1485        epoch: u32,
1486        corr: u64,
1487        message: impl Into<String>,
1488    ) -> Self {
1489        Self::Backend {
1490            channel,
1491            epoch,
1492            corr,
1493            message: message.into(),
1494        }
1495    }
1496
1497    pub fn route_error(
1498        channel: u16,
1499        corr: u64,
1500        code: impl Into<String>,
1501        message: impl Into<String>,
1502    ) -> Self {
1503        Self::route_error_with_epoch(channel, 0, corr, code, message)
1504    }
1505
1506    pub fn route_error_with_epoch(
1507        channel: u16,
1508        epoch: u32,
1509        corr: u64,
1510        code: impl Into<String>,
1511        message: impl Into<String>,
1512    ) -> Self {
1513        Self::RouteError {
1514            channel,
1515            epoch,
1516            corr,
1517            code: code.into(),
1518            message: message.into(),
1519        }
1520    }
1521
1522    /// Translate route failures that belong on the wire into an `ERROR` frame.
1523    pub fn to_error_frame(&self) -> Option<Frame> {
1524        match self {
1525            Self::UnknownChannel {
1526                channel,
1527                epoch,
1528                corr,
1529            } => error_frame(
1530                *channel,
1531                *epoch,
1532                *corr,
1533                error_codes::UNKNOWN_CHANNEL,
1534                format!("unknown channel {channel}"),
1535            ),
1536            Self::StaleRouteEpoch {
1537                channel,
1538                epoch,
1539                corr,
1540            } => error_frame(
1541                *channel,
1542                *epoch,
1543                *corr,
1544                error_codes::STALE_ROUTE_EPOCH,
1545                format!("stale route epoch for channel {channel}"),
1546            ),
1547            Self::Backend {
1548                channel,
1549                epoch,
1550                corr,
1551                message,
1552            } => error_frame(*channel, *epoch, *corr, "backend_error", message.clone()),
1553            Self::RouteError {
1554                channel,
1555                epoch,
1556                corr,
1557                code,
1558                message,
1559            } => error_frame(*channel, *epoch, *corr, code, message.clone()),
1560            Self::ReservedChannelZero
1561            | Self::DuplicateChannel { .. }
1562            | Self::FrameBuild(_)
1563            | Self::Forwarding(_) => None,
1564        }
1565    }
1566}
1567
1568fn error_frame(channel: u16, epoch: u32, corr: u64, code: &str, message: String) -> Option<Frame> {
1569    let body = serde_json::to_vec(&ErrorBody {
1570        code: code.to_string(),
1571        message,
1572        detail: None,
1573    })
1574    .ok()?;
1575
1576    Frame::build(
1577        FrameType::Error,
1578        Flags::new(false, Priority::Passive, false),
1579        channel,
1580        epoch,
1581        corr,
1582        body,
1583    )
1584    .ok()
1585}
1586
1587fn frame_too_large_error(
1588    channel: u16,
1589    epoch: u32,
1590    corr: u64,
1591    bytes: usize,
1592    limit: usize,
1593) -> Result<Frame, RouterError> {
1594    let body = serde_json::to_vec(&ErrorBody {
1595        code: error_codes::FRAME_TOO_LARGE.to_string(),
1596        message: format!(
1597            "module frame of {bytes} bytes exceeds client egress limit of {limit} bytes"
1598        ),
1599        detail: Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit })),
1600    })
1601    .expect("serializing a frame size error");
1602    Frame::build(
1603        FrameType::Error,
1604        Flags::new(false, Priority::Passive, false).with_daemon_origin(),
1605        channel,
1606        epoch,
1607        corr,
1608        body,
1609    )
1610    .map_err(RouterError::FrameBuild)
1611}
1612
1613impl fmt::Display for RouterError {
1614    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1615        match self {
1616            Self::ReservedChannelZero => write!(f, "channel 0 is reserved for subc"),
1617            Self::DuplicateChannel { channel } => {
1618                write!(f, "backend already registered for channel {channel}")
1619            }
1620            Self::UnknownChannel { channel, corr, .. } => {
1621                write!(f, "unknown channel {channel} for corr {corr}")
1622            }
1623            Self::StaleRouteEpoch { channel, corr, .. } => {
1624                write!(f, "stale route epoch for channel {channel} corr {corr}")
1625            }
1626            Self::Backend {
1627                channel,
1628                corr,
1629                message,
1630                ..
1631            } => write!(
1632                f,
1633                "backend error on channel {channel} corr {corr}: {message}"
1634            ),
1635            Self::RouteError {
1636                channel,
1637                corr,
1638                code,
1639                message,
1640                ..
1641            } => write!(
1642                f,
1643                "route error {code} on channel {channel} corr {corr}: {message}"
1644            ),
1645            Self::FrameBuild(err) => write!(f, "failed to build routed frame: {err}"),
1646            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
1647        }
1648    }
1649}
1650
1651impl Error for RouterError {
1652    fn source(&self) -> Option<&(dyn Error + 'static)> {
1653        match self {
1654            Self::FrameBuild(err) => Some(err),
1655            Self::Forwarding(err) => Some(err),
1656            Self::ReservedChannelZero
1657            | Self::DuplicateChannel { .. }
1658            | Self::UnknownChannel { .. }
1659            | Self::StaleRouteEpoch { .. }
1660            | Self::Backend { .. }
1661            | Self::RouteError { .. } => None,
1662        }
1663    }
1664}
1665
1666#[cfg(test)]
1667mod tests {
1668    use super::*;
1669    use crate::{
1670        forwarding::RouteBindRelayOutcome,
1671        supervise::{ModuleSpec, RestartPolicy, Supervisor, SupervisorHandle},
1672        ControlHandler, Registry,
1673    };
1674    use std::{
1675        sync::{mpsc as std_mpsc, Arc},
1676        time::Duration,
1677    };
1678    use subc_control::ModuleProtocol;
1679    use subc_protocol::{manifest::Concurrency, ErrorBody, Flags, FrameType, Priority};
1680    use tokio::sync::mpsc;
1681
1682    pub(crate) use crate::router::test_log::{captured_logs, log_capture};
1683
1684    fn logged_millis(logs: &str, field: &str) -> u64 {
1685        logs.split_whitespace()
1686            .find_map(|part| part.strip_prefix(field))
1687            .and_then(|value| value.parse().ok())
1688            .unwrap_or_else(|| panic!("missing numeric {field} in logs: {logs}"))
1689    }
1690
1691    fn request(channel: u16, corr: u64, body: &[u8]) -> Frame {
1692        Frame::build(
1693            FrameType::Request,
1694            Flags::new(true, Priority::Interactive, false),
1695            channel,
1696            0,
1697            corr,
1698            body.to_vec(),
1699        )
1700        .unwrap()
1701    }
1702
1703    fn ping(corr: u64) -> Frame {
1704        Frame::build(
1705            FrameType::Ping,
1706            Flags::new(false, Priority::Passive, false),
1707            0,
1708            0,
1709            corr,
1710            Vec::new(),
1711        )
1712        .unwrap()
1713    }
1714
1715    fn route_ctx() -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
1716        let (tx, rx) = mpsc::channel(8);
1717        (
1718            RouteCtx {
1719                connection_id: ConnectionId::LOCAL,
1720                egress: FrameSink::new(tx),
1721            },
1722            rx,
1723        )
1724    }
1725
1726    #[tokio::test]
1727    async fn echo_backend_returns_response_with_byte_identical_body() {
1728        let mut router = Router::with_default_self_handler();
1729        router.register_backend(7, EchoBackend).unwrap();
1730        let (ctx, mut rx) = route_ctx();
1731        let body = b"{not parsed}\0\xff";
1732
1733        router
1734            .route_for_connection(&ctx, request(7, 123, body))
1735            .await
1736            .unwrap();
1737        let response = rx.recv().await.unwrap();
1738
1739        assert_eq!(response.header.ty, FrameType::Response);
1740        assert_eq!(response.header.channel, 7);
1741        assert_eq!(response.header.corr, 123);
1742        assert_eq!(response.body, body);
1743        assert!(rx.try_recv().is_err());
1744    }
1745
1746    #[tokio::test]
1747    async fn unknown_channel_emits_canonical_error_frame() {
1748        let router = Router::with_default_self_handler();
1749        let (ctx, mut rx) = route_ctx();
1750
1751        router
1752            .route_for_connection(&ctx, request(99, 5, b"payload"))
1753            .await
1754            .unwrap();
1755        let error_frame = rx.recv().await.unwrap();
1756
1757        assert_eq!(error_frame.header.ty, FrameType::Error);
1758        assert_eq!(error_frame.header.channel, 99);
1759        assert_eq!(error_frame.header.corr, 5);
1760        let body: ErrorBody = serde_json::from_slice(&error_frame.body).unwrap();
1761        assert_eq!(body.code, "unknown_channel");
1762        assert_eq!(body.message, "unknown channel 99");
1763    }
1764
1765    #[tokio::test]
1766    async fn channel_zero_uses_control_handler_not_backend_registry() {
1767        let mut router = Router::with_default_self_handler();
1768        router.register_backend(1, EchoBackend).unwrap();
1769        let (ctx, mut rx) = route_ctx();
1770
1771        router.route_for_connection(&ctx, ping(77)).await.unwrap();
1772        let response = rx.recv().await.unwrap();
1773
1774        assert_eq!(response.header.ty, FrameType::Pong);
1775        assert_eq!(response.header.channel, 0);
1776        assert_eq!(response.header.corr, 77);
1777        assert!(response.body.is_empty());
1778    }
1779
1780    #[tokio::test]
1781    async fn slow_control_dispatch_logs_decoded_op_and_elapsed_time() {
1782        let control = Arc::new(
1783            ControlHandler::new(Arc::new(Registry::default()))
1784                .with_control_dispatch_delay(Duration::from_millis(1050)),
1785        );
1786        let router = Router::with_control_handler(control);
1787        let (ctx, mut rx) = route_ctx();
1788        let (output, guard) = log_capture(tracing::Level::WARN);
1789
1790        router
1791            .route_for_connection(&ctx, request(0, 41, br#"{"op":"server.describe"}"#))
1792            .await
1793            .expect("slow request routes");
1794        assert!(rx.recv().await.is_some(), "request receives a response");
1795        drop(guard);
1796
1797        let logs = captured_logs(&output);
1798        assert!(logs.contains("slow control dispatch"));
1799        assert!(logs.contains("op=server.describe"));
1800        assert!(logs.contains("connection_id=0"));
1801        assert!(logs.contains("corr=41"));
1802        assert!(
1803            logged_millis(&logs, "elapsed_ms=") >= 1050,
1804            "elapsed must include the injected handler delay: {logs}"
1805        );
1806    }
1807
1808    #[tokio::test]
1809    async fn fast_control_dispatch_emits_arrival_without_slow_warning() {
1810        let router = Router::with_default_self_handler();
1811        let (ctx, mut rx) = route_ctx();
1812        let (output, guard) = log_capture(tracing::Level::DEBUG);
1813
1814        router
1815            .route_for_connection(&ctx, request(0, 42, br#"{"op":"server.describe"}"#))
1816            .await
1817            .expect("fast request routes");
1818        assert!(rx.recv().await.is_some(), "request receives a response");
1819        drop(guard);
1820
1821        let logs = captured_logs(&output);
1822        assert!(logs.contains("control dispatch op=server.describe connection_id=0 corr=42"));
1823        assert!(!logs.contains("slow control dispatch"));
1824    }
1825
1826    #[tokio::test]
1827    async fn control_dispatch_arrival_is_hidden_at_info() {
1828        let router = Router::with_default_self_handler();
1829        let (ctx, mut rx) = route_ctx();
1830        let (output, guard) = log_capture(tracing::Level::INFO);
1831
1832        router
1833            .route_for_connection(&ctx, request(0, 43, br#"{"op":"server.describe"}"#))
1834            .await
1835            .expect("fast request routes");
1836        assert!(rx.recv().await.is_some(), "request receives a response");
1837        drop(guard);
1838
1839        assert!(
1840            !captured_logs(&output).contains("control dispatch"),
1841            "arrival logging must stay hidden at INFO"
1842        );
1843    }
1844
1845    #[tokio::test]
1846    async fn supervisor_list_logs_contended_snapshot_lock_only() {
1847        let registry = Arc::new(Registry::default());
1848        let handle = SupervisorHandle::new();
1849        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
1850            .with_handle(handle.clone());
1851        let module = supervisor
1852            .supervise_configured(
1853                ModuleSpec {
1854                    module_id: "held-module".to_string(),
1855                    program: "test-module".into(),
1856                    args: Vec::new(),
1857                    env: Vec::new(),
1858                    reserved: false,
1859                    reserved_prefixes: Vec::new(),
1860                    protocol: ModuleProtocol::Subc,
1861                    overlap: Default::default(),
1862                },
1863                false,
1864            )
1865            .expect("disabled test module is supervised");
1866        let router = Router::with_control_handler(Arc::new(
1867            ControlHandler::new(Arc::clone(&registry)).with_supervisor(handle),
1868        ));
1869        let (ctx, mut rx) = route_ctx();
1870        let (acquired, ready) = std_mpsc::channel();
1871        let holder = module.hold_snapshot_for_test(acquired, Duration::from_millis(400));
1872        ready.recv().expect("holder acquired snapshot lock");
1873        let (output, guard) = log_capture(tracing::Level::WARN);
1874
1875        router
1876            .route_for_connection(&ctx, request(0, 44, br#"{"op":"supervisor.list"}"#))
1877            .await
1878            .expect("list request routes after the lock releases");
1879        assert!(
1880            rx.recv().await.is_some(),
1881            "list request receives a response"
1882        );
1883        holder.join().expect("snapshot holder exits cleanly");
1884        drop(guard);
1885
1886        let logs = captured_logs(&output);
1887        assert!(logs.contains("slow snapshot lock"));
1888        assert!(logs.contains("module_id=held-module"));
1889        assert!(logs.contains("caller=list"));
1890        assert!(
1891            logged_millis(&logs, "waited_ms=") >= 250,
1892            "wait must exceed the slow-lock threshold: {logs}"
1893        );
1894
1895        let (output, guard) = log_capture(tracing::Level::WARN);
1896        router
1897            .route_for_connection(&ctx, request(0, 45, br#"{"op":"supervisor.list"}"#))
1898            .await
1899            .expect("uncontended list request routes");
1900        assert!(
1901            rx.recv().await.is_some(),
1902            "uncontended list receives a response"
1903        );
1904        drop(guard);
1905        assert!(
1906            !captured_logs(&output).contains("slow snapshot lock"),
1907            "uncontended list acquisition must not warn"
1908        );
1909    }
1910
1911    #[tokio::test]
1912    async fn full_module_to_client_sink_requests_client_close_without_erroring_module() {
1913        let forwarding = Arc::new(ForwardingTable::default());
1914        let control = Arc::new(ControlHandler::with_forwarding(
1915            Arc::new(crate::Registry::default()),
1916            Arc::clone(&forwarding),
1917        ));
1918        let router = Router::with_control_handler(control);
1919        let module_connection = ConnectionId::new(10);
1920        let client_connection = ConnectionId::new(20);
1921        let mut close_receiver = forwarding.register_connection_close(client_connection);
1922        let (module_tx, _module_rx) = mpsc::channel(1);
1923        forwarding
1924            .register_module_connection(
1925                module_connection,
1926                "full-sink-provider".to_string(),
1927                1,
1928                Concurrency::ModuleManaged,
1929                FrameSink::new(module_tx),
1930            )
1931            .unwrap();
1932        let (client_tx, mut client_rx) = mpsc::channel(1);
1933        let pending = forwarding
1934            .begin_route_bind_relay_for_test(
1935                client_connection,
1936                FrameSink::new(client_tx),
1937                700,
1938                "full-sink-provider",
1939            )
1940            .unwrap();
1941        forwarding
1942            .complete_pending_relay(
1943                module_connection,
1944                pending.corr,
1945                RouteBindRelayOutcome::Accepted,
1946            )
1947            .unwrap();
1948
1949        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
1950        let module_ctx = RouteCtx {
1951            connection_id: module_connection,
1952            egress: FrameSink::new(module_egress_tx),
1953        };
1954        let terminal = Frame::build(
1955            FrameType::Response,
1956            Flags::new(false, Priority::Interactive, true),
1957            pending.module_channel,
1958            pending.module_epoch,
1959            701,
1960            b"terminal".to_vec(),
1961        )
1962        .unwrap();
1963
1964        router
1965            .route_for_connection(&module_ctx, terminal)
1966            .await
1967            .unwrap();
1968        let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
1969            .await
1970            .expect("close request should be sent for the full client sink")
1971            .expect("close sender should include a reason");
1972        assert!(
1973            reason
1974                .to_string()
1975                .contains("module_to_client_delivery_failed"),
1976            "unexpected close reason: {reason}"
1977        );
1978        assert_eq!(client_rx.try_recv().unwrap().header.corr, 700);
1979        assert!(client_rx.try_recv().is_err());
1980        assert_eq!(
1981            router.counters.snapshot()["client_egress_close_delivery_failed"],
1982            1
1983        );
1984    }
1985
1986    /// A terminal frame ends the request at the module even when the client
1987    /// cannot take it, so the drain must stop counting it at once rather than
1988    /// when the client connection's cleanup later removes the route.
1989    #[tokio::test]
1990    async fn terminal_frame_releases_its_credit_even_when_client_delivery_fails() {
1991        let forwarding = Arc::new(ForwardingTable::default());
1992        let control = Arc::new(ControlHandler::with_forwarding(
1993            Arc::new(crate::Registry::default()),
1994            Arc::clone(&forwarding),
1995        ));
1996        let router = Router::with_control_handler(control);
1997        let module_connection = ConnectionId::new(11);
1998        let client_connection = ConnectionId::new(21);
1999        let _close_receiver = forwarding.register_connection_close(client_connection);
2000        let (module_tx, _module_rx) = mpsc::channel(1);
2001        forwarding
2002            .register_module_connection(
2003                module_connection,
2004                "credit-provider".to_string(),
2005                1,
2006                Concurrency::ModuleManaged,
2007                FrameSink::new(module_tx),
2008            )
2009            .unwrap();
2010        // Capacity one, filled by the route.open response, so the terminal
2011        // frame below cannot be enqueued for the client.
2012        let (client_tx, _client_rx) = mpsc::channel(1);
2013        let pending = forwarding
2014            .begin_route_bind_relay_for_test(
2015                client_connection,
2016                FrameSink::new(client_tx),
2017                800,
2018                "credit-provider",
2019            )
2020            .unwrap();
2021        forwarding
2022            .complete_pending_relay(
2023                module_connection,
2024                pending.corr,
2025                RouteBindRelayOutcome::Accepted,
2026            )
2027            .unwrap();
2028        let DataRoute::Client(DataRouteState::Bound(route)) = forwarding
2029            .lookup_data_route(
2030                client_connection,
2031                pending.client_channel,
2032                pending.client_epoch,
2033            )
2034            .unwrap()
2035        else {
2036            panic!("expected a bound client route");
2037        };
2038        route.flow.acquire_tagged(801, false).await.unwrap();
2039        assert_eq!(route.flow.drain_in_flight(), 1);
2040
2041        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2042        let module_ctx = RouteCtx {
2043            connection_id: module_connection,
2044            egress: FrameSink::new(module_egress_tx),
2045        };
2046        let terminal = Frame::build(
2047            FrameType::Response,
2048            Flags::new(false, Priority::Interactive, true),
2049            pending.module_channel,
2050            pending.module_epoch,
2051            801,
2052            b"terminal".to_vec(),
2053        )
2054        .unwrap();
2055        router
2056            .route_for_connection(&module_ctx, terminal)
2057            .await
2058            .unwrap();
2059
2060        assert_eq!(
2061            router.counters.snapshot()["client_egress_close_delivery_failed"],
2062            1,
2063            "the client delivery must have failed for this test to mean anything"
2064        );
2065        assert_eq!(
2066            route.flow.drain_in_flight(),
2067            0,
2068            "the module's terminal frame must release its credit even though the client could not take it"
2069        );
2070    }
2071
2072    #[tokio::test]
2073    async fn full_route_goodbye_sink_requests_target_close_without_erroring_module() {
2074        let forwarding = Arc::new(ForwardingTable::default());
2075        let control = Arc::new(ControlHandler::with_forwarding(
2076            Arc::new(crate::Registry::default()),
2077            Arc::clone(&forwarding),
2078        ));
2079        let router = Router::with_control_handler(control);
2080        let module_connection = ConnectionId::new(30);
2081        let client_connection = ConnectionId::new(40);
2082        let mut close_receiver = forwarding.register_connection_close(client_connection);
2083        let (module_tx, _module_rx) = mpsc::channel(1);
2084        forwarding
2085            .register_module_connection(
2086                module_connection,
2087                "goodbye-full-provider".to_string(),
2088                1,
2089                Concurrency::ModuleManaged,
2090                FrameSink::new(module_tx),
2091            )
2092            .unwrap();
2093        let (client_tx, mut client_rx) = mpsc::channel(1);
2094        let pending = forwarding
2095            .begin_route_bind_relay_for_test(
2096                client_connection,
2097                FrameSink::new(client_tx),
2098                800,
2099                "goodbye-full-provider",
2100            )
2101            .unwrap();
2102        forwarding
2103            .complete_pending_relay(
2104                module_connection,
2105                pending.corr,
2106                RouteBindRelayOutcome::Accepted,
2107            )
2108            .unwrap();
2109
2110        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2111        let module_ctx = RouteCtx {
2112            connection_id: module_connection,
2113            egress: FrameSink::new(module_egress_tx),
2114        };
2115        let goodbye = Frame::build(
2116            FrameType::Goodbye,
2117            Flags::new(false, Priority::Passive, true),
2118            pending.module_channel,
2119            pending.module_epoch,
2120            801,
2121            Vec::new(),
2122        )
2123        .unwrap();
2124
2125        router
2126            .route_for_connection(&module_ctx, goodbye)
2127            .await
2128            .unwrap();
2129        let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2130            .await
2131            .expect("close request should be sent for the full GOODBYE sink")
2132            .expect("close sender should include a reason");
2133        assert!(
2134            reason.to_string().contains("route_goodbye_delivery_failed"),
2135            "unexpected close reason: {reason}"
2136        );
2137        assert_eq!(client_rx.try_recv().unwrap().header.corr, 800);
2138        assert!(client_rx.try_recv().is_err());
2139        assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 1);
2140        assert_eq!(router.counters.snapshot()["route_released_epoch_fenced"], 1);
2141    }
2142
2143    /// One module and one client connection with `routes` routes bound between
2144    /// them. The client uses a real connection egress queue (the same byte
2145    /// budget and frame-count backstop as a live connection), and its route.open
2146    /// responses are drained so the queue starts empty.
2147    async fn multi_route_client(
2148        module_id: &str,
2149        module_connection: ConnectionId,
2150        client_connection: ConnectionId,
2151        routes: usize,
2152    ) -> (
2153        Router,
2154        FrameSink,
2155        mpsc::Receiver<OutboundFrame>,
2156        RouteCtx,
2157        Vec<crate::forwarding::PendingRouteBindRelay>,
2158        ConnectionCloseReceiver,
2159    ) {
2160        let forwarding = Arc::new(ForwardingTable::default());
2161        let control = Arc::new(ControlHandler::with_forwarding(
2162            Arc::new(crate::Registry::default()),
2163            Arc::clone(&forwarding),
2164        ));
2165        let router = Router::with_control_handler(control);
2166        let close_receiver = forwarding.register_connection_close(client_connection);
2167        let (module_tx, _module_rx) = mpsc::channel(8);
2168        forwarding
2169            .register_module_connection(
2170                module_connection,
2171                module_id.to_string(),
2172                1,
2173                Concurrency::ModuleManaged,
2174                FrameSink::new(module_tx),
2175            )
2176            .unwrap();
2177        let (client_sink, mut client_rx) = crate::server::connection_egress();
2178        let mut bound = Vec::with_capacity(routes);
2179        for index in 0..routes {
2180            let pending = forwarding
2181                .begin_route_bind_relay_for_test(
2182                    client_connection,
2183                    client_sink.clone(),
2184                    900 + index as u64,
2185                    module_id,
2186                )
2187                .unwrap();
2188            forwarding
2189                .complete_pending_relay(
2190                    module_connection,
2191                    pending.corr,
2192                    RouteBindRelayOutcome::Accepted,
2193                )
2194                .unwrap();
2195            assert_eq!(
2196                client_rx.recv().await.unwrap().header.corr,
2197                900 + index as u64
2198            );
2199            bound.push(pending);
2200        }
2201        let (module_egress_tx, _module_egress_rx) = mpsc::channel(8);
2202        let module_ctx = RouteCtx {
2203            connection_id: module_connection,
2204            egress: FrameSink::new(module_egress_tx),
2205        };
2206        (
2207            router,
2208            client_sink,
2209            client_rx,
2210            module_ctx,
2211            bound,
2212            close_receiver,
2213        )
2214    }
2215
2216    /// Per-frame cost of the egress sink's admission and release accounting:
2217    /// one million 200-byte frames enqueued with `try_send` and taken off the
2218    /// queue the way the connection writer does, in batches of 1,000 so the
2219    /// queue stays well inside its budget. Prints nanoseconds per frame; it is
2220    /// a measurement, not a gate. Run with
2221    /// `cargo test --release -p subc-daemon --lib egress_sink_per_frame_cost -- --ignored --nocapture`.
2222    #[test]
2223    #[ignore = "timing measurement, run on demand"]
2224    fn egress_sink_per_frame_cost() {
2225        const FRAMES: usize = 1_000_000;
2226        const BATCH: usize = 1_000;
2227        let (sink, mut rx) = crate::server::connection_egress();
2228        let template = stream_frame(9, 1, 0, vec![b't'; 200]);
2229        let started = Instant::now();
2230        for _ in 0..FRAMES / BATCH {
2231            for _ in 0..BATCH {
2232                sink.try_send(template.clone()).unwrap();
2233            }
2234            for _ in 0..BATCH {
2235                let outbound = rx.try_recv().unwrap();
2236                if let Some(charge) = &outbound.charge {
2237                    charge.taken_by_writer();
2238                }
2239                drop(std::hint::black_box(outbound));
2240            }
2241        }
2242        let elapsed = started.elapsed();
2243        assert_eq!(sink.backlog().queued_bytes, 0);
2244        println!(
2245            "egress sink: {FRAMES} frames in {elapsed:?}, {:.1} ns/frame",
2246            elapsed.as_nanos() as f64 / FRAMES as f64
2247        );
2248    }
2249
2250    fn stream_frame(channel: u16, epoch: u32, corr: u64, body: Vec<u8>) -> Frame {
2251        Frame::build(
2252            FrameType::StreamData,
2253            Flags::new(false, Priority::Interactive, false),
2254            channel,
2255            epoch,
2256            corr,
2257            body,
2258        )
2259        .unwrap()
2260    }
2261
2262    fn module_route_for_test(
2263        router: &Router,
2264        ctx: &RouteCtx,
2265        route: &crate::forwarding::PendingRouteBindRelay,
2266    ) -> Arc<RouteBinding> {
2267        match router
2268            .forwarding
2269            .lookup_data_route(ctx.connection_id, route.module_channel, route.module_epoch)
2270            .unwrap()
2271        {
2272            DataRoute::Module(DataRouteState::Bound(binding)) => binding,
2273            other => panic!("expected a bound module route, got {other:?}"),
2274        }
2275    }
2276
2277    fn assert_frame_size_error(frame: &Frame, channel: u16, epoch: u32, corr: u64, bytes: usize) {
2278        assert_eq!(frame.header.ty, FrameType::Error);
2279        assert_eq!(frame.header.channel, channel);
2280        assert_eq!(frame.header.epoch, epoch);
2281        assert_eq!(frame.header.corr, corr);
2282        assert!(frame.header.flags.is_daemon_origin());
2283        let body: ErrorBody = serde_json::from_slice(&frame.body).unwrap();
2284        assert_eq!(body.code, error_codes::FRAME_TOO_LARGE);
2285        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2286        assert!(body.message.contains(&bytes.to_string()));
2287        assert!(body.message.contains(&limit.to_string()));
2288        assert_eq!(
2289            body.detail,
2290            Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit }))
2291        );
2292    }
2293
2294    #[tokio::test]
2295    async fn module_egress_response_oversized_reply_keeps_connection_and_other_route() {
2296        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
2297        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2298            "oversize-provider",
2299            ConnectionId::new(80),
2300            ConnectionId::new(81),
2301            2,
2302        )
2303        .await;
2304        let binding = module_route_for_test(&router, &ctx, &routes[0]);
2305        binding.flow.acquire_tagged(101, false).await.unwrap();
2306        module_route_for_test(&router, &ctx, &routes[1])
2307            .flow
2308            .acquire_tagged(102, false)
2309            .await
2310            .unwrap();
2311        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2312        let oversized = Frame::build(
2313            FrameType::Response,
2314            Flags::new(false, Priority::Interactive, true),
2315            routes[0].module_channel,
2316            routes[0].module_epoch,
2317            101,
2318            vec![b'x'; limit - subc_protocol::HEADER_LEN + 1],
2319        )
2320        .unwrap();
2321        assert_eq!(sink.backlog().queued_bytes, 0);
2322        router.route_for_connection(&ctx, oversized).await.unwrap();
2323
2324        // Reproduce a large reply followed by ordinary traffic on another route
2325        // before the client reads anything. The large reply must not monopolize
2326        // the queue and make that second delivery close the shared connection.
2327        router
2328            .route_for_connection(
2329                &ctx,
2330                route_frame(
2331                    FrameType::Response,
2332                    routes[1].module_channel,
2333                    routes[1].module_epoch,
2334                    102,
2335                ),
2336            )
2337            .await
2338            .unwrap();
2339        assert!(
2340            close.try_recv().is_err(),
2341            "an oversized reply must not close the multiplexed client"
2342        );
2343        let error = rx.try_recv().expect("the waiting client gets an ERROR");
2344        assert_frame_size_error(
2345            &error,
2346            routes[0].client_channel,
2347            routes[0].client_epoch,
2348            101,
2349            limit + 1,
2350        );
2351        let second = rx.try_recv().expect("the other route's reply arrives");
2352        assert_eq!(second.header.ty, FrameType::Response);
2353        assert_eq!(second.header.channel, routes[1].client_channel);
2354        assert_eq!(second.header.epoch, routes[1].client_epoch);
2355        assert_eq!(second.header.corr, 102);
2356        assert_eq!(second.body, b"route-body");
2357        assert_eq!(binding.flow.drain_in_flight(), 0);
2358        for route in &routes {
2359            module_route_for_test(&router, &ctx, route);
2360        }
2361        assert!(rx.try_recv().is_err());
2362        drop((error, second));
2363        assert_eq!(sink.backlog().queued_bytes, 0);
2364        assert_eq!(
2365            router.counters.snapshot()["client_egress_close_delivery_failed"],
2366            0
2367        );
2368
2369        let captured = test_log::captured_logs(&logs);
2370        let warnings: Vec<_> = captured
2371            .lines()
2372            .filter(|line| line.contains("WARN"))
2373            .collect();
2374        assert_eq!(warnings.len(), 1, "{captured}");
2375        let warning = warnings[0];
2376        for expected in [
2377            "module_id=oversize-provider".to_string(),
2378            "client_principal=direct".to_string(),
2379            format!("client_channel={}", routes[0].client_channel),
2380            format!("frame_bytes={}", limit + 1),
2381            format!("byte_limit={limit}"),
2382        ] {
2383            assert!(warning.contains(&expected), "missing {expected}: {warning}");
2384        }
2385    }
2386
2387    async fn poll_once<F: std::future::Future>(
2388        mut future: std::pin::Pin<&mut F>,
2389    ) -> std::task::Poll<F::Output> {
2390        std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
2391    }
2392
2393    fn fill_client_budget(sink: &FrameSink, route: &crate::forwarding::PendingRouteBindRelay) {
2394        sink.try_send(
2395            Frame::build(
2396                FrameType::Response,
2397                Flags::new(false, Priority::Interactive, true),
2398                route.client_channel,
2399                route.client_epoch,
2400                100,
2401                vec![
2402                    b'x';
2403                    crate::server::CONNECTION_EGRESS_BYTE_BUDGET - subc_protocol::HEADER_LEN
2404                ],
2405            )
2406            .unwrap(),
2407        )
2408        .unwrap();
2409        assert_eq!(
2410            sink.backlog().queued_bytes,
2411            crate::server::CONNECTION_EGRESS_BYTE_BUDGET
2412        );
2413    }
2414
2415    async fn assert_full_client_burst_survives(ty: FrameType, oversized: bool) {
2416        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2417            "draining-burst",
2418            ConnectionId::new(92),
2419            ConnectionId::new(93),
2420            2,
2421        )
2422        .await;
2423        fill_client_budget(&sink, &routes[0]);
2424        let frame = if oversized {
2425            Frame::build(
2426                ty,
2427                Flags::new(false, Priority::Interactive, false),
2428                routes[1].module_channel,
2429                routes[1].module_epoch,
2430                101,
2431                vec![b's'; crate::server::CONNECTION_EGRESS_BYTE_BUDGET],
2432            )
2433            .unwrap()
2434        } else {
2435            route_frame(ty, routes[1].module_channel, routes[1].module_epoch, 101)
2436        };
2437        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2438        assert!(poll_once(delivery.as_mut()).await.is_pending());
2439        assert!(
2440            close.try_recv().is_err(),
2441            "fresh bursts must not close: {ty:?}"
2442        );
2443        tokio::time::advance(Duration::from_millis(50)).await;
2444        let queued = rx.try_recv().unwrap();
2445        queued.charge.as_ref().unwrap().taken_by_writer();
2446        drop(queued);
2447        assert!(matches!(
2448            poll_once(delivery.as_mut()).await,
2449            std::task::Poll::Ready(Ok(()))
2450        ));
2451        let delivered = rx
2452            .try_recv()
2453            .expect("the burst frame must be delivered after draining");
2454        assert_eq!(delivered.header.channel, routes[1].client_channel);
2455        assert_eq!(delivered.header.epoch, routes[1].client_epoch);
2456        assert_eq!(delivered.header.corr, 101);
2457        if oversized {
2458            assert_frame_size_error(
2459                &delivered,
2460                routes[1].client_channel,
2461                routes[1].client_epoch,
2462                101,
2463                crate::server::CONNECTION_EGRESS_BYTE_BUDGET + subc_protocol::HEADER_LEN,
2464            );
2465        } else {
2466            assert_eq!(delivered.header.ty, ty);
2467        }
2468        assert!(rx.try_recv().is_err());
2469        assert!(close.try_recv().is_err());
2470        assert_eq!(
2471            router.counters.snapshot()["client_egress_close_delivery_failed"],
2472            0
2473        );
2474        assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 0);
2475    }
2476
2477    #[tokio::test(start_paused = true)]
2478    async fn module_egress_fresh_full_queue_burst_survives() {
2479        for ty in [
2480            FrameType::Response,
2481            FrameType::StreamData,
2482            FrameType::Request,
2483            FrameType::Push,
2484        ] {
2485            assert_full_client_burst_survives(ty, false).await;
2486        }
2487    }
2488
2489    #[tokio::test(start_paused = true)]
2490    async fn module_goodbye_fresh_full_queue_burst_survives() {
2491        assert_full_client_burst_survives(FrameType::Goodbye, false).await;
2492    }
2493
2494    #[tokio::test(start_paused = true)]
2495    async fn frame_count_backstop_also_allows_a_draining_burst() {
2496        let (tx, mut rx) = mpsc::channel(1);
2497        let sink = FrameSink::with_byte_budget(tx, 2_000);
2498        sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
2499            .unwrap();
2500        let mut delivery = Box::pin(sink.send_to_client(stream_frame(1, 1, 2, vec![b'y'; 100])));
2501        assert!(poll_once(delivery.as_mut()).await.is_pending());
2502        assert!(
2503            sink.backlog().queued_bytes < 2_000,
2504            "only frame capacity is exhausted"
2505        );
2506        tokio::time::advance(Duration::from_millis(50)).await;
2507        drop(rx.try_recv().unwrap());
2508        assert!(matches!(
2509            poll_once(delivery.as_mut()).await,
2510            std::task::Poll::Ready(Ok(()))
2511        ));
2512        assert_eq!(rx.try_recv().unwrap().header.corr, 2);
2513    }
2514
2515    #[tokio::test(start_paused = true)]
2516    async fn oversized_reply_replacement_fresh_full_queue_burst_survives() {
2517        for ty in [FrameType::Response, FrameType::StreamData] {
2518            assert_full_client_burst_survives(ty, true).await;
2519        }
2520    }
2521
2522    #[tokio::test(start_paused = true)]
2523    async fn module_egress_stopped_reader_closes_without_waiting() {
2524        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
2525        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2526            "oldest-stopped-reader",
2527            ConnectionId::new(94),
2528            ConnectionId::new(95),
2529            2,
2530        )
2531        .await;
2532        fill_client_budget(&sink, &routes[0]);
2533        // The writer has taken the head but cannot finish writing it. Its
2534        // bytes stay charged, and the atomic age still diagnoses that stall.
2535        let writing = rx.try_recv().unwrap();
2536        writing.charge.as_ref().unwrap().taken_by_writer();
2537        tokio::time::advance(Duration::from_millis(251)).await;
2538        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(251)));
2539        assert_eq!(sink.backlog().queued_bytes, queued_frame_bytes(&writing));
2540        let started = tokio::time::Instant::now();
2541        let frame = stream_frame(
2542            routes[0].module_channel,
2543            routes[0].module_epoch,
2544            101,
2545            b"secret-body-never-log".to_vec(),
2546        );
2547        let flags = frame.header.flags.0;
2548        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2549        assert!(
2550            matches!(
2551                poll_once(delivery.as_mut()).await,
2552                std::task::Poll::Ready(Ok(()))
2553            ),
2554            "an oldest frame older than 250 ms must close immediately, without a new wait"
2555        );
2556        assert_eq!(started.elapsed(), Duration::ZERO);
2557        assert!(close
2558            .try_recv()
2559            .unwrap()
2560            .to_string()
2561            .contains("module_to_client_delivery_failed"));
2562        let captured = test_log::captured_logs(&logs);
2563        for expected in [
2564            "frame_type=Some(StreamData)".to_string(),
2565            format!("frame_flags=Some({flags})"),
2566            "oldest_queued_ms=251".to_string(),
2567            "close_cause=\"oldest_frame_older_than_250_ms\"".to_string(),
2568        ] {
2569            assert!(
2570                captured.contains(&expected),
2571                "missing {expected}: {captured}"
2572            );
2573        }
2574        assert!(!captured.contains("secret-body-never-log"));
2575    }
2576
2577    #[tokio::test(start_paused = true)]
2578    async fn client_gone_mid_wait_resumes_module_reader_immediately() {
2579        for writer_gone in [false, true] {
2580            let (router, sink, mut rx, ctx, routes, _close) = multi_route_client(
2581                "gone-during-burst",
2582                ConnectionId::new(96),
2583                ConnectionId::new(97),
2584                1,
2585            )
2586            .await;
2587            fill_client_budget(&sink, &routes[0]);
2588            let frame = route_frame(
2589                FrameType::Response,
2590                routes[0].module_channel,
2591                routes[0].module_epoch,
2592                101,
2593            );
2594            let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2595            assert!(poll_once(delivery.as_mut()).await.is_pending());
2596            tokio::time::advance(Duration::from_millis(50)).await;
2597            let started = tokio::time::Instant::now();
2598            if writer_gone {
2599                rx.close();
2600            } else {
2601                sink.close_client_delivery();
2602            }
2603            assert!(
2604                matches!(
2605                    poll_once(delivery.as_mut()).await,
2606                    std::task::Poll::Ready(Ok(()))
2607                ),
2608                "client teardown must wake the module reader before the writer drains"
2609            );
2610            assert_eq!(started.elapsed(), Duration::ZERO);
2611            assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
2612            assert_eq!(rx.try_recv().unwrap().header.corr, 100);
2613            assert!(rx.try_recv().is_err());
2614        }
2615    }
2616
2617    #[tokio::test(start_paused = true)]
2618    async fn cancelled_module_delivery_releases_waiter_without_closing_client() {
2619        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2620            "cancelled-burst",
2621            ConnectionId::new(98),
2622            ConnectionId::new(99),
2623            1,
2624        )
2625        .await;
2626        fill_client_budget(&sink, &routes[0]);
2627        let frame = route_frame(
2628            FrameType::Response,
2629            routes[0].module_channel,
2630            routes[0].module_epoch,
2631            101,
2632        );
2633        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2634        assert!(poll_once(delivery.as_mut()).await.is_pending());
2635        assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 1);
2636        drop(delivery);
2637        assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
2638        tokio::time::advance(Duration::from_secs(1)).await;
2639        assert!(close.try_recv().is_err());
2640        drop(rx.try_recv().unwrap());
2641        assert_eq!(sink.backlog().queued_bytes, 0);
2642        assert!(sink.backlog().oldest_age.is_none());
2643    }
2644
2645    #[tokio::test(start_paused = true)]
2646    async fn egress_atomic_age_follows_writer_and_clears_when_drained() {
2647        let (sink, mut rx) = crate::server::connection_egress();
2648        sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
2649            .unwrap();
2650        tokio::time::advance(Duration::from_millis(200)).await;
2651        sink.try_send(stream_frame(2, 1, 2, vec![b'y'; 100]))
2652            .unwrap();
2653        let writing = rx.try_recv().unwrap();
2654        writing.charge.as_ref().unwrap().taken_by_writer();
2655        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
2656        drop(writing);
2657        // Between frames the lock-free marker can still name the previous
2658        // head. The next writer take advances it without scanning the queue.
2659        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
2660        tokio::time::advance(Duration::from_millis(50)).await;
2661        let next = rx.try_recv().unwrap();
2662        next.charge.as_ref().unwrap().taken_by_writer();
2663        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(50)));
2664        assert!(
2665            sink.backlog().queued_bytes > 0,
2666            "in-flight bytes remain charged until written"
2667        );
2668        drop(next);
2669        assert_eq!(sink.backlog().queued_bytes, 0);
2670        assert_eq!(sink.backlog().oldest_age, None);
2671    }
2672
2673    #[tokio::test(start_paused = true)]
2674    async fn module_egress_response_full_client_waits_only_remaining_grace() {
2675        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2676            "backpressure-provider",
2677            ConnectionId::new(82),
2678            ConnectionId::new(83),
2679            2,
2680        )
2681        .await;
2682        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2683        // Exactly the limit, including its header, must be admitted to an empty
2684        // queue. Leave it unread to deterministically exhaust only byte capacity.
2685        router
2686            .route_for_connection(
2687                &ctx,
2688                Frame::build(
2689                    FrameType::Response,
2690                    Flags::new(false, Priority::Interactive, true),
2691                    routes[0].module_channel,
2692                    routes[0].module_epoch,
2693                    201,
2694                    vec![b'x'; limit - subc_protocol::HEADER_LEN],
2695                )
2696                .unwrap(),
2697            )
2698            .await
2699            .unwrap();
2700        assert_eq!(sink.backlog().queued_bytes, limit);
2701        assert_eq!(sink.backlog().queued_frames, 1);
2702        let ordinary = route_frame(
2703            FrameType::Response,
2704            routes[1].module_channel,
2705            routes[1].module_epoch,
2706            202,
2707        );
2708        assert!(queued_frame_bytes(&ordinary) < limit);
2709        // A fitting reply gets only the unused portion of the oldest frame's
2710        // 250 ms grace, not an immediate close or a fresh 250 ms wait.
2711        tokio::time::advance(Duration::from_millis(100)).await;
2712        let mut delivery = Box::pin(router.route_for_connection(&ctx, ordinary));
2713        assert!(poll_once(delivery.as_mut()).await.is_pending());
2714        assert!(close.try_recv().is_err());
2715        tokio::time::advance(Duration::from_millis(149)).await;
2716        assert!(poll_once(delivery.as_mut()).await.is_pending());
2717        assert!(close.try_recv().is_err());
2718        tokio::time::advance(Duration::from_millis(1)).await;
2719        assert!(matches!(
2720            poll_once(delivery.as_mut()).await,
2721            std::task::Poll::Ready(Ok(()))
2722        ));
2723        drop(delivery);
2724        let reason = close.try_recv().expect("the full client must close");
2725        assert!(reason
2726            .to_string()
2727            .contains("module_to_client_delivery_failed"));
2728        assert!(!reason.to_string().contains("frame_too_large"));
2729        assert_eq!(
2730            router.counters.snapshot()["client_egress_close_delivery_failed"],
2731            1
2732        );
2733        let admitted = rx.try_recv().expect("the budget-sized frame was admitted");
2734        assert_eq!(admitted.header.ty, FrameType::Response);
2735        assert_eq!(admitted.header.corr, 201);
2736        assert!(
2737            rx.try_recv().is_err(),
2738            "no oversized-frame ERROR is substituted for backpressure"
2739        );
2740    }
2741
2742    #[tokio::test(start_paused = true)]
2743    async fn oversized_reply_to_a_full_client_waits_only_remaining_grace() {
2744        let (router, sink, _rx, ctx, routes, mut close) = multi_route_client(
2745            "full-oversize-provider",
2746            ConnectionId::new(90),
2747            ConnectionId::new(91),
2748            2,
2749        )
2750        .await;
2751        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2752        // Fill the client's queue to its budget and leave it unread.
2753        router
2754            .route_for_connection(
2755                &ctx,
2756                Frame::build(
2757                    FrameType::Response,
2758                    Flags::new(false, Priority::Interactive, true),
2759                    routes[0].module_channel,
2760                    routes[0].module_epoch,
2761                    501,
2762                    vec![b'x'; limit - subc_protocol::HEADER_LEN],
2763                )
2764                .unwrap(),
2765            )
2766            .await
2767            .unwrap();
2768        assert_eq!(sink.backlog().queued_bytes, limit);
2769        let oversized = Frame::build(
2770            FrameType::Response,
2771            Flags::new(false, Priority::Interactive, true),
2772            routes[1].module_channel,
2773            routes[1].module_epoch,
2774            502,
2775            vec![b'y'; limit - subc_protocol::HEADER_LEN + 1],
2776        )
2777        .unwrap();
2778        // The replacement ERROR gets the same bounded burst grace as the
2779        // ordinary reply; oversized payloads themselves are never queued.
2780        tokio::time::advance(Duration::from_millis(80)).await;
2781        let mut delivery = Box::pin(router.route_for_connection(&ctx, oversized));
2782        assert!(poll_once(delivery.as_mut()).await.is_pending());
2783        assert!(close.try_recv().is_err());
2784        tokio::time::advance(Duration::from_millis(169)).await;
2785        assert!(poll_once(delivery.as_mut()).await.is_pending());
2786        assert!(close.try_recv().is_err());
2787        tokio::time::advance(Duration::from_millis(1)).await;
2788        assert!(matches!(
2789            poll_once(delivery.as_mut()).await,
2790            std::task::Poll::Ready(Ok(()))
2791        ));
2792        drop(delivery);
2793        let reason = close.try_recv().expect("the full client must close");
2794        assert!(reason
2795            .to_string()
2796            .contains("module_to_client_delivery_failed"));
2797    }
2798
2799    #[tokio::test]
2800    async fn oversized_reverse_request_errors_module_without_reaching_client() {
2801        let (router, _sink, mut rx, mut ctx, routes, mut close) = multi_route_client(
2802            "reverse-oversize-provider",
2803            ConnectionId::new(84),
2804            ConnectionId::new(85),
2805            2,
2806        )
2807        .await;
2808        let (module_tx, mut module_rx) = mpsc::channel(8);
2809        ctx.egress = FrameSink::new(module_tx);
2810        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2811        router
2812            .route_for_connection(
2813                &ctx,
2814                Frame::build(
2815                    FrameType::Request,
2816                    Flags::new(false, Priority::Interactive, false),
2817                    routes[0].module_channel,
2818                    routes[0].module_epoch,
2819                    301,
2820                    vec![b'r'; limit - subc_protocol::HEADER_LEN + 1],
2821                )
2822                .unwrap(),
2823            )
2824            .await
2825            .unwrap();
2826        assert!(
2827            rx.try_recv().is_err(),
2828            "the reverse request never reaches the client"
2829        );
2830        let error = module_rx.try_recv().expect("the module gets the refusal");
2831        assert_frame_size_error(
2832            &error,
2833            routes[0].module_channel,
2834            routes[0].module_epoch,
2835            301,
2836            limit + 1,
2837        );
2838        assert!(module_rx.try_recv().is_err());
2839        for (index, route) in routes.iter().enumerate() {
2840            module_route_for_test(&router, &ctx, route);
2841            router
2842                .route_for_connection(
2843                    &ctx,
2844                    route_frame(
2845                        FrameType::Response,
2846                        route.module_channel,
2847                        route.module_epoch,
2848                        302 + index as u64,
2849                    ),
2850                )
2851                .await
2852                .unwrap();
2853            let reply = rx.try_recv().expect("both routes still deliver");
2854            assert_eq!(reply.header.ty, FrameType::Response);
2855            assert_eq!(reply.header.channel, route.client_channel);
2856        }
2857        assert!(close.try_recv().is_err());
2858        assert_eq!(
2859            router.counters.snapshot()["client_egress_close_delivery_failed"],
2860            0
2861        );
2862    }
2863
2864    #[tokio::test]
2865    async fn oversized_push_closes_client_with_precise_warn() {
2866        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
2867        let (router, _sink, mut rx, ctx, routes, mut close) = multi_route_client(
2868            "push-oversize-provider",
2869            ConnectionId::new(86),
2870            ConnectionId::new(87),
2871            2,
2872        )
2873        .await;
2874        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2875        router
2876            .route_for_connection(
2877                &ctx,
2878                Frame::build(
2879                    FrameType::Push,
2880                    Flags::new(false, Priority::Interactive, false),
2881                    routes[0].module_channel,
2882                    routes[0].module_epoch,
2883                    0,
2884                    vec![b'p'; limit - subc_protocol::HEADER_LEN + 1],
2885                )
2886                .unwrap(),
2887            )
2888            .await
2889            .unwrap();
2890        let reason = close
2891            .try_recv()
2892            .expect("unsolicited oversize still closes the client");
2893        assert!(reason
2894            .to_string()
2895            .contains("frame_too_large: oversized push"));
2896        assert!(
2897            rx.try_recv().is_err(),
2898            "no part of the oversized push is forwarded"
2899        );
2900        let captured = test_log::captured_logs(&logs);
2901        let warnings: Vec<_> = captured
2902            .lines()
2903            .filter(|line| line.contains("WARN"))
2904            .collect();
2905        assert_eq!(warnings.len(), 1, "{captured}");
2906        let warning = warnings[0];
2907        for expected in [
2908            "oversized push".to_string(),
2909            "module_id=\"push-oversize-provider\"".to_string(),
2910            "principals=direct".to_string(),
2911            format!("client_channel={}", routes[0].client_channel),
2912            format!("size={}", limit + 1),
2913            format!("limit={limit}"),
2914        ] {
2915            assert!(warning.contains(&expected), "missing {expected}: {warning}");
2916        }
2917        assert_eq!(
2918            router.counters.snapshot()["client_egress_close_delivery_failed"],
2919            1
2920        );
2921    }
2922
2923    #[tokio::test]
2924    async fn oversized_stream_and_error_frames_end_request_even_with_queued_traffic() {
2925        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2926            "stream-oversize-provider",
2927            ConnectionId::new(88),
2928            ConnectionId::new(89),
2929            2,
2930        )
2931        .await;
2932        let binding = module_route_for_test(&router, &ctx, &routes[0]);
2933        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2934        for ty in [
2935            FrameType::StreamData,
2936            FrameType::StreamEnd,
2937            FrameType::Error,
2938        ] {
2939            binding.flow.acquire_tagged(401, false).await.unwrap();
2940            sink.try_send(route_frame(
2941                FrameType::Response,
2942                routes[1].client_channel,
2943                routes[1].client_epoch,
2944                402,
2945            ))
2946            .unwrap();
2947            router
2948                .route_for_connection(
2949                    &ctx,
2950                    Frame::build(
2951                        ty,
2952                        Flags::new(false, Priority::Interactive, false),
2953                        routes[0].module_channel,
2954                        routes[0].module_epoch,
2955                        401,
2956                        vec![b's'; limit - subc_protocol::HEADER_LEN + 1],
2957                    )
2958                    .unwrap(),
2959                )
2960                .await
2961                .unwrap();
2962            assert!(close.try_recv().is_err());
2963            assert_eq!(rx.try_recv().unwrap().header.corr, 402);
2964            let error = rx
2965                .try_recv()
2966                .expect("every correlated oversize frame becomes ERROR");
2967            assert_frame_size_error(
2968                &error,
2969                routes[0].client_channel,
2970                routes[0].client_epoch,
2971                401,
2972                limit + 1,
2973            );
2974            assert_eq!(binding.flow.drain_in_flight(), 0, "{ty:?} ends the request");
2975            assert!(rx.try_recv().is_err());
2976        }
2977    }
2978
2979    /// An awaited send (a control or error reply) is never refused by the byte
2980    /// budget: it parks until the writer frees bytes. If a release could miss a
2981    /// parked sender, that reply would hang for the connection's lifetime, so
2982    /// this pins the wakeup: the send stays parked while the queue is full and
2983    /// completes as soon as one frame leaves it.
2984    #[tokio::test]
2985    async fn awaited_send_parked_behind_a_full_byte_budget_wakes_when_bytes_free() {
2986        let (tx, mut rx) = mpsc::channel(1024);
2987        let sink = FrameSink::with_byte_budget(tx, 2_000);
2988        let mut queued = 0u64;
2989        while sink
2990            .try_send(stream_frame(7, 1, queued, vec![b'x'; 200]))
2991            .is_ok()
2992        {
2993            queued += 1;
2994        }
2995        assert!(queued > 0, "the budget admitted nothing");
2996
2997        let parked = tokio::spawn({
2998            let sink = sink.clone();
2999            async move { sink.send(stream_frame(0, 0, 999, vec![b'r'; 200])).await }
3000        });
3001        tokio::time::sleep(Duration::from_millis(100)).await;
3002        assert!(
3003            !parked.is_finished(),
3004            "the awaited send must wait while the byte budget is full"
3005        );
3006
3007        // The writer takes one frame and drops it, releasing its bytes.
3008        drop(rx.recv().await.expect("a queued frame"));
3009        tokio::time::timeout(Duration::from_secs(2), parked)
3010            .await
3011            .expect("the parked send was never woken after bytes were freed")
3012            .unwrap()
3013            .unwrap();
3014    }
3015
3016    /// A client multiplexing several token streams pauses its reader while the
3017    /// module keeps producing small frames. Far more frames than the old
3018    /// 64-frame queue allowed, but far fewer bytes than the budget, must all be
3019    /// held without closing the connection and delivered in order afterwards.
3020    #[tokio::test]
3021    async fn paused_client_reader_keeps_connection_through_small_frame_burst() {
3022        const ROUTES: usize = 4;
3023        const FRAMES_PER_ROUTE: usize = 250;
3024        let (router, client_sink, mut client_rx, module_ctx, routes, mut close_receiver) =
3025            multi_route_client(
3026                "burst-provider",
3027                ConnectionId::new(60),
3028                ConnectionId::new(61),
3029                ROUTES,
3030            )
3031            .await;
3032
3033        // The reader is paused: nothing is received until every frame is sent.
3034        for seq in 0..FRAMES_PER_ROUTE as u64 {
3035            for (index, route) in routes.iter().enumerate() {
3036                let body = format!("route-{index}-token-{seq:05}-{}", "t".repeat(170));
3037                router
3038                    .route_for_connection(
3039                        &module_ctx,
3040                        stream_frame(
3041                            route.module_channel,
3042                            route.module_epoch,
3043                            seq,
3044                            body.into_bytes(),
3045                        ),
3046                    )
3047                    .await
3048                    .unwrap();
3049            }
3050        }
3051
3052        let backlog = client_sink.backlog();
3053        assert_eq!(backlog.queued_frames, ROUTES * FRAMES_PER_ROUTE);
3054        assert!(backlog.queued_bytes < crate::server::CONNECTION_EGRESS_BYTE_BUDGET);
3055        assert!(
3056            close_receiver.try_recv().is_err(),
3057            "a paused reader under the byte budget must not be closed"
3058        );
3059        assert_eq!(
3060            router.counters.snapshot()["client_egress_close_delivery_failed"],
3061            0
3062        );
3063
3064        // The reader resumes: every frame arrives, in order within each route.
3065        let mut next_seq = vec![0u64; ROUTES];
3066        for _ in 0..ROUTES * FRAMES_PER_ROUTE {
3067            let frame = client_rx.try_recv().expect("every queued frame arrives");
3068            let index = routes
3069                .iter()
3070                .position(|route| route.client_channel == frame.header.channel)
3071                .expect("frame arrives on one of the bound client channels");
3072            assert_eq!(frame.header.corr, next_seq[index], "route {index} order");
3073            let expected_prefix = format!("route-{index}-token-{:05}-", next_seq[index]);
3074            assert!(frame.body.starts_with(expected_prefix.as_bytes()));
3075            next_seq[index] += 1;
3076        }
3077        assert!(client_rx.try_recv().is_err());
3078        assert_eq!(next_seq, vec![FRAMES_PER_ROUTE as u64; ROUTES]);
3079        assert_eq!(client_sink.backlog().queued_bytes, 0);
3080    }
3081
3082    /// A client that never reads exhausts its byte budget and the oldest
3083    /// frame's 250 ms grace, then closes with one header-only diagnosis.
3084    #[tokio::test(start_paused = true)]
3085    async fn never_reading_client_is_closed_at_byte_budget_with_warn_diagnosis() {
3086        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3087        const BODY: usize = 16 * 1024;
3088        let (router, client_sink, _client_rx, module_ctx, routes, mut close_receiver) =
3089            multi_route_client(
3090                "stuck-reader-provider",
3091                ConnectionId::new(70),
3092                ConnectionId::new(71),
3093                2,
3094            )
3095            .await;
3096
3097        let mut admitted = 0usize;
3098        let mut sent = 0u64;
3099        let started = tokio::time::Instant::now();
3100        while router.counters.snapshot()["client_egress_close_delivery_failed"] == 0 {
3101            assert!(sent < 1_000, "the byte budget never refused a frame");
3102            let route = &routes[(sent % 2) as usize];
3103            router
3104                .route_for_connection(
3105                    &module_ctx,
3106                    stream_frame(
3107                        route.module_channel,
3108                        route.module_epoch,
3109                        sent,
3110                        vec![b'z'; BODY],
3111                    ),
3112                )
3113                .await
3114                .unwrap();
3115            sent += 1;
3116            admitted = client_sink.backlog().queued_frames;
3117        }
3118        assert_eq!(
3119            started.elapsed(),
3120            crate::server::CONNECTION_EGRESS_STALL_GRACE
3121        );
3122        // The budget, not the frame-count backstop, did the refusing.
3123        let frame_bytes = subc_protocol::HEADER_LEN + BODY;
3124        assert_eq!(
3125            admitted,
3126            crate::server::CONNECTION_EGRESS_BYTE_BUDGET / frame_bytes
3127        );
3128        let reason = close_receiver
3129            .try_recv()
3130            .expect("the client connection must be asked to close");
3131        assert!(reason
3132            .to_string()
3133            .contains("module_to_client_delivery_failed"));
3134
3135        // A second refused frame for the same connection adds no second WARN.
3136        router
3137            .route_for_connection(
3138                &module_ctx,
3139                stream_frame(
3140                    routes[0].module_channel,
3141                    routes[0].module_epoch,
3142                    sent,
3143                    vec![b'z'; BODY],
3144                ),
3145            )
3146            .await
3147            .unwrap();
3148
3149        let captured = test_log::captured_logs(&logs);
3150        let warn_lines = captured
3151            .lines()
3152            .filter(|line| {
3153                line.contains("closing client connection: its egress queue could not take a frame")
3154            })
3155            .collect::<Vec<_>>();
3156        assert_eq!(warn_lines.len(), 1, "exactly one WARN, got: {captured}");
3157        let line = warn_lines[0];
3158        assert!(line.contains("WARN"), "{line}");
3159        assert!(line.contains("connection_id=71"), "{line}");
3160        assert!(
3161            line.contains("module_id=\"stuck-reader-provider\""),
3162            "{line}"
3163        );
3164        assert!(line.contains("client_channel="), "{line}");
3165        assert!(line.contains("principals=direct"), "{line}");
3166        let queued_bytes: usize = line
3167            .split("queued_bytes=")
3168            .nth(1)
3169            .and_then(|rest| rest.split_whitespace().next())
3170            .and_then(|value| value.parse().ok())
3171            .expect("queued_bytes is logged");
3172        assert_eq!(queued_bytes, admitted * frame_bytes);
3173        assert!(
3174            line.contains(&format!("queued_frames={admitted}")),
3175            "{line}"
3176        );
3177        assert!(line.contains("oldest_queued_ms="), "{line}");
3178        assert!(line.contains("oldest_queued_ms=250"), "{line}");
3179        assert!(line.contains("frame_type=Some(StreamData)"), "{line}");
3180        assert!(
3181            line.contains(&format!(
3182                "frame_flags=Some({})",
3183                Flags::new(false, Priority::Interactive, false).0
3184            )),
3185            "{line}"
3186        );
3187        assert!(
3188            line.contains("close_cause=\"waited_250_ms_without_room\""),
3189            "{line}"
3190        );
3191        assert!(
3192            !line.contains(&"z".repeat(20)),
3193            "body bytes must never be logged: {line}"
3194        );
3195    }
3196
3197    fn route_frame(ty: FrameType, channel: u16, epoch: u32, corr: u64) -> Frame {
3198        Frame::build(
3199            ty,
3200            Flags::new(false, Priority::Interactive, false),
3201            channel,
3202            epoch,
3203            corr,
3204            if ty == FrameType::Request || ty == FrameType::Response {
3205                b"route-body".to_vec()
3206            } else {
3207                Vec::new()
3208            },
3209        )
3210        .unwrap()
3211    }
3212
3213    type DynamicRouteFixture = (
3214        Router,
3215        Arc<ForwardingTable>,
3216        RouteCtx,
3217        mpsc::Receiver<crate::router::OutboundFrame>,
3218        RouteCtx,
3219        mpsc::Receiver<crate::router::OutboundFrame>,
3220        mpsc::Receiver<crate::router::OutboundFrame>,
3221        crate::forwarding::PendingRouteBindRelay,
3222    );
3223
3224    fn dynamic_route_fixture(commit: bool) -> DynamicRouteFixture {
3225        let forwarding = Arc::new(ForwardingTable::default());
3226        let control = Arc::new(crate::ControlHandler::with_forwarding(
3227            Arc::new(crate::Registry::default()),
3228            Arc::clone(&forwarding),
3229        ));
3230        let router = Router::with_control_handler(control);
3231        let module_connection = ConnectionId::new(500);
3232        let client_connection = ConnectionId::new(501);
3233        let (module_tx, module_rx) = mpsc::channel(8);
3234        forwarding
3235            .register_module_connection(
3236                module_connection,
3237                "epoch-router".into(),
3238                2,
3239                Concurrency::ModuleManaged,
3240                FrameSink::new(module_tx),
3241            )
3242            .unwrap();
3243        let (client_tx, client_rx) = mpsc::channel(8);
3244        let client_sink = FrameSink::new(client_tx);
3245        let pending = forwarding
3246            .begin_route_bind_relay_for_test(
3247                client_connection,
3248                client_sink.clone(),
3249                700,
3250                "epoch-router",
3251            )
3252            .unwrap();
3253        if commit {
3254            forwarding
3255                .complete_pending_relay(
3256                    module_connection,
3257                    pending.corr,
3258                    RouteBindRelayOutcome::Accepted,
3259                )
3260                .unwrap();
3261        }
3262        let (module_egress_tx, module_egress_rx) = mpsc::channel(8);
3263        (
3264            router,
3265            forwarding,
3266            RouteCtx {
3267                connection_id: client_connection,
3268                egress: client_sink,
3269            },
3270            client_rx,
3271            RouteCtx {
3272                connection_id: module_connection,
3273                egress: FrameSink::new(module_egress_tx),
3274            },
3275            module_egress_rx,
3276            module_rx,
3277            pending,
3278        )
3279    }
3280
3281    #[tokio::test]
3282    async fn route_epochs_validate_both_directions_and_rewrite_to_peer_handle() {
3283        let (
3284            router,
3285            _forwarding,
3286            client_ctx,
3287            mut client_rx,
3288            module_ctx,
3289            _module_egress_rx,
3290            mut module_rx,
3291            pending,
3292        ) = dynamic_route_fixture(true);
3293        let route_open = client_rx.recv().await.unwrap();
3294        assert_eq!(route_open.header.corr, 700);
3295
3296        router
3297            .route_for_connection(
3298                &client_ctx,
3299                route_frame(
3300                    FrameType::Request,
3301                    pending.client_channel,
3302                    pending.client_epoch,
3303                    701,
3304                ),
3305            )
3306            .await
3307            .unwrap();
3308        let forwarded = module_rx.recv().await.unwrap();
3309        assert_eq!(forwarded.header.channel, pending.module_channel);
3310        assert_eq!(forwarded.header.epoch, pending.module_epoch);
3311
3312        router
3313            .route_for_connection(
3314                &module_ctx,
3315                route_frame(
3316                    FrameType::Response,
3317                    pending.module_channel,
3318                    pending.module_epoch,
3319                    701,
3320                ),
3321            )
3322            .await
3323            .unwrap();
3324        let delivered = client_rx.recv().await.unwrap();
3325        assert_eq!(delivered.header.channel, pending.client_channel);
3326        assert_eq!(delivered.header.epoch, pending.client_epoch);
3327
3328        router
3329            .route_for_connection(
3330                &client_ctx,
3331                route_frame(
3332                    FrameType::Request,
3333                    pending.client_channel,
3334                    pending.client_epoch + 1,
3335                    702,
3336                ),
3337            )
3338            .await
3339            .unwrap();
3340        router
3341            .route_for_connection(
3342                &module_ctx,
3343                route_frame(
3344                    FrameType::Response,
3345                    pending.module_channel,
3346                    pending.module_epoch + 1,
3347                    703,
3348                ),
3349            )
3350            .await
3351            .unwrap();
3352        let stale_error = client_rx.recv().await.unwrap();
3353        assert_eq!(stale_error.header.ty, FrameType::Error);
3354        assert_eq!(stale_error.header.channel, pending.client_channel);
3355        assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
3356        assert_eq!(stale_error.header.corr, 702);
3357        let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
3358        assert_eq!(body.code, "stale_route_epoch");
3359        assert!(module_rx.try_recv().is_err());
3360        assert!(client_rx.try_recv().is_err());
3361        let counters = router.counters.snapshot();
3362        assert_eq!(counters["client_frames_dropped_stale_route"], 1);
3363        assert_eq!(counters["module_frames_dropped_no_route"], 1);
3364    }
3365
3366    #[tokio::test]
3367    async fn accepted_route_publishes_route_open_before_immediate_reverse_request() {
3368        let (
3369            router,
3370            _,
3371            _client_ctx,
3372            mut client_rx,
3373            module_ctx,
3374            _module_egress_rx,
3375            _module_rx,
3376            pending,
3377        ) = dynamic_route_fixture(true);
3378        router
3379            .route_for_connection(
3380                &module_ctx,
3381                route_frame(
3382                    FrameType::Request,
3383                    pending.module_channel,
3384                    pending.module_epoch,
3385                    800,
3386                ),
3387            )
3388            .await
3389            .unwrap();
3390
3391        let first = client_rx.recv().await.unwrap();
3392        let second = client_rx.recv().await.unwrap();
3393        assert_eq!(first.header.channel, 0);
3394        assert_eq!(first.header.corr, 700);
3395        assert_eq!(second.header.channel, pending.client_channel);
3396        assert_eq!(second.header.epoch, pending.client_epoch);
3397        assert_eq!(second.header.corr, 800);
3398    }
3399
3400    #[tokio::test]
3401    async fn reserved_slot_ingress_errors_only_matching_client_requests() {
3402        let (
3403            router,
3404            _forwarding,
3405            client_ctx,
3406            mut client_rx,
3407            _module_ctx,
3408            _module_egress_rx,
3409            mut module_rx,
3410            pending,
3411        ) = dynamic_route_fixture(false);
3412        router
3413            .route_for_connection(
3414                &client_ctx,
3415                route_frame(
3416                    FrameType::Request,
3417                    pending.client_channel,
3418                    pending.client_epoch,
3419                    900,
3420                ),
3421            )
3422            .await
3423            .unwrap();
3424        let error = client_rx.recv().await.unwrap();
3425        assert_eq!(error.header.ty, FrameType::Error);
3426        assert_eq!(error.header.channel, pending.client_channel);
3427        assert_eq!(error.header.epoch, pending.client_epoch);
3428        assert_eq!(error.header.corr, 900);
3429
3430        router
3431            .route_for_connection(
3432                &client_ctx,
3433                route_frame(
3434                    FrameType::Response,
3435                    pending.client_channel,
3436                    pending.client_epoch,
3437                    901,
3438                ),
3439            )
3440            .await
3441            .unwrap();
3442        router
3443            .route_for_connection(
3444                &client_ctx,
3445                route_frame(
3446                    FrameType::Request,
3447                    pending.client_channel,
3448                    pending.client_epoch + 1,
3449                    902,
3450                ),
3451            )
3452            .await
3453            .unwrap();
3454        let stale_error = client_rx.recv().await.unwrap();
3455        assert_eq!(stale_error.header.ty, FrameType::Error);
3456        assert_eq!(stale_error.header.channel, pending.client_channel);
3457        assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
3458        assert_eq!(stale_error.header.corr, 902);
3459        let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
3460        assert_eq!(body.code, "stale_route_epoch");
3461        assert!(module_rx.try_recv().is_err());
3462        let counters = router.counters.snapshot();
3463        assert_eq!(counters["client_frames_dropped_stale_route"], 1);
3464        assert_eq!(counters["module_frames_dropped_no_route"], 0);
3465    }
3466
3467    #[tokio::test]
3468    async fn dropped_module_route_goodbye_increments_counter() {
3469        let (
3470            router,
3471            _forwarding,
3472            client_ctx,
3473            mut client_rx,
3474            _module_ctx,
3475            _module_egress_rx,
3476            mut module_rx,
3477            pending,
3478        ) = dynamic_route_fixture(true);
3479        let _ = client_rx.recv().await;
3480        module_rx.close();
3481
3482        router
3483            .route_for_connection(
3484                &client_ctx,
3485                route_frame(
3486                    FrameType::Goodbye,
3487                    pending.client_channel,
3488                    pending.client_epoch,
3489                    999,
3490                ),
3491            )
3492            .await
3493            .unwrap();
3494
3495        let counters = router.counters.snapshot();
3496        assert_eq!(counters["goodbye_relay_module_dropped"], 1);
3497        assert_eq!(
3498            counters["goodbye_relay_module_dropped_by_module"],
3499            serde_json::json!({ "epoch-router": 1 })
3500        );
3501        assert_eq!(counters["route_released_epoch_fenced"], 1);
3502    }
3503
3504    #[tokio::test]
3505    async fn module_request_on_stale_epoch_receives_stale_route_epoch() {
3506        let (
3507            router,
3508            _forwarding,
3509            _client_ctx,
3510            _client_rx,
3511            module_ctx,
3512            mut module_egress_rx,
3513            mut module_rx,
3514            pending,
3515        ) = dynamic_route_fixture(true);
3516
3517        router
3518            .route_for_connection(
3519                &module_ctx,
3520                route_frame(
3521                    FrameType::Request,
3522                    pending.module_channel,
3523                    pending.module_epoch + 1,
3524                    1_000,
3525                ),
3526            )
3527            .await
3528            .unwrap();
3529
3530        let error = module_egress_rx.try_recv().unwrap();
3531        assert_eq!(error.header.ty, FrameType::Error);
3532        assert_eq!(error.header.channel, pending.module_channel);
3533        assert_eq!(error.header.epoch, pending.module_epoch + 1);
3534        assert_eq!(error.header.corr, 1_000);
3535        let body: ErrorBody = serde_json::from_slice(&error.body).unwrap();
3536        assert_eq!(body.code, "stale_route_epoch");
3537        assert!(module_rx.try_recv().is_err());
3538        let counters = router.counters.snapshot();
3539        assert_eq!(counters["module_requests_dropped_stale_route"], 1);
3540        assert_eq!(counters["module_frames_dropped_no_route"], 0);
3541    }
3542
3543    #[tokio::test]
3544    async fn module_request_on_reserved_or_absent_route_receives_unknown_channel() {
3545        let (
3546            reserved_router,
3547            _forwarding,
3548            _client_ctx,
3549            _client_rx,
3550            reserved_module_ctx,
3551            mut reserved_module_egress_rx,
3552            _module_rx,
3553            reserved,
3554        ) = dynamic_route_fixture(false);
3555        reserved_router
3556            .route_for_connection(
3557                &reserved_module_ctx,
3558                route_frame(
3559                    FrameType::Request,
3560                    reserved.module_channel,
3561                    reserved.module_epoch,
3562                    1_001,
3563                ),
3564            )
3565            .await
3566            .unwrap();
3567        let reserved_error = reserved_module_egress_rx.try_recv().unwrap();
3568        let reserved_body: ErrorBody = serde_json::from_slice(&reserved_error.body).unwrap();
3569        assert_eq!(reserved_error.header.ty, FrameType::Error);
3570        assert_eq!(reserved_error.header.channel, reserved.module_channel);
3571        assert_eq!(reserved_error.header.epoch, reserved.module_epoch);
3572        assert_eq!(reserved_error.header.corr, 1_001);
3573        assert_eq!(reserved_body.code, "unknown_channel");
3574        assert_eq!(
3575            reserved_router.counters.snapshot()["module_requests_dropped_stale_route"],
3576            1
3577        );
3578
3579        let (
3580            absent_router,
3581            _forwarding,
3582            _client_ctx,
3583            _client_rx,
3584            absent_module_ctx,
3585            mut absent_module_egress_rx,
3586            _module_rx,
3587            absent,
3588        ) = dynamic_route_fixture(false);
3589        absent_router
3590            .route_for_connection(
3591                &absent_module_ctx,
3592                route_frame(
3593                    FrameType::Request,
3594                    absent.module_channel + 1,
3595                    absent.module_epoch,
3596                    1_002,
3597                ),
3598            )
3599            .await
3600            .unwrap();
3601        let absent_error = absent_module_egress_rx.try_recv().unwrap();
3602        let absent_body: ErrorBody = serde_json::from_slice(&absent_error.body).unwrap();
3603        assert_eq!(absent_error.header.ty, FrameType::Error);
3604        assert_eq!(absent_error.header.channel, absent.module_channel + 1);
3605        assert_eq!(absent_error.header.epoch, absent.module_epoch);
3606        assert_eq!(absent_error.header.corr, 1_002);
3607        assert_eq!(absent_body.code, "unknown_channel");
3608        assert_eq!(
3609            absent_router.counters.snapshot()["module_requests_dropped_stale_route"],
3610            1
3611        );
3612    }
3613
3614    #[tokio::test]
3615    async fn non_request_module_frame_on_dead_route_is_counted_without_error() {
3616        let (
3617            router,
3618            forwarding,
3619            client_ctx,
3620            mut client_rx,
3621            module_ctx,
3622            mut module_egress_rx,
3623            mut module_rx,
3624            pending,
3625        ) = dynamic_route_fixture(true);
3626        let (other_module_tx, _other_module_rx) = mpsc::channel(8);
3627        forwarding
3628            .register_module_connection(
3629                ConnectionId::new(502),
3630                "other-module".into(),
3631                2,
3632                Concurrency::ModuleManaged,
3633                FrameSink::new(other_module_tx),
3634            )
3635            .unwrap();
3636        let _ = client_rx.recv().await.unwrap();
3637
3638        router
3639            .route_for_connection(
3640                &client_ctx,
3641                route_frame(
3642                    FrameType::Goodbye,
3643                    pending.client_channel,
3644                    pending.client_epoch,
3645                    1_003,
3646                ),
3647            )
3648            .await
3649            .unwrap();
3650        let _ = module_rx.recv().await.unwrap();
3651
3652        router
3653            .route_for_connection(
3654                &module_ctx,
3655                route_frame(
3656                    FrameType::StreamData,
3657                    pending.module_channel,
3658                    pending.module_epoch,
3659                    1_004,
3660                ),
3661            )
3662            .await
3663            .unwrap();
3664
3665        // No ERROR goes back for a non-request frame. The one reply is the
3666        // route GOODBYE telling the module to let go of the released route.
3667        let reply = module_egress_rx.try_recv().unwrap();
3668        assert_eq!(reply.header.ty, FrameType::Goodbye);
3669        assert!(module_egress_rx.try_recv().is_err());
3670        let counters = router.counters.snapshot();
3671        assert_eq!(counters["module_frames_dropped_no_route"], 1);
3672        assert_eq!(
3673            counters["module_frames_dropped_no_route_by_module"],
3674            serde_json::json!({ "epoch-router": 1 })
3675        );
3676        assert_eq!(counters["module_requests_dropped_stale_route"], 0);
3677    }
3678
3679    /// Drain whatever the module connection's queue holds right now, the way a
3680    /// module's reader would before it stalls.
3681    fn drain_now(rx: &mut mpsc::Receiver<OutboundFrame>) {
3682        while rx.try_recv().is_ok() {}
3683    }
3684
3685    /// A module that stops reading for a moment when a client closes one of
3686    /// its routes still learns the route is gone: the GOODBYE its full egress
3687    /// queue refused is delivered as soon as it reads again, rather than
3688    /// dropped, which would leave the module holding the route for the rest of
3689    /// its connection.
3690    #[tokio::test]
3691    async fn route_goodbye_refused_by_stalled_module_is_delivered_when_it_resumes_reading() {
3692        const BUDGET: usize = 4_096;
3693        let forwarding = Arc::new(ForwardingTable::default());
3694        let control = Arc::new(ControlHandler::with_forwarding(
3695            Arc::new(Registry::default()),
3696            Arc::clone(&forwarding),
3697        ));
3698        let router = Router::with_control_handler(control);
3699        let module_connection = ConnectionId::new(80);
3700        let client_connection = ConnectionId::new(81);
3701        let (module_tx, mut module_rx) = mpsc::channel(64);
3702        let module_sink = FrameSink::with_byte_budget(module_tx, BUDGET);
3703        forwarding
3704            .register_module_connection(
3705                module_connection,
3706                "stalled-provider".into(),
3707                2,
3708                Concurrency::ModuleManaged,
3709                module_sink.clone(),
3710            )
3711            .unwrap();
3712        let (client_tx, mut client_rx) = mpsc::channel(8);
3713        let client_sink = FrameSink::new(client_tx);
3714        let pending = forwarding
3715            .begin_route_bind_relay_for_test(
3716                client_connection,
3717                client_sink.clone(),
3718                1_100,
3719                "stalled-provider",
3720            )
3721            .unwrap();
3722        forwarding
3723            .complete_pending_relay(
3724                module_connection,
3725                pending.corr,
3726                RouteBindRelayOutcome::Accepted,
3727            )
3728            .unwrap();
3729        let _ = client_rx.recv().await.unwrap();
3730        drain_now(&mut module_rx);
3731
3732        // The module stalls: its queue holds more than the byte budget, so the
3733        // queue refuses anything further.
3734        module_sink
3735            .try_send(stream_frame(9, 1, 0, vec![b'f'; BUDGET]))
3736            .unwrap();
3737        assert!(module_sink
3738            .try_send(stream_frame(9, 1, 1, Vec::new()))
3739            .is_err());
3740
3741        let client_ctx = RouteCtx {
3742            connection_id: client_connection,
3743            egress: client_sink,
3744        };
3745        router
3746            .route_for_connection(
3747                &client_ctx,
3748                route_frame(
3749                    FrameType::Goodbye,
3750                    pending.client_channel,
3751                    pending.client_epoch,
3752                    1_101,
3753                ),
3754            )
3755            .await
3756            .unwrap();
3757        tokio::task::yield_now().await;
3758        assert_eq!(
3759            router.counters.snapshot()["goodbye_relay_module_dropped"],
3760            0,
3761            "a GOODBYE refused by a momentarily full module queue must not be dropped"
3762        );
3763
3764        // The module reads again: the filler comes off, then the GOODBYE.
3765        let filler = module_rx.recv().await.unwrap();
3766        assert_eq!(filler.header.ty, FrameType::StreamData);
3767        drop(filler);
3768        let goodbye = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
3769            .await
3770            .expect("the refused GOODBYE must be delivered once the module frees room")
3771            .unwrap();
3772        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
3773        assert_eq!(goodbye.header.channel, pending.module_channel);
3774        assert_eq!(goodbye.header.epoch, pending.module_epoch);
3775        assert_eq!(
3776            router.counters.snapshot()["goodbye_relay_module_dropped"],
3777            0
3778        );
3779    }
3780
3781    /// A module still sending on a route the daemon released is told, with a
3782    /// route GOODBYE for exactly the (channel, epoch) it sent on. The same
3783    /// happens for a stale epoch on a channel whose route moved on and for a
3784    /// channel that never had a route; only the first is counted as traffic on
3785    /// a released route.
3786    #[tokio::test]
3787    async fn module_frame_on_route_the_daemon_does_not_hold_is_answered_with_goodbye() {
3788        let (
3789            router,
3790            _forwarding,
3791            client_ctx,
3792            mut client_rx,
3793            module_ctx,
3794            mut module_egress_rx,
3795            mut module_rx,
3796            pending,
3797        ) = dynamic_route_fixture(true);
3798        let _ = client_rx.recv().await.unwrap();
3799        router
3800            .route_for_connection(
3801                &client_ctx,
3802                route_frame(
3803                    FrameType::Goodbye,
3804                    pending.client_channel,
3805                    pending.client_epoch,
3806                    1_200,
3807                ),
3808            )
3809            .await
3810            .unwrap();
3811        drain_now(&mut module_rx);
3812
3813        let cases = [
3814            // Released route: the daemon allocated this (channel, epoch).
3815            (pending.module_channel, pending.module_epoch),
3816            // A channel the daemon never allocated on this connection.
3817            (pending.module_channel + 1, 1),
3818        ];
3819        for (channel, epoch) in cases {
3820            router
3821                .route_for_connection(
3822                    &module_ctx,
3823                    route_frame(FrameType::StreamData, channel, epoch, 1_201),
3824                )
3825                .await
3826                .unwrap();
3827            let reply = module_egress_rx
3828                .try_recv()
3829                .expect("a frame on a route the daemon does not hold is answered");
3830            assert_eq!(reply.header.ty, FrameType::Goodbye);
3831            assert_eq!(reply.header.channel, channel);
3832            assert_eq!(reply.header.epoch, epoch);
3833            assert_eq!(reply.header.corr, 0);
3834            assert!(module_egress_rx.try_recv().is_err());
3835        }
3836
3837        // A GOODBYE from the module for a route the daemon already released
3838        // is not answered: the module is letting go already.
3839        router
3840            .route_for_connection(
3841                &module_ctx,
3842                route_frame(FrameType::Goodbye, pending.module_channel + 2, 1, 0),
3843            )
3844            .await
3845            .unwrap();
3846        assert!(module_egress_rx.try_recv().is_err());
3847
3848        let counters = router.counters.snapshot();
3849        assert_eq!(counters["module_frames_dropped_no_route"], 3);
3850        assert_eq!(counters["module_frames_dropped_released_route"], 1);
3851        assert_eq!(
3852            counters["module_frames_dropped_released_route_by_module"],
3853            serde_json::json!({ "epoch-router": 1 })
3854        );
3855        assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
3856    }
3857
3858    /// A chatty orphan (a streaming module still producing on a route it
3859    /// missed the GOODBYE for) is answered once per interval, not once per
3860    /// frame, and answered again once the interval has passed.
3861    #[tokio::test(start_paused = true)]
3862    async fn burst_of_orphan_module_frames_is_answered_once_per_interval() {
3863        let (
3864            router,
3865            _forwarding,
3866            client_ctx,
3867            mut client_rx,
3868            module_ctx,
3869            mut module_egress_rx,
3870            mut module_rx,
3871            pending,
3872        ) = dynamic_route_fixture(true);
3873        let _ = client_rx.recv().await.unwrap();
3874        router
3875            .route_for_connection(
3876                &client_ctx,
3877                route_frame(
3878                    FrameType::Goodbye,
3879                    pending.client_channel,
3880                    pending.client_epoch,
3881                    1_300,
3882                ),
3883            )
3884            .await
3885            .unwrap();
3886        drain_now(&mut module_rx);
3887
3888        let send_burst = |corr: u64| {
3889            route_frame(
3890                FrameType::StreamData,
3891                pending.module_channel,
3892                pending.module_epoch,
3893                corr,
3894            )
3895        };
3896        for corr in 0..20 {
3897            router
3898                .route_for_connection(&module_ctx, send_burst(corr))
3899                .await
3900                .unwrap();
3901        }
3902        let mut replies = 0;
3903        while let Ok(reply) = module_egress_rx.try_recv() {
3904            assert_eq!(reply.header.ty, FrameType::Goodbye);
3905            replies += 1;
3906        }
3907        assert_eq!(replies, 1, "a burst within the interval gets one GOODBYE");
3908
3909        tokio::time::advance(ORPHAN_ROUTE_GOODBYE_INTERVAL).await;
3910        router
3911            .route_for_connection(&module_ctx, send_burst(20))
3912            .await
3913            .unwrap();
3914        let retry = module_egress_rx
3915            .try_recv()
3916            .expect("the first orphan frame after the interval is answered again");
3917        assert_eq!(retry.header.channel, pending.module_channel);
3918        assert_eq!(retry.header.epoch, pending.module_epoch);
3919
3920        let counters = router.counters.snapshot();
3921        assert_eq!(counters["module_frames_dropped_no_route"], 21);
3922        assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
3923    }
3924
3925    /// The rate-limit memory is per connection and goes away with it.
3926    #[test]
3927    fn orphan_goodbye_rate_limit_state_is_released_with_the_connection() {
3928        let router = Router::with_default_self_handler();
3929        let connection = router.begin_connection();
3930        let id = connection.id();
3931        assert!(router.orphan_goodbyes.claim(id, 7));
3932        assert!(!router.orphan_goodbyes.claim(id, 7));
3933        assert!(router.orphan_goodbyes.claim(id, 8));
3934        drop(connection);
3935        assert!(router
3936            .orphan_goodbyes
3937            .last_sent
3938            .lock()
3939            .unwrap()
3940            .get(&id)
3941            .is_none());
3942    }
3943
3944    #[test]
3945    fn channel_zero_cannot_be_registered_as_backend() {
3946        let mut router = Router::with_default_self_handler();
3947
3948        let err = router.register_backend(0, EchoBackend).unwrap_err();
3949
3950        assert_eq!(err, RouterError::ReservedChannelZero);
3951    }
3952}