Skip to main content

subc_daemon/
router.rs

1use std::{
2    collections::HashMap,
3    error::Error,
4    fmt,
5    sync::{
6        atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
7        Arc, Mutex,
8    },
9    time::{Duration, Instant},
10};
11
12use subc_protocol::{error_codes, ErrorBody, Flags, FrameType, Priority};
13use tokio::sync::{mpsc, Notify};
14use tracing::{debug, warn};
15
16use crate::{
17    control::ControlHandler,
18    forwarding::{
19        principal_label, CloseReason, ConnectionCloseReceiver, DataRoute, DataRouteState,
20        ForwardingError, ForwardingTable, RouteBinding, RouteRelease, UndeliveredFrame,
21    },
22    registry::ConnectionId,
23    DaemonCounters, Frame, FrameBuildError,
24};
25
26/// One queued outbound frame plus the instant it entered the writer queue.
27///
28/// The stamp exists for the reply-path half of slow-control diagnosis: a
29/// handler can finish in microseconds while the reply sits in this queue
30/// waiting for the writer task to be scheduled, and without a per-item stamp
31/// that wait is invisible to every other timing point (the client's round
32/// trip is the only witness, and it cannot say which side ate the time).
33/// Constructed exclusively inside [`FrameSink`] so no caller can forget it.
34#[derive(Debug)]
35pub struct OutboundFrame {
36    pub frame: Frame,
37    pub enqueued_at: std::time::Instant,
38    pub(crate) flushed: Option<tokio::sync::oneshot::Sender<()>>,
39    /// This frame's share of the connection's queued-byte count. Dropping the
40    /// frame (after the writer has written it, or when the queue itself is
41    /// dropped with the frame still in it) gives the bytes back. `None` only for
42    /// frames built outside a [`FrameSink`], which exist in tests alone.
43    pub(crate) charge: Option<EgressCharge>,
44}
45
46impl OutboundFrame {
47    fn charged(frame: Frame, charge: EgressCharge) -> Self {
48        Self {
49            frame,
50            enqueued_at: charge.enqueued_at,
51            flushed: None,
52            charge: Some(charge),
53        }
54    }
55}
56
57/// Bytes a frame occupies in the connection's egress queue for budget
58/// purposes: the fixed envelope header plus the body.
59fn queued_frame_bytes(frame: &Frame) -> usize {
60    subc_protocol::HEADER_LEN + frame.body.len()
61}
62
63/// A point-in-time view of one connection's egress queue, for diagnosing why a
64/// frame did not fit.
65#[derive(Debug, Clone, Copy, PartialEq, Eq)]
66pub(crate) struct EgressBacklog {
67    pub queued_bytes: usize,
68    pub queued_frames: usize,
69    /// Now minus the enqueue time of the frame the writer most recently took
70    /// off the queue (the frame it is writing, or has just written), or, if
71    /// the writer has taken nothing since the queue was last empty, of the
72    /// frame that made the queue non-empty. While the writer is blocked on a
73    /// frame this is exactly the oldest frame's age; between frames it can be
74    /// one frame older than the true head. `None` when nothing is queued.
75    pub oldest_age: Option<Duration>,
76}
77
78/// Queued-byte accounting shared by every clone of one connection's
79/// [`FrameSink`] and by every frame that sink has admitted. Its per-frame
80/// accounting uses atomics: this is on the path of every frame a module sends
81/// to a client, so it takes no lock per frame.
82#[derive(Debug)]
83struct EgressAccounting {
84    byte_budget: usize,
85    queued_bytes: AtomicUsize,
86    queued_frames: AtomicUsize,
87    /// Origin for the nanosecond timestamps in `oldest_enqueued_nanos`.
88    time_base: Instant,
89    /// Enqueue time, as nanoseconds after `time_base` plus one, of the frame
90    /// described by [`EgressBacklog::oldest_age`]; 0 means none.
91    oldest_enqueued_nanos: AtomicU64,
92    /// Awaited senders currently parked on `freed`. A release only pays for a
93    /// notification when this is non-zero.
94    waiters: AtomicUsize,
95    /// Woken when a charged frame releases its bytes while a sender waits, so
96    /// the awaited send can re-check for room.
97    freed: Notify,
98    client_delivery_closed: AtomicBool,
99    client_delivery_stopped: Notify,
100}
101
102impl EgressAccounting {
103    fn new(byte_budget: usize) -> Self {
104        Self {
105            byte_budget,
106            queued_bytes: AtomicUsize::new(0),
107            queued_frames: AtomicUsize::new(0),
108            time_base: tokio::time::Instant::now().into_std(),
109            oldest_enqueued_nanos: AtomicU64::new(0),
110            waiters: AtomicUsize::new(0),
111            freed: Notify::new(),
112            client_delivery_closed: AtomicBool::new(false),
113            client_delivery_stopped: Notify::new(),
114        }
115    }
116
117    fn stamp(&self, at: Instant) -> u64 {
118        (at.saturating_duration_since(self.time_base).as_nanos() as u64).saturating_add(1)
119    }
120
121    /// Charge `bytes` if they fit in the budget. A frame larger than the whole
122    /// budget (bodies may be up to 64 MiB) is still admitted into an EMPTY
123    /// queue, since it could otherwise never be sent at all; it simply has the
124    /// queue to itself until it is written.
125    fn try_charge(self: &Arc<Self>, bytes: usize) -> Option<EgressCharge> {
126        // SeqCst pairs with `release`: either this load sees bytes a release
127        // just freed, or that release sees this sender counted in `waiters`.
128        let mut current = self.queued_bytes.load(Ordering::SeqCst);
129        loop {
130            if current != 0 && current.saturating_add(bytes) > self.byte_budget {
131                return None;
132            }
133            match self.queued_bytes.compare_exchange_weak(
134                current,
135                current + bytes,
136                Ordering::SeqCst,
137                Ordering::SeqCst,
138            ) {
139                Ok(_) => return Some(self.record(bytes)),
140                Err(actual) => current = actual,
141            }
142        }
143    }
144
145    /// Charge `bytes` regardless of the budget. Used only for a frame whose
146    /// queue slot was reserved in advance, which must be sendable.
147    fn charge_unconditionally(self: &Arc<Self>, bytes: usize) -> EgressCharge {
148        self.queued_bytes.fetch_add(bytes, Ordering::SeqCst);
149        self.record(bytes)
150    }
151
152    fn record(self: &Arc<Self>, bytes: usize) -> EgressCharge {
153        let enqueued_at = tokio::time::Instant::now().into_std();
154        let stamp = self.stamp(enqueued_at);
155        if self.queued_frames.fetch_add(1, Ordering::AcqRel) == 0 {
156            // This frame made the queue non-empty, so it is the head until the
157            // writer takes something.
158            self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
159        }
160        EgressCharge {
161            accounting: Arc::clone(self),
162            bytes,
163            stamp,
164            enqueued_at,
165        }
166    }
167
168    /// The writer has taken the frame stamped `stamp` off the queue.
169    fn taken_by_writer(&self, stamp: u64) {
170        self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
171    }
172
173    fn release(&self, bytes: usize, stamp: u64) {
174        self.queued_bytes.fetch_sub(bytes, Ordering::SeqCst);
175        if self.queued_frames.fetch_sub(1, Ordering::AcqRel) == 1 {
176            // The queue drained. Clear the marker only if it still names this
177            // frame, so a frame admitted in the meantime keeps its stamp.
178            let _ = self.oldest_enqueued_nanos.compare_exchange(
179                stamp,
180                0,
181                Ordering::AcqRel,
182                Ordering::Acquire,
183            );
184        }
185        if self.waiters.load(Ordering::SeqCst) != 0 {
186            self.freed.notify_waiters();
187        }
188    }
189
190    fn backlog(&self) -> EgressBacklog {
191        let queued_frames = self.queued_frames.load(Ordering::Acquire);
192        let stamp = self.oldest_enqueued_nanos.load(Ordering::Acquire);
193        let oldest_age = (queued_frames != 0 && stamp != 0).then(|| {
194            let enqueued = self.time_base + Duration::from_nanos(stamp - 1);
195            tokio::time::Instant::from_std(enqueued).elapsed()
196        });
197        EgressBacklog {
198            queued_bytes: self.queued_bytes.load(Ordering::Acquire),
199            queued_frames,
200            oldest_age,
201        }
202    }
203}
204
205/// One admitted frame's claim on its connection's egress byte budget,
206/// returned when the frame is dropped.
207#[derive(Debug)]
208pub(crate) struct EgressCharge {
209    accounting: Arc<EgressAccounting>,
210    bytes: usize,
211    stamp: u64,
212    enqueued_at: Instant,
213}
214
215impl EgressCharge {
216    /// Called by the connection writer when it takes this frame off the queue,
217    /// so the backlog's oldest-age figure follows the writer.
218    pub(crate) fn taken_by_writer(&self) {
219        self.accounting.taken_by_writer(self.stamp);
220    }
221}
222
223impl Drop for EgressCharge {
224    fn drop(&mut self) {
225        self.accounting.release(self.bytes, self.stamp);
226    }
227}
228
229/// A queue slot reserved ahead of time (a pending `route.open` holds one until
230/// its module answers). Sending through it never waits and never fails for
231/// lack of room: the slot is already held, and its frame is charged to the
232/// byte count outside the budget check. At most
233/// `MAX_PENDING_ROUTE_OPENS_PER_CONNECTION` such frames exist per connection.
234#[derive(Debug)]
235pub(crate) struct EgressPermit {
236    permit: mpsc::OwnedPermit<OutboundFrame>,
237    accounting: Arc<EgressAccounting>,
238}
239
240impl EgressPermit {
241    /// Enqueue `frame` in the reserved slot. Returns true when the connection's
242    /// writer had already gone away, meaning the frame will never be written.
243    pub(crate) fn send(self, frame: Frame) -> bool {
244        let charge = self
245            .accounting
246            .charge_unconditionally(queued_frame_bytes(&frame));
247        let sender = self.permit.send(OutboundFrame::charged(frame, charge));
248        sender.is_closed()
249    }
250}
251
252/// An `OutboundFrame` is a stamped `Frame`; deref keeps every existing frame
253/// read (headers, bodies, assertions) working on queued items unchanged.
254impl std::ops::Deref for OutboundFrame {
255    type Target = Frame;
256
257    fn deref(&self) -> &Frame {
258        &self.frame
259    }
260}
261
262/// Shared tracing-capture helpers for timing-observability tests across
263/// modules (router dispatch, server reply path). Test-only.
264#[cfg(test)]
265pub(crate) mod test_log {
266    use std::{
267        io::Write,
268        sync::{Arc, Mutex, OnceLock},
269    };
270
271    #[derive(Clone)]
272    struct TestLogWriter(Arc<Mutex<Vec<u8>>>);
273
274    impl Write for TestLogWriter {
275        fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
276            self.0
277                .lock()
278                .expect("test log capture is not poisoned")
279                .extend(buffer);
280            Ok(buffer.len())
281        }
282
283        fn flush(&mut self) -> std::io::Result<()> {
284            Ok(())
285        }
286    }
287
288    pub(crate) fn log_capture(
289        level: tracing::Level,
290    ) -> (Arc<Mutex<Vec<u8>>>, tracing::dispatcher::DefaultGuard) {
291        // tracing-core's single-dispatch fast path registers callsites using
292        // the emitting thread's default subscriber. A parallel test with no
293        // subscriber can therefore cache `never` for a callsite this capture
294        // needs. Keep a second dispatch alive so registration considers all
295        // live subscribers instead. This registry is never installed as a
296        // default: each test still records only into its own thread's writer.
297        static INTEREST_ANCHOR: OnceLock<tracing::Dispatch> = OnceLock::new();
298        INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
299        let output = Arc::new(Mutex::new(Vec::new()));
300        let writer = Arc::clone(&output);
301        let subscriber = tracing_subscriber::fmt()
302            .with_max_level(level)
303            .with_ansi(false)
304            .without_time()
305            .with_target(false)
306            .with_writer(move || TestLogWriter(Arc::clone(&writer)))
307            .finish();
308        let guard = tracing::subscriber::set_default(subscriber);
309        (output, guard)
310    }
311
312    pub(crate) fn captured_logs(output: &Arc<Mutex<Vec<u8>>>) -> String {
313        String::from_utf8(
314            output
315                .lock()
316                .expect("test log capture is not poisoned")
317                .clone(),
318        )
319        .expect("tracing output is UTF-8")
320    }
321
322    #[cfg(test)]
323    mod tests {
324        use super::*;
325
326        fn emit_capture_probe(source: &str) {
327            tracing::error!(source, "parallel capture probe");
328        }
329
330        #[test]
331        fn parallel_callsite_registration_keeps_the_test_capture_enabled() {
332            let (output, _guard) = log_capture(tracing::Level::ERROR);
333            // First register the very same callsite on a thread without a
334            // default subscriber, while the test's capture remains live.
335            std::thread::spawn(|| emit_capture_probe("other-thread"))
336                .join()
337                .expect("unsubscribed thread exits");
338            emit_capture_probe("captured-thread");
339
340            let logs = captured_logs(&output);
341            assert!(
342                logs.contains("captured-thread"),
343                "capture was disabled: {logs}"
344            );
345            assert!(!logs.contains("other-thread"), "capture leaked: {logs}");
346        }
347    }
348}
349
350/// Cheaply cloneable handle to one connection's bounded outbound frame queue.
351///
352/// Backends emit responses, streaming frames, and future PUSH frames through this
353/// single path. The queue is bounded twice: by queued BYTES (the budget that
354/// matters for memory and for how long a slow reader may pause), and by the
355/// `mpsc` channel's frame count (a backstop). The socket layer owns the sole
356/// receiver/writer.
357#[derive(Debug, Clone)]
358pub struct FrameSink {
359    tx: mpsc::Sender<OutboundFrame>,
360    accounting: Arc<EgressAccounting>,
361}
362
363impl FrameSink {
364    /// A sink with the standard per-connection byte budget
365    /// ([`crate::server::CONNECTION_EGRESS_BYTE_BUDGET`]); the frame-count bound
366    /// is whatever capacity `tx`'s channel was created with.
367    pub fn new(tx: mpsc::Sender<OutboundFrame>) -> Self {
368        Self::with_byte_budget(tx, crate::server::CONNECTION_EGRESS_BYTE_BUDGET)
369    }
370
371    pub(crate) fn with_byte_budget(tx: mpsc::Sender<OutboundFrame>, byte_budget: usize) -> Self {
372        Self {
373            tx,
374            accounting: Arc::new(EgressAccounting::new(byte_budget)),
375        }
376    }
377
378    /// Wait until the frame's bytes fit the budget, then charge them. Awaited
379    /// senders (control replies, error replies, shutdown notices) are never
380    /// refused by the byte budget: they wait for the writer to free bytes, just
381    /// as they wait for a free slot, which is the same backpressure they had
382    /// when the queue was bounded by frame count alone. Returns `None` if the
383    /// writer goes away while waiting.
384    async fn charge_waiting(&self, bytes: usize) -> Option<EgressCharge> {
385        if let Some(charge) = self.accounting.try_charge(bytes) {
386            return Some(charge);
387        }
388        // Counted as a waiter for as long as this future is parked, including
389        // when it is cancelled, so releases notify only while someone waits.
390        struct Waiting<'a>(&'a AtomicUsize);
391        impl Drop for Waiting<'_> {
392            fn drop(&mut self) {
393                self.0.fetch_sub(1, Ordering::SeqCst);
394            }
395        }
396        self.accounting.waiters.fetch_add(1, Ordering::SeqCst);
397        let _waiting = Waiting(&self.accounting.waiters);
398        loop {
399            let freed = self.accounting.freed.notified();
400            tokio::pin!(freed);
401            // Register interest before checking, so a release that happens
402            // between the check and the await still wakes this sender.
403            freed.as_mut().enable();
404            if let Some(charge) = self.accounting.try_charge(bytes) {
405                return Some(charge);
406            }
407            tokio::select! {
408                _ = &mut freed => {}
409                _ = self.tx.closed() => return None,
410            }
411        }
412    }
413
414    pub async fn send(&self, frame: Frame) -> Result<(), RouterError> {
415        let channel = frame.header.channel;
416        let epoch = frame.header.epoch;
417        let corr = frame.header.corr;
418        let closed =
419            || RouterError::backend_with_epoch(channel, epoch, corr, "connection writer closed");
420        let charge = self
421            .charge_waiting(queued_frame_bytes(&frame))
422            .await
423            .ok_or_else(closed)?;
424        self.tx
425            .send(OutboundFrame::charged(frame, charge))
426            .await
427            .map_err(|_| closed())
428    }
429
430    /// Shutdown notices must leave the socket writer before an idle daemon exits.
431    /// Queue admission alone does not prove this; the writer acknowledges flush.
432    ///
433    /// The local endpoint uses it the same way for its clients-only refusal: the
434    /// connection closes right after the refusal is sent, so the writer must
435    /// flush it first.
436    #[cfg(unix)]
437    pub async fn send_flushed(&self, frame: Frame) -> Result<(), RouterError> {
438        let (tx, rx) = tokio::sync::oneshot::channel();
439        let charge = self
440            .charge_waiting(queued_frame_bytes(&frame))
441            .await
442            .ok_or_else(|| RouterError::backend(0, 0, "connection writer closed"))?;
443        let mut outbound = OutboundFrame::charged(frame, charge);
444        outbound.flushed = Some(tx);
445        self.tx
446            .send(outbound)
447            .await
448            .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
449        rx.await
450            .map_err(|_| RouterError::backend(0, 0, "connection flush failed"))
451    }
452
453    pub(crate) async fn reserve_owned(&self) -> Result<EgressPermit, RouterError> {
454        let permit = self
455            .tx
456            .clone()
457            .reserve_owned()
458            .await
459            .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
460        Ok(EgressPermit {
461            permit,
462            accounting: Arc::clone(&self.accounting),
463        })
464    }
465
466    #[cfg(test)]
467    pub(crate) fn try_reserve_owned(&self) -> Result<EgressPermit, RouterError> {
468        let permit = self
469            .tx
470            .clone()
471            .try_reserve_owned()
472            .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
473        Ok(EgressPermit {
474            permit,
475            accounting: Arc::clone(&self.accounting),
476        })
477    }
478
479    pub(crate) fn is_closed(&self) -> bool {
480        self.tx.is_closed()
481    }
482
483    /// What the connection's egress queue holds right now.
484    pub(crate) fn backlog(&self) -> EgressBacklog {
485        self.accounting.backlog()
486    }
487
488    /// Stop module deliveries as soon as the client reader exits, without
489    /// waiting for the socket writer's separate teardown drain grace.
490    pub(crate) fn close_client_delivery(&self) {
491        self.accounting
492            .client_delivery_closed
493            .store(true, Ordering::Release);
494        self.accounting.client_delivery_stopped.notify_waiters();
495    }
496
497    /// A full queue can be a short fanout burst rather than a stopped reader.
498    /// Spend only the grace the atomic backlog age has not already used.
499    async fn send_to_client(&self, frame: Frame) -> Result<(), ClientDeliveryFailure> {
500        let frame_type = frame.header.ty;
501        let flags = frame.header.flags;
502        let stopped = self.accounting.client_delivery_stopped.notified();
503        tokio::pin!(stopped);
504        stopped.as_mut().enable();
505        let failure = |cause, error| ClientDeliveryFailure {
506            diagnosis: ClientDeliveryDiagnosis {
507                frame_type,
508                flags,
509                cause,
510            },
511            error,
512        };
513        let closed = || {
514            RouterError::backend_with_epoch(
515                frame.header.channel,
516                frame.header.epoch,
517                frame.header.corr,
518                "client connection closed",
519            )
520        };
521        if self
522            .accounting
523            .client_delivery_closed
524            .load(Ordering::Acquire)
525        {
526            return Err(failure("client_closed", closed()));
527        }
528        let (frame, error) = match self.try_send_owned(frame) {
529            Ok(()) => return Ok(()),
530            Err(refused) => refused,
531        };
532        if self.is_closed() {
533            return Err(failure("client_closed", error));
534        }
535        let grace = crate::server::CONNECTION_EGRESS_STALL_GRACE;
536        let age = self.backlog().oldest_age.unwrap_or_default();
537        if age > grace {
538            return Err(failure("oldest_frame_older_than_250_ms", error));
539        }
540        let result = tokio::select! {
541            biased;
542            _ = &mut stopped => return Err(failure("client_closed", error)),
543            result = tokio::time::timeout(grace.saturating_sub(age), self.send(frame)) => result,
544        };
545        match result {
546            Ok(Ok(())) => Ok(()),
547            Ok(Err(error)) => Err(failure("client_closed", error)),
548            Err(_) => Err(failure("waited_250_ms_without_room", error)),
549        }
550    }
551
552    /// Enqueue without waiting. Fails when the frame's bytes would take the
553    /// queue past its byte budget, when the channel's frame-count backstop is
554    /// full, or when the writer is gone.
555    pub(crate) fn try_send(&self, frame: Frame) -> Result<(), RouterError> {
556        self.try_send_owned(frame).map_err(|(_, error)| error)
557    }
558
559    fn try_send_owned(&self, frame: Frame) -> Result<(), (Frame, RouterError)> {
560        let channel = frame.header.channel;
561        let epoch = frame.header.epoch;
562        let corr = frame.header.corr;
563        let unavailable = |why: String| {
564            RouterError::backend_with_epoch(
565                channel,
566                epoch,
567                corr,
568                format!("connection writer unavailable: {why}"),
569            )
570        };
571        let bytes = queued_frame_bytes(&frame);
572        let Some(charge) = self.accounting.try_charge(bytes) else {
573            if self.tx.is_closed() {
574                return Err((frame, unavailable("channel closed".to_string())));
575            }
576            return Err((
577                frame,
578                unavailable(format!(
579                    "egress byte budget exhausted ({} queued bytes, frame of {bytes} bytes, budget {})",
580                    self.accounting.queued_bytes.load(Ordering::Acquire),
581                    self.accounting.byte_budget
582                )),
583            ));
584        };
585        // Return the refused frame for a bounded retry; dropping the rest of
586        // the outbound wrapper returns its charge.
587        self.tx
588            .try_send(OutboundFrame::charged(frame, charge))
589            .map_err(|err| {
590                let error = unavailable(err.to_string());
591                (err.into_inner().frame, error)
592            })
593    }
594}
595
596#[derive(Debug)]
597struct ClientDeliveryFailure {
598    diagnosis: ClientDeliveryDiagnosis,
599    error: RouterError,
600}
601
602/// Header facts only: payloads can contain secrets and must never enter logs.
603#[derive(Debug)]
604pub(crate) struct ClientDeliveryDiagnosis {
605    pub frame_type: FrameType,
606    pub flags: Flags,
607    pub cause: &'static str,
608}
609
610impl fmt::Display for ClientDeliveryFailure {
611    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
612        self.error.fmt(f)
613    }
614}
615
616/// Minimum time between two route GOODBYEs the daemon sends one module
617/// connection for one channel in answer to the module's frames on a (channel,
618/// epoch) the daemon holds no route for.
619///
620/// A module that missed a GOODBYE keeps sending on that route (a streaming
621/// module, many frames a second), and every one of those frames lands here. One
622/// answer is enough when it arrives, so answering each frame would only flood
623/// the module's egress queue at the moment it is already behind. Five seconds
624/// is long enough for an answer queued behind a deep backlog to reach the module
625/// and take effect before a second is sent, and short enough that an answer
626/// the queue refused is retried by the next orphan frame well within a minute.
627const ORPHAN_ROUTE_GOODBYE_INTERVAL: Duration = Duration::from_secs(5);
628
629/// Once one connection has this many remembered channels, entries older than
630/// [`ORPHAN_ROUTE_GOODBYE_INTERVAL`] are pruned before another is added. An
631/// expired entry has no effect on rate limiting, so pruning never changes a
632/// decision; it keeps a module that orphaned many channels long ago from
633/// pinning memory for all of them. The map is bounded in any case by the
634/// 16-bit channel space and is dropped whole when the connection ends.
635const ORPHAN_ROUTE_GOODBYE_PRUNE_AT: usize = 256;
636
637/// When each module connection was last sent an orphan-route GOODBYE, per
638/// channel. Shared by the [`Router`] (which consults it) and every
639/// [`RouterConnection`] (which removes its own entry when the connection ends).
640#[derive(Debug, Default)]
641struct OrphanGoodbyeLimiter {
642    last_sent: Mutex<HashMap<ConnectionId, HashMap<u16, tokio::time::Instant>>>,
643}
644
645impl OrphanGoodbyeLimiter {
646    /// True, and the send recorded, when `channel` on `connection_id` has not
647    /// been answered within the interval.
648    fn claim(&self, connection_id: ConnectionId, channel: u16) -> bool {
649        let now = tokio::time::Instant::now();
650        let mut last_sent = self
651            .last_sent
652            .lock()
653            .unwrap_or_else(std::sync::PoisonError::into_inner);
654        let channels = last_sent.entry(connection_id).or_default();
655        if channels.get(&channel).is_some_and(|sent| {
656            now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
657        }) {
658            return false;
659        }
660        if channels.len() >= ORPHAN_ROUTE_GOODBYE_PRUNE_AT {
661            channels.retain(|_, sent| {
662                now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
663            });
664        }
665        channels.insert(channel, now);
666        true
667    }
668
669    fn forget_connection(&self, connection_id: ConnectionId) {
670        self.last_sent
671            .lock()
672            .unwrap_or_else(std::sync::PoisonError::into_inner)
673            .remove(&connection_id);
674    }
675}
676
677/// Per-route context shared with backends besides the frame itself.
678#[derive(Debug, Clone)]
679pub struct RouteCtx {
680    pub connection_id: ConnectionId,
681    pub egress: FrameSink,
682}
683
684/// Closed set of data-plane backends for non-zero channels.
685///
686/// Channel 0 is structurally special and remains `Router::control`, not an
687/// enum variant. Static/test backends live in `Router::backends`; the forwarding
688/// backend is selected dynamically only when a per-connection forwarding binding exists.
689#[derive(Debug, Clone)]
690pub enum Backend {
691    Echo(EchoBackend),
692    Forward(ForwardBackend),
693}
694
695impl From<EchoBackend> for Backend {
696    fn from(backend: EchoBackend) -> Self {
697        Self::Echo(backend)
698    }
699}
700
701impl From<ForwardBackend> for Backend {
702    fn from(backend: ForwardBackend) -> Self {
703        Self::Forward(backend)
704    }
705}
706
707impl Backend {
708    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
709        match self {
710            Self::Echo(backend) => backend.handle(ctx, frame).await,
711            Self::Forward(backend) => backend.handle(ctx, frame).await,
712        }
713    }
714}
715
716/// I/O-agnostic splice router keyed by envelope `channel`.
717///
718/// Channel 0 is reserved for subc itself and is always dispatched to the
719/// dedicated control handler. Other channels must be explicitly registered.
720/// Unknown client-originated non-zero channels are translated to canonical JSON `ERROR` frames on
721/// the connection sink so the peer can continue using the same socket. Module-originated frames for
722/// released route channels are logged and dropped as the channel-gone race backstop.
723///
724/// DATA-PLANE BODIES ARE NEVER DECODED HERE, and the consequence is worth stating
725/// because it looks like a guarantee and is not. Additive fields in a request or
726/// response body reach the far side untouched — not because anything permits them,
727/// but because routing reads only the 21-byte header and treats the body as opaque
728/// bytes. That is a performance property, so **nothing prevents it from changing**:
729/// a future reason to inspect a body would convert a wire-transparent path into a
730/// filtering one, and nobody would think of it as a contract change.
731///
732/// So when a peer asks whether subc sees an additive field, the answer is per-path
733/// and this path's zero means WE NEVER LOOK rather than WE LOOK AT EVERYTHING. The
734/// control plane (typed enums at the frame boundary) and the MCP gateway (envelope
735/// unwrap plus a named struct) both narrow; only this one does not.
736pub struct Router {
737    backends: HashMap<u16, Backend>,
738    control: Arc<ControlHandler>,
739    forwarding: Arc<ForwardingTable>,
740    forward_backend: ForwardBackend,
741    counters: DaemonCounters,
742    next_connection_id: AtomicU64,
743    orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
744}
745
746impl Router {
747    pub fn with_control_handler(control: Arc<ControlHandler>) -> Self {
748        // The handler the router serves is the one a swap must tell when it
749        // promotes a candidate; this is where it first sits behind an `Arc`.
750        control.install_swap_promotion_observer();
751        let forwarding = control.forwarding();
752        let counters = control.counters();
753        Self {
754            backends: HashMap::new(),
755            control,
756            forwarding: Arc::clone(&forwarding),
757            forward_backend: ForwardBackend::new(forwarding),
758            counters,
759            // ConnectionId::LOCAL is 0; real socket ids start at 1 and never collide.
760            next_connection_id: AtomicU64::new(1),
761            orphan_goodbyes: Arc::default(),
762        }
763    }
764
765    pub fn with_default_self_handler() -> Self {
766        Self::with_control_handler(Arc::new(ControlHandler::default()))
767    }
768
769    pub fn forwarding(&self) -> Arc<ForwardingTable> {
770        Arc::clone(&self.forwarding)
771    }
772
773    pub fn register_backend(
774        &mut self,
775        channel: u16,
776        backend: impl Into<Backend>,
777    ) -> Result<(), RouterError> {
778        self.register_backend_arc(channel, Arc::new(backend.into()))
779    }
780
781    pub(crate) fn register_backend_arc(
782        &mut self,
783        channel: u16,
784        backend: Arc<Backend>,
785    ) -> Result<(), RouterError> {
786        if channel == 0 {
787            return Err(RouterError::ReservedChannelZero);
788        }
789        if self.backends.contains_key(&channel) {
790            return Err(RouterError::DuplicateChannel { channel });
791        }
792        self.backends.insert(channel, backend.as_ref().clone());
793        Ok(())
794    }
795
796    /// Start a connection-scoped routing context. Dropping the guard releases
797    /// any control-plane registrations owned by the connection.
798    fn record_module_frame_drop(&self, connection_id: ConnectionId) -> Result<(), RouterError> {
799        let module_id = self
800            .forwarding
801            .module_id_for_connection(connection_id)
802            .map_err(RouterError::Forwarding)?;
803        self.counters
804            .increment_module_frames_dropped_no_route(module_id.as_deref());
805        Ok(())
806    }
807
808    /// A module sent a non-request frame on a (channel, epoch) the daemon holds
809    /// no route for: most often a route the daemon released whose GOODBYE the
810    /// module never received, so the module still believes it is open and keeps
811    /// sending. Count the drop, and tell the module to let go of exactly that
812    /// (channel, epoch) with a route GOODBYE, at most once per
813    /// [`ORPHAN_ROUTE_GOODBYE_INTERVAL`] per connection and channel.
814    ///
815    /// The answer uses `try_send`: it is a best-effort nudge, and if the queue
816    /// refuses it, the module's next frame on that route after the interval
817    /// asks again. A GOODBYE from the module is not answered, since the module
818    /// is already letting go of the route.
819    fn handle_orphan_module_frame(&self, ctx: &RouteCtx, frame: &Frame) -> Result<(), RouterError> {
820        let channel = frame.header.channel;
821        let epoch = frame.header.epoch;
822        let module_id = self
823            .forwarding
824            .module_id_for_connection(ctx.connection_id)
825            .map_err(RouterError::Forwarding)?;
826        self.counters
827            .increment_module_frames_dropped_no_route(module_id.as_deref());
828        if self
829            .forwarding
830            .module_route_epoch_was_allocated(ctx.connection_id, channel, epoch)
831            .map_err(RouterError::Forwarding)?
832        {
833            self.counters
834                .increment_module_frames_dropped_released_route(module_id.as_deref());
835        }
836        if frame.header.ty == FrameType::Goodbye
837            || !self.orphan_goodbyes.claim(ctx.connection_id, channel)
838        {
839            return Ok(());
840        }
841        let goodbye = Frame::build_with_version(
842            frame.header.ver,
843            FrameType::Goodbye,
844            Flags::new(false, Priority::Passive, false),
845            channel,
846            epoch,
847            0,
848            Vec::new(),
849        )
850        .map_err(RouterError::FrameBuild)?;
851        match ctx.egress.try_send(goodbye) {
852            Ok(()) => {
853                self.counters.increment_module_orphan_route_goodbyes_sent();
854                debug!(
855                    connection_id = ctx.connection_id.get(),
856                    module_id = module_id.as_deref().unwrap_or("unknown"),
857                    channel,
858                    epoch,
859                    "answered module frame on a route the daemon does not hold with a route GOODBYE"
860                );
861            }
862            Err(err) => {
863                self.forwarding.warn_frame_drop(
864                    ctx.connection_id,
865                    channel,
866                    epoch,
867                    "daemon_to_module",
868                    if ctx.egress.is_closed() {
869                        "endpoint_gone"
870                    } else {
871                        "egress_budget"
872                    },
873                );
874                debug!(
875                connection_id = ctx.connection_id.get(),
876                module_id = module_id.as_deref().unwrap_or("unknown"),
877                channel,
878                epoch,
879                error = %err,
880                "could not enqueue route GOODBYE for module frame on a route the daemon does not hold; the next such frame retries"
881                );
882            }
883        }
884        Ok(())
885    }
886
887    pub fn begin_connection(&self) -> RouterConnection {
888        let raw = self.next_connection_id.fetch_add(1, Ordering::Relaxed);
889        let id = ConnectionId::new(raw);
890        let close_receiver = self.forwarding.register_connection_close(id);
891        RouterConnection {
892            id,
893            control_handler: Arc::clone(&self.control),
894            forwarding: Arc::clone(&self.forwarding),
895            close_receiver: Some(close_receiver),
896            orphan_goodbyes: Arc::clone(&self.orphan_goodbyes),
897        }
898    }
899
900    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
901        self.control.route_open_target(frame)
902    }
903
904    pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
905        self.control.record_local_connection(connection);
906    }
907
908    pub(crate) fn record_captured_peer(
909        &self,
910        connection: ConnectionId,
911        peer: subc_os::local_peer::CapturedPeer,
912    ) {
913        self.control.record_captured_peer(connection, peer);
914    }
915
916    pub(crate) fn record_caller_endpoints(
917        &self,
918        connection: ConnectionId,
919        endpoints: Option<(std::net::SocketAddr, std::net::SocketAddr)>,
920    ) {
921        self.control.record_caller_endpoints(connection, endpoints);
922    }
923
924    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
925        self.control.defers_privileged_control(ctx, frame)
926    }
927
928    pub(crate) fn route_open_capacity_refusal(
929        &self,
930        ctx: &RouteCtx,
931        frame: &Frame,
932        target_module_id: &str,
933        in_flight: usize,
934        limit: usize,
935    ) -> Result<Frame, RouterError> {
936        self.control
937            .route_open_capacity_refusal(ctx, frame, target_module_id, in_flight, limit)
938    }
939
940    pub async fn route_for_connection(
941        &self,
942        ctx: &RouteCtx,
943        frame: Frame,
944    ) -> Result<(), RouterError> {
945        self.route_for_connection_started(ctx, frame, None).await
946    }
947
948    pub(crate) async fn observe_connection_wait<F: std::future::Future>(
949        &self,
950        ctx: &RouteCtx,
951        kind: &'static str,
952        future: F,
953    ) -> F::Output {
954        self.counters
955            .observe_reader_wait(
956                || {
957                    let module = self
958                        .forwarding
959                        .module_id_for_connection(ctx.connection_id)
960                        .ok()
961                        .flatten();
962                    let principal = if module.is_some() { "module" } else { "direct" };
963                    (
964                        module.unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
965                        principal.into(),
966                    )
967                },
968                kind,
969                future,
970            )
971            .await
972    }
973
974    pub(crate) async fn route_for_connection_started(
975        &self,
976        ctx: &RouteCtx,
977        frame: Frame,
978        dispatch_started_at: Option<Instant>,
979    ) -> Result<(), RouterError> {
980        let channel = frame.header.channel;
981        let epoch = frame.header.epoch;
982        let corr = frame.header.corr;
983        if channel == 0 {
984            debug!(
985                connection_id = ctx.connection_id.get(),
986                corr,
987                frame_type = ?frame.header.ty,
988                "routing control frame"
989            );
990            // The connection loop dispatches every control operation directly
991            // except `route.open`, activity writes and privileged requests. Each
992            // task passes a timestamp captured before
993            // spawn, so slow-dispatch timing includes scheduler delay but no
994            // wait in an application-owned queue.
995            let dispatch_started_at = (frame.header.ty == FrameType::Request)
996                .then(|| dispatch_started_at.unwrap_or_else(Instant::now));
997            let responses = self
998                .control
999                .handle_control_frame_timed(ctx, frame, dispatch_started_at)
1000                .await?;
1001            for response in responses {
1002                self.observe_connection_wait(ctx, "reply_egress", ctx.egress.send(response))
1003                    .await?;
1004            }
1005            return Ok(());
1006        }
1007
1008        let data_route = self
1009            .forwarding
1010            .lookup_data_route(ctx.connection_id, channel, epoch)
1011            .map_err(RouterError::Forwarding)?;
1012
1013        match data_route {
1014            DataRoute::Module(DataRouteState::EpochMismatch) => {
1015                self.forwarding.warn_frame_drop(
1016                    ctx.connection_id,
1017                    channel,
1018                    epoch,
1019                    "module_to_client",
1020                    "stale_epoch",
1021                );
1022                if frame.header.ty == FrameType::Request {
1023                    self.counters
1024                        .increment_module_requests_dropped_stale_route();
1025                    let err = RouterError::StaleRouteEpoch {
1026                        channel,
1027                        epoch,
1028                        corr,
1029                    };
1030                    if let Some(error_frame) = err.to_error_frame() {
1031                        self.observe_connection_wait(
1032                            ctx,
1033                            "reply_egress",
1034                            ctx.egress.send(error_frame),
1035                        )
1036                        .await?;
1037                    }
1038                } else {
1039                    self.handle_orphan_module_frame(ctx, &frame)?;
1040                }
1041                debug!(
1042                    connection_id = ctx.connection_id.get(),
1043                    channel, epoch, corr, "dropping module frame for stale route epoch"
1044                );
1045                return Ok(());
1046            }
1047            DataRoute::Module(DataRouteState::Reserved) => {
1048                self.forwarding.warn_frame_drop(
1049                    ctx.connection_id,
1050                    channel,
1051                    epoch,
1052                    "module_to_client",
1053                    "missing_binding",
1054                );
1055                if frame.header.ty == FrameType::Request {
1056                    self.counters
1057                        .increment_module_requests_dropped_stale_route();
1058                    let err = RouterError::UnknownChannel {
1059                        channel,
1060                        epoch,
1061                        corr,
1062                    };
1063                    if let Some(error_frame) = err.to_error_frame() {
1064                        self.observe_connection_wait(
1065                            ctx,
1066                            "reply_egress",
1067                            ctx.egress.send(error_frame),
1068                        )
1069                        .await?;
1070                    }
1071                } else {
1072                    self.record_module_frame_drop(ctx.connection_id)?;
1073                }
1074                debug!(
1075                    connection_id = ctx.connection_id.get(),
1076                    channel, epoch, corr, "dropping module frame for reserved route handle"
1077                );
1078                return Ok(());
1079            }
1080            DataRoute::Module(DataRouteState::Absent) => {
1081                self.forwarding.warn_frame_drop(
1082                    ctx.connection_id,
1083                    channel,
1084                    epoch,
1085                    "module_to_client",
1086                    "unknown_channel",
1087                );
1088                if frame.header.ty == FrameType::Request {
1089                    self.counters
1090                        .increment_module_requests_dropped_stale_route();
1091                    let err = RouterError::UnknownChannel {
1092                        channel,
1093                        epoch,
1094                        corr,
1095                    };
1096                    if let Some(error_frame) = err.to_error_frame() {
1097                        self.observe_connection_wait(
1098                            ctx,
1099                            "reply_egress",
1100                            ctx.egress.send(error_frame),
1101                        )
1102                        .await?;
1103                    }
1104                } else {
1105                    self.handle_orphan_module_frame(ctx, &frame)?;
1106                }
1107                debug!(
1108                    connection_id = ctx.connection_id.get(),
1109                    channel, epoch, corr, "dropping module frame for absent route handle"
1110                );
1111                return Ok(());
1112            }
1113            DataRoute::Module(DataRouteState::Bound(route)) => {
1114                if frame.header.ty == FrameType::Goodbye {
1115                    if let RouteRelease::Removed(target) = self
1116                        .forwarding
1117                        .release_module_route(ctx.connection_id, channel, epoch)
1118                        .map_err(RouterError::Forwarding)?
1119                    {
1120                        let mut goodbye = frame;
1121                        goodbye.header.channel = target.channel;
1122                        goodbye.header.epoch = target.epoch;
1123                        // A module must never be able to make a frame look
1124                        // daemon-authored, so the origin bit is cleared on
1125                        // every module frame relayed to a client.
1126                        goodbye.header.flags = goodbye.header.flags.without_daemon_origin();
1127                        if let Err(err) = self
1128                            .observe_connection_wait(
1129                                ctx,
1130                                "module_to_client",
1131                                target.sink.send_to_client(goodbye),
1132                            )
1133                            .await
1134                        {
1135                            if !target.close_on_delivery_failure() {
1136                                self.forwarding.warn_frame_drop(
1137                                    target.connection_id,
1138                                    target.channel,
1139                                    target.epoch,
1140                                    "module_to_client",
1141                                    err.diagnosis.cause,
1142                                );
1143                            }
1144                            if target.close_on_delivery_failure()
1145                                && self
1146                                    .forwarding
1147                                    .escalate_client_delivery_failure_diagnosed(
1148                                        target.connection_id,
1149                                        target.channel,
1150                                        target.epoch,
1151                                        CloseReason::new(
1152                                            "route_goodbye_delivery_failed",
1153                                            format!(
1154                                                "failed to enqueue route GOODBYE for client channel {}: {err}",
1155                                                target.channel
1156                                            ),
1157                                        ),
1158                                        UndeliveredFrame {
1159                                            module_id: Some(&route.module_id),
1160                                            sink: &target.sink,
1161                                        },
1162                                        Some(&err.diagnosis),
1163                                    )
1164                                    .map_err(RouterError::Forwarding)?
1165                            {
1166                                self.counters.increment_goodbye_relay_client_failed();
1167                            }
1168                        }
1169                    } else {
1170                        self.forwarding.warn_frame_drop(
1171                            ctx.connection_id,
1172                            channel,
1173                            epoch,
1174                            "module_to_client",
1175                            "missing_binding",
1176                        );
1177                    }
1178                    return Ok(());
1179                }
1180
1181                // Accounting allows an oversized frame to occupy an empty queue
1182                // on other send paths. Refuse it at this splice before admission,
1183                // so the next ordinary reply cannot close a multiplexed client.
1184                let bytes = queued_frame_bytes(&frame);
1185                let limit = route.client_sink.accounting.byte_budget;
1186                if bytes > limit {
1187                    let ty = frame.header.ty;
1188                    let flags = frame.header.flags;
1189                    drop(frame);
1190                    return self
1191                        .refuse_oversized_module_frame(ctx, &route, (ty, flags), corr, bytes)
1192                        .await;
1193                }
1194
1195                // A terminal frame ends the request at the module whether or not
1196                // the client can still take it, so its credit is released before
1197                // delivery is attempted. Releasing only after a successful
1198                // enqueue would leave a drain counting a finished request until
1199                // the client connection's cleanup removes the route.
1200                let releases_credit = is_terminal_frame(frame.header.ty);
1201                if releases_credit {
1202                    route.flow.release_corr(corr);
1203                }
1204                let mut frame = frame;
1205                frame.header.channel = route.client_channel;
1206                frame.header.epoch = route.client_epoch;
1207                // Responses, errors, stream data and ends, and reverse
1208                // requests all pass through here. A module must never be able
1209                // to make a frame look daemon-authored, so the origin bit a
1210                // module may have set is cleared; every other flag bit and
1211                // the body travel unchanged.
1212                frame.header.flags = frame.header.flags.without_daemon_origin();
1213                if let Err(err) = self
1214                    .counters
1215                    .observe_reader_wait(
1216                        || (route.module_id.clone(), principal_label(&route.principal)),
1217                        "module_to_client",
1218                        route.client_sink.send_to_client(frame),
1219                    )
1220                    .await
1221                {
1222                    if self
1223                        .forwarding
1224                        .escalate_client_delivery_failure_diagnosed(
1225                            route.client_connection_id,
1226                            route.client_channel,
1227                            route.client_epoch,
1228                            CloseReason::new(
1229                                "module_to_client_delivery_failed",
1230                                format!(
1231                                    "failed to enqueue module frame for client channel {} corr {corr}: {err}",
1232                                    route.client_channel
1233                                ),
1234                            ),
1235                            UndeliveredFrame {
1236                                module_id: Some(&route.module_id),
1237                                sink: &route.client_sink,
1238                            },
1239                            Some(&err.diagnosis),
1240                        )
1241                        .map_err(RouterError::Forwarding)?
1242                    {
1243                        self.counters
1244                            .increment_client_egress_close_delivery_failed();
1245                    }
1246                    return Ok(());
1247                }
1248                return Ok(());
1249            }
1250            DataRoute::Client(DataRouteState::EpochMismatch) => {
1251                self.forwarding.warn_frame_drop(
1252                    ctx.connection_id,
1253                    channel,
1254                    epoch,
1255                    "client_to_module",
1256                    "stale_epoch",
1257                );
1258                if frame.header.ty == FrameType::Request {
1259                    self.counters.increment_client_frames_dropped_stale_route();
1260                    // Dropped before forwarding; a re-bind retry cannot double-execute this request.
1261                    let err = RouterError::StaleRouteEpoch {
1262                        channel,
1263                        epoch,
1264                        corr,
1265                    };
1266                    if let Some(error_frame) = err.to_error_frame() {
1267                        self.observe_connection_wait(
1268                            ctx,
1269                            "reply_egress",
1270                            ctx.egress.send(error_frame),
1271                        )
1272                        .await?;
1273                    }
1274                }
1275                debug!(
1276                    connection_id = ctx.connection_id.get(),
1277                    channel, epoch, corr, "dropping client frame for stale route epoch"
1278                );
1279                return Ok(());
1280            }
1281            DataRoute::Client(DataRouteState::Reserved) => {
1282                self.forwarding.warn_frame_drop(
1283                    ctx.connection_id,
1284                    channel,
1285                    epoch,
1286                    "client_to_module",
1287                    "missing_binding",
1288                );
1289                if frame.header.ty == FrameType::Request {
1290                    let err = RouterError::UnknownChannel {
1291                        channel,
1292                        epoch,
1293                        corr,
1294                    };
1295                    if let Some(error_frame) = err.to_error_frame() {
1296                        self.observe_connection_wait(
1297                            ctx,
1298                            "reply_egress",
1299                            ctx.egress.send(error_frame),
1300                        )
1301                        .await?;
1302                    }
1303                }
1304                return Ok(());
1305            }
1306            DataRoute::Client(DataRouteState::Bound(route)) => {
1307                if frame.header.ty == FrameType::Goodbye {
1308                    let _ = self
1309                        .control
1310                        .handle_route_goodbye(ctx.connection_id, channel, epoch)?;
1311                    return Ok(());
1312                }
1313                return self.forward_backend.handle_bound(frame, route).await;
1314            }
1315            DataRoute::Client(DataRouteState::Absent) => {}
1316        }
1317
1318        if let Some(backend) = self.backends.get(&channel) {
1319            return backend.handle(ctx.clone(), frame).await.inspect_err(|_| {
1320                self.forwarding.warn_frame_drop(
1321                    ctx.connection_id,
1322                    channel,
1323                    epoch,
1324                    "client_to_backend",
1325                    "endpoint_gone",
1326                );
1327            });
1328        }
1329        self.forwarding.warn_frame_drop(
1330            ctx.connection_id,
1331            channel,
1332            epoch,
1333            "client_to_module",
1334            "unknown_channel",
1335        );
1336        if frame.header.ty == FrameType::Request {
1337            let err = RouterError::UnknownChannel {
1338                channel,
1339                epoch,
1340                corr,
1341            };
1342            if let Some(error_frame) = err.to_error_frame() {
1343                self.observe_connection_wait(ctx, "reply_egress", ctx.egress.send(error_frame))
1344                    .await?;
1345            }
1346        }
1347        Ok(())
1348    }
1349
1350    async fn refuse_oversized_module_frame(
1351        &self,
1352        ctx: &RouteCtx,
1353        route: &RouteBinding,
1354        header: (FrameType, Flags),
1355        corr: u64,
1356        bytes: usize,
1357    ) -> Result<(), RouterError> {
1358        let (ty, flags) = header;
1359        let limit = route.client_sink.accounting.byte_budget;
1360        if ty == FrameType::Push {
1361            // Deployed Rust clients cannot decode a new route.closed reason.
1362            // Until that vocabulary can be extended compatibly, an oversized
1363            // unsolicited push still closes the connection, with its cause named.
1364            let diagnosis = ClientDeliveryDiagnosis {
1365                frame_type: ty,
1366                flags,
1367                cause: "frame_too_large",
1368            };
1369            if self
1370                .forwarding
1371                .escalate_client_delivery_failure_diagnosed(
1372                    route.client_connection_id,
1373                    route.client_channel,
1374                    route.client_epoch,
1375                    CloseReason::new(
1376                        error_codes::FRAME_TOO_LARGE,
1377                        format!(
1378                            "oversized push: size={bytes} limit={limit} (header plus body bytes)"
1379                        ),
1380                    ),
1381                    UndeliveredFrame {
1382                        module_id: Some(&route.module_id),
1383                        sink: &route.client_sink,
1384                    },
1385                    Some(&diagnosis),
1386                )
1387                .map_err(RouterError::Forwarding)?
1388            {
1389                self.counters
1390                    .increment_client_egress_close_delivery_failed();
1391            }
1392            return Ok(());
1393        }
1394
1395        warn!(
1396            connection_id = route.client_connection_id.get(),
1397            module_id = %route.module_id,
1398            client_principal = %principal_label(&route.principal),
1399            client_channel = route.client_channel,
1400            frame_type = ?ty,
1401            frame_bytes = bytes,
1402            byte_limit = limit,
1403            "refusing oversized module-to-client frame with ERROR frame_too_large"
1404        );
1405        if ty == FrameType::Request {
1406            // A reverse request belongs to the module, not to a waiting client.
1407            // Answer its sender without forwarding any part of it to the client.
1408            let error = frame_too_large_error(
1409                route.module_channel,
1410                route.module_epoch,
1411                corr,
1412                bytes,
1413                limit,
1414            )?;
1415            self.observe_connection_wait(ctx, "reply_egress", ctx.egress.send(error))
1416                .await?;
1417        } else {
1418            // A refused stream chunk also ends the waiting request. Releasing
1419            // its credit prevents a finished request from keeping a drain alive.
1420            route.flow.release_corr(corr);
1421            let error = frame_too_large_error(
1422                route.client_channel,
1423                route.client_epoch,
1424                corr,
1425                bytes,
1426                limit,
1427            )?;
1428            // The replacement shares the reply's bounded burst grace, so a
1429            // stopped client cannot indefinitely stall the module frame reader.
1430            if let Err(err) = self
1431                .counters
1432                .observe_reader_wait(
1433                    || (route.module_id.clone(), principal_label(&route.principal)),
1434                    "module_to_client",
1435                    route.client_sink.send_to_client(error),
1436                )
1437                .await
1438            {
1439                if self
1440                    .forwarding
1441                    .escalate_client_delivery_failure_diagnosed(
1442                        route.client_connection_id,
1443                        route.client_channel,
1444                        route.client_epoch,
1445                        CloseReason::new(
1446                            "module_to_client_delivery_failed",
1447                            format!(
1448                                "failed to enqueue frame_too_large for client channel {} corr {corr}: {err}",
1449                                route.client_channel
1450                            ),
1451                        ),
1452                        UndeliveredFrame {
1453                            module_id: Some(&route.module_id),
1454                            sink: &route.client_sink,
1455                        },
1456                        Some(&err.diagnosis),
1457                    )
1458                    .map_err(RouterError::Forwarding)?
1459                {
1460                    self.counters
1461                        .increment_client_egress_close_delivery_failed();
1462                }
1463            }
1464        }
1465        Ok(())
1466    }
1467}
1468
1469impl Default for Router {
1470    fn default() -> Self {
1471        Self::with_default_self_handler()
1472    }
1473}
1474
1475/// Connection-scoped cleanup guard returned by [`Router::begin_connection`].
1476#[must_use]
1477pub struct RouterConnection {
1478    id: ConnectionId,
1479    control_handler: Arc<ControlHandler>,
1480    forwarding: Arc<ForwardingTable>,
1481    close_receiver: Option<ConnectionCloseReceiver>,
1482    orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
1483}
1484
1485impl RouterConnection {
1486    pub fn id(&self) -> ConnectionId {
1487        self.id
1488    }
1489
1490    pub(crate) fn take_close_receiver(&mut self) -> ConnectionCloseReceiver {
1491        self.close_receiver
1492            .take()
1493            .expect("connection close receiver can only be taken once")
1494    }
1495}
1496
1497impl Drop for RouterConnection {
1498    fn drop(&mut self) {
1499        self.forwarding.unregister_connection_close(self.id);
1500        self.orphan_goodbyes.forget_connection(self.id);
1501        // GOODBYE (explicit) and connection-drop cleanup both call the same
1502        // idempotent deregistration path.
1503        let _ = self.control_handler.cleanup_connection(self.id);
1504    }
1505}
1506
1507/// Minimal in-memory backend used by tests and early wiring: it emits a
1508/// `RESPONSE` on the same channel/correlation id with the exact same body bytes.
1509#[derive(Debug, Default, Clone, Copy)]
1510pub struct EchoBackend;
1511
1512impl EchoBackend {
1513    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1514        let response = Frame::build_with_version(
1515            frame.header.ver,
1516            FrameType::Response,
1517            frame.header.flags,
1518            frame.header.channel,
1519            frame.header.epoch,
1520            frame.header.corr,
1521            frame.body,
1522        )
1523        .map_err(RouterError::FrameBuild)?;
1524        ctx.egress.send(response).await
1525    }
1526}
1527
1528/// Data-plane backend that splices client frames to the module connection bound at attach time.
1529#[derive(Debug, Clone)]
1530pub struct ForwardBackend {
1531    forwarding: Arc<ForwardingTable>,
1532}
1533
1534impl ForwardBackend {
1535    pub fn new(forwarding: Arc<ForwardingTable>) -> Self {
1536        Self { forwarding }
1537    }
1538
1539    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1540        let channel = frame.header.channel;
1541        let corr = frame.header.corr;
1542        let route = match self
1543            .forwarding
1544            .lookup_data_route(ctx.connection_id, channel, frame.header.epoch)
1545            .map_err(RouterError::Forwarding)?
1546        {
1547            DataRoute::Client(DataRouteState::Bound(route)) => route,
1548            DataRoute::Client(_) | DataRoute::Module(_) => {
1549                self.forwarding.warn_frame_drop(
1550                    ctx.connection_id,
1551                    channel,
1552                    frame.header.epoch,
1553                    "client_to_module",
1554                    "missing_binding",
1555                );
1556                return Err(RouterError::UnknownChannel {
1557                    channel,
1558                    epoch: frame.header.epoch,
1559                    corr,
1560                });
1561            }
1562        };
1563        self.handle_bound(frame, route).await
1564    }
1565
1566    pub(crate) async fn handle_bound(
1567        &self,
1568        frame: Frame,
1569        route: Arc<RouteBinding>,
1570    ) -> Result<(), RouterError> {
1571        let channel = frame.header.channel;
1572        let corr = frame.header.corr;
1573        let frame_type = frame.header.ty;
1574
1575        // CANCEL and other non-REQUEST frames bypass the request-credit window;
1576        // the original request's credit is freed only by the module's terminal frame.
1577        let acquired_credit = frame_type == FrameType::Request;
1578        if acquired_credit && route.flow.busy_refusals() {
1579            use crate::forwarding::RequestAdmission;
1580            match self
1581                .forwarding
1582                .try_admit_request(
1583                    &route,
1584                    frame.header.epoch,
1585                    corr,
1586                    frame.header.flags.is_subscription(),
1587                )
1588                .map_err(RouterError::Forwarding)?
1589            {
1590                RequestAdmission::Admitted => {}
1591                RequestAdmission::Busy(detail) => {
1592                    let principal = principal_label(&route.principal);
1593                    let counters = self.forwarding.counters_ref();
1594                    counters.increment_route_busy(&route.module_id, &principal);
1595                    if route.flow.should_warn_busy() {
1596                        warn!(module_id = %route.module_id, client_principal = %principal, client_channel = channel,
1597                            route_epoch = route.client_epoch, ?detail, "request refused: route_busy");
1598                    }
1599                    let error = RouterError::RouteBusy {
1600                        channel,
1601                        epoch: frame.header.epoch,
1602                        corr,
1603                        detail: Box::new(detail),
1604                    };
1605                    let refusal = error
1606                        .to_error_frame()
1607                        .expect("route_busy is a serializable wire error");
1608                    counters
1609                        .observe_reader_wait(
1610                            || (route.module_id.clone(), principal),
1611                            "route_busy_egress",
1612                            route.client_sink.send(refusal),
1613                        )
1614                        .await?;
1615                    return Ok(());
1616                }
1617                RequestAdmission::ModuleReloading => {
1618                    return Err(RouterError::route_error_with_epoch(
1619                        channel,
1620                        frame.header.epoch,
1621                        corr,
1622                        error_codes::MODULE_RELOADING,
1623                        format!("module endpoint for route channel {channel} is reloading"),
1624                    ));
1625                }
1626                RequestAdmission::Stale => {
1627                    return Err(RouterError::StaleRouteEpoch {
1628                        channel,
1629                        epoch: frame.header.epoch,
1630                        corr,
1631                    })
1632                }
1633            }
1634        } else if acquired_credit {
1635            let counters = self.forwarding.counters_ref();
1636            if let Err(err) = counters
1637                .observe_reader_wait(
1638                    || (route.module_id.clone(), principal_label(&route.principal)),
1639                    "legacy_credit",
1640                    route
1641                        .flow
1642                        .acquire_tagged(corr, frame.header.flags.is_subscription()),
1643                )
1644                .await
1645            {
1646                self.forwarding.warn_frame_drop(
1647                    route.client_connection_id,
1648                    channel,
1649                    frame.header.epoch,
1650                    "client_to_module",
1651                    "endpoint_gone",
1652                );
1653                // `module_reloading` here is answered BEFORE the frame is
1654                // forwarded, so the module never sees the request and callers
1655                // may re-dispatch it after reopening the route. That is a wire
1656                // guarantee documented on `error_codes::MODULE_RELOADING`; do
1657                // not emit this code for a request that already reached
1658                // `module_sink.send` below.
1659                if self
1660                    .forwarding
1661                    .endpoint_is_draining(route.module_endpoint)
1662                    .map_err(RouterError::Forwarding)?
1663                {
1664                    return Err(RouterError::route_error_with_epoch(
1665                        channel,
1666                        frame.header.epoch,
1667                        corr,
1668                        "module_reloading",
1669                        format!("module endpoint for route channel {channel} is reloading"),
1670                    ));
1671                }
1672                return Err(RouterError::backend_with_epoch(
1673                    channel,
1674                    frame.header.epoch,
1675                    corr,
1676                    format!("{err} for route channel {channel}"),
1677                ));
1678            }
1679        }
1680
1681        let mut frame = frame;
1682        frame.header.channel = route.module_channel;
1683        frame.header.epoch = route.module_epoch;
1684        // The origin bit is the daemon's own signal to clients. A client
1685        // frame carrying it must not reach a module appearing daemon-authored,
1686        // so it is cleared here; every other flag bit travels unchanged.
1687        frame.header.flags = frame.header.flags.without_daemon_origin();
1688        let counters = self.forwarding.counters_ref();
1689        let result = counters
1690            .observe_reader_wait(
1691                || (route.module_id.clone(), principal_label(&route.principal)),
1692                "module_queue",
1693                route.module_sink.send(frame),
1694            )
1695            .await
1696            .map_err(|err| {
1697                self.forwarding.warn_frame_drop(
1698                    route.client_connection_id,
1699                    channel,
1700                    route.client_epoch,
1701                    "client_to_module",
1702                    "endpoint_gone",
1703                );
1704                RouterError::backend_with_epoch(channel, route.client_epoch, corr, err.to_string())
1705            });
1706        if acquired_credit && result.is_err() {
1707            route.flow.release_corr(corr);
1708        }
1709        result
1710    }
1711}
1712
1713fn is_terminal_frame(frame_type: FrameType) -> bool {
1714    matches!(
1715        frame_type,
1716        FrameType::Response | FrameType::Error | FrameType::StreamEnd
1717    )
1718}
1719
1720/// Typed router errors. Routable failures can be translated to canonical JSON
1721/// `ERROR` frames with [`RouterError::to_error_frame`].
1722#[derive(Debug, Clone, PartialEq, Eq)]
1723pub enum RouterError {
1724    ReservedChannelZero,
1725    DuplicateChannel {
1726        channel: u16,
1727    },
1728    UnknownChannel {
1729        channel: u16,
1730        epoch: u32,
1731        corr: u64,
1732    },
1733    StaleRouteEpoch {
1734        channel: u16,
1735        epoch: u32,
1736        corr: u64,
1737    },
1738    RouteBusy {
1739        channel: u16,
1740        epoch: u32,
1741        corr: u64,
1742        detail: Box<subc_protocol::RouteBusyDetail>,
1743    },
1744    Backend {
1745        channel: u16,
1746        epoch: u32,
1747        corr: u64,
1748        message: String,
1749    },
1750    RouteError {
1751        channel: u16,
1752        epoch: u32,
1753        corr: u64,
1754        code: String,
1755        message: String,
1756    },
1757    FrameBuild(FrameBuildError),
1758    Forwarding(ForwardingError),
1759}
1760
1761impl RouterError {
1762    pub fn backend(channel: u16, corr: u64, message: impl Into<String>) -> Self {
1763        Self::backend_with_epoch(channel, 0, corr, message)
1764    }
1765
1766    pub fn backend_with_epoch(
1767        channel: u16,
1768        epoch: u32,
1769        corr: u64,
1770        message: impl Into<String>,
1771    ) -> Self {
1772        Self::Backend {
1773            channel,
1774            epoch,
1775            corr,
1776            message: message.into(),
1777        }
1778    }
1779
1780    pub fn route_error(
1781        channel: u16,
1782        corr: u64,
1783        code: impl Into<String>,
1784        message: impl Into<String>,
1785    ) -> Self {
1786        Self::route_error_with_epoch(channel, 0, corr, code, message)
1787    }
1788
1789    pub fn route_error_with_epoch(
1790        channel: u16,
1791        epoch: u32,
1792        corr: u64,
1793        code: impl Into<String>,
1794        message: impl Into<String>,
1795    ) -> Self {
1796        Self::RouteError {
1797            channel,
1798            epoch,
1799            corr,
1800            code: code.into(),
1801            message: message.into(),
1802        }
1803    }
1804
1805    /// Translate route failures that belong on the wire into an `ERROR` frame.
1806    pub fn to_error_frame(&self) -> Option<Frame> {
1807        match self {
1808            Self::RouteBusy {
1809                channel,
1810                epoch,
1811                corr,
1812                detail,
1813            } => {
1814                let body = serde_json::to_vec(&ErrorBody {
1815                    code: error_codes::ROUTE_BUSY.into(),
1816                    message: "route has no free request or watch credit; retry on the same route"
1817                        .into(),
1818                    detail: Some(serde_json::to_value(detail).ok()?),
1819                })
1820                .ok()?;
1821                Frame::build(
1822                    FrameType::Error,
1823                    daemon_error_flags(),
1824                    *channel,
1825                    *epoch,
1826                    *corr,
1827                    body,
1828                )
1829                .ok()
1830            }
1831            Self::UnknownChannel {
1832                channel,
1833                epoch,
1834                corr,
1835            } => error_frame(
1836                *channel,
1837                *epoch,
1838                *corr,
1839                error_codes::UNKNOWN_CHANNEL,
1840                format!("unknown channel {channel}"),
1841            ),
1842            Self::StaleRouteEpoch {
1843                channel,
1844                epoch,
1845                corr,
1846            } => error_frame(
1847                *channel,
1848                *epoch,
1849                *corr,
1850                error_codes::STALE_ROUTE_EPOCH,
1851                format!("stale route epoch for channel {channel}"),
1852            ),
1853            Self::Backend {
1854                channel,
1855                epoch,
1856                corr,
1857                message,
1858            } => error_frame(*channel, *epoch, *corr, "backend_error", message.clone()),
1859            Self::RouteError {
1860                channel,
1861                epoch,
1862                corr,
1863                code,
1864                message,
1865            } => error_frame(*channel, *epoch, *corr, code, message.clone()),
1866            Self::ReservedChannelZero
1867            | Self::DuplicateChannel { .. }
1868            | Self::FrameBuild(_)
1869            | Self::Forwarding(_) => None,
1870        }
1871    }
1872}
1873
1874/// Flags for an ERROR frame the daemon authors itself.
1875///
1876/// Bit 6 (`FLAG_DAEMON_ORIGIN`) tells the client the daemon wrote this
1877/// error, never a module. What a code then proves is per code: on a bound
1878/// route the daemon writes `module_reloading`, `unknown_channel` and
1879/// `stale_route_epoch` only before forwarding, so with the bit they mean the
1880/// request was not delivered, while `frame_too_large` replaces a reply the
1881/// module already produced. Every daemon-authored ERROR goes through here;
1882/// frames a module sent keep their own flags with the bit cleared at the
1883/// splice.
1884/// Nothing else about the flags differs: passive priority, not binary, not last.
1885pub(crate) fn daemon_error_flags() -> Flags {
1886    Flags::new(false, Priority::Passive, false).with_daemon_origin()
1887}
1888
1889fn error_frame(channel: u16, epoch: u32, corr: u64, code: &str, message: String) -> Option<Frame> {
1890    let body = serde_json::to_vec(&ErrorBody {
1891        code: code.to_string(),
1892        message,
1893        detail: None,
1894    })
1895    .ok()?;
1896
1897    Frame::build(
1898        FrameType::Error,
1899        daemon_error_flags(),
1900        channel,
1901        epoch,
1902        corr,
1903        body,
1904    )
1905    .ok()
1906}
1907
1908fn frame_too_large_error(
1909    channel: u16,
1910    epoch: u32,
1911    corr: u64,
1912    bytes: usize,
1913    limit: usize,
1914) -> Result<Frame, RouterError> {
1915    let body = serde_json::to_vec(&ErrorBody {
1916        code: error_codes::FRAME_TOO_LARGE.to_string(),
1917        message: format!(
1918            "module frame of {bytes} bytes exceeds client egress limit of {limit} bytes"
1919        ),
1920        detail: Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit })),
1921    })
1922    .expect("serializing a frame size error");
1923    Frame::build(
1924        FrameType::Error,
1925        daemon_error_flags(),
1926        channel,
1927        epoch,
1928        corr,
1929        body,
1930    )
1931    .map_err(RouterError::FrameBuild)
1932}
1933
1934impl fmt::Display for RouterError {
1935    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1936        match self {
1937            Self::ReservedChannelZero => write!(f, "channel 0 is reserved for subc"),
1938            Self::DuplicateChannel { channel } => {
1939                write!(f, "backend already registered for channel {channel}")
1940            }
1941            Self::UnknownChannel { channel, corr, .. } => {
1942                write!(f, "unknown channel {channel} for corr {corr}")
1943            }
1944            Self::StaleRouteEpoch { channel, corr, .. } => {
1945                write!(f, "stale route epoch for channel {channel} corr {corr}")
1946            }
1947            Self::RouteBusy { channel, corr, .. } => {
1948                write!(f, "route busy on channel {channel} corr {corr}")
1949            }
1950            Self::Backend {
1951                channel,
1952                corr,
1953                message,
1954                ..
1955            } => write!(
1956                f,
1957                "backend error on channel {channel} corr {corr}: {message}"
1958            ),
1959            Self::RouteError {
1960                channel,
1961                corr,
1962                code,
1963                message,
1964                ..
1965            } => write!(
1966                f,
1967                "route error {code} on channel {channel} corr {corr}: {message}"
1968            ),
1969            Self::FrameBuild(err) => write!(f, "failed to build routed frame: {err}"),
1970            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
1971        }
1972    }
1973}
1974
1975impl Error for RouterError {
1976    fn source(&self) -> Option<&(dyn Error + 'static)> {
1977        match self {
1978            Self::FrameBuild(err) => Some(err),
1979            Self::Forwarding(err) => Some(err),
1980            Self::ReservedChannelZero
1981            | Self::DuplicateChannel { .. }
1982            | Self::UnknownChannel { .. }
1983            | Self::StaleRouteEpoch { .. }
1984            | Self::RouteBusy { .. }
1985            | Self::Backend { .. }
1986            | Self::RouteError { .. } => None,
1987        }
1988    }
1989}
1990
1991#[cfg(test)]
1992mod tests {
1993    include!("route_busy_router_tests.rs");
1994    use super::*;
1995    use crate::{
1996        forwarding::RouteBindRelayOutcome,
1997        supervise::{ModuleSpec, RestartPolicy, Supervisor, SupervisorHandle},
1998        ControlHandler, Registry,
1999    };
2000    use std::{
2001        sync::{mpsc as std_mpsc, Arc},
2002        time::Duration,
2003    };
2004    use subc_control::ModuleProtocol;
2005    use subc_protocol::{manifest::Concurrency, ErrorBody, Flags, FrameType, Priority};
2006    use tokio::sync::mpsc;
2007
2008    pub(crate) use crate::router::test_log::{captured_logs, log_capture};
2009
2010    fn logged_millis(logs: &str, field: &str) -> u64 {
2011        logs.split_whitespace()
2012            .find_map(|part| part.strip_prefix(field))
2013            .and_then(|value| value.parse().ok())
2014            .unwrap_or_else(|| panic!("missing numeric {field} in logs: {logs}"))
2015    }
2016
2017    fn request(channel: u16, corr: u64, body: &[u8]) -> Frame {
2018        Frame::build(
2019            FrameType::Request,
2020            Flags::new(true, Priority::Interactive, false),
2021            channel,
2022            0,
2023            corr,
2024            body.to_vec(),
2025        )
2026        .unwrap()
2027    }
2028
2029    fn ping(corr: u64) -> Frame {
2030        Frame::build(
2031            FrameType::Ping,
2032            Flags::new(false, Priority::Passive, false),
2033            0,
2034            0,
2035            corr,
2036            Vec::new(),
2037        )
2038        .unwrap()
2039    }
2040
2041    fn route_ctx() -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
2042        let (tx, rx) = mpsc::channel(8);
2043        (
2044            RouteCtx {
2045                connection_id: ConnectionId::LOCAL,
2046                egress: FrameSink::new(tx),
2047            },
2048            rx,
2049        )
2050    }
2051
2052    #[tokio::test]
2053    async fn echo_backend_returns_response_with_byte_identical_body() {
2054        let mut router = Router::with_default_self_handler();
2055        router.register_backend(7, EchoBackend).unwrap();
2056        let (ctx, mut rx) = route_ctx();
2057        let body = b"{not parsed}\0\xff";
2058
2059        router
2060            .route_for_connection(&ctx, request(7, 123, body))
2061            .await
2062            .unwrap();
2063        let response = rx.recv().await.unwrap();
2064
2065        assert_eq!(response.header.ty, FrameType::Response);
2066        assert_eq!(response.header.channel, 7);
2067        assert_eq!(response.header.corr, 123);
2068        assert_eq!(response.body, body);
2069        assert!(rx.try_recv().is_err());
2070    }
2071
2072    #[tokio::test]
2073    async fn unknown_channel_emits_canonical_error_frame() {
2074        let router = Router::with_default_self_handler();
2075        let (ctx, mut rx) = route_ctx();
2076
2077        router
2078            .route_for_connection(&ctx, request(99, 5, b"payload"))
2079            .await
2080            .unwrap();
2081        let error_frame = rx.recv().await.unwrap();
2082
2083        assert_eq!(error_frame.header.ty, FrameType::Error);
2084        assert_eq!(error_frame.header.channel, 99);
2085        assert_eq!(error_frame.header.corr, 5);
2086        let body: ErrorBody = serde_json::from_slice(&error_frame.body).unwrap();
2087        assert_eq!(body.code, "unknown_channel");
2088        assert_eq!(body.message, "unknown channel 99");
2089    }
2090
2091    /// Every ERROR the daemon authors carries the daemon-origin bit, so a
2092    /// client can tell a daemon refusal from a module's reply. Both the
2093    /// unknown-channel and the stale-epoch errors below are received on the
2094    /// client's own egress, not merely built.
2095    #[tokio::test]
2096    async fn daemon_authored_data_plane_errors_carry_daemon_origin() {
2097        let router = Router::with_default_self_handler();
2098        let (ctx, mut rx) = route_ctx();
2099        router
2100            .route_for_connection(&ctx, request(99, 5, b"payload"))
2101            .await
2102            .unwrap();
2103        let unknown = rx.recv().await.unwrap();
2104        assert_eq!(unknown.header.ty, FrameType::Error);
2105        let body: ErrorBody = serde_json::from_slice(&unknown.body).unwrap();
2106        assert_eq!(body.code, "unknown_channel");
2107        assert!(
2108            unknown.header.flags.is_daemon_origin(),
2109            "unknown_channel must carry daemon origin: {:?}",
2110            unknown.header.flags
2111        );
2112
2113        let (
2114            router,
2115            _forwarding,
2116            client_ctx,
2117            mut client_rx,
2118            _module_ctx,
2119            _module_egress_rx,
2120            _module_rx,
2121            pending,
2122        ) = dynamic_route_fixture(true);
2123        let _ = client_rx.recv().await;
2124        router
2125            .route_for_connection(
2126                &client_ctx,
2127                route_frame(
2128                    FrameType::Request,
2129                    pending.client_channel,
2130                    pending.client_epoch + 1,
2131                    702,
2132                ),
2133            )
2134            .await
2135            .unwrap();
2136        let stale = client_rx.recv().await.unwrap();
2137        assert_eq!(stale.header.ty, FrameType::Error);
2138        let body: ErrorBody = serde_json::from_slice(&stale.body).unwrap();
2139        assert_eq!(body.code, "stale_route_epoch");
2140        assert!(
2141            stale.header.flags.is_daemon_origin(),
2142            "stale_route_epoch must carry daemon origin: {:?}",
2143            stale.header.flags
2144        );
2145    }
2146
2147    /// A module ERROR that spoofs a daemon code with the daemon-origin bit
2148    /// set must reach the client with the bit cleared and the body
2149    /// byte-identical: the bit means the daemon wrote the frame, and a module
2150    /// must never be able to claim that.
2151    #[tokio::test]
2152    async fn module_error_with_daemon_bit_is_stripped_before_client_delivery() {
2153        let (
2154            router,
2155            _forwarding,
2156            client_ctx,
2157            mut client_rx,
2158            module_ctx,
2159            _module_egress_rx,
2160            mut module_rx,
2161            pending,
2162        ) = dynamic_route_fixture(true);
2163        let _ = client_rx.recv().await;
2164        // Open flow credit the way a real request would, so the terminal
2165        // error exercises the credit-release path too.
2166        router
2167            .route_for_connection(
2168                &client_ctx,
2169                route_frame(
2170                    FrameType::Request,
2171                    pending.client_channel,
2172                    pending.client_epoch,
2173                    701,
2174                ),
2175            )
2176            .await
2177            .unwrap();
2178        let _ = module_rx.recv().await;
2179
2180        let body = serde_json::to_vec(&ErrorBody::new("module_reloading", "try again")).unwrap();
2181        let spoof = Frame::build(
2182            FrameType::Error,
2183            Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2184            pending.module_channel,
2185            pending.module_epoch,
2186            701,
2187            body.clone(),
2188        )
2189        .unwrap();
2190        router
2191            .route_for_connection(&module_ctx, spoof)
2192            .await
2193            .unwrap();
2194        let delivered = client_rx.recv().await.unwrap();
2195        assert_eq!(delivered.header.ty, FrameType::Error);
2196        assert_eq!(delivered.header.channel, pending.client_channel);
2197        assert_eq!(delivered.header.epoch, pending.client_epoch);
2198        assert_eq!(delivered.header.corr, 701);
2199        assert!(
2200            !delivered.header.flags.is_daemon_origin(),
2201            "module-set daemon origin must be cleared: {:?}",
2202            delivered.header.flags
2203        );
2204        assert_eq!(delivered.body, body, "the body travels byte-identical");
2205    }
2206
2207    /// Clearing the origin bit strips exactly that bit: priority, LAST and
2208    /// subscription survive the splice unchanged.
2209    #[tokio::test]
2210    async fn module_response_with_daemon_bit_is_stripped_preserving_other_flags() {
2211        use subc_protocol::FLAG_SUBSCRIPTION;
2212        let (
2213            router,
2214            _forwarding,
2215            client_ctx,
2216            mut client_rx,
2217            module_ctx,
2218            _module_egress_rx,
2219            mut module_rx,
2220            pending,
2221        ) = dynamic_route_fixture(true);
2222        let _ = client_rx.recv().await;
2223        router
2224            .route_for_connection(
2225                &client_ctx,
2226                route_frame(
2227                    FrameType::Request,
2228                    pending.client_channel,
2229                    pending.client_epoch,
2230                    701,
2231                ),
2232            )
2233            .await
2234            .unwrap();
2235        let _ = module_rx.recv().await;
2236
2237        let sent_flags = Flags(
2238            Flags::new(true, Priority::Background, true).0
2239                | FLAG_SUBSCRIPTION
2240                | subc_protocol::FLAG_DAEMON_ORIGIN,
2241        );
2242        assert!(sent_flags.is_daemon_origin());
2243        let response = Frame::build(
2244            FrameType::Response,
2245            sent_flags,
2246            pending.module_channel,
2247            pending.module_epoch,
2248            701,
2249            b"route-body".to_vec(),
2250        )
2251        .unwrap();
2252        router
2253            .route_for_connection(&module_ctx, response)
2254            .await
2255            .unwrap();
2256        let delivered = client_rx.recv().await.unwrap();
2257        assert_eq!(delivered.header.ty, FrameType::Response);
2258        assert!(
2259            !delivered.header.flags.is_daemon_origin(),
2260            "module-set daemon origin must be cleared: {:?}",
2261            delivered.header.flags
2262        );
2263        assert!(
2264            delivered.header.flags.is_binary(),
2265            "binary flag must survive: {:?}",
2266            delivered.header.flags
2267        );
2268        assert!(
2269            delivered.header.flags.is_last(),
2270            "LAST flag must survive: {:?}",
2271            delivered.header.flags
2272        );
2273        assert!(
2274            delivered.header.flags.is_subscription(),
2275            "subscription flag must survive: {:?}",
2276            delivered.header.flags
2277        );
2278        assert_eq!(
2279            delivered.header.flags.priority(),
2280            Some(Priority::Background),
2281            "priority must survive"
2282        );
2283    }
2284
2285    /// A client frame carrying the origin bit must not reach the module
2286    /// looking daemon-authored either: the splice clears it symmetrically.
2287    #[tokio::test]
2288    async fn client_frame_with_daemon_bit_is_stripped_before_module_delivery() {
2289        let (
2290            router,
2291            _forwarding,
2292            client_ctx,
2293            mut client_rx,
2294            _module_ctx,
2295            _module_egress_rx,
2296            mut module_rx,
2297            pending,
2298        ) = dynamic_route_fixture(true);
2299        let _ = client_rx.recv().await;
2300        let spoof = Frame::build(
2301            FrameType::Request,
2302            Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2303            pending.client_channel,
2304            pending.client_epoch,
2305            701,
2306            b"route-body".to_vec(),
2307        )
2308        .unwrap();
2309        router
2310            .route_for_connection(&client_ctx, spoof)
2311            .await
2312            .unwrap();
2313        let delivered = module_rx.recv().await.unwrap();
2314        assert_eq!(delivered.header.ty, FrameType::Request);
2315        assert!(
2316            !delivered.header.flags.is_daemon_origin(),
2317            "client-set daemon origin must be cleared: {:?}",
2318            delivered.header.flags
2319        );
2320    }
2321
2322    #[tokio::test]
2323    async fn channel_zero_uses_control_handler_not_backend_registry() {
2324        let mut router = Router::with_default_self_handler();
2325        router.register_backend(1, EchoBackend).unwrap();
2326        let (ctx, mut rx) = route_ctx();
2327
2328        router.route_for_connection(&ctx, ping(77)).await.unwrap();
2329        let response = rx.recv().await.unwrap();
2330
2331        assert_eq!(response.header.ty, FrameType::Pong);
2332        assert_eq!(response.header.channel, 0);
2333        assert_eq!(response.header.corr, 77);
2334        assert!(response.body.is_empty());
2335    }
2336
2337    #[tokio::test]
2338    async fn slow_control_dispatch_logs_decoded_op_and_elapsed_time() {
2339        let control = Arc::new(
2340            ControlHandler::new(Arc::new(Registry::default()))
2341                .with_control_dispatch_delay(Duration::from_millis(1050)),
2342        );
2343        let router = Router::with_control_handler(control);
2344        let (ctx, mut rx) = route_ctx();
2345        let (output, guard) = log_capture(tracing::Level::WARN);
2346
2347        router
2348            .route_for_connection(&ctx, request(0, 41, br#"{"op":"server.describe"}"#))
2349            .await
2350            .expect("slow request routes");
2351        assert!(rx.recv().await.is_some(), "request receives a response");
2352        drop(guard);
2353
2354        let logs = captured_logs(&output);
2355        assert!(logs.contains("slow control dispatch"));
2356        assert!(logs.contains("op=server.describe"));
2357        assert!(logs.contains("connection_id=0"));
2358        assert!(logs.contains("corr=41"));
2359        assert!(
2360            logged_millis(&logs, "elapsed_ms=") >= 1050,
2361            "elapsed must include the injected handler delay: {logs}"
2362        );
2363    }
2364
2365    #[tokio::test]
2366    async fn fast_control_dispatch_emits_arrival_without_slow_warning() {
2367        let router = Router::with_default_self_handler();
2368        let (ctx, mut rx) = route_ctx();
2369        let (output, guard) = log_capture(tracing::Level::DEBUG);
2370
2371        router
2372            .route_for_connection(&ctx, request(0, 42, br#"{"op":"server.describe"}"#))
2373            .await
2374            .expect("fast request routes");
2375        assert!(rx.recv().await.is_some(), "request receives a response");
2376        drop(guard);
2377
2378        let logs = captured_logs(&output);
2379        assert!(logs.contains("control dispatch op=server.describe connection_id=0 corr=42"));
2380        assert!(!logs.contains("slow control dispatch"));
2381    }
2382
2383    #[tokio::test]
2384    async fn control_dispatch_arrival_is_hidden_at_info() {
2385        let router = Router::with_default_self_handler();
2386        let (ctx, mut rx) = route_ctx();
2387        let (output, guard) = log_capture(tracing::Level::INFO);
2388
2389        router
2390            .route_for_connection(&ctx, request(0, 43, br#"{"op":"server.describe"}"#))
2391            .await
2392            .expect("fast request routes");
2393        assert!(rx.recv().await.is_some(), "request receives a response");
2394        drop(guard);
2395
2396        assert!(
2397            !captured_logs(&output).contains("control dispatch"),
2398            "arrival logging must stay hidden at INFO"
2399        );
2400    }
2401
2402    #[tokio::test]
2403    async fn supervisor_list_logs_contended_snapshot_lock_only() {
2404        let registry = Arc::new(Registry::default());
2405        let handle = SupervisorHandle::new();
2406        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
2407            .with_handle(handle.clone());
2408        let module = supervisor
2409            .supervise_configured(
2410                ModuleSpec {
2411                    module_id: "held-module".to_string(),
2412                    program: "test-module".into(),
2413                    args: Vec::new(),
2414                    env: Vec::new(),
2415                    reserved: false,
2416                    reserved_prefixes: Vec::new(),
2417                    protocol: ModuleProtocol::Subc,
2418                    overlap: Default::default(),
2419                },
2420                false,
2421            )
2422            .expect("disabled test module is supervised");
2423        let router = Router::with_control_handler(Arc::new(
2424            ControlHandler::new(Arc::clone(&registry)).with_supervisor(handle),
2425        ));
2426        let (ctx, mut rx) = route_ctx();
2427        let (acquired, ready) = std_mpsc::channel();
2428        // Park the disabled supervisor actor before holding its snapshot. The
2429        // detached list reader, not actor startup, must be the lock contender.
2430        tokio::task::yield_now().await;
2431        let holder = module.hold_snapshot_for_test(acquired, Duration::from_millis(400));
2432        ready.recv().expect("holder acquired snapshot lock");
2433        let (output, guard) = log_capture(tracing::Level::WARN);
2434
2435        router
2436            .route_for_connection(&ctx, request(0, 44, br#"{"op":"supervisor.list"}"#))
2437            .await
2438            .expect("list request routes after the lock releases");
2439        assert!(
2440            rx.recv().await.is_some(),
2441            "list request receives a response"
2442        );
2443        holder.join().expect("snapshot holder exits cleanly");
2444        drop(guard);
2445
2446        let logs = captured_logs(&output);
2447        assert!(logs.contains("slow snapshot lock"));
2448        assert!(logs.contains("module_id=held-module"));
2449        assert!(logs.contains("caller=list"));
2450        assert!(
2451            logged_millis(&logs, "waited_ms=") >= 250,
2452            "wait must exceed the slow-lock threshold: {logs}"
2453        );
2454
2455        let (output, guard) = log_capture(tracing::Level::WARN);
2456        router
2457            .route_for_connection(&ctx, request(0, 45, br#"{"op":"supervisor.list"}"#))
2458            .await
2459            .expect("uncontended list request routes");
2460        assert!(
2461            rx.recv().await.is_some(),
2462            "uncontended list receives a response"
2463        );
2464        drop(guard);
2465        assert!(
2466            !captured_logs(&output).contains("slow snapshot lock"),
2467            "uncontended list acquisition must not warn"
2468        );
2469    }
2470
2471    #[tokio::test]
2472    async fn full_module_to_client_sink_requests_client_close_without_erroring_module() {
2473        let forwarding = Arc::new(ForwardingTable::default());
2474        let control = Arc::new(ControlHandler::with_forwarding(
2475            Arc::new(crate::Registry::default()),
2476            Arc::clone(&forwarding),
2477        ));
2478        let router = Router::with_control_handler(control);
2479        let module_connection = ConnectionId::new(10);
2480        let client_connection = ConnectionId::new(20);
2481        let mut close_receiver = forwarding.register_connection_close(client_connection);
2482        let (module_tx, _module_rx) = mpsc::channel(1);
2483        forwarding
2484            .register_module_connection(
2485                module_connection,
2486                "full-sink-provider".to_string(),
2487                1,
2488                Concurrency::ModuleManaged,
2489                FrameSink::new(module_tx),
2490            )
2491            .unwrap();
2492        let (client_tx, mut client_rx) = mpsc::channel(1);
2493        let pending = forwarding
2494            .begin_route_bind_relay_for_test(
2495                client_connection,
2496                FrameSink::new(client_tx),
2497                700,
2498                "full-sink-provider",
2499            )
2500            .unwrap();
2501        forwarding
2502            .complete_pending_relay(
2503                module_connection,
2504                pending.corr,
2505                RouteBindRelayOutcome::Accepted,
2506            )
2507            .unwrap();
2508
2509        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2510        let module_ctx = RouteCtx {
2511            connection_id: module_connection,
2512            egress: FrameSink::new(module_egress_tx),
2513        };
2514        let terminal = Frame::build(
2515            FrameType::Response,
2516            Flags::new(false, Priority::Interactive, true),
2517            pending.module_channel,
2518            pending.module_epoch,
2519            701,
2520            b"terminal".to_vec(),
2521        )
2522        .unwrap();
2523
2524        router
2525            .route_for_connection(&module_ctx, terminal)
2526            .await
2527            .unwrap();
2528        let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2529            .await
2530            .expect("close request should be sent for the full client sink")
2531            .expect("close sender should include a reason");
2532        assert!(
2533            reason
2534                .to_string()
2535                .contains("module_to_client_delivery_failed"),
2536            "unexpected close reason: {reason}"
2537        );
2538        assert_eq!(client_rx.try_recv().unwrap().header.corr, 700);
2539        assert!(client_rx.try_recv().is_err());
2540        assert_eq!(
2541            router.counters.snapshot()["client_egress_close_delivery_failed"],
2542            1
2543        );
2544    }
2545
2546    /// A terminal frame ends the request at the module even when the client
2547    /// cannot take it, so the drain must stop counting it at once rather than
2548    /// when the client connection's cleanup later removes the route.
2549    #[tokio::test]
2550    async fn terminal_frame_releases_its_credit_even_when_client_delivery_fails() {
2551        let forwarding = Arc::new(ForwardingTable::default());
2552        let control = Arc::new(ControlHandler::with_forwarding(
2553            Arc::new(crate::Registry::default()),
2554            Arc::clone(&forwarding),
2555        ));
2556        let router = Router::with_control_handler(control);
2557        let module_connection = ConnectionId::new(11);
2558        let client_connection = ConnectionId::new(21);
2559        let _close_receiver = forwarding.register_connection_close(client_connection);
2560        let (module_tx, _module_rx) = mpsc::channel(1);
2561        forwarding
2562            .register_module_connection(
2563                module_connection,
2564                "credit-provider".to_string(),
2565                1,
2566                Concurrency::ModuleManaged,
2567                FrameSink::new(module_tx),
2568            )
2569            .unwrap();
2570        // Capacity one, filled by the route.open response, so the terminal
2571        // frame below cannot be enqueued for the client.
2572        let (client_tx, _client_rx) = mpsc::channel(1);
2573        let pending = forwarding
2574            .begin_route_bind_relay_for_test(
2575                client_connection,
2576                FrameSink::new(client_tx),
2577                800,
2578                "credit-provider",
2579            )
2580            .unwrap();
2581        forwarding
2582            .complete_pending_relay(
2583                module_connection,
2584                pending.corr,
2585                RouteBindRelayOutcome::Accepted,
2586            )
2587            .unwrap();
2588        let DataRoute::Client(DataRouteState::Bound(route)) = forwarding
2589            .lookup_data_route(
2590                client_connection,
2591                pending.client_channel,
2592                pending.client_epoch,
2593            )
2594            .unwrap()
2595        else {
2596            panic!("expected a bound client route");
2597        };
2598        route.flow.acquire_tagged(801, false).await.unwrap();
2599        assert_eq!(route.flow.drain_in_flight(), 1);
2600
2601        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2602        let module_ctx = RouteCtx {
2603            connection_id: module_connection,
2604            egress: FrameSink::new(module_egress_tx),
2605        };
2606        let terminal = Frame::build(
2607            FrameType::Response,
2608            Flags::new(false, Priority::Interactive, true),
2609            pending.module_channel,
2610            pending.module_epoch,
2611            801,
2612            b"terminal".to_vec(),
2613        )
2614        .unwrap();
2615        router
2616            .route_for_connection(&module_ctx, terminal)
2617            .await
2618            .unwrap();
2619
2620        assert_eq!(
2621            router.counters.snapshot()["client_egress_close_delivery_failed"],
2622            1,
2623            "the client delivery must have failed for this test to mean anything"
2624        );
2625        assert_eq!(
2626            route.flow.drain_in_flight(),
2627            0,
2628            "the module's terminal frame must release its credit even though the client could not take it"
2629        );
2630    }
2631
2632    #[tokio::test]
2633    async fn full_route_goodbye_sink_requests_target_close_without_erroring_module() {
2634        let forwarding = Arc::new(ForwardingTable::default());
2635        let control = Arc::new(ControlHandler::with_forwarding(
2636            Arc::new(crate::Registry::default()),
2637            Arc::clone(&forwarding),
2638        ));
2639        let router = Router::with_control_handler(control);
2640        let module_connection = ConnectionId::new(30);
2641        let client_connection = ConnectionId::new(40);
2642        let mut close_receiver = forwarding.register_connection_close(client_connection);
2643        let (module_tx, _module_rx) = mpsc::channel(1);
2644        forwarding
2645            .register_module_connection(
2646                module_connection,
2647                "goodbye-full-provider".to_string(),
2648                1,
2649                Concurrency::ModuleManaged,
2650                FrameSink::new(module_tx),
2651            )
2652            .unwrap();
2653        let (client_tx, mut client_rx) = mpsc::channel(1);
2654        let pending = forwarding
2655            .begin_route_bind_relay_for_test(
2656                client_connection,
2657                FrameSink::new(client_tx),
2658                800,
2659                "goodbye-full-provider",
2660            )
2661            .unwrap();
2662        forwarding
2663            .complete_pending_relay(
2664                module_connection,
2665                pending.corr,
2666                RouteBindRelayOutcome::Accepted,
2667            )
2668            .unwrap();
2669
2670        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2671        let module_ctx = RouteCtx {
2672            connection_id: module_connection,
2673            egress: FrameSink::new(module_egress_tx),
2674        };
2675        let goodbye = Frame::build(
2676            FrameType::Goodbye,
2677            Flags::new(false, Priority::Passive, true),
2678            pending.module_channel,
2679            pending.module_epoch,
2680            801,
2681            Vec::new(),
2682        )
2683        .unwrap();
2684
2685        router
2686            .route_for_connection(&module_ctx, goodbye)
2687            .await
2688            .unwrap();
2689        let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2690            .await
2691            .expect("close request should be sent for the full GOODBYE sink")
2692            .expect("close sender should include a reason");
2693        assert!(
2694            reason.to_string().contains("route_goodbye_delivery_failed"),
2695            "unexpected close reason: {reason}"
2696        );
2697        assert_eq!(client_rx.try_recv().unwrap().header.corr, 800);
2698        assert!(client_rx.try_recv().is_err());
2699        assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 1);
2700        assert_eq!(router.counters.snapshot()["route_released_epoch_fenced"], 1);
2701    }
2702
2703    /// One module and one client connection with `routes` routes bound between
2704    /// them. The client uses a real connection egress queue (the same byte
2705    /// budget and frame-count backstop as a live connection), and its route.open
2706    /// responses are drained so the queue starts empty.
2707    async fn multi_route_client(
2708        module_id: &str,
2709        module_connection: ConnectionId,
2710        client_connection: ConnectionId,
2711        routes: usize,
2712    ) -> (
2713        Router,
2714        FrameSink,
2715        mpsc::Receiver<OutboundFrame>,
2716        RouteCtx,
2717        Vec<crate::forwarding::PendingRouteBindRelay>,
2718        ConnectionCloseReceiver,
2719    ) {
2720        let forwarding = Arc::new(ForwardingTable::default());
2721        let control = Arc::new(ControlHandler::with_forwarding(
2722            Arc::new(crate::Registry::default()),
2723            Arc::clone(&forwarding),
2724        ));
2725        let router = Router::with_control_handler(control);
2726        let close_receiver = forwarding.register_connection_close(client_connection);
2727        let (module_tx, _module_rx) = mpsc::channel(8);
2728        forwarding
2729            .register_module_connection(
2730                module_connection,
2731                module_id.to_string(),
2732                1,
2733                Concurrency::ModuleManaged,
2734                FrameSink::new(module_tx),
2735            )
2736            .unwrap();
2737        let (client_sink, mut client_rx) = crate::server::connection_egress();
2738        let mut bound = Vec::with_capacity(routes);
2739        for index in 0..routes {
2740            let pending = forwarding
2741                .begin_route_bind_relay_for_test(
2742                    client_connection,
2743                    client_sink.clone(),
2744                    900 + index as u64,
2745                    module_id,
2746                )
2747                .unwrap();
2748            forwarding
2749                .complete_pending_relay(
2750                    module_connection,
2751                    pending.corr,
2752                    RouteBindRelayOutcome::Accepted,
2753                )
2754                .unwrap();
2755            assert_eq!(
2756                client_rx.recv().await.unwrap().header.corr,
2757                900 + index as u64
2758            );
2759            bound.push(pending);
2760        }
2761        let (module_egress_tx, _module_egress_rx) = mpsc::channel(8);
2762        let module_ctx = RouteCtx {
2763            connection_id: module_connection,
2764            egress: FrameSink::new(module_egress_tx),
2765        };
2766        (
2767            router,
2768            client_sink,
2769            client_rx,
2770            module_ctx,
2771            bound,
2772            close_receiver,
2773        )
2774    }
2775
2776    /// Per-frame cost of the egress sink's admission and release accounting:
2777    /// one million 200-byte frames enqueued with `try_send` and taken off the
2778    /// queue the way the connection writer does, in batches of 1,000 so the
2779    /// queue stays well inside its budget. Prints nanoseconds per frame; it is
2780    /// a measurement, not a gate. Run with
2781    /// `cargo test --release -p subc-daemon --lib egress_sink_per_frame_cost -- --ignored --nocapture`.
2782    #[test]
2783    #[ignore = "timing measurement, run on demand"]
2784    fn egress_sink_per_frame_cost() {
2785        const FRAMES: usize = 1_000_000;
2786        const BATCH: usize = 1_000;
2787        let (sink, mut rx) = crate::server::connection_egress();
2788        let template = stream_frame(9, 1, 0, vec![b't'; 200]);
2789        let started = Instant::now();
2790        for _ in 0..FRAMES / BATCH {
2791            for _ in 0..BATCH {
2792                sink.try_send(template.clone()).unwrap();
2793            }
2794            for _ in 0..BATCH {
2795                let outbound = rx.try_recv().unwrap();
2796                if let Some(charge) = &outbound.charge {
2797                    charge.taken_by_writer();
2798                }
2799                drop(std::hint::black_box(outbound));
2800            }
2801        }
2802        let elapsed = started.elapsed();
2803        assert_eq!(sink.backlog().queued_bytes, 0);
2804        println!(
2805            "egress sink: {FRAMES} frames in {elapsed:?}, {:.1} ns/frame",
2806            elapsed.as_nanos() as f64 / FRAMES as f64
2807        );
2808    }
2809
2810    fn stream_frame(channel: u16, epoch: u32, corr: u64, body: Vec<u8>) -> Frame {
2811        Frame::build(
2812            FrameType::StreamData,
2813            Flags::new(false, Priority::Interactive, false),
2814            channel,
2815            epoch,
2816            corr,
2817            body,
2818        )
2819        .unwrap()
2820    }
2821
2822    fn module_route_for_test(
2823        router: &Router,
2824        ctx: &RouteCtx,
2825        route: &crate::forwarding::PendingRouteBindRelay,
2826    ) -> Arc<RouteBinding> {
2827        match router
2828            .forwarding
2829            .lookup_data_route(ctx.connection_id, route.module_channel, route.module_epoch)
2830            .unwrap()
2831        {
2832            DataRoute::Module(DataRouteState::Bound(binding)) => binding,
2833            other => panic!("expected a bound module route, got {other:?}"),
2834        }
2835    }
2836
2837    fn assert_frame_size_error(frame: &Frame, channel: u16, epoch: u32, corr: u64, bytes: usize) {
2838        assert_eq!(frame.header.ty, FrameType::Error);
2839        assert_eq!(frame.header.channel, channel);
2840        assert_eq!(frame.header.epoch, epoch);
2841        assert_eq!(frame.header.corr, corr);
2842        assert!(frame.header.flags.is_daemon_origin());
2843        let body: ErrorBody = serde_json::from_slice(&frame.body).unwrap();
2844        assert_eq!(body.code, error_codes::FRAME_TOO_LARGE);
2845        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2846        assert!(body.message.contains(&bytes.to_string()));
2847        assert!(body.message.contains(&limit.to_string()));
2848        assert_eq!(
2849            body.detail,
2850            Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit }))
2851        );
2852    }
2853
2854    #[tokio::test]
2855    async fn module_egress_response_oversized_reply_keeps_connection_and_other_route() {
2856        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
2857        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2858            "oversize-provider",
2859            ConnectionId::new(80),
2860            ConnectionId::new(81),
2861            2,
2862        )
2863        .await;
2864        let binding = module_route_for_test(&router, &ctx, &routes[0]);
2865        binding.flow.acquire_tagged(101, false).await.unwrap();
2866        module_route_for_test(&router, &ctx, &routes[1])
2867            .flow
2868            .acquire_tagged(102, false)
2869            .await
2870            .unwrap();
2871        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2872        let oversized = Frame::build(
2873            FrameType::Response,
2874            Flags::new(false, Priority::Interactive, true),
2875            routes[0].module_channel,
2876            routes[0].module_epoch,
2877            101,
2878            vec![b'x'; limit - subc_protocol::HEADER_LEN + 1],
2879        )
2880        .unwrap();
2881        assert_eq!(sink.backlog().queued_bytes, 0);
2882        router.route_for_connection(&ctx, oversized).await.unwrap();
2883
2884        // Reproduce a large reply followed by ordinary traffic on another route
2885        // before the client reads anything. The large reply must not monopolize
2886        // the queue and make that second delivery close the shared connection.
2887        router
2888            .route_for_connection(
2889                &ctx,
2890                route_frame(
2891                    FrameType::Response,
2892                    routes[1].module_channel,
2893                    routes[1].module_epoch,
2894                    102,
2895                ),
2896            )
2897            .await
2898            .unwrap();
2899        assert!(
2900            close.try_recv().is_err(),
2901            "an oversized reply must not close the multiplexed client"
2902        );
2903        let error = rx.try_recv().expect("the waiting client gets an ERROR");
2904        assert_frame_size_error(
2905            &error,
2906            routes[0].client_channel,
2907            routes[0].client_epoch,
2908            101,
2909            limit + 1,
2910        );
2911        let second = rx.try_recv().expect("the other route's reply arrives");
2912        assert_eq!(second.header.ty, FrameType::Response);
2913        assert_eq!(second.header.channel, routes[1].client_channel);
2914        assert_eq!(second.header.epoch, routes[1].client_epoch);
2915        assert_eq!(second.header.corr, 102);
2916        assert_eq!(second.body, b"route-body");
2917        assert_eq!(binding.flow.drain_in_flight(), 0);
2918        for route in &routes {
2919            module_route_for_test(&router, &ctx, route);
2920        }
2921        assert!(rx.try_recv().is_err());
2922        drop((error, second));
2923        assert_eq!(sink.backlog().queued_bytes, 0);
2924        assert_eq!(
2925            router.counters.snapshot()["client_egress_close_delivery_failed"],
2926            0
2927        );
2928
2929        let captured = test_log::captured_logs(&logs);
2930        let warnings: Vec<_> = captured
2931            .lines()
2932            .filter(|line| line.contains("WARN"))
2933            .collect();
2934        assert_eq!(warnings.len(), 1, "{captured}");
2935        let warning = warnings[0];
2936        for expected in [
2937            "module_id=oversize-provider".to_string(),
2938            "client_principal=direct".to_string(),
2939            format!("client_channel={}", routes[0].client_channel),
2940            format!("frame_bytes={}", limit + 1),
2941            format!("byte_limit={limit}"),
2942        ] {
2943            assert!(warning.contains(&expected), "missing {expected}: {warning}");
2944        }
2945    }
2946
2947    async fn poll_once<F: std::future::Future>(
2948        mut future: std::pin::Pin<&mut F>,
2949    ) -> std::task::Poll<F::Output> {
2950        std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
2951    }
2952
2953    fn fill_client_budget(sink: &FrameSink, route: &crate::forwarding::PendingRouteBindRelay) {
2954        sink.try_send(
2955            Frame::build(
2956                FrameType::Response,
2957                Flags::new(false, Priority::Interactive, true),
2958                route.client_channel,
2959                route.client_epoch,
2960                100,
2961                vec![
2962                    b'x';
2963                    crate::server::CONNECTION_EGRESS_BYTE_BUDGET - subc_protocol::HEADER_LEN
2964                ],
2965            )
2966            .unwrap(),
2967        )
2968        .unwrap();
2969        assert_eq!(
2970            sink.backlog().queued_bytes,
2971            crate::server::CONNECTION_EGRESS_BYTE_BUDGET
2972        );
2973    }
2974
2975    async fn assert_full_client_burst_survives(ty: FrameType, oversized: bool) {
2976        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2977            "draining-burst",
2978            ConnectionId::new(92),
2979            ConnectionId::new(93),
2980            2,
2981        )
2982        .await;
2983        fill_client_budget(&sink, &routes[0]);
2984        let frame = if oversized {
2985            Frame::build(
2986                ty,
2987                Flags::new(false, Priority::Interactive, false),
2988                routes[1].module_channel,
2989                routes[1].module_epoch,
2990                101,
2991                vec![b's'; crate::server::CONNECTION_EGRESS_BYTE_BUDGET],
2992            )
2993            .unwrap()
2994        } else {
2995            route_frame(ty, routes[1].module_channel, routes[1].module_epoch, 101)
2996        };
2997        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2998        assert!(poll_once(delivery.as_mut()).await.is_pending());
2999        assert!(
3000            close.try_recv().is_err(),
3001            "fresh bursts must not close: {ty:?}"
3002        );
3003        tokio::time::advance(Duration::from_millis(50)).await;
3004        let queued = rx.try_recv().unwrap();
3005        queued.charge.as_ref().unwrap().taken_by_writer();
3006        drop(queued);
3007        assert!(matches!(
3008            poll_once(delivery.as_mut()).await,
3009            std::task::Poll::Ready(Ok(()))
3010        ));
3011        let delivered = rx
3012            .try_recv()
3013            .expect("the burst frame must be delivered after draining");
3014        assert_eq!(delivered.header.channel, routes[1].client_channel);
3015        assert_eq!(delivered.header.epoch, routes[1].client_epoch);
3016        assert_eq!(delivered.header.corr, 101);
3017        if oversized {
3018            assert_frame_size_error(
3019                &delivered,
3020                routes[1].client_channel,
3021                routes[1].client_epoch,
3022                101,
3023                crate::server::CONNECTION_EGRESS_BYTE_BUDGET + subc_protocol::HEADER_LEN,
3024            );
3025        } else {
3026            assert_eq!(delivered.header.ty, ty);
3027        }
3028        assert!(rx.try_recv().is_err());
3029        assert!(close.try_recv().is_err());
3030        assert_eq!(
3031            router.counters.snapshot()["client_egress_close_delivery_failed"],
3032            0
3033        );
3034        assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 0);
3035    }
3036
3037    #[tokio::test(start_paused = true)]
3038    async fn module_egress_fresh_full_queue_burst_survives() {
3039        for ty in [
3040            FrameType::Response,
3041            FrameType::StreamData,
3042            FrameType::Request,
3043            FrameType::Push,
3044        ] {
3045            assert_full_client_burst_survives(ty, false).await;
3046        }
3047    }
3048
3049    #[tokio::test(start_paused = true)]
3050    async fn module_goodbye_fresh_full_queue_burst_survives() {
3051        assert_full_client_burst_survives(FrameType::Goodbye, false).await;
3052    }
3053
3054    #[tokio::test(start_paused = true)]
3055    async fn frame_count_backstop_also_allows_a_draining_burst() {
3056        let (tx, mut rx) = mpsc::channel(1);
3057        let sink = FrameSink::with_byte_budget(tx, 2_000);
3058        sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
3059            .unwrap();
3060        let mut delivery = Box::pin(sink.send_to_client(stream_frame(1, 1, 2, vec![b'y'; 100])));
3061        assert!(poll_once(delivery.as_mut()).await.is_pending());
3062        assert!(
3063            sink.backlog().queued_bytes < 2_000,
3064            "only frame capacity is exhausted"
3065        );
3066        tokio::time::advance(Duration::from_millis(50)).await;
3067        drop(rx.try_recv().unwrap());
3068        assert!(matches!(
3069            poll_once(delivery.as_mut()).await,
3070            std::task::Poll::Ready(Ok(()))
3071        ));
3072        assert_eq!(rx.try_recv().unwrap().header.corr, 2);
3073    }
3074
3075    #[tokio::test(start_paused = true)]
3076    async fn oversized_reply_replacement_fresh_full_queue_burst_survives() {
3077        for ty in [FrameType::Response, FrameType::StreamData] {
3078            assert_full_client_burst_survives(ty, true).await;
3079        }
3080    }
3081
3082    #[tokio::test(start_paused = true)]
3083    async fn module_egress_stopped_reader_closes_without_waiting() {
3084        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3085        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3086            "oldest-stopped-reader",
3087            ConnectionId::new(94),
3088            ConnectionId::new(95),
3089            2,
3090        )
3091        .await;
3092        fill_client_budget(&sink, &routes[0]);
3093        // The writer has taken the head but cannot finish writing it. Its
3094        // bytes stay charged, and the atomic age still diagnoses that stall.
3095        let writing = rx.try_recv().unwrap();
3096        writing.charge.as_ref().unwrap().taken_by_writer();
3097        tokio::time::advance(Duration::from_millis(251)).await;
3098        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(251)));
3099        assert_eq!(sink.backlog().queued_bytes, queued_frame_bytes(&writing));
3100        let started = tokio::time::Instant::now();
3101        let frame = stream_frame(
3102            routes[0].module_channel,
3103            routes[0].module_epoch,
3104            101,
3105            b"secret-body-never-log".to_vec(),
3106        );
3107        let flags = frame.header.flags.0;
3108        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3109        assert!(
3110            matches!(
3111                poll_once(delivery.as_mut()).await,
3112                std::task::Poll::Ready(Ok(()))
3113            ),
3114            "an oldest frame older than 250 ms must close immediately, without a new wait"
3115        );
3116        assert_eq!(started.elapsed(), Duration::ZERO);
3117        assert!(close
3118            .try_recv()
3119            .unwrap()
3120            .to_string()
3121            .contains("module_to_client_delivery_failed"));
3122        let captured = test_log::captured_logs(&logs);
3123        for expected in [
3124            "frame_type=Some(StreamData)".to_string(),
3125            format!("frame_flags=Some({flags})"),
3126            "oldest_queued_ms=251".to_string(),
3127            "close_cause=\"oldest_frame_older_than_250_ms\"".to_string(),
3128        ] {
3129            assert!(
3130                captured.contains(&expected),
3131                "missing {expected}: {captured}"
3132            );
3133        }
3134        assert!(!captured.contains("secret-body-never-log"));
3135    }
3136
3137    #[tokio::test(start_paused = true)]
3138    async fn client_gone_mid_wait_resumes_module_reader_immediately() {
3139        for writer_gone in [false, true] {
3140            let (router, sink, mut rx, ctx, routes, _close) = multi_route_client(
3141                "gone-during-burst",
3142                ConnectionId::new(96),
3143                ConnectionId::new(97),
3144                1,
3145            )
3146            .await;
3147            fill_client_budget(&sink, &routes[0]);
3148            let frame = route_frame(
3149                FrameType::Response,
3150                routes[0].module_channel,
3151                routes[0].module_epoch,
3152                101,
3153            );
3154            let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3155            assert!(poll_once(delivery.as_mut()).await.is_pending());
3156            tokio::time::advance(Duration::from_millis(50)).await;
3157            let started = tokio::time::Instant::now();
3158            if writer_gone {
3159                rx.close();
3160            } else {
3161                sink.close_client_delivery();
3162            }
3163            assert!(
3164                matches!(
3165                    poll_once(delivery.as_mut()).await,
3166                    std::task::Poll::Ready(Ok(()))
3167                ),
3168                "client teardown must wake the module reader before the writer drains"
3169            );
3170            assert_eq!(started.elapsed(), Duration::ZERO);
3171            assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
3172            assert_eq!(rx.try_recv().unwrap().header.corr, 100);
3173            assert!(rx.try_recv().is_err());
3174        }
3175    }
3176
3177    #[tokio::test(start_paused = true)]
3178    async fn cancelled_module_delivery_releases_waiter_without_closing_client() {
3179        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3180            "cancelled-burst",
3181            ConnectionId::new(98),
3182            ConnectionId::new(99),
3183            1,
3184        )
3185        .await;
3186        fill_client_budget(&sink, &routes[0]);
3187        let frame = route_frame(
3188            FrameType::Response,
3189            routes[0].module_channel,
3190            routes[0].module_epoch,
3191            101,
3192        );
3193        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3194        assert!(poll_once(delivery.as_mut()).await.is_pending());
3195        assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 1);
3196        drop(delivery);
3197        assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
3198        tokio::time::advance(Duration::from_secs(1)).await;
3199        assert!(close.try_recv().is_err());
3200        drop(rx.try_recv().unwrap());
3201        assert_eq!(sink.backlog().queued_bytes, 0);
3202        assert!(sink.backlog().oldest_age.is_none());
3203    }
3204
3205    #[tokio::test(start_paused = true)]
3206    async fn egress_atomic_age_follows_writer_and_clears_when_drained() {
3207        let (sink, mut rx) = crate::server::connection_egress();
3208        sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
3209            .unwrap();
3210        tokio::time::advance(Duration::from_millis(200)).await;
3211        sink.try_send(stream_frame(2, 1, 2, vec![b'y'; 100]))
3212            .unwrap();
3213        let writing = rx.try_recv().unwrap();
3214        writing.charge.as_ref().unwrap().taken_by_writer();
3215        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3216        drop(writing);
3217        // Between frames the lock-free marker can still name the previous
3218        // head. The next writer take advances it without scanning the queue.
3219        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3220        tokio::time::advance(Duration::from_millis(50)).await;
3221        let next = rx.try_recv().unwrap();
3222        next.charge.as_ref().unwrap().taken_by_writer();
3223        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(50)));
3224        assert!(
3225            sink.backlog().queued_bytes > 0,
3226            "in-flight bytes remain charged until written"
3227        );
3228        drop(next);
3229        assert_eq!(sink.backlog().queued_bytes, 0);
3230        assert_eq!(sink.backlog().oldest_age, None);
3231    }
3232
3233    #[tokio::test(start_paused = true)]
3234    async fn module_egress_response_full_client_waits_only_remaining_grace() {
3235        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3236            "backpressure-provider",
3237            ConnectionId::new(82),
3238            ConnectionId::new(83),
3239            2,
3240        )
3241        .await;
3242        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3243        // Exactly the limit, including its header, must be admitted to an empty
3244        // queue. Leave it unread to deterministically exhaust only byte capacity.
3245        router
3246            .route_for_connection(
3247                &ctx,
3248                Frame::build(
3249                    FrameType::Response,
3250                    Flags::new(false, Priority::Interactive, true),
3251                    routes[0].module_channel,
3252                    routes[0].module_epoch,
3253                    201,
3254                    vec![b'x'; limit - subc_protocol::HEADER_LEN],
3255                )
3256                .unwrap(),
3257            )
3258            .await
3259            .unwrap();
3260        assert_eq!(sink.backlog().queued_bytes, limit);
3261        assert_eq!(sink.backlog().queued_frames, 1);
3262        let ordinary = route_frame(
3263            FrameType::Response,
3264            routes[1].module_channel,
3265            routes[1].module_epoch,
3266            202,
3267        );
3268        assert!(queued_frame_bytes(&ordinary) < limit);
3269        // A fitting reply gets only the unused portion of the oldest frame's
3270        // 250 ms grace, not an immediate close or a fresh 250 ms wait.
3271        tokio::time::advance(Duration::from_millis(100)).await;
3272        let mut delivery = Box::pin(router.route_for_connection(&ctx, ordinary));
3273        assert!(poll_once(delivery.as_mut()).await.is_pending());
3274        assert!(close.try_recv().is_err());
3275        tokio::time::advance(Duration::from_millis(149)).await;
3276        assert!(poll_once(delivery.as_mut()).await.is_pending());
3277        assert!(close.try_recv().is_err());
3278        tokio::time::advance(Duration::from_millis(1)).await;
3279        assert!(matches!(
3280            poll_once(delivery.as_mut()).await,
3281            std::task::Poll::Ready(Ok(()))
3282        ));
3283        drop(delivery);
3284        let reason = close.try_recv().expect("the full client must close");
3285        assert!(reason
3286            .to_string()
3287            .contains("module_to_client_delivery_failed"));
3288        assert!(!reason.to_string().contains("frame_too_large"));
3289        assert_eq!(
3290            router.counters.snapshot()["client_egress_close_delivery_failed"],
3291            1
3292        );
3293        let admitted = rx.try_recv().expect("the budget-sized frame was admitted");
3294        assert_eq!(admitted.header.ty, FrameType::Response);
3295        assert_eq!(admitted.header.corr, 201);
3296        assert!(
3297            rx.try_recv().is_err(),
3298            "no oversized-frame ERROR is substituted for backpressure"
3299        );
3300    }
3301
3302    #[tokio::test(start_paused = true)]
3303    async fn oversized_reply_to_a_full_client_waits_only_remaining_grace() {
3304        let (router, sink, _rx, ctx, routes, mut close) = multi_route_client(
3305            "full-oversize-provider",
3306            ConnectionId::new(90),
3307            ConnectionId::new(91),
3308            2,
3309        )
3310        .await;
3311        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3312        // Fill the client's queue to its budget and leave it unread.
3313        router
3314            .route_for_connection(
3315                &ctx,
3316                Frame::build(
3317                    FrameType::Response,
3318                    Flags::new(false, Priority::Interactive, true),
3319                    routes[0].module_channel,
3320                    routes[0].module_epoch,
3321                    501,
3322                    vec![b'x'; limit - subc_protocol::HEADER_LEN],
3323                )
3324                .unwrap(),
3325            )
3326            .await
3327            .unwrap();
3328        assert_eq!(sink.backlog().queued_bytes, limit);
3329        let oversized = Frame::build(
3330            FrameType::Response,
3331            Flags::new(false, Priority::Interactive, true),
3332            routes[1].module_channel,
3333            routes[1].module_epoch,
3334            502,
3335            vec![b'y'; limit - subc_protocol::HEADER_LEN + 1],
3336        )
3337        .unwrap();
3338        // The replacement ERROR gets the same bounded burst grace as the
3339        // ordinary reply; oversized payloads themselves are never queued.
3340        tokio::time::advance(Duration::from_millis(80)).await;
3341        let mut delivery = Box::pin(router.route_for_connection(&ctx, oversized));
3342        assert!(poll_once(delivery.as_mut()).await.is_pending());
3343        assert!(close.try_recv().is_err());
3344        tokio::time::advance(Duration::from_millis(169)).await;
3345        assert!(poll_once(delivery.as_mut()).await.is_pending());
3346        assert!(close.try_recv().is_err());
3347        tokio::time::advance(Duration::from_millis(1)).await;
3348        assert!(matches!(
3349            poll_once(delivery.as_mut()).await,
3350            std::task::Poll::Ready(Ok(()))
3351        ));
3352        drop(delivery);
3353        let reason = close.try_recv().expect("the full client must close");
3354        assert!(reason
3355            .to_string()
3356            .contains("module_to_client_delivery_failed"));
3357    }
3358
3359    #[tokio::test]
3360    async fn oversized_reverse_request_errors_module_without_reaching_client() {
3361        let (router, _sink, mut rx, mut ctx, routes, mut close) = multi_route_client(
3362            "reverse-oversize-provider",
3363            ConnectionId::new(84),
3364            ConnectionId::new(85),
3365            2,
3366        )
3367        .await;
3368        let (module_tx, mut module_rx) = mpsc::channel(8);
3369        ctx.egress = FrameSink::new(module_tx);
3370        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3371        router
3372            .route_for_connection(
3373                &ctx,
3374                Frame::build(
3375                    FrameType::Request,
3376                    Flags::new(false, Priority::Interactive, false),
3377                    routes[0].module_channel,
3378                    routes[0].module_epoch,
3379                    301,
3380                    vec![b'r'; limit - subc_protocol::HEADER_LEN + 1],
3381                )
3382                .unwrap(),
3383            )
3384            .await
3385            .unwrap();
3386        assert!(
3387            rx.try_recv().is_err(),
3388            "the reverse request never reaches the client"
3389        );
3390        let error = module_rx.try_recv().expect("the module gets the refusal");
3391        assert_frame_size_error(
3392            &error,
3393            routes[0].module_channel,
3394            routes[0].module_epoch,
3395            301,
3396            limit + 1,
3397        );
3398        assert!(module_rx.try_recv().is_err());
3399        for (index, route) in routes.iter().enumerate() {
3400            module_route_for_test(&router, &ctx, route);
3401            router
3402                .route_for_connection(
3403                    &ctx,
3404                    route_frame(
3405                        FrameType::Response,
3406                        route.module_channel,
3407                        route.module_epoch,
3408                        302 + index as u64,
3409                    ),
3410                )
3411                .await
3412                .unwrap();
3413            let reply = rx.try_recv().expect("both routes still deliver");
3414            assert_eq!(reply.header.ty, FrameType::Response);
3415            assert_eq!(reply.header.channel, route.client_channel);
3416        }
3417        assert!(close.try_recv().is_err());
3418        assert_eq!(
3419            router.counters.snapshot()["client_egress_close_delivery_failed"],
3420            0
3421        );
3422    }
3423
3424    #[tokio::test]
3425    async fn oversized_push_closes_client_with_precise_warn() {
3426        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3427        let (router, _sink, mut rx, ctx, routes, mut close) = multi_route_client(
3428            "push-oversize-provider",
3429            ConnectionId::new(86),
3430            ConnectionId::new(87),
3431            2,
3432        )
3433        .await;
3434        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3435        router
3436            .route_for_connection(
3437                &ctx,
3438                Frame::build(
3439                    FrameType::Push,
3440                    Flags::new(false, Priority::Interactive, false),
3441                    routes[0].module_channel,
3442                    routes[0].module_epoch,
3443                    0,
3444                    vec![b'p'; limit - subc_protocol::HEADER_LEN + 1],
3445                )
3446                .unwrap(),
3447            )
3448            .await
3449            .unwrap();
3450        let reason = close
3451            .try_recv()
3452            .expect("unsolicited oversize still closes the client");
3453        assert!(reason
3454            .to_string()
3455            .contains("frame_too_large: oversized push"));
3456        assert!(
3457            rx.try_recv().is_err(),
3458            "no part of the oversized push is forwarded"
3459        );
3460        let captured = test_log::captured_logs(&logs);
3461        let warnings: Vec<_> = captured
3462            .lines()
3463            .filter(|line| line.contains("WARN"))
3464            .collect();
3465        assert_eq!(warnings.len(), 1, "{captured}");
3466        let warning = warnings[0];
3467        for expected in [
3468            "oversized push".to_string(),
3469            "module_id=\"push-oversize-provider\"".to_string(),
3470            "principals=direct".to_string(),
3471            format!("client_channel={}", routes[0].client_channel),
3472            format!("size={}", limit + 1),
3473            format!("limit={limit}"),
3474        ] {
3475            assert!(warning.contains(&expected), "missing {expected}: {warning}");
3476        }
3477        assert_eq!(
3478            router.counters.snapshot()["client_egress_close_delivery_failed"],
3479            1
3480        );
3481    }
3482
3483    #[tokio::test]
3484    async fn oversized_stream_and_error_frames_end_request_even_with_queued_traffic() {
3485        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3486            "stream-oversize-provider",
3487            ConnectionId::new(88),
3488            ConnectionId::new(89),
3489            2,
3490        )
3491        .await;
3492        let binding = module_route_for_test(&router, &ctx, &routes[0]);
3493        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3494        for ty in [
3495            FrameType::StreamData,
3496            FrameType::StreamEnd,
3497            FrameType::Error,
3498        ] {
3499            binding.flow.acquire_tagged(401, false).await.unwrap();
3500            sink.try_send(route_frame(
3501                FrameType::Response,
3502                routes[1].client_channel,
3503                routes[1].client_epoch,
3504                402,
3505            ))
3506            .unwrap();
3507            router
3508                .route_for_connection(
3509                    &ctx,
3510                    Frame::build(
3511                        ty,
3512                        Flags::new(false, Priority::Interactive, false),
3513                        routes[0].module_channel,
3514                        routes[0].module_epoch,
3515                        401,
3516                        vec![b's'; limit - subc_protocol::HEADER_LEN + 1],
3517                    )
3518                    .unwrap(),
3519                )
3520                .await
3521                .unwrap();
3522            assert!(close.try_recv().is_err());
3523            assert_eq!(rx.try_recv().unwrap().header.corr, 402);
3524            let error = rx
3525                .try_recv()
3526                .expect("every correlated oversize frame becomes ERROR");
3527            assert_frame_size_error(
3528                &error,
3529                routes[0].client_channel,
3530                routes[0].client_epoch,
3531                401,
3532                limit + 1,
3533            );
3534            assert_eq!(binding.flow.drain_in_flight(), 0, "{ty:?} ends the request");
3535            assert!(rx.try_recv().is_err());
3536        }
3537    }
3538
3539    /// An awaited send (a control or error reply) is never refused by the byte
3540    /// budget: it parks until the writer frees bytes. If a release could miss a
3541    /// parked sender, that reply would hang for the connection's lifetime, so
3542    /// this pins the wakeup: the send stays parked while the queue is full and
3543    /// completes as soon as one frame leaves it.
3544    #[tokio::test]
3545    async fn awaited_send_parked_behind_a_full_byte_budget_wakes_when_bytes_free() {
3546        let (tx, mut rx) = mpsc::channel(1024);
3547        let sink = FrameSink::with_byte_budget(tx, 2_000);
3548        let mut queued = 0u64;
3549        while sink
3550            .try_send(stream_frame(7, 1, queued, vec![b'x'; 200]))
3551            .is_ok()
3552        {
3553            queued += 1;
3554        }
3555        assert!(queued > 0, "the budget admitted nothing");
3556
3557        let parked = tokio::spawn({
3558            let sink = sink.clone();
3559            async move { sink.send(stream_frame(0, 0, 999, vec![b'r'; 200])).await }
3560        });
3561        tokio::time::sleep(Duration::from_millis(100)).await;
3562        assert!(
3563            !parked.is_finished(),
3564            "the awaited send must wait while the byte budget is full"
3565        );
3566
3567        // The writer takes one frame and drops it, releasing its bytes.
3568        drop(rx.recv().await.expect("a queued frame"));
3569        tokio::time::timeout(Duration::from_secs(2), parked)
3570            .await
3571            .expect("the parked send was never woken after bytes were freed")
3572            .unwrap()
3573            .unwrap();
3574    }
3575
3576    /// A client multiplexing several token streams pauses its reader while the
3577    /// module keeps producing small frames. Far more frames than the old
3578    /// 64-frame queue allowed, but far fewer bytes than the budget, must all be
3579    /// held without closing the connection and delivered in order afterwards.
3580    #[tokio::test]
3581    async fn paused_client_reader_keeps_connection_through_small_frame_burst() {
3582        const ROUTES: usize = 4;
3583        const FRAMES_PER_ROUTE: usize = 250;
3584        let (router, client_sink, mut client_rx, module_ctx, routes, mut close_receiver) =
3585            multi_route_client(
3586                "burst-provider",
3587                ConnectionId::new(60),
3588                ConnectionId::new(61),
3589                ROUTES,
3590            )
3591            .await;
3592
3593        // The reader is paused: nothing is received until every frame is sent.
3594        for seq in 0..FRAMES_PER_ROUTE as u64 {
3595            for (index, route) in routes.iter().enumerate() {
3596                let body = format!("route-{index}-token-{seq:05}-{}", "t".repeat(170));
3597                router
3598                    .route_for_connection(
3599                        &module_ctx,
3600                        stream_frame(
3601                            route.module_channel,
3602                            route.module_epoch,
3603                            seq,
3604                            body.into_bytes(),
3605                        ),
3606                    )
3607                    .await
3608                    .unwrap();
3609            }
3610        }
3611
3612        let backlog = client_sink.backlog();
3613        assert_eq!(backlog.queued_frames, ROUTES * FRAMES_PER_ROUTE);
3614        assert!(backlog.queued_bytes < crate::server::CONNECTION_EGRESS_BYTE_BUDGET);
3615        assert!(
3616            close_receiver.try_recv().is_err(),
3617            "a paused reader under the byte budget must not be closed"
3618        );
3619        assert_eq!(
3620            router.counters.snapshot()["client_egress_close_delivery_failed"],
3621            0
3622        );
3623
3624        // The reader resumes: every frame arrives, in order within each route.
3625        let mut next_seq = vec![0u64; ROUTES];
3626        for _ in 0..ROUTES * FRAMES_PER_ROUTE {
3627            let frame = client_rx.try_recv().expect("every queued frame arrives");
3628            let index = routes
3629                .iter()
3630                .position(|route| route.client_channel == frame.header.channel)
3631                .expect("frame arrives on one of the bound client channels");
3632            assert_eq!(frame.header.corr, next_seq[index], "route {index} order");
3633            let expected_prefix = format!("route-{index}-token-{:05}-", next_seq[index]);
3634            assert!(frame.body.starts_with(expected_prefix.as_bytes()));
3635            next_seq[index] += 1;
3636        }
3637        assert!(client_rx.try_recv().is_err());
3638        assert_eq!(next_seq, vec![FRAMES_PER_ROUTE as u64; ROUTES]);
3639        assert_eq!(client_sink.backlog().queued_bytes, 0);
3640    }
3641
3642    /// A client that never reads exhausts its byte budget and the oldest
3643    /// frame's 250 ms grace, then closes with one header-only diagnosis.
3644    #[tokio::test(start_paused = true)]
3645    async fn never_reading_client_is_closed_at_byte_budget_with_warn_diagnosis() {
3646        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3647        const BODY: usize = 16 * 1024;
3648        let (router, client_sink, _client_rx, module_ctx, routes, mut close_receiver) =
3649            multi_route_client(
3650                "stuck-reader-provider",
3651                ConnectionId::new(70),
3652                ConnectionId::new(71),
3653                2,
3654            )
3655            .await;
3656
3657        let mut admitted = 0usize;
3658        let mut sent = 0u64;
3659        let started = tokio::time::Instant::now();
3660        while router.counters.snapshot()["client_egress_close_delivery_failed"] == 0 {
3661            assert!(sent < 1_000, "the byte budget never refused a frame");
3662            let route = &routes[(sent % 2) as usize];
3663            router
3664                .route_for_connection(
3665                    &module_ctx,
3666                    stream_frame(
3667                        route.module_channel,
3668                        route.module_epoch,
3669                        sent,
3670                        vec![b'z'; BODY],
3671                    ),
3672                )
3673                .await
3674                .unwrap();
3675            sent += 1;
3676            admitted = client_sink.backlog().queued_frames;
3677        }
3678        assert_eq!(
3679            started.elapsed(),
3680            crate::server::CONNECTION_EGRESS_STALL_GRACE
3681        );
3682        // The budget, not the frame-count backstop, did the refusing.
3683        let frame_bytes = subc_protocol::HEADER_LEN + BODY;
3684        assert_eq!(
3685            admitted,
3686            crate::server::CONNECTION_EGRESS_BYTE_BUDGET / frame_bytes
3687        );
3688        let reason = close_receiver
3689            .try_recv()
3690            .expect("the client connection must be asked to close");
3691        assert!(reason
3692            .to_string()
3693            .contains("module_to_client_delivery_failed"));
3694
3695        // A second refused frame for the same connection adds no second WARN.
3696        router
3697            .route_for_connection(
3698                &module_ctx,
3699                stream_frame(
3700                    routes[0].module_channel,
3701                    routes[0].module_epoch,
3702                    sent,
3703                    vec![b'z'; BODY],
3704                ),
3705            )
3706            .await
3707            .unwrap();
3708
3709        let captured = test_log::captured_logs(&logs);
3710        let warn_lines = captured
3711            .lines()
3712            .filter(|line| {
3713                line.contains("closing client connection: its egress queue could not take a frame")
3714            })
3715            .collect::<Vec<_>>();
3716        assert_eq!(warn_lines.len(), 1, "exactly one WARN, got: {captured}");
3717        let line = warn_lines[0];
3718        assert!(line.contains("WARN"), "{line}");
3719        assert!(line.contains("connection_id=71"), "{line}");
3720        assert!(
3721            line.contains("module_id=\"stuck-reader-provider\""),
3722            "{line}"
3723        );
3724        assert!(line.contains("client_channel="), "{line}");
3725        assert!(line.contains("principals=direct"), "{line}");
3726        let queued_bytes: usize = line
3727            .split("queued_bytes=")
3728            .nth(1)
3729            .and_then(|rest| rest.split_whitespace().next())
3730            .and_then(|value| value.parse().ok())
3731            .expect("queued_bytes is logged");
3732        assert_eq!(queued_bytes, admitted * frame_bytes);
3733        assert!(
3734            line.contains(&format!("queued_frames={admitted}")),
3735            "{line}"
3736        );
3737        assert!(line.contains("oldest_queued_ms="), "{line}");
3738        assert!(line.contains("oldest_queued_ms=250"), "{line}");
3739        assert!(line.contains("frame_type=Some(StreamData)"), "{line}");
3740        assert!(
3741            line.contains(&format!(
3742                "frame_flags=Some({})",
3743                Flags::new(false, Priority::Interactive, false).0
3744            )),
3745            "{line}"
3746        );
3747        assert!(
3748            line.contains("close_cause=\"waited_250_ms_without_room\""),
3749            "{line}"
3750        );
3751        assert!(
3752            !line.contains(&"z".repeat(20)),
3753            "body bytes must never be logged: {line}"
3754        );
3755    }
3756
3757    fn route_frame(ty: FrameType, channel: u16, epoch: u32, corr: u64) -> Frame {
3758        Frame::build(
3759            ty,
3760            Flags::new(false, Priority::Interactive, false),
3761            channel,
3762            epoch,
3763            corr,
3764            if ty == FrameType::Request || ty == FrameType::Response {
3765                b"route-body".to_vec()
3766            } else {
3767                Vec::new()
3768            },
3769        )
3770        .unwrap()
3771    }
3772
3773    type DynamicRouteFixture = (
3774        Router,
3775        Arc<ForwardingTable>,
3776        RouteCtx,
3777        mpsc::Receiver<crate::router::OutboundFrame>,
3778        RouteCtx,
3779        mpsc::Receiver<crate::router::OutboundFrame>,
3780        mpsc::Receiver<crate::router::OutboundFrame>,
3781        crate::forwarding::PendingRouteBindRelay,
3782    );
3783
3784    fn dynamic_route_fixture(commit: bool) -> DynamicRouteFixture {
3785        dynamic_route_fixture_with_busy(commit, false)
3786    }
3787
3788    fn dynamic_route_fixture_with_busy(commit: bool, busy_refusals: bool) -> DynamicRouteFixture {
3789        let forwarding = Arc::new(ForwardingTable::default());
3790        let control = Arc::new(crate::ControlHandler::with_forwarding(
3791            Arc::new(crate::Registry::default()),
3792            Arc::clone(&forwarding),
3793        ));
3794        let router = Router::with_control_handler(control);
3795        let module_connection = ConnectionId::new(500);
3796        let client_connection = ConnectionId::new(501);
3797        let (module_tx, module_rx) = mpsc::channel(if busy_refusals { 64 } else { 8 });
3798        forwarding
3799            .register_module_connection(
3800                module_connection,
3801                "epoch-router".into(),
3802                2,
3803                Concurrency::ModuleManaged,
3804                FrameSink::new(module_tx),
3805            )
3806            .unwrap();
3807        let (client_tx, client_rx) = mpsc::channel(if busy_refusals { 64 } else { 8 });
3808        let client_sink = FrameSink::new(client_tx);
3809        let pending = forwarding
3810            .begin_route_bind_relay_with_busy_for_test(
3811                client_connection,
3812                client_sink.clone(),
3813                700,
3814                "epoch-router",
3815                busy_refusals,
3816            )
3817            .unwrap();
3818        if commit {
3819            forwarding
3820                .complete_pending_relay(
3821                    module_connection,
3822                    pending.corr,
3823                    RouteBindRelayOutcome::Accepted,
3824                )
3825                .unwrap();
3826        }
3827        let (module_egress_tx, module_egress_rx) = mpsc::channel(8);
3828        (
3829            router,
3830            forwarding,
3831            RouteCtx {
3832                connection_id: client_connection,
3833                egress: client_sink,
3834            },
3835            client_rx,
3836            RouteCtx {
3837                connection_id: module_connection,
3838                egress: FrameSink::new(module_egress_tx),
3839            },
3840            module_egress_rx,
3841            module_rx,
3842            pending,
3843        )
3844    }
3845
3846    #[tokio::test]
3847    async fn route_epochs_validate_both_directions_and_rewrite_to_peer_handle() {
3848        let (
3849            router,
3850            _forwarding,
3851            client_ctx,
3852            mut client_rx,
3853            module_ctx,
3854            _module_egress_rx,
3855            mut module_rx,
3856            pending,
3857        ) = dynamic_route_fixture(true);
3858        let route_open = client_rx.recv().await.unwrap();
3859        assert_eq!(route_open.header.corr, 700);
3860
3861        router
3862            .route_for_connection(
3863                &client_ctx,
3864                route_frame(
3865                    FrameType::Request,
3866                    pending.client_channel,
3867                    pending.client_epoch,
3868                    701,
3869                ),
3870            )
3871            .await
3872            .unwrap();
3873        let forwarded = module_rx.recv().await.unwrap();
3874        assert_eq!(forwarded.header.channel, pending.module_channel);
3875        assert_eq!(forwarded.header.epoch, pending.module_epoch);
3876
3877        router
3878            .route_for_connection(
3879                &module_ctx,
3880                route_frame(
3881                    FrameType::Response,
3882                    pending.module_channel,
3883                    pending.module_epoch,
3884                    701,
3885                ),
3886            )
3887            .await
3888            .unwrap();
3889        let delivered = client_rx.recv().await.unwrap();
3890        assert_eq!(delivered.header.channel, pending.client_channel);
3891        assert_eq!(delivered.header.epoch, pending.client_epoch);
3892
3893        router
3894            .route_for_connection(
3895                &client_ctx,
3896                route_frame(
3897                    FrameType::Request,
3898                    pending.client_channel,
3899                    pending.client_epoch + 1,
3900                    702,
3901                ),
3902            )
3903            .await
3904            .unwrap();
3905        router
3906            .route_for_connection(
3907                &module_ctx,
3908                route_frame(
3909                    FrameType::Response,
3910                    pending.module_channel,
3911                    pending.module_epoch + 1,
3912                    703,
3913                ),
3914            )
3915            .await
3916            .unwrap();
3917        let stale_error = client_rx.recv().await.unwrap();
3918        assert_eq!(stale_error.header.ty, FrameType::Error);
3919        assert_eq!(stale_error.header.channel, pending.client_channel);
3920        assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
3921        assert_eq!(stale_error.header.corr, 702);
3922        let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
3923        assert_eq!(body.code, "stale_route_epoch");
3924        assert!(module_rx.try_recv().is_err());
3925        assert!(client_rx.try_recv().is_err());
3926        let counters = router.counters.snapshot();
3927        assert_eq!(counters["client_frames_dropped_stale_route"], 1);
3928        assert_eq!(counters["module_frames_dropped_no_route"], 1);
3929    }
3930
3931    #[tokio::test]
3932    async fn accepted_route_publishes_route_open_before_immediate_reverse_request() {
3933        let (
3934            router,
3935            _,
3936            _client_ctx,
3937            mut client_rx,
3938            module_ctx,
3939            _module_egress_rx,
3940            _module_rx,
3941            pending,
3942        ) = dynamic_route_fixture(true);
3943        router
3944            .route_for_connection(
3945                &module_ctx,
3946                route_frame(
3947                    FrameType::Request,
3948                    pending.module_channel,
3949                    pending.module_epoch,
3950                    800,
3951                ),
3952            )
3953            .await
3954            .unwrap();
3955
3956        let first = client_rx.recv().await.unwrap();
3957        let second = client_rx.recv().await.unwrap();
3958        assert_eq!(first.header.channel, 0);
3959        assert_eq!(first.header.corr, 700);
3960        assert_eq!(second.header.channel, pending.client_channel);
3961        assert_eq!(second.header.epoch, pending.client_epoch);
3962        assert_eq!(second.header.corr, 800);
3963    }
3964
3965    #[tokio::test]
3966    async fn reserved_slot_ingress_errors_only_matching_client_requests() {
3967        let (
3968            router,
3969            _forwarding,
3970            client_ctx,
3971            mut client_rx,
3972            _module_ctx,
3973            _module_egress_rx,
3974            mut module_rx,
3975            pending,
3976        ) = dynamic_route_fixture(false);
3977        router
3978            .route_for_connection(
3979                &client_ctx,
3980                route_frame(
3981                    FrameType::Request,
3982                    pending.client_channel,
3983                    pending.client_epoch,
3984                    900,
3985                ),
3986            )
3987            .await
3988            .unwrap();
3989        let error = client_rx.recv().await.unwrap();
3990        assert_eq!(error.header.ty, FrameType::Error);
3991        assert_eq!(error.header.channel, pending.client_channel);
3992        assert_eq!(error.header.epoch, pending.client_epoch);
3993        assert_eq!(error.header.corr, 900);
3994
3995        router
3996            .route_for_connection(
3997                &client_ctx,
3998                route_frame(
3999                    FrameType::Response,
4000                    pending.client_channel,
4001                    pending.client_epoch,
4002                    901,
4003                ),
4004            )
4005            .await
4006            .unwrap();
4007        router
4008            .route_for_connection(
4009                &client_ctx,
4010                route_frame(
4011                    FrameType::Request,
4012                    pending.client_channel,
4013                    pending.client_epoch + 1,
4014                    902,
4015                ),
4016            )
4017            .await
4018            .unwrap();
4019        let stale_error = client_rx.recv().await.unwrap();
4020        assert_eq!(stale_error.header.ty, FrameType::Error);
4021        assert_eq!(stale_error.header.channel, pending.client_channel);
4022        assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
4023        assert_eq!(stale_error.header.corr, 902);
4024        let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
4025        assert_eq!(body.code, "stale_route_epoch");
4026        assert!(module_rx.try_recv().is_err());
4027        let counters = router.counters.snapshot();
4028        assert_eq!(counters["client_frames_dropped_stale_route"], 1);
4029        assert_eq!(counters["module_frames_dropped_no_route"], 0);
4030    }
4031
4032    #[tokio::test]
4033    async fn dropped_module_route_goodbye_increments_counter() {
4034        let (
4035            router,
4036            _forwarding,
4037            client_ctx,
4038            mut client_rx,
4039            _module_ctx,
4040            _module_egress_rx,
4041            mut module_rx,
4042            pending,
4043        ) = dynamic_route_fixture(true);
4044        let _ = client_rx.recv().await;
4045        module_rx.close();
4046
4047        router
4048            .route_for_connection(
4049                &client_ctx,
4050                route_frame(
4051                    FrameType::Goodbye,
4052                    pending.client_channel,
4053                    pending.client_epoch,
4054                    999,
4055                ),
4056            )
4057            .await
4058            .unwrap();
4059
4060        let counters = router.counters.snapshot();
4061        assert_eq!(counters["goodbye_relay_module_dropped"], 1);
4062        assert_eq!(
4063            counters["goodbye_relay_module_dropped_by_module"],
4064            serde_json::json!({ "epoch-router": 1 })
4065        );
4066        assert_eq!(counters["route_released_epoch_fenced"], 1);
4067    }
4068
4069    #[tokio::test]
4070    async fn module_request_on_stale_epoch_receives_stale_route_epoch() {
4071        let (
4072            router,
4073            _forwarding,
4074            _client_ctx,
4075            _client_rx,
4076            module_ctx,
4077            mut module_egress_rx,
4078            mut module_rx,
4079            pending,
4080        ) = dynamic_route_fixture(true);
4081
4082        router
4083            .route_for_connection(
4084                &module_ctx,
4085                route_frame(
4086                    FrameType::Request,
4087                    pending.module_channel,
4088                    pending.module_epoch + 1,
4089                    1_000,
4090                ),
4091            )
4092            .await
4093            .unwrap();
4094
4095        let error = module_egress_rx.try_recv().unwrap();
4096        assert_eq!(error.header.ty, FrameType::Error);
4097        assert_eq!(error.header.channel, pending.module_channel);
4098        assert_eq!(error.header.epoch, pending.module_epoch + 1);
4099        assert_eq!(error.header.corr, 1_000);
4100        let body: ErrorBody = serde_json::from_slice(&error.body).unwrap();
4101        assert_eq!(body.code, "stale_route_epoch");
4102        assert!(module_rx.try_recv().is_err());
4103        let counters = router.counters.snapshot();
4104        assert_eq!(counters["module_requests_dropped_stale_route"], 1);
4105        assert_eq!(counters["module_frames_dropped_no_route"], 0);
4106    }
4107
4108    #[tokio::test(start_paused = true)]
4109    async fn stale_epoch_drop_warns_once_per_minute_with_suppressed_count() {
4110        let (output, _guard) = test_log::log_capture(tracing::Level::WARN);
4111        let (
4112            router,
4113            _forwarding,
4114            client_ctx,
4115            mut client_rx,
4116            _module_ctx,
4117            _module_egress_rx,
4118            mut module_rx,
4119            pending,
4120        ) = dynamic_route_fixture(true);
4121        let _ = client_rx.recv().await;
4122        for corr in 1..=3 {
4123            router
4124                .route_for_connection(
4125                    &client_ctx,
4126                    route_frame(
4127                        FrameType::Request,
4128                        pending.client_channel,
4129                        pending.client_epoch + 1,
4130                        corr,
4131                    ),
4132                )
4133                .await
4134                .unwrap();
4135            assert_eq!(client_rx.recv().await.unwrap().header.ty, FrameType::Error);
4136        }
4137        assert!(module_rx.try_recv().is_err());
4138        let logs = test_log::captured_logs(&output);
4139        assert_eq!(
4140            logs.matches("dropping data-plane frame").count(),
4141            1,
4142            "{logs}"
4143        );
4144        assert!(logs.contains("WARN"), "{logs}");
4145        assert!(
4146            logs.contains(&format!("connection_id={}", client_ctx.connection_id.get())),
4147            "{logs}"
4148        );
4149        assert!(
4150            logs.contains(&format!("channel={}", pending.client_channel)),
4151            "{logs}"
4152        );
4153        assert!(
4154            logs.contains(&format!("epoch={}", pending.client_epoch + 1)),
4155            "{logs}"
4156        );
4157        assert!(
4158            logs.contains("client_to_module") && logs.contains("stale_epoch"),
4159            "{logs}"
4160        );
4161        assert!(logs.contains("suppressed=0"), "{logs}");
4162
4163        tokio::time::advance(Duration::from_secs(60)).await;
4164        router
4165            .route_for_connection(
4166                &client_ctx,
4167                route_frame(
4168                    FrameType::Request,
4169                    pending.client_channel,
4170                    pending.client_epoch + 2,
4171                    4,
4172                ),
4173            )
4174            .await
4175            .unwrap();
4176        let logs = test_log::captured_logs(&output);
4177        assert_eq!(
4178            logs.matches("dropping data-plane frame").count(),
4179            2,
4180            "{logs}"
4181        );
4182        assert!(logs.contains("suppressed=2"), "{logs}");
4183    }
4184
4185    #[tokio::test]
4186    async fn module_request_on_reserved_or_absent_route_receives_unknown_channel() {
4187        let (
4188            reserved_router,
4189            _forwarding,
4190            _client_ctx,
4191            _client_rx,
4192            reserved_module_ctx,
4193            mut reserved_module_egress_rx,
4194            _module_rx,
4195            reserved,
4196        ) = dynamic_route_fixture(false);
4197        reserved_router
4198            .route_for_connection(
4199                &reserved_module_ctx,
4200                route_frame(
4201                    FrameType::Request,
4202                    reserved.module_channel,
4203                    reserved.module_epoch,
4204                    1_001,
4205                ),
4206            )
4207            .await
4208            .unwrap();
4209        let reserved_error = reserved_module_egress_rx.try_recv().unwrap();
4210        let reserved_body: ErrorBody = serde_json::from_slice(&reserved_error.body).unwrap();
4211        assert_eq!(reserved_error.header.ty, FrameType::Error);
4212        assert_eq!(reserved_error.header.channel, reserved.module_channel);
4213        assert_eq!(reserved_error.header.epoch, reserved.module_epoch);
4214        assert_eq!(reserved_error.header.corr, 1_001);
4215        assert_eq!(reserved_body.code, "unknown_channel");
4216        assert_eq!(
4217            reserved_router.counters.snapshot()["module_requests_dropped_stale_route"],
4218            1
4219        );
4220
4221        let (
4222            absent_router,
4223            _forwarding,
4224            _client_ctx,
4225            _client_rx,
4226            absent_module_ctx,
4227            mut absent_module_egress_rx,
4228            _module_rx,
4229            absent,
4230        ) = dynamic_route_fixture(false);
4231        absent_router
4232            .route_for_connection(
4233                &absent_module_ctx,
4234                route_frame(
4235                    FrameType::Request,
4236                    absent.module_channel + 1,
4237                    absent.module_epoch,
4238                    1_002,
4239                ),
4240            )
4241            .await
4242            .unwrap();
4243        let absent_error = absent_module_egress_rx.try_recv().unwrap();
4244        let absent_body: ErrorBody = serde_json::from_slice(&absent_error.body).unwrap();
4245        assert_eq!(absent_error.header.ty, FrameType::Error);
4246        assert_eq!(absent_error.header.channel, absent.module_channel + 1);
4247        assert_eq!(absent_error.header.epoch, absent.module_epoch);
4248        assert_eq!(absent_error.header.corr, 1_002);
4249        assert_eq!(absent_body.code, "unknown_channel");
4250        assert_eq!(
4251            absent_router.counters.snapshot()["module_requests_dropped_stale_route"],
4252            1
4253        );
4254    }
4255
4256    #[tokio::test]
4257    async fn non_request_module_frame_on_dead_route_is_counted_without_error() {
4258        let (
4259            router,
4260            forwarding,
4261            client_ctx,
4262            mut client_rx,
4263            module_ctx,
4264            mut module_egress_rx,
4265            mut module_rx,
4266            pending,
4267        ) = dynamic_route_fixture(true);
4268        let (other_module_tx, _other_module_rx) = mpsc::channel(8);
4269        forwarding
4270            .register_module_connection(
4271                ConnectionId::new(502),
4272                "other-module".into(),
4273                2,
4274                Concurrency::ModuleManaged,
4275                FrameSink::new(other_module_tx),
4276            )
4277            .unwrap();
4278        let _ = client_rx.recv().await.unwrap();
4279
4280        router
4281            .route_for_connection(
4282                &client_ctx,
4283                route_frame(
4284                    FrameType::Goodbye,
4285                    pending.client_channel,
4286                    pending.client_epoch,
4287                    1_003,
4288                ),
4289            )
4290            .await
4291            .unwrap();
4292        let _ = module_rx.recv().await.unwrap();
4293
4294        router
4295            .route_for_connection(
4296                &module_ctx,
4297                route_frame(
4298                    FrameType::StreamData,
4299                    pending.module_channel,
4300                    pending.module_epoch,
4301                    1_004,
4302                ),
4303            )
4304            .await
4305            .unwrap();
4306
4307        // No ERROR goes back for a non-request frame. The one reply is the
4308        // route GOODBYE telling the module to let go of the released route.
4309        let reply = module_egress_rx.try_recv().unwrap();
4310        assert_eq!(reply.header.ty, FrameType::Goodbye);
4311        assert!(module_egress_rx.try_recv().is_err());
4312        let counters = router.counters.snapshot();
4313        assert_eq!(counters["module_frames_dropped_no_route"], 1);
4314        assert_eq!(
4315            counters["module_frames_dropped_no_route_by_module"],
4316            serde_json::json!({ "epoch-router": 1 })
4317        );
4318        assert_eq!(counters["module_requests_dropped_stale_route"], 0);
4319    }
4320
4321    /// Drain whatever the module connection's queue holds right now, the way a
4322    /// module's reader would before it stalls.
4323    fn drain_now(rx: &mut mpsc::Receiver<OutboundFrame>) {
4324        while rx.try_recv().is_ok() {}
4325    }
4326
4327    /// A module that stops reading for a moment when a client closes one of
4328    /// its routes still learns the route is gone: the GOODBYE its full egress
4329    /// queue refused is delivered as soon as it reads again, rather than
4330    /// dropped, which would leave the module holding the route for the rest of
4331    /// its connection.
4332    #[tokio::test]
4333    async fn route_goodbye_refused_by_stalled_module_is_delivered_when_it_resumes_reading() {
4334        const BUDGET: usize = 4_096;
4335        let forwarding = Arc::new(ForwardingTable::default());
4336        let control = Arc::new(ControlHandler::with_forwarding(
4337            Arc::new(Registry::default()),
4338            Arc::clone(&forwarding),
4339        ));
4340        let router = Router::with_control_handler(control);
4341        let module_connection = ConnectionId::new(80);
4342        let client_connection = ConnectionId::new(81);
4343        let (module_tx, mut module_rx) = mpsc::channel(64);
4344        let module_sink = FrameSink::with_byte_budget(module_tx, BUDGET);
4345        forwarding
4346            .register_module_connection(
4347                module_connection,
4348                "stalled-provider".into(),
4349                2,
4350                Concurrency::ModuleManaged,
4351                module_sink.clone(),
4352            )
4353            .unwrap();
4354        let (client_tx, mut client_rx) = mpsc::channel(8);
4355        let client_sink = FrameSink::new(client_tx);
4356        let pending = forwarding
4357            .begin_route_bind_relay_for_test(
4358                client_connection,
4359                client_sink.clone(),
4360                1_100,
4361                "stalled-provider",
4362            )
4363            .unwrap();
4364        forwarding
4365            .complete_pending_relay(
4366                module_connection,
4367                pending.corr,
4368                RouteBindRelayOutcome::Accepted,
4369            )
4370            .unwrap();
4371        let _ = client_rx.recv().await.unwrap();
4372        drain_now(&mut module_rx);
4373
4374        // The module stalls: its queue holds more than the byte budget, so the
4375        // queue refuses anything further.
4376        module_sink
4377            .try_send(stream_frame(9, 1, 0, vec![b'f'; BUDGET]))
4378            .unwrap();
4379        assert!(module_sink
4380            .try_send(stream_frame(9, 1, 1, Vec::new()))
4381            .is_err());
4382
4383        let client_ctx = RouteCtx {
4384            connection_id: client_connection,
4385            egress: client_sink,
4386        };
4387        router
4388            .route_for_connection(
4389                &client_ctx,
4390                route_frame(
4391                    FrameType::Goodbye,
4392                    pending.client_channel,
4393                    pending.client_epoch,
4394                    1_101,
4395                ),
4396            )
4397            .await
4398            .unwrap();
4399        tokio::task::yield_now().await;
4400        assert_eq!(
4401            router.counters.snapshot()["goodbye_relay_module_dropped"],
4402            0,
4403            "a GOODBYE refused by a momentarily full module queue must not be dropped"
4404        );
4405
4406        // The module reads again: the filler comes off, then the GOODBYE.
4407        let filler = module_rx.recv().await.unwrap();
4408        assert_eq!(filler.header.ty, FrameType::StreamData);
4409        drop(filler);
4410        let goodbye = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
4411            .await
4412            .expect("the refused GOODBYE must be delivered once the module frees room")
4413            .unwrap();
4414        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
4415        assert_eq!(goodbye.header.channel, pending.module_channel);
4416        assert_eq!(goodbye.header.epoch, pending.module_epoch);
4417        assert_eq!(
4418            router.counters.snapshot()["goodbye_relay_module_dropped"],
4419            0
4420        );
4421    }
4422
4423    /// A module still sending on a route the daemon released is told, with a
4424    /// route GOODBYE for exactly the (channel, epoch) it sent on. The same
4425    /// happens for a stale epoch on a channel whose route moved on and for a
4426    /// channel that never had a route; only the first is counted as traffic on
4427    /// a released route.
4428    #[tokio::test]
4429    async fn module_frame_on_route_the_daemon_does_not_hold_is_answered_with_goodbye() {
4430        let (
4431            router,
4432            _forwarding,
4433            client_ctx,
4434            mut client_rx,
4435            module_ctx,
4436            mut module_egress_rx,
4437            mut module_rx,
4438            pending,
4439        ) = dynamic_route_fixture(true);
4440        let _ = client_rx.recv().await.unwrap();
4441        router
4442            .route_for_connection(
4443                &client_ctx,
4444                route_frame(
4445                    FrameType::Goodbye,
4446                    pending.client_channel,
4447                    pending.client_epoch,
4448                    1_200,
4449                ),
4450            )
4451            .await
4452            .unwrap();
4453        drain_now(&mut module_rx);
4454
4455        let cases = [
4456            // Released route: the daemon allocated this (channel, epoch).
4457            (pending.module_channel, pending.module_epoch),
4458            // A channel the daemon never allocated on this connection.
4459            (pending.module_channel + 1, 1),
4460        ];
4461        for (channel, epoch) in cases {
4462            router
4463                .route_for_connection(
4464                    &module_ctx,
4465                    route_frame(FrameType::StreamData, channel, epoch, 1_201),
4466                )
4467                .await
4468                .unwrap();
4469            let reply = module_egress_rx
4470                .try_recv()
4471                .expect("a frame on a route the daemon does not hold is answered");
4472            assert_eq!(reply.header.ty, FrameType::Goodbye);
4473            assert_eq!(reply.header.channel, channel);
4474            assert_eq!(reply.header.epoch, epoch);
4475            assert_eq!(reply.header.corr, 0);
4476            assert!(module_egress_rx.try_recv().is_err());
4477        }
4478
4479        // A GOODBYE from the module for a route the daemon already released
4480        // is not answered: the module is letting go already.
4481        router
4482            .route_for_connection(
4483                &module_ctx,
4484                route_frame(FrameType::Goodbye, pending.module_channel + 2, 1, 0),
4485            )
4486            .await
4487            .unwrap();
4488        assert!(module_egress_rx.try_recv().is_err());
4489
4490        let counters = router.counters.snapshot();
4491        assert_eq!(counters["module_frames_dropped_no_route"], 3);
4492        assert_eq!(counters["module_frames_dropped_released_route"], 1);
4493        assert_eq!(
4494            counters["module_frames_dropped_released_route_by_module"],
4495            serde_json::json!({ "epoch-router": 1 })
4496        );
4497        assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4498    }
4499
4500    /// A chatty orphan (a streaming module still producing on a route it
4501    /// missed the GOODBYE for) is answered once per interval, not once per
4502    /// frame, and answered again once the interval has passed.
4503    #[tokio::test(start_paused = true)]
4504    async fn burst_of_orphan_module_frames_is_answered_once_per_interval() {
4505        let (
4506            router,
4507            _forwarding,
4508            client_ctx,
4509            mut client_rx,
4510            module_ctx,
4511            mut module_egress_rx,
4512            mut module_rx,
4513            pending,
4514        ) = dynamic_route_fixture(true);
4515        let _ = client_rx.recv().await.unwrap();
4516        router
4517            .route_for_connection(
4518                &client_ctx,
4519                route_frame(
4520                    FrameType::Goodbye,
4521                    pending.client_channel,
4522                    pending.client_epoch,
4523                    1_300,
4524                ),
4525            )
4526            .await
4527            .unwrap();
4528        drain_now(&mut module_rx);
4529
4530        let send_burst = |corr: u64| {
4531            route_frame(
4532                FrameType::StreamData,
4533                pending.module_channel,
4534                pending.module_epoch,
4535                corr,
4536            )
4537        };
4538        for corr in 0..20 {
4539            router
4540                .route_for_connection(&module_ctx, send_burst(corr))
4541                .await
4542                .unwrap();
4543        }
4544        let mut replies = 0;
4545        while let Ok(reply) = module_egress_rx.try_recv() {
4546            assert_eq!(reply.header.ty, FrameType::Goodbye);
4547            replies += 1;
4548        }
4549        assert_eq!(replies, 1, "a burst within the interval gets one GOODBYE");
4550
4551        tokio::time::advance(ORPHAN_ROUTE_GOODBYE_INTERVAL).await;
4552        router
4553            .route_for_connection(&module_ctx, send_burst(20))
4554            .await
4555            .unwrap();
4556        let retry = module_egress_rx
4557            .try_recv()
4558            .expect("the first orphan frame after the interval is answered again");
4559        assert_eq!(retry.header.channel, pending.module_channel);
4560        assert_eq!(retry.header.epoch, pending.module_epoch);
4561
4562        let counters = router.counters.snapshot();
4563        assert_eq!(counters["module_frames_dropped_no_route"], 21);
4564        assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4565    }
4566
4567    /// The rate-limit memory is per connection and goes away with it.
4568    #[test]
4569    fn orphan_goodbye_rate_limit_state_is_released_with_the_connection() {
4570        let router = Router::with_default_self_handler();
4571        let connection = router.begin_connection();
4572        let id = connection.id();
4573        assert!(router.orphan_goodbyes.claim(id, 7));
4574        assert!(!router.orphan_goodbyes.claim(id, 7));
4575        assert!(router.orphan_goodbyes.claim(id, 8));
4576        drop(connection);
4577        assert!(router
4578            .orphan_goodbyes
4579            .last_sent
4580            .lock()
4581            .unwrap()
4582            .get(&id)
4583            .is_none());
4584    }
4585
4586    #[test]
4587    fn channel_zero_cannot_be_registered_as_backend() {
4588        let mut router = Router::with_default_self_handler();
4589
4590        let err = router.register_backend(0, EchoBackend).unwrap_err();
4591
4592        assert_eq!(err, RouterError::ReservedChannelZero);
4593    }
4594}