Skip to main content

subc_daemon/
router.rs

1use std::{
2    collections::HashMap,
3    error::Error,
4    fmt,
5    sync::{
6        atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
7        Arc, Mutex,
8    },
9    time::{Duration, Instant},
10};
11
12use subc_protocol::{error_codes, ErrorBody, Flags, FrameType, Priority};
13use tokio::sync::{mpsc, Notify};
14use tracing::{debug, warn};
15
16use crate::{
17    control::ControlHandler,
18    forwarding::{
19        principal_label, CloseReason, ConnectionCloseReceiver, DataRoute, DataRouteState,
20        ForwardingError, ForwardingTable, RouteBinding, RouteRelease, UndeliveredFrame,
21    },
22    registry::ConnectionId,
23    DaemonCounters, Frame, FrameBuildError,
24};
25
26/// One queued outbound frame plus the instant it entered the writer queue.
27///
28/// The stamp exists for the reply-path half of slow-control diagnosis: a
29/// handler can finish in microseconds while the reply sits in this queue
30/// waiting for the writer task to be scheduled, and without a per-item stamp
31/// that wait is invisible to every other timing point (the client's round
32/// trip is the only witness, and it cannot say which side ate the time).
33/// Constructed exclusively inside [`FrameSink`] so no caller can forget it.
34#[derive(Debug)]
35pub struct OutboundFrame {
36    pub frame: Frame,
37    pub enqueued_at: std::time::Instant,
38    pub(crate) flushed: Option<tokio::sync::oneshot::Sender<()>>,
39    /// This frame's share of the connection's queued-byte count. Dropping the
40    /// frame (after the writer has written it, or when the queue itself is
41    /// dropped with the frame still in it) gives the bytes back. `None` only for
42    /// frames built outside a [`FrameSink`], which exist in tests alone.
43    pub(crate) charge: Option<EgressCharge>,
44}
45
46impl OutboundFrame {
47    fn charged(frame: Frame, charge: EgressCharge) -> Self {
48        Self {
49            frame,
50            enqueued_at: charge.enqueued_at,
51            flushed: None,
52            charge: Some(charge),
53        }
54    }
55}
56
57/// Bytes a frame occupies in the connection's egress queue for budget
58/// purposes: the fixed envelope header plus the body.
59fn queued_frame_bytes(frame: &Frame) -> usize {
60    subc_protocol::HEADER_LEN + frame.body.len()
61}
62
63/// A point-in-time view of one connection's egress queue, for diagnosing why a
64/// frame did not fit.
65#[derive(Debug, Clone, Copy, PartialEq, Eq)]
66pub(crate) struct EgressBacklog {
67    pub queued_bytes: usize,
68    pub queued_frames: usize,
69    /// Now minus the enqueue time of the frame the writer most recently took
70    /// off the queue (the frame it is writing, or has just written), or, if
71    /// the writer has taken nothing since the queue was last empty, of the
72    /// frame that made the queue non-empty. While the writer is blocked on a
73    /// frame this is exactly the oldest frame's age; between frames it can be
74    /// one frame older than the true head. `None` when nothing is queued.
75    pub oldest_age: Option<Duration>,
76}
77
78/// Queued-byte accounting shared by every clone of one connection's
79/// [`FrameSink`] and by every frame that sink has admitted. Its per-frame
80/// accounting uses atomics: this is on the path of every frame a module sends
81/// to a client, so it takes no lock per frame.
82#[derive(Debug)]
83struct EgressAccounting {
84    byte_budget: usize,
85    queued_bytes: AtomicUsize,
86    queued_frames: AtomicUsize,
87    /// Origin for the nanosecond timestamps in `oldest_enqueued_nanos`.
88    time_base: Instant,
89    /// Enqueue time, as nanoseconds after `time_base` plus one, of the frame
90    /// described by [`EgressBacklog::oldest_age`]; 0 means none.
91    oldest_enqueued_nanos: AtomicU64,
92    /// Awaited senders currently parked on `freed`. A release only pays for a
93    /// notification when this is non-zero.
94    waiters: AtomicUsize,
95    /// Woken when a charged frame releases its bytes while a sender waits, so
96    /// the awaited send can re-check for room.
97    freed: Notify,
98    client_delivery_closed: AtomicBool,
99    client_delivery_stopped: Notify,
100}
101
102impl EgressAccounting {
103    fn new(byte_budget: usize) -> Self {
104        Self {
105            byte_budget,
106            queued_bytes: AtomicUsize::new(0),
107            queued_frames: AtomicUsize::new(0),
108            time_base: tokio::time::Instant::now().into_std(),
109            oldest_enqueued_nanos: AtomicU64::new(0),
110            waiters: AtomicUsize::new(0),
111            freed: Notify::new(),
112            client_delivery_closed: AtomicBool::new(false),
113            client_delivery_stopped: Notify::new(),
114        }
115    }
116
117    fn stamp(&self, at: Instant) -> u64 {
118        (at.saturating_duration_since(self.time_base).as_nanos() as u64).saturating_add(1)
119    }
120
121    /// Charge `bytes` if they fit in the budget. A frame larger than the whole
122    /// budget (bodies may be up to 64 MiB) is still admitted into an EMPTY
123    /// queue, since it could otherwise never be sent at all; it simply has the
124    /// queue to itself until it is written.
125    fn try_charge(self: &Arc<Self>, bytes: usize) -> Option<EgressCharge> {
126        // SeqCst pairs with `release`: either this load sees bytes a release
127        // just freed, or that release sees this sender counted in `waiters`.
128        let mut current = self.queued_bytes.load(Ordering::SeqCst);
129        loop {
130            if current != 0 && current.saturating_add(bytes) > self.byte_budget {
131                return None;
132            }
133            match self.queued_bytes.compare_exchange_weak(
134                current,
135                current + bytes,
136                Ordering::SeqCst,
137                Ordering::SeqCst,
138            ) {
139                Ok(_) => return Some(self.record(bytes)),
140                Err(actual) => current = actual,
141            }
142        }
143    }
144
145    /// Charge `bytes` regardless of the budget. Used only for a frame whose
146    /// queue slot was reserved in advance, which must be sendable.
147    fn charge_unconditionally(self: &Arc<Self>, bytes: usize) -> EgressCharge {
148        self.queued_bytes.fetch_add(bytes, Ordering::SeqCst);
149        self.record(bytes)
150    }
151
152    fn record(self: &Arc<Self>, bytes: usize) -> EgressCharge {
153        let enqueued_at = tokio::time::Instant::now().into_std();
154        let stamp = self.stamp(enqueued_at);
155        if self.queued_frames.fetch_add(1, Ordering::AcqRel) == 0 {
156            // This frame made the queue non-empty, so it is the head until the
157            // writer takes something.
158            self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
159        }
160        EgressCharge {
161            accounting: Arc::clone(self),
162            bytes,
163            stamp,
164            enqueued_at,
165        }
166    }
167
168    /// The writer has taken the frame stamped `stamp` off the queue.
169    fn taken_by_writer(&self, stamp: u64) {
170        self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
171    }
172
173    fn release(&self, bytes: usize, stamp: u64) {
174        self.queued_bytes.fetch_sub(bytes, Ordering::SeqCst);
175        if self.queued_frames.fetch_sub(1, Ordering::AcqRel) == 1 {
176            // The queue drained. Clear the marker only if it still names this
177            // frame, so a frame admitted in the meantime keeps its stamp.
178            let _ = self.oldest_enqueued_nanos.compare_exchange(
179                stamp,
180                0,
181                Ordering::AcqRel,
182                Ordering::Acquire,
183            );
184        }
185        if self.waiters.load(Ordering::SeqCst) != 0 {
186            self.freed.notify_waiters();
187        }
188    }
189
190    fn backlog(&self) -> EgressBacklog {
191        let queued_frames = self.queued_frames.load(Ordering::Acquire);
192        let stamp = self.oldest_enqueued_nanos.load(Ordering::Acquire);
193        let oldest_age = (queued_frames != 0 && stamp != 0).then(|| {
194            let enqueued = self.time_base + Duration::from_nanos(stamp - 1);
195            tokio::time::Instant::from_std(enqueued).elapsed()
196        });
197        EgressBacklog {
198            queued_bytes: self.queued_bytes.load(Ordering::Acquire),
199            queued_frames,
200            oldest_age,
201        }
202    }
203}
204
205/// One admitted frame's claim on its connection's egress byte budget,
206/// returned when the frame is dropped.
207#[derive(Debug)]
208pub(crate) struct EgressCharge {
209    accounting: Arc<EgressAccounting>,
210    bytes: usize,
211    stamp: u64,
212    enqueued_at: Instant,
213}
214
215impl EgressCharge {
216    /// Called by the connection writer when it takes this frame off the queue,
217    /// so the backlog's oldest-age figure follows the writer.
218    pub(crate) fn taken_by_writer(&self) {
219        self.accounting.taken_by_writer(self.stamp);
220    }
221}
222
223impl Drop for EgressCharge {
224    fn drop(&mut self) {
225        self.accounting.release(self.bytes, self.stamp);
226    }
227}
228
229/// A queue slot reserved ahead of time (a pending `route.open` holds one until
230/// its module answers). Sending through it never waits and never fails for
231/// lack of room: the slot is already held, and its frame is charged to the
232/// byte count outside the budget check. At most
233/// `MAX_PENDING_ROUTE_OPENS_PER_CONNECTION` such frames exist per connection.
234#[derive(Debug)]
235pub(crate) struct EgressPermit {
236    permit: mpsc::OwnedPermit<OutboundFrame>,
237    accounting: Arc<EgressAccounting>,
238}
239
240impl EgressPermit {
241    /// Enqueue `frame` in the reserved slot. Returns true when the connection's
242    /// writer had already gone away, meaning the frame will never be written.
243    pub(crate) fn send(self, frame: Frame) -> bool {
244        let charge = self
245            .accounting
246            .charge_unconditionally(queued_frame_bytes(&frame));
247        let sender = self.permit.send(OutboundFrame::charged(frame, charge));
248        sender.is_closed()
249    }
250}
251
252/// An `OutboundFrame` is a stamped `Frame`; deref keeps every existing frame
253/// read (headers, bodies, assertions) working on queued items unchanged.
254impl std::ops::Deref for OutboundFrame {
255    type Target = Frame;
256
257    fn deref(&self) -> &Frame {
258        &self.frame
259    }
260}
261
262/// Shared tracing-capture helpers for timing-observability tests across
263/// modules (router dispatch, server reply path). Test-only.
264#[cfg(test)]
265pub(crate) mod test_log {
266    use std::{
267        io::Write,
268        sync::{Arc, Mutex, OnceLock},
269    };
270
271    #[derive(Clone)]
272    struct TestLogWriter(Arc<Mutex<Vec<u8>>>);
273
274    impl Write for TestLogWriter {
275        fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
276            self.0
277                .lock()
278                .expect("test log capture is not poisoned")
279                .extend(buffer);
280            Ok(buffer.len())
281        }
282
283        fn flush(&mut self) -> std::io::Result<()> {
284            Ok(())
285        }
286    }
287
288    pub(crate) fn log_capture(
289        level: tracing::Level,
290    ) -> (Arc<Mutex<Vec<u8>>>, tracing::dispatcher::DefaultGuard) {
291        // tracing-core's single-dispatch fast path registers callsites using
292        // the emitting thread's default subscriber. A parallel test with no
293        // subscriber can therefore cache `never` for a callsite this capture
294        // needs. Keep a second dispatch alive so registration considers all
295        // live subscribers instead. This registry is never installed as a
296        // default: each test still records only into its own thread's writer.
297        static INTEREST_ANCHOR: OnceLock<tracing::Dispatch> = OnceLock::new();
298        INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
299        let output = Arc::new(Mutex::new(Vec::new()));
300        let writer = Arc::clone(&output);
301        let subscriber = tracing_subscriber::fmt()
302            .with_max_level(level)
303            .with_ansi(false)
304            .without_time()
305            .with_target(false)
306            .with_writer(move || TestLogWriter(Arc::clone(&writer)))
307            .finish();
308        let guard = tracing::subscriber::set_default(subscriber);
309        (output, guard)
310    }
311
312    pub(crate) fn captured_logs(output: &Arc<Mutex<Vec<u8>>>) -> String {
313        String::from_utf8(
314            output
315                .lock()
316                .expect("test log capture is not poisoned")
317                .clone(),
318        )
319        .expect("tracing output is UTF-8")
320    }
321
322    #[cfg(test)]
323    mod tests {
324        use super::*;
325
326        fn emit_capture_probe(source: &str) {
327            tracing::error!(source, "parallel capture probe");
328        }
329
330        #[test]
331        fn parallel_callsite_registration_keeps_the_test_capture_enabled() {
332            let (output, _guard) = log_capture(tracing::Level::ERROR);
333            // First register the very same callsite on a thread without a
334            // default subscriber, while the test's capture remains live.
335            std::thread::spawn(|| emit_capture_probe("other-thread"))
336                .join()
337                .expect("unsubscribed thread exits");
338            emit_capture_probe("captured-thread");
339
340            let logs = captured_logs(&output);
341            assert!(
342                logs.contains("captured-thread"),
343                "capture was disabled: {logs}"
344            );
345            assert!(!logs.contains("other-thread"), "capture leaked: {logs}");
346        }
347    }
348}
349
350/// Cheaply cloneable handle to one connection's bounded outbound frame queue.
351///
352/// Backends emit responses, streaming frames, and future PUSH frames through this
353/// single path. The queue is bounded twice: by queued BYTES (the budget that
354/// matters for memory and for how long a slow reader may pause), and by the
355/// `mpsc` channel's frame count (a backstop). The socket layer owns the sole
356/// receiver/writer.
357#[derive(Debug, Clone)]
358pub struct FrameSink {
359    tx: mpsc::Sender<OutboundFrame>,
360    accounting: Arc<EgressAccounting>,
361}
362
363impl FrameSink {
364    /// A sink with the standard per-connection byte budget
365    /// ([`crate::server::CONNECTION_EGRESS_BYTE_BUDGET`]); the frame-count bound
366    /// is whatever capacity `tx`'s channel was created with.
367    pub fn new(tx: mpsc::Sender<OutboundFrame>) -> Self {
368        Self::with_byte_budget(tx, crate::server::CONNECTION_EGRESS_BYTE_BUDGET)
369    }
370
371    pub(crate) fn with_byte_budget(tx: mpsc::Sender<OutboundFrame>, byte_budget: usize) -> Self {
372        Self {
373            tx,
374            accounting: Arc::new(EgressAccounting::new(byte_budget)),
375        }
376    }
377
378    /// Wait until the frame's bytes fit the budget, then charge them. Awaited
379    /// senders (control replies, error replies, shutdown notices) are never
380    /// refused by the byte budget: they wait for the writer to free bytes, just
381    /// as they wait for a free slot, which is the same backpressure they had
382    /// when the queue was bounded by frame count alone. Returns `None` if the
383    /// writer goes away while waiting.
384    async fn charge_waiting(&self, bytes: usize) -> Option<EgressCharge> {
385        if let Some(charge) = self.accounting.try_charge(bytes) {
386            return Some(charge);
387        }
388        // Counted as a waiter for as long as this future is parked, including
389        // when it is cancelled, so releases notify only while someone waits.
390        struct Waiting<'a>(&'a AtomicUsize);
391        impl Drop for Waiting<'_> {
392            fn drop(&mut self) {
393                self.0.fetch_sub(1, Ordering::SeqCst);
394            }
395        }
396        self.accounting.waiters.fetch_add(1, Ordering::SeqCst);
397        let _waiting = Waiting(&self.accounting.waiters);
398        loop {
399            let freed = self.accounting.freed.notified();
400            tokio::pin!(freed);
401            // Register interest before checking, so a release that happens
402            // between the check and the await still wakes this sender.
403            freed.as_mut().enable();
404            if let Some(charge) = self.accounting.try_charge(bytes) {
405                return Some(charge);
406            }
407            tokio::select! {
408                _ = &mut freed => {}
409                _ = self.tx.closed() => return None,
410            }
411        }
412    }
413
414    pub async fn send(&self, frame: Frame) -> Result<(), RouterError> {
415        let channel = frame.header.channel;
416        let epoch = frame.header.epoch;
417        let corr = frame.header.corr;
418        let closed =
419            || RouterError::backend_with_epoch(channel, epoch, corr, "connection writer closed");
420        let charge = self
421            .charge_waiting(queued_frame_bytes(&frame))
422            .await
423            .ok_or_else(closed)?;
424        self.tx
425            .send(OutboundFrame::charged(frame, charge))
426            .await
427            .map_err(|_| closed())
428    }
429
430    /// Shutdown notices must leave the socket writer before an idle daemon exits.
431    /// Queue admission alone does not prove this; the writer acknowledges flush.
432    ///
433    /// The local endpoint uses it the same way for its clients-only refusal: the
434    /// connection closes right after the refusal is sent, so the writer must
435    /// flush it first.
436    #[cfg(unix)]
437    pub async fn send_flushed(&self, frame: Frame) -> Result<(), RouterError> {
438        let (tx, rx) = tokio::sync::oneshot::channel();
439        let charge = self
440            .charge_waiting(queued_frame_bytes(&frame))
441            .await
442            .ok_or_else(|| RouterError::backend(0, 0, "connection writer closed"))?;
443        let mut outbound = OutboundFrame::charged(frame, charge);
444        outbound.flushed = Some(tx);
445        self.tx
446            .send(outbound)
447            .await
448            .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
449        rx.await
450            .map_err(|_| RouterError::backend(0, 0, "connection flush failed"))
451    }
452
453    pub(crate) async fn reserve_owned(&self) -> Result<EgressPermit, RouterError> {
454        let permit = self
455            .tx
456            .clone()
457            .reserve_owned()
458            .await
459            .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
460        Ok(EgressPermit {
461            permit,
462            accounting: Arc::clone(&self.accounting),
463        })
464    }
465
466    #[cfg(test)]
467    pub(crate) fn try_reserve_owned(&self) -> Result<EgressPermit, RouterError> {
468        let permit = self
469            .tx
470            .clone()
471            .try_reserve_owned()
472            .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
473        Ok(EgressPermit {
474            permit,
475            accounting: Arc::clone(&self.accounting),
476        })
477    }
478
479    pub(crate) fn is_closed(&self) -> bool {
480        self.tx.is_closed()
481    }
482
483    /// What the connection's egress queue holds right now.
484    pub(crate) fn backlog(&self) -> EgressBacklog {
485        self.accounting.backlog()
486    }
487
488    /// Stop module deliveries as soon as the client reader exits, without
489    /// waiting for the socket writer's separate teardown drain grace.
490    pub(crate) fn close_client_delivery(&self) {
491        self.accounting
492            .client_delivery_closed
493            .store(true, Ordering::Release);
494        self.accounting.client_delivery_stopped.notify_waiters();
495    }
496
497    /// A full queue can be a short fanout burst rather than a stopped reader.
498    /// Spend only the grace the atomic backlog age has not already used.
499    async fn send_to_client(&self, frame: Frame) -> Result<(), ClientDeliveryFailure> {
500        let frame_type = frame.header.ty;
501        let flags = frame.header.flags;
502        let stopped = self.accounting.client_delivery_stopped.notified();
503        tokio::pin!(stopped);
504        stopped.as_mut().enable();
505        let failure = |cause, error| ClientDeliveryFailure {
506            diagnosis: ClientDeliveryDiagnosis {
507                frame_type,
508                flags,
509                cause,
510            },
511            error,
512        };
513        let closed = || {
514            RouterError::backend_with_epoch(
515                frame.header.channel,
516                frame.header.epoch,
517                frame.header.corr,
518                "client connection closed",
519            )
520        };
521        if self
522            .accounting
523            .client_delivery_closed
524            .load(Ordering::Acquire)
525        {
526            return Err(failure("client_closed", closed()));
527        }
528        let (frame, error) = match self.try_send_owned(frame) {
529            Ok(()) => return Ok(()),
530            Err(refused) => refused,
531        };
532        if self.is_closed() {
533            return Err(failure("client_closed", error));
534        }
535        let grace = crate::server::CONNECTION_EGRESS_STALL_GRACE;
536        let age = self.backlog().oldest_age.unwrap_or_default();
537        if age > grace {
538            return Err(failure("oldest_frame_older_than_250_ms", error));
539        }
540        let result = tokio::select! {
541            biased;
542            _ = &mut stopped => return Err(failure("client_closed", error)),
543            result = tokio::time::timeout(grace.saturating_sub(age), self.send(frame)) => result,
544        };
545        match result {
546            Ok(Ok(())) => Ok(()),
547            Ok(Err(error)) => Err(failure("client_closed", error)),
548            Err(_) => Err(failure("waited_250_ms_without_room", error)),
549        }
550    }
551
552    /// Enqueue without waiting. Fails when the frame's bytes would take the
553    /// queue past its byte budget, when the channel's frame-count backstop is
554    /// full, or when the writer is gone.
555    pub(crate) fn try_send(&self, frame: Frame) -> Result<(), RouterError> {
556        self.try_send_owned(frame).map_err(|(_, error)| error)
557    }
558
559    fn try_send_owned(&self, frame: Frame) -> Result<(), (Frame, RouterError)> {
560        let channel = frame.header.channel;
561        let epoch = frame.header.epoch;
562        let corr = frame.header.corr;
563        let unavailable = |why: String| {
564            RouterError::backend_with_epoch(
565                channel,
566                epoch,
567                corr,
568                format!("connection writer unavailable: {why}"),
569            )
570        };
571        let bytes = queued_frame_bytes(&frame);
572        let Some(charge) = self.accounting.try_charge(bytes) else {
573            if self.tx.is_closed() {
574                return Err((frame, unavailable("channel closed".to_string())));
575            }
576            return Err((
577                frame,
578                unavailable(format!(
579                    "egress byte budget exhausted ({} queued bytes, frame of {bytes} bytes, budget {})",
580                    self.accounting.queued_bytes.load(Ordering::Acquire),
581                    self.accounting.byte_budget
582                )),
583            ));
584        };
585        // Return the refused frame for a bounded retry; dropping the rest of
586        // the outbound wrapper returns its charge.
587        self.tx
588            .try_send(OutboundFrame::charged(frame, charge))
589            .map_err(|err| {
590                let error = unavailable(err.to_string());
591                (err.into_inner().frame, error)
592            })
593    }
594}
595
596#[derive(Debug)]
597struct ClientDeliveryFailure {
598    diagnosis: ClientDeliveryDiagnosis,
599    error: RouterError,
600}
601
602/// Header facts only: payloads can contain secrets and must never enter logs.
603#[derive(Debug)]
604pub(crate) struct ClientDeliveryDiagnosis {
605    pub frame_type: FrameType,
606    pub flags: Flags,
607    pub cause: &'static str,
608}
609
610impl fmt::Display for ClientDeliveryFailure {
611    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
612        self.error.fmt(f)
613    }
614}
615
616/// Minimum time between two route GOODBYEs the daemon sends one module
617/// connection for one channel in answer to the module's frames on a (channel,
618/// epoch) the daemon holds no route for.
619///
620/// A module that missed a GOODBYE keeps sending on that route (a streaming
621/// module, many frames a second), and every one of those frames lands here. One
622/// answer is enough when it arrives, so answering each frame would only flood
623/// the module's egress queue at the moment it is already behind. Five seconds
624/// is long enough for an answer queued behind a deep backlog to reach the module
625/// and take effect before a second is sent, and short enough that an answer
626/// the queue refused is retried by the next orphan frame well within a minute.
627const ORPHAN_ROUTE_GOODBYE_INTERVAL: Duration = Duration::from_secs(5);
628
629/// Once one connection has this many remembered channels, entries older than
630/// [`ORPHAN_ROUTE_GOODBYE_INTERVAL`] are pruned before another is added. An
631/// expired entry has no effect on rate limiting, so pruning never changes a
632/// decision; it keeps a module that orphaned many channels long ago from
633/// pinning memory for all of them. The map is bounded in any case by the
634/// 16-bit channel space and is dropped whole when the connection ends.
635const ORPHAN_ROUTE_GOODBYE_PRUNE_AT: usize = 256;
636
637/// When each module connection was last sent an orphan-route GOODBYE, per
638/// channel. Shared by the [`Router`] (which consults it) and every
639/// [`RouterConnection`] (which removes its own entry when the connection ends).
640#[derive(Debug, Default)]
641struct OrphanGoodbyeLimiter {
642    last_sent: Mutex<HashMap<ConnectionId, HashMap<u16, tokio::time::Instant>>>,
643}
644
645impl OrphanGoodbyeLimiter {
646    /// True, and the send recorded, when `channel` on `connection_id` has not
647    /// been answered within the interval.
648    fn claim(&self, connection_id: ConnectionId, channel: u16) -> bool {
649        let now = tokio::time::Instant::now();
650        let mut last_sent = self
651            .last_sent
652            .lock()
653            .unwrap_or_else(std::sync::PoisonError::into_inner);
654        let channels = last_sent.entry(connection_id).or_default();
655        if channels.get(&channel).is_some_and(|sent| {
656            now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
657        }) {
658            return false;
659        }
660        if channels.len() >= ORPHAN_ROUTE_GOODBYE_PRUNE_AT {
661            channels.retain(|_, sent| {
662                now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
663            });
664        }
665        channels.insert(channel, now);
666        true
667    }
668
669    fn forget_connection(&self, connection_id: ConnectionId) {
670        self.last_sent
671            .lock()
672            .unwrap_or_else(std::sync::PoisonError::into_inner)
673            .remove(&connection_id);
674    }
675}
676
677/// Per-route context shared with backends besides the frame itself.
678#[derive(Debug, Clone)]
679pub struct RouteCtx {
680    pub connection_id: ConnectionId,
681    pub egress: FrameSink,
682}
683
684/// Closed set of data-plane backends for non-zero channels.
685///
686/// Channel 0 is structurally special and remains `Router::control`, not an
687/// enum variant. Static/test backends live in `Router::backends`; the forwarding
688/// backend is selected dynamically only when a per-connection forwarding binding exists.
689#[derive(Debug, Clone)]
690pub enum Backend {
691    Echo(EchoBackend),
692    Forward(ForwardBackend),
693}
694
695impl From<EchoBackend> for Backend {
696    fn from(backend: EchoBackend) -> Self {
697        Self::Echo(backend)
698    }
699}
700
701impl From<ForwardBackend> for Backend {
702    fn from(backend: ForwardBackend) -> Self {
703        Self::Forward(backend)
704    }
705}
706
707impl Backend {
708    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
709        match self {
710            Self::Echo(backend) => backend.handle(ctx, frame).await,
711            Self::Forward(backend) => backend.handle(ctx, frame).await,
712        }
713    }
714}
715
716/// I/O-agnostic splice router keyed by envelope `channel`.
717///
718/// Channel 0 is reserved for subc itself and is always dispatched to the
719/// dedicated control handler. Other channels must be explicitly registered.
720/// Unknown client-originated non-zero channels are translated to canonical JSON `ERROR` frames on
721/// the connection sink so the peer can continue using the same socket. Module-originated frames for
722/// released route channels are logged and dropped as the channel-gone race backstop.
723///
724/// DATA-PLANE BODIES ARE NEVER DECODED HERE, and the consequence is worth stating
725/// because it looks like a guarantee and is not. Additive fields in a request or
726/// response body reach the far side untouched — not because anything permits them,
727/// but because routing reads only the 21-byte header and treats the body as opaque
728/// bytes. That is a performance property, so **nothing prevents it from changing**:
729/// a future reason to inspect a body would convert a wire-transparent path into a
730/// filtering one, and nobody would think of it as a contract change.
731///
732/// So when a peer asks whether subc sees an additive field, the answer is per-path
733/// and this path's zero means WE NEVER LOOK rather than WE LOOK AT EVERYTHING. The
734/// control plane (typed enums at the frame boundary) and the MCP gateway (envelope
735/// unwrap plus a named struct) both narrow; only this one does not.
736pub struct Router {
737    backends: HashMap<u16, Backend>,
738    control: Arc<ControlHandler>,
739    forwarding: Arc<ForwardingTable>,
740    forward_backend: ForwardBackend,
741    counters: DaemonCounters,
742    next_connection_id: AtomicU64,
743    orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
744}
745
746impl Router {
747    pub fn with_control_handler(control: Arc<ControlHandler>) -> Self {
748        // The handler the router serves is the one a swap must tell when it
749        // promotes a candidate; this is where it first sits behind an `Arc`.
750        control.install_swap_promotion_observer();
751        let forwarding = control.forwarding();
752        let counters = control.counters();
753        Self {
754            backends: HashMap::new(),
755            control,
756            forwarding: Arc::clone(&forwarding),
757            forward_backend: ForwardBackend::new(forwarding),
758            counters,
759            // ConnectionId::LOCAL is 0; real socket ids start at 1 and never collide.
760            next_connection_id: AtomicU64::new(1),
761            orphan_goodbyes: Arc::default(),
762        }
763    }
764
765    pub fn with_default_self_handler() -> Self {
766        Self::with_control_handler(Arc::new(ControlHandler::default()))
767    }
768
769    pub fn forwarding(&self) -> Arc<ForwardingTable> {
770        Arc::clone(&self.forwarding)
771    }
772
773    pub fn register_backend(
774        &mut self,
775        channel: u16,
776        backend: impl Into<Backend>,
777    ) -> Result<(), RouterError> {
778        self.register_backend_arc(channel, Arc::new(backend.into()))
779    }
780
781    pub(crate) fn register_backend_arc(
782        &mut self,
783        channel: u16,
784        backend: Arc<Backend>,
785    ) -> Result<(), RouterError> {
786        if channel == 0 {
787            return Err(RouterError::ReservedChannelZero);
788        }
789        if self.backends.contains_key(&channel) {
790            return Err(RouterError::DuplicateChannel { channel });
791        }
792        self.backends.insert(channel, backend.as_ref().clone());
793        Ok(())
794    }
795
796    /// Start a connection-scoped routing context. Dropping the guard releases
797    /// any control-plane registrations owned by the connection.
798    fn record_module_frame_drop(&self, connection_id: ConnectionId) -> Result<(), RouterError> {
799        let module_id = self
800            .forwarding
801            .module_id_for_connection(connection_id)
802            .map_err(RouterError::Forwarding)?;
803        self.counters
804            .increment_module_frames_dropped_no_route(module_id.as_deref());
805        Ok(())
806    }
807
808    /// A module sent a non-request frame on a (channel, epoch) the daemon holds
809    /// no route for: most often a route the daemon released whose GOODBYE the
810    /// module never received, so the module still believes it is open and keeps
811    /// sending. Count the drop, and tell the module to let go of exactly that
812    /// (channel, epoch) with a route GOODBYE, at most once per
813    /// [`ORPHAN_ROUTE_GOODBYE_INTERVAL`] per connection and channel.
814    ///
815    /// The answer uses `try_send`: it is a best-effort nudge, and if the queue
816    /// refuses it, the module's next frame on that route after the interval
817    /// asks again. A GOODBYE from the module is not answered, since the module
818    /// is already letting go of the route.
819    fn handle_orphan_module_frame(&self, ctx: &RouteCtx, frame: &Frame) -> Result<(), RouterError> {
820        let channel = frame.header.channel;
821        let epoch = frame.header.epoch;
822        let module_id = self
823            .forwarding
824            .module_id_for_connection(ctx.connection_id)
825            .map_err(RouterError::Forwarding)?;
826        self.counters
827            .increment_module_frames_dropped_no_route(module_id.as_deref());
828        if self
829            .forwarding
830            .module_route_epoch_was_allocated(ctx.connection_id, channel, epoch)
831            .map_err(RouterError::Forwarding)?
832        {
833            self.counters
834                .increment_module_frames_dropped_released_route(module_id.as_deref());
835        }
836        if frame.header.ty == FrameType::Goodbye
837            || !self.orphan_goodbyes.claim(ctx.connection_id, channel)
838        {
839            return Ok(());
840        }
841        let goodbye = Frame::build_with_version(
842            frame.header.ver,
843            FrameType::Goodbye,
844            Flags::new(false, Priority::Passive, false),
845            channel,
846            epoch,
847            0,
848            Vec::new(),
849        )
850        .map_err(RouterError::FrameBuild)?;
851        match ctx.egress.try_send(goodbye) {
852            Ok(()) => {
853                self.counters.increment_module_orphan_route_goodbyes_sent();
854                debug!(
855                    connection_id = ctx.connection_id.get(),
856                    module_id = module_id.as_deref().unwrap_or("unknown"),
857                    channel,
858                    epoch,
859                    "answered module frame on a route the daemon does not hold with a route GOODBYE"
860                );
861            }
862            Err(err) => {
863                self.forwarding.warn_frame_drop(
864                    ctx.connection_id,
865                    channel,
866                    epoch,
867                    "daemon_to_module",
868                    if ctx.egress.is_closed() {
869                        "endpoint_gone"
870                    } else {
871                        "egress_budget"
872                    },
873                );
874                debug!(
875                connection_id = ctx.connection_id.get(),
876                module_id = module_id.as_deref().unwrap_or("unknown"),
877                channel,
878                epoch,
879                error = %err,
880                "could not enqueue route GOODBYE for module frame on a route the daemon does not hold; the next such frame retries"
881                );
882            }
883        }
884        Ok(())
885    }
886
887    pub fn begin_connection(&self) -> RouterConnection {
888        let raw = self.next_connection_id.fetch_add(1, Ordering::Relaxed);
889        let id = ConnectionId::new(raw);
890        let close_receiver = self.forwarding.register_connection_close(id);
891        RouterConnection {
892            id,
893            control_handler: Arc::clone(&self.control),
894            forwarding: Arc::clone(&self.forwarding),
895            close_receiver: Some(close_receiver),
896            orphan_goodbyes: Arc::clone(&self.orphan_goodbyes),
897        }
898    }
899
900    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
901        self.control.route_open_target(frame)
902    }
903
904    pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
905        self.control.record_local_connection(connection);
906    }
907
908    pub(crate) fn record_captured_peer(
909        &self,
910        connection: ConnectionId,
911        peer: subc_os::local_peer::CapturedPeer,
912    ) {
913        self.control.record_captured_peer(connection, peer);
914    }
915
916    pub(crate) fn record_caller_endpoints(
917        &self,
918        connection: ConnectionId,
919        endpoints: Option<(std::net::SocketAddr, std::net::SocketAddr)>,
920    ) {
921        self.control.record_caller_endpoints(connection, endpoints);
922    }
923
924    pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
925        self.control.defers_privileged_control(ctx, frame)
926    }
927
928    pub(crate) fn route_open_capacity_refusal(
929        &self,
930        ctx: &RouteCtx,
931        frame: &Frame,
932        target_module_id: &str,
933        in_flight: usize,
934        limit: usize,
935    ) -> Result<Frame, RouterError> {
936        self.control
937            .route_open_capacity_refusal(ctx, frame, target_module_id, in_flight, limit)
938    }
939
940    pub async fn route_for_connection(
941        &self,
942        ctx: &RouteCtx,
943        frame: Frame,
944    ) -> Result<(), RouterError> {
945        self.route_for_connection_started(ctx, frame, None).await
946    }
947
948    pub(crate) async fn observe_connection_wait<F: std::future::Future>(
949        &self,
950        ctx: &RouteCtx,
951        kind: &'static str,
952        future: F,
953    ) -> F::Output {
954        self.counters
955            .observe_reader_wait(
956                || {
957                    let module = self
958                        .forwarding
959                        .module_id_for_connection(ctx.connection_id)
960                        .ok()
961                        .flatten();
962                    let principal = if module.is_some() { "module" } else { "direct" };
963                    (
964                        module.unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
965                        principal.into(),
966                    )
967                },
968                kind,
969                future,
970            )
971            .await
972    }
973
974    pub(crate) async fn route_for_connection_started(
975        &self,
976        ctx: &RouteCtx,
977        frame: Frame,
978        dispatch_started_at: Option<Instant>,
979    ) -> Result<(), RouterError> {
980        let channel = frame.header.channel;
981        let epoch = frame.header.epoch;
982        let corr = frame.header.corr;
983        if channel == 0 {
984            debug!(
985                connection_id = ctx.connection_id.get(),
986                corr,
987                frame_type = ?frame.header.ty,
988                "routing control frame"
989            );
990            // The connection loop dispatches every control operation directly
991            // except `route.open`, activity writes and privileged requests. Each
992            // task passes a timestamp captured before
993            // spawn, so slow-dispatch timing includes scheduler delay but no
994            // wait in an application-owned queue.
995            let dispatch_started_at = (frame.header.ty == FrameType::Request)
996                .then(|| dispatch_started_at.unwrap_or_else(Instant::now));
997            let responses = self
998                .control
999                .handle_control_frame_timed(ctx, frame, dispatch_started_at)
1000                .await?;
1001            for response in responses {
1002                self.observe_connection_wait(ctx, "reply_egress", ctx.egress.send(response))
1003                    .await?;
1004            }
1005            return Ok(());
1006        }
1007
1008        let data_route = self
1009            .forwarding
1010            .lookup_data_route(ctx.connection_id, channel, epoch)
1011            .map_err(RouterError::Forwarding)?;
1012
1013        match data_route {
1014            DataRoute::Module(DataRouteState::EpochMismatch) => {
1015                self.forwarding.warn_frame_drop(
1016                    ctx.connection_id,
1017                    channel,
1018                    epoch,
1019                    "module_to_client",
1020                    "stale_epoch",
1021                );
1022                if frame.header.ty == FrameType::Request {
1023                    self.counters
1024                        .increment_module_requests_dropped_stale_route();
1025                    let err = RouterError::StaleRouteEpoch {
1026                        channel,
1027                        epoch,
1028                        corr,
1029                    };
1030                    if let Some(error_frame) = err.to_error_frame() {
1031                        self.observe_connection_wait(
1032                            ctx,
1033                            "reply_egress",
1034                            ctx.egress.send(error_frame),
1035                        )
1036                        .await?;
1037                    }
1038                } else {
1039                    self.handle_orphan_module_frame(ctx, &frame)?;
1040                }
1041                debug!(
1042                    connection_id = ctx.connection_id.get(),
1043                    channel, epoch, corr, "dropping module frame for stale route epoch"
1044                );
1045                return Ok(());
1046            }
1047            DataRoute::Module(DataRouteState::Reserved) => {
1048                self.forwarding.warn_frame_drop(
1049                    ctx.connection_id,
1050                    channel,
1051                    epoch,
1052                    "module_to_client",
1053                    "missing_binding",
1054                );
1055                if frame.header.ty == FrameType::Request {
1056                    self.counters
1057                        .increment_module_requests_dropped_stale_route();
1058                    let err = RouterError::UnknownChannel {
1059                        channel,
1060                        epoch,
1061                        corr,
1062                    };
1063                    if let Some(error_frame) = err.to_error_frame() {
1064                        self.observe_connection_wait(
1065                            ctx,
1066                            "reply_egress",
1067                            ctx.egress.send(error_frame),
1068                        )
1069                        .await?;
1070                    }
1071                } else {
1072                    self.record_module_frame_drop(ctx.connection_id)?;
1073                }
1074                debug!(
1075                    connection_id = ctx.connection_id.get(),
1076                    channel, epoch, corr, "dropping module frame for reserved route handle"
1077                );
1078                return Ok(());
1079            }
1080            DataRoute::Module(DataRouteState::Absent) => {
1081                self.forwarding.warn_frame_drop(
1082                    ctx.connection_id,
1083                    channel,
1084                    epoch,
1085                    "module_to_client",
1086                    "unknown_channel",
1087                );
1088                if frame.header.ty == FrameType::Request {
1089                    self.counters
1090                        .increment_module_requests_dropped_stale_route();
1091                    let err = RouterError::UnknownChannel {
1092                        channel,
1093                        epoch,
1094                        corr,
1095                    };
1096                    if let Some(error_frame) = err.to_error_frame() {
1097                        self.observe_connection_wait(
1098                            ctx,
1099                            "reply_egress",
1100                            ctx.egress.send(error_frame),
1101                        )
1102                        .await?;
1103                    }
1104                } else {
1105                    self.handle_orphan_module_frame(ctx, &frame)?;
1106                }
1107                debug!(
1108                    connection_id = ctx.connection_id.get(),
1109                    channel, epoch, corr, "dropping module frame for absent route handle"
1110                );
1111                return Ok(());
1112            }
1113            DataRoute::Module(DataRouteState::Bound(route)) => {
1114                if frame.header.ty == FrameType::Goodbye {
1115                    if let RouteRelease::Removed(target) = self
1116                        .forwarding
1117                        .release_module_route(ctx.connection_id, channel, epoch)
1118                        .map_err(RouterError::Forwarding)?
1119                    {
1120                        let mut goodbye = frame;
1121                        goodbye.header.channel = target.channel;
1122                        goodbye.header.epoch = target.epoch;
1123                        // A module must never be able to make a frame look
1124                        // daemon-authored, so the origin bit is cleared on
1125                        // every module frame relayed to a client.
1126                        goodbye.header.flags = goodbye.header.flags.without_daemon_origin();
1127                        if let Err(err) = self
1128                            .observe_connection_wait(
1129                                ctx,
1130                                "module_to_client",
1131                                target.sink.send_to_client(goodbye),
1132                            )
1133                            .await
1134                        {
1135                            if !target.close_on_delivery_failure() {
1136                                self.forwarding.warn_frame_drop(
1137                                    target.connection_id,
1138                                    target.channel,
1139                                    target.epoch,
1140                                    "module_to_client",
1141                                    err.diagnosis.cause,
1142                                );
1143                            }
1144                            if target.close_on_delivery_failure()
1145                                && self
1146                                    .forwarding
1147                                    .escalate_client_delivery_failure_diagnosed(
1148                                        target.connection_id,
1149                                        target.channel,
1150                                        target.epoch,
1151                                        CloseReason::new(
1152                                            "route_goodbye_delivery_failed",
1153                                            format!(
1154                                                "failed to enqueue route GOODBYE for client channel {}: {err}",
1155                                                target.channel
1156                                            ),
1157                                        ),
1158                                        UndeliveredFrame {
1159                                            module_id: Some(&route.module_id),
1160                                            sink: &target.sink,
1161                                        },
1162                                        Some(&err.diagnosis),
1163                                    )
1164                                    .map_err(RouterError::Forwarding)?
1165                            {
1166                                self.counters.increment_goodbye_relay_client_failed();
1167                            }
1168                        }
1169                    } else {
1170                        self.forwarding.warn_frame_drop(
1171                            ctx.connection_id,
1172                            channel,
1173                            epoch,
1174                            "module_to_client",
1175                            "missing_binding",
1176                        );
1177                    }
1178                    return Ok(());
1179                }
1180
1181                // Accounting allows an oversized frame to occupy an empty queue
1182                // on other send paths. Refuse it at this splice before admission,
1183                // so the next ordinary reply cannot close a multiplexed client.
1184                let bytes = queued_frame_bytes(&frame);
1185                let limit = route.client_sink.accounting.byte_budget;
1186                if bytes > limit {
1187                    let ty = frame.header.ty;
1188                    let flags = frame.header.flags;
1189                    drop(frame);
1190                    return self
1191                        .refuse_oversized_module_frame(ctx, &route, (ty, flags), corr, bytes)
1192                        .await;
1193                }
1194
1195                // A terminal frame ends the request at the module whether or not
1196                // the client can still take it, so its credit is released before
1197                // delivery is attempted. Releasing only after a successful
1198                // enqueue would leave a drain counting a finished request until
1199                // the client connection's cleanup removes the route.
1200                let releases_credit = is_terminal_frame(frame.header.ty);
1201                if releases_credit {
1202                    route.flow.release_corr(corr);
1203                }
1204                let mut frame = frame;
1205                frame.header.channel = route.client_channel;
1206                frame.header.epoch = route.client_epoch;
1207                // Responses, errors, stream data and ends, and reverse
1208                // requests all pass through here. A module must never be able
1209                // to make a frame look daemon-authored, so the origin bit a
1210                // module may have set is cleared; every other flag bit and
1211                // the body travel unchanged.
1212                frame.header.flags = frame.header.flags.without_daemon_origin();
1213                if let Err(err) = self
1214                    .counters
1215                    .observe_reader_wait(
1216                        || (route.module_id.clone(), principal_label(&route.principal)),
1217                        "module_to_client",
1218                        route.client_sink.send_to_client(frame),
1219                    )
1220                    .await
1221                {
1222                    if self
1223                        .forwarding
1224                        .escalate_client_delivery_failure_diagnosed(
1225                            route.client_connection_id,
1226                            route.client_channel,
1227                            route.client_epoch,
1228                            CloseReason::new(
1229                                "module_to_client_delivery_failed",
1230                                format!(
1231                                    "failed to enqueue module frame for client channel {} corr {corr}: {err}",
1232                                    route.client_channel
1233                                ),
1234                            ),
1235                            UndeliveredFrame {
1236                                module_id: Some(&route.module_id),
1237                                sink: &route.client_sink,
1238                            },
1239                            Some(&err.diagnosis),
1240                        )
1241                        .map_err(RouterError::Forwarding)?
1242                    {
1243                        self.counters
1244                            .increment_client_egress_close_delivery_failed();
1245                    }
1246                    return Ok(());
1247                }
1248                return Ok(());
1249            }
1250            DataRoute::Client(DataRouteState::EpochMismatch) => {
1251                self.forwarding.warn_frame_drop(
1252                    ctx.connection_id,
1253                    channel,
1254                    epoch,
1255                    "client_to_module",
1256                    "stale_epoch",
1257                );
1258                if frame.header.ty == FrameType::Request {
1259                    self.counters.increment_client_frames_dropped_stale_route();
1260                    // Dropped before forwarding; a re-bind retry cannot double-execute this request.
1261                    let err = RouterError::StaleRouteEpoch {
1262                        channel,
1263                        epoch,
1264                        corr,
1265                    };
1266                    if let Some(error_frame) = err.to_error_frame() {
1267                        self.observe_connection_wait(
1268                            ctx,
1269                            "reply_egress",
1270                            ctx.egress.send(error_frame),
1271                        )
1272                        .await?;
1273                    }
1274                }
1275                debug!(
1276                    connection_id = ctx.connection_id.get(),
1277                    channel, epoch, corr, "dropping client frame for stale route epoch"
1278                );
1279                return Ok(());
1280            }
1281            DataRoute::Client(DataRouteState::Reserved) => {
1282                self.forwarding.warn_frame_drop(
1283                    ctx.connection_id,
1284                    channel,
1285                    epoch,
1286                    "client_to_module",
1287                    "missing_binding",
1288                );
1289                if frame.header.ty == FrameType::Request {
1290                    let err = RouterError::UnknownChannel {
1291                        channel,
1292                        epoch,
1293                        corr,
1294                    };
1295                    if let Some(error_frame) = err.to_error_frame() {
1296                        self.observe_connection_wait(
1297                            ctx,
1298                            "reply_egress",
1299                            ctx.egress.send(error_frame),
1300                        )
1301                        .await?;
1302                    }
1303                }
1304                return Ok(());
1305            }
1306            DataRoute::Client(DataRouteState::Bound(route)) => {
1307                if frame.header.ty == FrameType::Goodbye {
1308                    let _ = self
1309                        .control
1310                        .handle_route_goodbye(ctx.connection_id, channel, epoch)?;
1311                    return Ok(());
1312                }
1313                return self.forward_backend.handle_bound(frame, route).await;
1314            }
1315            DataRoute::Client(DataRouteState::Absent) => {}
1316        }
1317
1318        if let Some(backend) = self.backends.get(&channel) {
1319            return backend.handle(ctx.clone(), frame).await.inspect_err(|_| {
1320                self.forwarding.warn_frame_drop(
1321                    ctx.connection_id,
1322                    channel,
1323                    epoch,
1324                    "client_to_backend",
1325                    "endpoint_gone",
1326                );
1327            });
1328        }
1329        self.forwarding.warn_frame_drop(
1330            ctx.connection_id,
1331            channel,
1332            epoch,
1333            "client_to_module",
1334            "unknown_channel",
1335        );
1336        if frame.header.ty == FrameType::Request {
1337            let err = RouterError::UnknownChannel {
1338                channel,
1339                epoch,
1340                corr,
1341            };
1342            if let Some(error_frame) = err.to_error_frame() {
1343                self.observe_connection_wait(ctx, "reply_egress", ctx.egress.send(error_frame))
1344                    .await?;
1345            }
1346        }
1347        Ok(())
1348    }
1349
1350    async fn refuse_oversized_module_frame(
1351        &self,
1352        ctx: &RouteCtx,
1353        route: &RouteBinding,
1354        header: (FrameType, Flags),
1355        corr: u64,
1356        bytes: usize,
1357    ) -> Result<(), RouterError> {
1358        let (ty, flags) = header;
1359        let limit = route.client_sink.accounting.byte_budget;
1360        if ty == FrameType::Push {
1361            // Deployed Rust clients cannot decode a new route.closed reason.
1362            // Until that vocabulary can be extended compatibly, an oversized
1363            // unsolicited push still closes the connection, with its cause named.
1364            let diagnosis = ClientDeliveryDiagnosis {
1365                frame_type: ty,
1366                flags,
1367                cause: "frame_too_large",
1368            };
1369            if self
1370                .forwarding
1371                .escalate_client_delivery_failure_diagnosed(
1372                    route.client_connection_id,
1373                    route.client_channel,
1374                    route.client_epoch,
1375                    CloseReason::new(
1376                        error_codes::FRAME_TOO_LARGE,
1377                        format!(
1378                            "oversized push: size={bytes} limit={limit} (header plus body bytes)"
1379                        ),
1380                    ),
1381                    UndeliveredFrame {
1382                        module_id: Some(&route.module_id),
1383                        sink: &route.client_sink,
1384                    },
1385                    Some(&diagnosis),
1386                )
1387                .map_err(RouterError::Forwarding)?
1388            {
1389                self.counters
1390                    .increment_client_egress_close_delivery_failed();
1391            }
1392            return Ok(());
1393        }
1394
1395        warn!(
1396            connection_id = route.client_connection_id.get(),
1397            module_id = %route.module_id,
1398            client_principal = %principal_label(&route.principal),
1399            client_channel = route.client_channel,
1400            frame_type = ?ty,
1401            frame_bytes = bytes,
1402            byte_limit = limit,
1403            "refusing oversized module-to-client frame with ERROR frame_too_large"
1404        );
1405        if ty == FrameType::Request {
1406            // A reverse request belongs to the module, not to a waiting client.
1407            // Answer its sender without forwarding any part of it to the client.
1408            let error = frame_too_large_error(
1409                route.module_channel,
1410                route.module_epoch,
1411                corr,
1412                bytes,
1413                limit,
1414            )?;
1415            self.observe_connection_wait(ctx, "reply_egress", ctx.egress.send(error))
1416                .await?;
1417        } else {
1418            // A refused stream chunk also ends the waiting request. Releasing
1419            // its credit prevents a finished request from keeping a drain alive.
1420            route.flow.release_corr(corr);
1421            let error = frame_too_large_error(
1422                route.client_channel,
1423                route.client_epoch,
1424                corr,
1425                bytes,
1426                limit,
1427            )?;
1428            // The replacement shares the reply's bounded burst grace, so a
1429            // stopped client cannot indefinitely stall the module frame reader.
1430            if let Err(err) = self
1431                .counters
1432                .observe_reader_wait(
1433                    || (route.module_id.clone(), principal_label(&route.principal)),
1434                    "module_to_client",
1435                    route.client_sink.send_to_client(error),
1436                )
1437                .await
1438            {
1439                if self
1440                    .forwarding
1441                    .escalate_client_delivery_failure_diagnosed(
1442                        route.client_connection_id,
1443                        route.client_channel,
1444                        route.client_epoch,
1445                        CloseReason::new(
1446                            "module_to_client_delivery_failed",
1447                            format!(
1448                                "failed to enqueue frame_too_large for client channel {} corr {corr}: {err}",
1449                                route.client_channel
1450                            ),
1451                        ),
1452                        UndeliveredFrame {
1453                            module_id: Some(&route.module_id),
1454                            sink: &route.client_sink,
1455                        },
1456                        Some(&err.diagnosis),
1457                    )
1458                    .map_err(RouterError::Forwarding)?
1459                {
1460                    self.counters
1461                        .increment_client_egress_close_delivery_failed();
1462                }
1463            }
1464        }
1465        Ok(())
1466    }
1467}
1468
1469impl Default for Router {
1470    fn default() -> Self {
1471        Self::with_default_self_handler()
1472    }
1473}
1474
1475/// Connection-scoped cleanup guard returned by [`Router::begin_connection`].
1476#[must_use]
1477pub struct RouterConnection {
1478    id: ConnectionId,
1479    control_handler: Arc<ControlHandler>,
1480    forwarding: Arc<ForwardingTable>,
1481    close_receiver: Option<ConnectionCloseReceiver>,
1482    orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
1483}
1484
1485impl RouterConnection {
1486    pub fn id(&self) -> ConnectionId {
1487        self.id
1488    }
1489
1490    pub(crate) fn take_close_receiver(&mut self) -> ConnectionCloseReceiver {
1491        self.close_receiver
1492            .take()
1493            .expect("connection close receiver can only be taken once")
1494    }
1495}
1496
1497impl Drop for RouterConnection {
1498    fn drop(&mut self) {
1499        self.forwarding.unregister_connection_close(self.id);
1500        self.orphan_goodbyes.forget_connection(self.id);
1501        // GOODBYE (explicit) and connection-drop cleanup both call the same
1502        // idempotent deregistration path.
1503        let _ = self.control_handler.cleanup_connection(self.id);
1504    }
1505}
1506
1507/// Minimal in-memory backend used by tests and early wiring: it emits a
1508/// `RESPONSE` on the same channel/correlation id with the exact same body bytes.
1509#[derive(Debug, Default, Clone, Copy)]
1510pub struct EchoBackend;
1511
1512impl EchoBackend {
1513    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1514        let response = Frame::build_with_version(
1515            frame.header.ver,
1516            FrameType::Response,
1517            frame.header.flags,
1518            frame.header.channel,
1519            frame.header.epoch,
1520            frame.header.corr,
1521            frame.body,
1522        )
1523        .map_err(RouterError::FrameBuild)?;
1524        ctx.egress.send(response).await
1525    }
1526}
1527
1528/// Data-plane backend that splices client frames to the module connection bound at attach time.
1529#[derive(Debug, Clone)]
1530pub struct ForwardBackend {
1531    forwarding: Arc<ForwardingTable>,
1532}
1533
1534impl ForwardBackend {
1535    pub fn new(forwarding: Arc<ForwardingTable>) -> Self {
1536        Self { forwarding }
1537    }
1538
1539    pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1540        let channel = frame.header.channel;
1541        let corr = frame.header.corr;
1542        let route = match self
1543            .forwarding
1544            .lookup_data_route(ctx.connection_id, channel, frame.header.epoch)
1545            .map_err(RouterError::Forwarding)?
1546        {
1547            DataRoute::Client(DataRouteState::Bound(route)) => route,
1548            DataRoute::Client(_) | DataRoute::Module(_) => {
1549                self.forwarding.warn_frame_drop(
1550                    ctx.connection_id,
1551                    channel,
1552                    frame.header.epoch,
1553                    "client_to_module",
1554                    "missing_binding",
1555                );
1556                return Err(RouterError::UnknownChannel {
1557                    channel,
1558                    epoch: frame.header.epoch,
1559                    corr,
1560                });
1561            }
1562        };
1563        self.handle_bound(frame, route).await
1564    }
1565
1566    pub(crate) async fn handle_bound(
1567        &self,
1568        frame: Frame,
1569        route: Arc<RouteBinding>,
1570    ) -> Result<(), RouterError> {
1571        let channel = frame.header.channel;
1572        let corr = frame.header.corr;
1573        let frame_type = frame.header.ty;
1574
1575        // CANCEL and other non-REQUEST frames bypass the request-credit window;
1576        // the original request's credit is freed only by the module's terminal frame.
1577        let acquired_credit = frame_type == FrameType::Request;
1578        if acquired_credit && route.flow.busy_refusals() {
1579            use crate::forwarding::RequestAdmission;
1580            match self
1581                .forwarding
1582                .try_admit_request(
1583                    &route,
1584                    frame.header.epoch,
1585                    corr,
1586                    frame.header.flags.is_subscription(),
1587                )
1588                .map_err(RouterError::Forwarding)?
1589            {
1590                RequestAdmission::Admitted => {}
1591                RequestAdmission::Busy(detail) => {
1592                    let principal = principal_label(&route.principal);
1593                    let counters = self.forwarding.counters_ref();
1594                    counters.increment_route_busy(&route.module_id, &principal);
1595                    if route.flow.should_warn_busy() {
1596                        warn!(module_id = %route.module_id, client_principal = %principal, client_channel = channel,
1597                            route_epoch = route.client_epoch, ?detail, "request refused: route_busy");
1598                    }
1599                    let error = RouterError::RouteBusy {
1600                        channel,
1601                        epoch: frame.header.epoch,
1602                        corr,
1603                        detail: Box::new(detail),
1604                    };
1605                    let refusal = error
1606                        .to_error_frame()
1607                        .expect("route_busy is a serializable wire error");
1608                    counters
1609                        .observe_reader_wait(
1610                            || (route.module_id.clone(), principal),
1611                            "route_busy_egress",
1612                            route.client_sink.send(refusal),
1613                        )
1614                        .await?;
1615                    return Ok(());
1616                }
1617                RequestAdmission::ModuleReloading => {
1618                    return Err(RouterError::route_error_with_epoch(
1619                        channel,
1620                        frame.header.epoch,
1621                        corr,
1622                        error_codes::MODULE_RELOADING,
1623                        format!("module endpoint for route channel {channel} is reloading"),
1624                    ));
1625                }
1626                RequestAdmission::Stale => {
1627                    return Err(RouterError::StaleRouteEpoch {
1628                        channel,
1629                        epoch: frame.header.epoch,
1630                        corr,
1631                    })
1632                }
1633            }
1634        } else if acquired_credit {
1635            let counters = self.forwarding.counters_ref();
1636            if let Err(err) = counters
1637                .observe_reader_wait(
1638                    || (route.module_id.clone(), principal_label(&route.principal)),
1639                    "legacy_credit",
1640                    route
1641                        .flow
1642                        .acquire_tagged(corr, frame.header.flags.is_subscription()),
1643                )
1644                .await
1645            {
1646                self.forwarding.warn_frame_drop(
1647                    route.client_connection_id,
1648                    channel,
1649                    frame.header.epoch,
1650                    "client_to_module",
1651                    "endpoint_gone",
1652                );
1653                // `module_reloading` here is answered BEFORE the frame is
1654                // forwarded, so the module never sees the request and callers
1655                // may re-dispatch it after reopening the route. That is a wire
1656                // guarantee documented on `error_codes::MODULE_RELOADING`; do
1657                // not emit this code for a request that already reached
1658                // `module_sink.send` below.
1659                if self
1660                    .forwarding
1661                    .endpoint_is_draining(route.module_endpoint)
1662                    .map_err(RouterError::Forwarding)?
1663                {
1664                    return Err(RouterError::route_error_with_epoch(
1665                        channel,
1666                        frame.header.epoch,
1667                        corr,
1668                        "module_reloading",
1669                        format!("module endpoint for route channel {channel} is reloading"),
1670                    ));
1671                }
1672                return Err(RouterError::backend_with_epoch(
1673                    channel,
1674                    frame.header.epoch,
1675                    corr,
1676                    format!("{err} for route channel {channel}"),
1677                ));
1678            }
1679        }
1680
1681        let mut frame = frame;
1682        frame.header.channel = route.module_channel;
1683        frame.header.epoch = route.module_epoch;
1684        // The origin bit is the daemon's own signal to clients. A client
1685        // frame carrying it must not reach a module appearing daemon-authored,
1686        // so it is cleared here; every other flag bit travels unchanged.
1687        frame.header.flags = frame.header.flags.without_daemon_origin();
1688        let counters = self.forwarding.counters_ref();
1689        let result = counters
1690            .observe_reader_wait(
1691                || (route.module_id.clone(), principal_label(&route.principal)),
1692                "module_queue",
1693                route.module_sink.send(frame),
1694            )
1695            .await
1696            .map_err(|err| {
1697                self.forwarding.warn_frame_drop(
1698                    route.client_connection_id,
1699                    channel,
1700                    route.client_epoch,
1701                    "client_to_module",
1702                    "endpoint_gone",
1703                );
1704                RouterError::backend_with_epoch(channel, route.client_epoch, corr, err.to_string())
1705            });
1706        if acquired_credit && result.is_err() {
1707            route.flow.release_corr(corr);
1708        }
1709        result
1710    }
1711}
1712
1713fn is_terminal_frame(frame_type: FrameType) -> bool {
1714    matches!(
1715        frame_type,
1716        FrameType::Response | FrameType::Error | FrameType::StreamEnd
1717    )
1718}
1719
1720/// Typed router errors. Routable failures can be translated to canonical JSON
1721/// `ERROR` frames with [`RouterError::to_error_frame`].
1722#[derive(Debug, Clone, PartialEq, Eq)]
1723pub enum RouterError {
1724    ReservedChannelZero,
1725    DuplicateChannel {
1726        channel: u16,
1727    },
1728    UnknownChannel {
1729        channel: u16,
1730        epoch: u32,
1731        corr: u64,
1732    },
1733    StaleRouteEpoch {
1734        channel: u16,
1735        epoch: u32,
1736        corr: u64,
1737    },
1738    RouteBusy {
1739        channel: u16,
1740        epoch: u32,
1741        corr: u64,
1742        detail: Box<subc_protocol::RouteBusyDetail>,
1743    },
1744    Backend {
1745        channel: u16,
1746        epoch: u32,
1747        corr: u64,
1748        message: String,
1749    },
1750    RouteError {
1751        channel: u16,
1752        epoch: u32,
1753        corr: u64,
1754        code: String,
1755        message: String,
1756    },
1757    FrameBuild(FrameBuildError),
1758    Forwarding(ForwardingError),
1759}
1760
1761impl RouterError {
1762    pub fn backend(channel: u16, corr: u64, message: impl Into<String>) -> Self {
1763        Self::backend_with_epoch(channel, 0, corr, message)
1764    }
1765
1766    pub fn backend_with_epoch(
1767        channel: u16,
1768        epoch: u32,
1769        corr: u64,
1770        message: impl Into<String>,
1771    ) -> Self {
1772        Self::Backend {
1773            channel,
1774            epoch,
1775            corr,
1776            message: message.into(),
1777        }
1778    }
1779
1780    pub fn route_error(
1781        channel: u16,
1782        corr: u64,
1783        code: impl Into<String>,
1784        message: impl Into<String>,
1785    ) -> Self {
1786        Self::route_error_with_epoch(channel, 0, corr, code, message)
1787    }
1788
1789    pub fn route_error_with_epoch(
1790        channel: u16,
1791        epoch: u32,
1792        corr: u64,
1793        code: impl Into<String>,
1794        message: impl Into<String>,
1795    ) -> Self {
1796        Self::RouteError {
1797            channel,
1798            epoch,
1799            corr,
1800            code: code.into(),
1801            message: message.into(),
1802        }
1803    }
1804
1805    /// Translate route failures that belong on the wire into an `ERROR` frame.
1806    pub fn to_error_frame(&self) -> Option<Frame> {
1807        match self {
1808            Self::RouteBusy {
1809                channel,
1810                epoch,
1811                corr,
1812                detail,
1813            } => {
1814                let body = serde_json::to_vec(&ErrorBody {
1815                    code: error_codes::ROUTE_BUSY.into(),
1816                    message: "route has no free request or watch credit; retry on the same route"
1817                        .into(),
1818                    detail: Some(serde_json::to_value(detail).ok()?),
1819                })
1820                .ok()?;
1821                Frame::build(
1822                    FrameType::Error,
1823                    daemon_error_flags(),
1824                    *channel,
1825                    *epoch,
1826                    *corr,
1827                    body,
1828                )
1829                .ok()
1830            }
1831            Self::UnknownChannel {
1832                channel,
1833                epoch,
1834                corr,
1835            } => error_frame(
1836                *channel,
1837                *epoch,
1838                *corr,
1839                error_codes::UNKNOWN_CHANNEL,
1840                format!("unknown channel {channel}"),
1841            ),
1842            Self::StaleRouteEpoch {
1843                channel,
1844                epoch,
1845                corr,
1846            } => error_frame(
1847                *channel,
1848                *epoch,
1849                *corr,
1850                error_codes::STALE_ROUTE_EPOCH,
1851                format!("stale route epoch for channel {channel}"),
1852            ),
1853            Self::Backend {
1854                channel,
1855                epoch,
1856                corr,
1857                message,
1858            } => error_frame(*channel, *epoch, *corr, "backend_error", message.clone()),
1859            Self::RouteError {
1860                channel,
1861                epoch,
1862                corr,
1863                code,
1864                message,
1865            } => error_frame(*channel, *epoch, *corr, code, message.clone()),
1866            Self::ReservedChannelZero
1867            | Self::DuplicateChannel { .. }
1868            | Self::FrameBuild(_)
1869            | Self::Forwarding(_) => None,
1870        }
1871    }
1872}
1873
1874/// Flags for an ERROR frame the daemon authors itself.
1875///
1876/// Bit 6 (`FLAG_DAEMON_ORIGIN`) tells the client the daemon wrote this
1877/// error, never a module. What a code then proves is per code: on a bound
1878/// route the daemon writes `module_reloading`, `unknown_channel` and
1879/// `stale_route_epoch` only before forwarding, so with the bit they mean the
1880/// request was not delivered, while `frame_too_large` replaces a reply the
1881/// module already produced. Every daemon-authored ERROR goes through here;
1882/// frames a module sent keep their own flags with the bit cleared at the
1883/// splice.
1884/// Nothing else about the flags differs: passive priority, not binary, not last.
1885pub(crate) fn daemon_error_flags() -> Flags {
1886    Flags::new(false, Priority::Passive, false).with_daemon_origin()
1887}
1888
1889fn error_frame(channel: u16, epoch: u32, corr: u64, code: &str, message: String) -> Option<Frame> {
1890    let body = serde_json::to_vec(&ErrorBody {
1891        code: code.to_string(),
1892        message,
1893        detail: None,
1894    })
1895    .ok()?;
1896
1897    Frame::build(
1898        FrameType::Error,
1899        daemon_error_flags(),
1900        channel,
1901        epoch,
1902        corr,
1903        body,
1904    )
1905    .ok()
1906}
1907
1908fn frame_too_large_error(
1909    channel: u16,
1910    epoch: u32,
1911    corr: u64,
1912    bytes: usize,
1913    limit: usize,
1914) -> Result<Frame, RouterError> {
1915    let body = serde_json::to_vec(&ErrorBody {
1916        code: error_codes::FRAME_TOO_LARGE.to_string(),
1917        message: format!(
1918            "module frame of {bytes} bytes exceeds client egress limit of {limit} bytes"
1919        ),
1920        detail: Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit })),
1921    })
1922    .expect("serializing a frame size error");
1923    Frame::build(
1924        FrameType::Error,
1925        daemon_error_flags(),
1926        channel,
1927        epoch,
1928        corr,
1929        body,
1930    )
1931    .map_err(RouterError::FrameBuild)
1932}
1933
1934impl fmt::Display for RouterError {
1935    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1936        match self {
1937            Self::ReservedChannelZero => write!(f, "channel 0 is reserved for subc"),
1938            Self::DuplicateChannel { channel } => {
1939                write!(f, "backend already registered for channel {channel}")
1940            }
1941            Self::UnknownChannel { channel, corr, .. } => {
1942                write!(f, "unknown channel {channel} for corr {corr}")
1943            }
1944            Self::StaleRouteEpoch { channel, corr, .. } => {
1945                write!(f, "stale route epoch for channel {channel} corr {corr}")
1946            }
1947            Self::RouteBusy { channel, corr, .. } => {
1948                write!(f, "route busy on channel {channel} corr {corr}")
1949            }
1950            Self::Backend {
1951                channel,
1952                corr,
1953                message,
1954                ..
1955            } => write!(
1956                f,
1957                "backend error on channel {channel} corr {corr}: {message}"
1958            ),
1959            Self::RouteError {
1960                channel,
1961                corr,
1962                code,
1963                message,
1964                ..
1965            } => write!(
1966                f,
1967                "route error {code} on channel {channel} corr {corr}: {message}"
1968            ),
1969            Self::FrameBuild(err) => write!(f, "failed to build routed frame: {err}"),
1970            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
1971        }
1972    }
1973}
1974
1975impl Error for RouterError {
1976    fn source(&self) -> Option<&(dyn Error + 'static)> {
1977        match self {
1978            Self::FrameBuild(err) => Some(err),
1979            Self::Forwarding(err) => Some(err),
1980            Self::ReservedChannelZero
1981            | Self::DuplicateChannel { .. }
1982            | Self::UnknownChannel { .. }
1983            | Self::StaleRouteEpoch { .. }
1984            | Self::RouteBusy { .. }
1985            | Self::Backend { .. }
1986            | Self::RouteError { .. } => None,
1987        }
1988    }
1989}
1990
1991#[cfg(test)]
1992mod tests {
1993    include!("route_busy_router_tests.rs");
1994    use super::*;
1995    use crate::{
1996        forwarding::RouteBindRelayOutcome,
1997        supervise::{ModuleSpec, RestartPolicy, Supervisor, SupervisorHandle},
1998        ControlHandler, Registry,
1999    };
2000    use std::{
2001        sync::{mpsc as std_mpsc, Arc},
2002        time::Duration,
2003    };
2004    use subc_control::ModuleProtocol;
2005    use subc_protocol::{manifest::Concurrency, ErrorBody, Flags, FrameType, Priority};
2006    use tokio::sync::mpsc;
2007
2008    pub(crate) use crate::router::test_log::{captured_logs, log_capture};
2009
2010    fn logged_millis(logs: &str, field: &str) -> u64 {
2011        logs.split_whitespace()
2012            .find_map(|part| part.strip_prefix(field))
2013            .and_then(|value| value.parse().ok())
2014            .unwrap_or_else(|| panic!("missing numeric {field} in logs: {logs}"))
2015    }
2016
2017    fn request(channel: u16, corr: u64, body: &[u8]) -> Frame {
2018        Frame::build(
2019            FrameType::Request,
2020            Flags::new(true, Priority::Interactive, false),
2021            channel,
2022            0,
2023            corr,
2024            body.to_vec(),
2025        )
2026        .unwrap()
2027    }
2028
2029    fn ping(corr: u64) -> Frame {
2030        Frame::build(
2031            FrameType::Ping,
2032            Flags::new(false, Priority::Passive, false),
2033            0,
2034            0,
2035            corr,
2036            Vec::new(),
2037        )
2038        .unwrap()
2039    }
2040
2041    fn route_ctx() -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
2042        let (tx, rx) = mpsc::channel(8);
2043        (
2044            RouteCtx {
2045                connection_id: ConnectionId::LOCAL,
2046                egress: FrameSink::new(tx),
2047            },
2048            rx,
2049        )
2050    }
2051
2052    #[tokio::test]
2053    async fn echo_backend_returns_response_with_byte_identical_body() {
2054        let mut router = Router::with_default_self_handler();
2055        router.register_backend(7, EchoBackend).unwrap();
2056        let (ctx, mut rx) = route_ctx();
2057        let body = b"{not parsed}\0\xff";
2058
2059        router
2060            .route_for_connection(&ctx, request(7, 123, body))
2061            .await
2062            .unwrap();
2063        let response = rx.recv().await.unwrap();
2064
2065        assert_eq!(response.header.ty, FrameType::Response);
2066        assert_eq!(response.header.channel, 7);
2067        assert_eq!(response.header.corr, 123);
2068        assert_eq!(response.body, body);
2069        assert!(rx.try_recv().is_err());
2070    }
2071
2072    #[tokio::test]
2073    async fn unknown_channel_emits_canonical_error_frame() {
2074        let router = Router::with_default_self_handler();
2075        let (ctx, mut rx) = route_ctx();
2076
2077        router
2078            .route_for_connection(&ctx, request(99, 5, b"payload"))
2079            .await
2080            .unwrap();
2081        let error_frame = rx.recv().await.unwrap();
2082
2083        assert_eq!(error_frame.header.ty, FrameType::Error);
2084        assert_eq!(error_frame.header.channel, 99);
2085        assert_eq!(error_frame.header.corr, 5);
2086        let body: ErrorBody = serde_json::from_slice(&error_frame.body).unwrap();
2087        assert_eq!(body.code, "unknown_channel");
2088        assert_eq!(body.message, "unknown channel 99");
2089    }
2090
2091    /// Every ERROR the daemon authors carries the daemon-origin bit, so a
2092    /// client can tell a daemon refusal from a module's reply. Both the
2093    /// unknown-channel and the stale-epoch errors below are received on the
2094    /// client's own egress, not merely built.
2095    #[tokio::test]
2096    async fn daemon_authored_data_plane_errors_carry_daemon_origin() {
2097        let router = Router::with_default_self_handler();
2098        let (ctx, mut rx) = route_ctx();
2099        router
2100            .route_for_connection(&ctx, request(99, 5, b"payload"))
2101            .await
2102            .unwrap();
2103        let unknown = rx.recv().await.unwrap();
2104        assert_eq!(unknown.header.ty, FrameType::Error);
2105        let body: ErrorBody = serde_json::from_slice(&unknown.body).unwrap();
2106        assert_eq!(body.code, "unknown_channel");
2107        assert!(
2108            unknown.header.flags.is_daemon_origin(),
2109            "unknown_channel must carry daemon origin: {:?}",
2110            unknown.header.flags
2111        );
2112
2113        let (
2114            router,
2115            _forwarding,
2116            client_ctx,
2117            mut client_rx,
2118            _module_ctx,
2119            _module_egress_rx,
2120            _module_rx,
2121            pending,
2122        ) = dynamic_route_fixture(true);
2123        let _ = client_rx.recv().await;
2124        router
2125            .route_for_connection(
2126                &client_ctx,
2127                route_frame(
2128                    FrameType::Request,
2129                    pending.client_channel,
2130                    pending.client_epoch + 1,
2131                    702,
2132                ),
2133            )
2134            .await
2135            .unwrap();
2136        let stale = client_rx.recv().await.unwrap();
2137        assert_eq!(stale.header.ty, FrameType::Error);
2138        let body: ErrorBody = serde_json::from_slice(&stale.body).unwrap();
2139        assert_eq!(body.code, "stale_route_epoch");
2140        assert!(
2141            stale.header.flags.is_daemon_origin(),
2142            "stale_route_epoch must carry daemon origin: {:?}",
2143            stale.header.flags
2144        );
2145    }
2146
2147    /// A module ERROR that spoofs a daemon code with the daemon-origin bit
2148    /// set must reach the client with the bit cleared and the body
2149    /// byte-identical: the bit means the daemon wrote the frame, and a module
2150    /// must never be able to claim that.
2151    #[tokio::test]
2152    async fn module_error_with_daemon_bit_is_stripped_before_client_delivery() {
2153        let (
2154            router,
2155            _forwarding,
2156            client_ctx,
2157            mut client_rx,
2158            module_ctx,
2159            _module_egress_rx,
2160            mut module_rx,
2161            pending,
2162        ) = dynamic_route_fixture(true);
2163        let _ = client_rx.recv().await;
2164        // Open flow credit the way a real request would, so the terminal
2165        // error exercises the credit-release path too.
2166        router
2167            .route_for_connection(
2168                &client_ctx,
2169                route_frame(
2170                    FrameType::Request,
2171                    pending.client_channel,
2172                    pending.client_epoch,
2173                    701,
2174                ),
2175            )
2176            .await
2177            .unwrap();
2178        let _ = module_rx.recv().await;
2179
2180        let body = serde_json::to_vec(&ErrorBody::new("module_reloading", "try again")).unwrap();
2181        let spoof = Frame::build(
2182            FrameType::Error,
2183            Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2184            pending.module_channel,
2185            pending.module_epoch,
2186            701,
2187            body.clone(),
2188        )
2189        .unwrap();
2190        router
2191            .route_for_connection(&module_ctx, spoof)
2192            .await
2193            .unwrap();
2194        let delivered = client_rx.recv().await.unwrap();
2195        assert_eq!(delivered.header.ty, FrameType::Error);
2196        assert_eq!(delivered.header.channel, pending.client_channel);
2197        assert_eq!(delivered.header.epoch, pending.client_epoch);
2198        assert_eq!(delivered.header.corr, 701);
2199        assert!(
2200            !delivered.header.flags.is_daemon_origin(),
2201            "module-set daemon origin must be cleared: {:?}",
2202            delivered.header.flags
2203        );
2204        assert_eq!(delivered.body, body, "the body travels byte-identical");
2205    }
2206
2207    /// Clearing the origin bit strips exactly that bit: priority, LAST and
2208    /// subscription survive the splice unchanged.
2209    #[tokio::test]
2210    async fn module_response_with_daemon_bit_is_stripped_preserving_other_flags() {
2211        use subc_protocol::FLAG_SUBSCRIPTION;
2212        let (
2213            router,
2214            _forwarding,
2215            client_ctx,
2216            mut client_rx,
2217            module_ctx,
2218            _module_egress_rx,
2219            mut module_rx,
2220            pending,
2221        ) = dynamic_route_fixture(true);
2222        let _ = client_rx.recv().await;
2223        router
2224            .route_for_connection(
2225                &client_ctx,
2226                route_frame(
2227                    FrameType::Request,
2228                    pending.client_channel,
2229                    pending.client_epoch,
2230                    701,
2231                ),
2232            )
2233            .await
2234            .unwrap();
2235        let _ = module_rx.recv().await;
2236
2237        let sent_flags = Flags(
2238            Flags::new(true, Priority::Background, true).0
2239                | FLAG_SUBSCRIPTION
2240                | subc_protocol::FLAG_DAEMON_ORIGIN,
2241        );
2242        assert!(sent_flags.is_daemon_origin());
2243        let response = Frame::build(
2244            FrameType::Response,
2245            sent_flags,
2246            pending.module_channel,
2247            pending.module_epoch,
2248            701,
2249            b"route-body".to_vec(),
2250        )
2251        .unwrap();
2252        router
2253            .route_for_connection(&module_ctx, response)
2254            .await
2255            .unwrap();
2256        let delivered = client_rx.recv().await.unwrap();
2257        assert_eq!(delivered.header.ty, FrameType::Response);
2258        assert!(
2259            !delivered.header.flags.is_daemon_origin(),
2260            "module-set daemon origin must be cleared: {:?}",
2261            delivered.header.flags
2262        );
2263        assert!(
2264            delivered.header.flags.is_binary(),
2265            "binary flag must survive: {:?}",
2266            delivered.header.flags
2267        );
2268        assert!(
2269            delivered.header.flags.is_last(),
2270            "LAST flag must survive: {:?}",
2271            delivered.header.flags
2272        );
2273        assert!(
2274            delivered.header.flags.is_subscription(),
2275            "subscription flag must survive: {:?}",
2276            delivered.header.flags
2277        );
2278        assert_eq!(
2279            delivered.header.flags.priority(),
2280            Some(Priority::Background),
2281            "priority must survive"
2282        );
2283    }
2284
2285    /// A client frame carrying the origin bit must not reach the module
2286    /// looking daemon-authored either: the splice clears it symmetrically.
2287    #[tokio::test]
2288    async fn client_frame_with_daemon_bit_is_stripped_before_module_delivery() {
2289        let (
2290            router,
2291            _forwarding,
2292            client_ctx,
2293            mut client_rx,
2294            _module_ctx,
2295            _module_egress_rx,
2296            mut module_rx,
2297            pending,
2298        ) = dynamic_route_fixture(true);
2299        let _ = client_rx.recv().await;
2300        let spoof = Frame::build(
2301            FrameType::Request,
2302            Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2303            pending.client_channel,
2304            pending.client_epoch,
2305            701,
2306            b"route-body".to_vec(),
2307        )
2308        .unwrap();
2309        router
2310            .route_for_connection(&client_ctx, spoof)
2311            .await
2312            .unwrap();
2313        let delivered = module_rx.recv().await.unwrap();
2314        assert_eq!(delivered.header.ty, FrameType::Request);
2315        assert!(
2316            !delivered.header.flags.is_daemon_origin(),
2317            "client-set daemon origin must be cleared: {:?}",
2318            delivered.header.flags
2319        );
2320    }
2321
2322    #[tokio::test]
2323    async fn channel_zero_uses_control_handler_not_backend_registry() {
2324        let mut router = Router::with_default_self_handler();
2325        router.register_backend(1, EchoBackend).unwrap();
2326        let (ctx, mut rx) = route_ctx();
2327
2328        router.route_for_connection(&ctx, ping(77)).await.unwrap();
2329        let response = rx.recv().await.unwrap();
2330
2331        assert_eq!(response.header.ty, FrameType::Pong);
2332        assert_eq!(response.header.channel, 0);
2333        assert_eq!(response.header.corr, 77);
2334        assert!(response.body.is_empty());
2335    }
2336
2337    #[tokio::test]
2338    async fn slow_control_dispatch_logs_decoded_op_and_elapsed_time() {
2339        let control = Arc::new(
2340            ControlHandler::new(Arc::new(Registry::default()))
2341                .with_control_dispatch_delay(Duration::from_millis(1050)),
2342        );
2343        let router = Router::with_control_handler(control);
2344        let (ctx, mut rx) = route_ctx();
2345        let (output, guard) = log_capture(tracing::Level::WARN);
2346
2347        router
2348            .route_for_connection(&ctx, request(0, 41, br#"{"op":"server.describe"}"#))
2349            .await
2350            .expect("slow request routes");
2351        assert!(rx.recv().await.is_some(), "request receives a response");
2352        drop(guard);
2353
2354        let logs = captured_logs(&output);
2355        assert!(logs.contains("slow control dispatch"));
2356        assert!(logs.contains("op=server.describe"));
2357        assert!(logs.contains("connection_id=0"));
2358        assert!(logs.contains("corr=41"));
2359        assert!(
2360            logged_millis(&logs, "elapsed_ms=") >= 1050,
2361            "elapsed must include the injected handler delay: {logs}"
2362        );
2363    }
2364
2365    #[tokio::test]
2366    async fn fast_control_dispatch_emits_arrival_without_slow_warning() {
2367        let router = Router::with_default_self_handler();
2368        let (ctx, mut rx) = route_ctx();
2369        let (output, guard) = log_capture(tracing::Level::DEBUG);
2370
2371        router
2372            .route_for_connection(&ctx, request(0, 42, br#"{"op":"server.describe"}"#))
2373            .await
2374            .expect("fast request routes");
2375        assert!(rx.recv().await.is_some(), "request receives a response");
2376        drop(guard);
2377
2378        let logs = captured_logs(&output);
2379        assert!(logs.contains("control dispatch op=server.describe connection_id=0 corr=42"));
2380        assert!(!logs.contains("slow control dispatch"));
2381    }
2382
2383    #[tokio::test]
2384    async fn control_dispatch_arrival_is_hidden_at_info() {
2385        let router = Router::with_default_self_handler();
2386        let (ctx, mut rx) = route_ctx();
2387        let (output, guard) = log_capture(tracing::Level::INFO);
2388
2389        router
2390            .route_for_connection(&ctx, request(0, 43, br#"{"op":"server.describe"}"#))
2391            .await
2392            .expect("fast request routes");
2393        assert!(rx.recv().await.is_some(), "request receives a response");
2394        drop(guard);
2395
2396        assert!(
2397            !captured_logs(&output).contains("control dispatch"),
2398            "arrival logging must stay hidden at INFO"
2399        );
2400    }
2401
2402    #[tokio::test]
2403    async fn supervisor_list_logs_contended_snapshot_lock_only() {
2404        let registry = Arc::new(Registry::default());
2405        let handle = SupervisorHandle::new();
2406        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
2407            .with_handle(handle.clone());
2408        let module = supervisor
2409            .supervise_configured(
2410                ModuleSpec {
2411                    carry_containment: false,
2412                    module_id: "held-module".to_string(),
2413                    program: "test-module".into(),
2414                    args: Vec::new(),
2415                    env: Vec::new(),
2416                    reserved: false,
2417                    reserved_prefixes: Vec::new(),
2418                    protocol: ModuleProtocol::Subc,
2419                    overlap: Default::default(),
2420                },
2421                false,
2422            )
2423            .expect("disabled test module is supervised");
2424        let router = Router::with_control_handler(Arc::new(
2425            ControlHandler::new(Arc::clone(&registry)).with_supervisor(handle),
2426        ));
2427        let (ctx, mut rx) = route_ctx();
2428        let (acquired, ready) = std_mpsc::channel();
2429        // Park the disabled supervisor actor before holding its snapshot. The
2430        // detached list reader, not actor startup, must be the lock contender.
2431        tokio::task::yield_now().await;
2432        let holder = module.hold_snapshot_for_test(acquired, Duration::from_millis(400));
2433        ready.recv().expect("holder acquired snapshot lock");
2434        let (output, guard) = log_capture(tracing::Level::WARN);
2435
2436        router
2437            .route_for_connection(&ctx, request(0, 44, br#"{"op":"supervisor.list"}"#))
2438            .await
2439            .expect("list request routes after the lock releases");
2440        assert!(
2441            rx.recv().await.is_some(),
2442            "list request receives a response"
2443        );
2444        holder.join().expect("snapshot holder exits cleanly");
2445        drop(guard);
2446
2447        let logs = captured_logs(&output);
2448        assert!(logs.contains("slow snapshot lock"));
2449        assert!(logs.contains("module_id=held-module"));
2450        assert!(logs.contains("caller=list"));
2451        assert!(
2452            logged_millis(&logs, "waited_ms=") >= 250,
2453            "wait must exceed the slow-lock threshold: {logs}"
2454        );
2455
2456        let (output, guard) = log_capture(tracing::Level::WARN);
2457        router
2458            .route_for_connection(&ctx, request(0, 45, br#"{"op":"supervisor.list"}"#))
2459            .await
2460            .expect("uncontended list request routes");
2461        assert!(
2462            rx.recv().await.is_some(),
2463            "uncontended list receives a response"
2464        );
2465        drop(guard);
2466        assert!(
2467            !captured_logs(&output).contains("slow snapshot lock"),
2468            "uncontended list acquisition must not warn"
2469        );
2470    }
2471
2472    #[tokio::test]
2473    async fn full_module_to_client_sink_requests_client_close_without_erroring_module() {
2474        let forwarding = Arc::new(ForwardingTable::default());
2475        let control = Arc::new(ControlHandler::with_forwarding(
2476            Arc::new(crate::Registry::default()),
2477            Arc::clone(&forwarding),
2478        ));
2479        let router = Router::with_control_handler(control);
2480        let module_connection = ConnectionId::new(10);
2481        let client_connection = ConnectionId::new(20);
2482        let mut close_receiver = forwarding.register_connection_close(client_connection);
2483        let (module_tx, _module_rx) = mpsc::channel(1);
2484        forwarding
2485            .register_module_connection(
2486                module_connection,
2487                "full-sink-provider".to_string(),
2488                1,
2489                Concurrency::ModuleManaged,
2490                FrameSink::new(module_tx),
2491            )
2492            .unwrap();
2493        let (client_tx, mut client_rx) = mpsc::channel(1);
2494        let pending = forwarding
2495            .begin_route_bind_relay_for_test(
2496                client_connection,
2497                FrameSink::new(client_tx),
2498                700,
2499                "full-sink-provider",
2500            )
2501            .unwrap();
2502        forwarding
2503            .complete_pending_relay(
2504                module_connection,
2505                pending.corr,
2506                RouteBindRelayOutcome::Accepted,
2507            )
2508            .unwrap();
2509
2510        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2511        let module_ctx = RouteCtx {
2512            connection_id: module_connection,
2513            egress: FrameSink::new(module_egress_tx),
2514        };
2515        let terminal = Frame::build(
2516            FrameType::Response,
2517            Flags::new(false, Priority::Interactive, true),
2518            pending.module_channel,
2519            pending.module_epoch,
2520            701,
2521            b"terminal".to_vec(),
2522        )
2523        .unwrap();
2524
2525        router
2526            .route_for_connection(&module_ctx, terminal)
2527            .await
2528            .unwrap();
2529        let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2530            .await
2531            .expect("close request should be sent for the full client sink")
2532            .expect("close sender should include a reason");
2533        assert!(
2534            reason
2535                .to_string()
2536                .contains("module_to_client_delivery_failed"),
2537            "unexpected close reason: {reason}"
2538        );
2539        assert_eq!(client_rx.try_recv().unwrap().header.corr, 700);
2540        assert!(client_rx.try_recv().is_err());
2541        assert_eq!(
2542            router.counters.snapshot()["client_egress_close_delivery_failed"],
2543            1
2544        );
2545    }
2546
2547    /// A terminal frame ends the request at the module even when the client
2548    /// cannot take it, so the drain must stop counting it at once rather than
2549    /// when the client connection's cleanup later removes the route.
2550    #[tokio::test]
2551    async fn terminal_frame_releases_its_credit_even_when_client_delivery_fails() {
2552        let forwarding = Arc::new(ForwardingTable::default());
2553        let control = Arc::new(ControlHandler::with_forwarding(
2554            Arc::new(crate::Registry::default()),
2555            Arc::clone(&forwarding),
2556        ));
2557        let router = Router::with_control_handler(control);
2558        let module_connection = ConnectionId::new(11);
2559        let client_connection = ConnectionId::new(21);
2560        let _close_receiver = forwarding.register_connection_close(client_connection);
2561        let (module_tx, _module_rx) = mpsc::channel(1);
2562        forwarding
2563            .register_module_connection(
2564                module_connection,
2565                "credit-provider".to_string(),
2566                1,
2567                Concurrency::ModuleManaged,
2568                FrameSink::new(module_tx),
2569            )
2570            .unwrap();
2571        // Capacity one, filled by the route.open response, so the terminal
2572        // frame below cannot be enqueued for the client.
2573        let (client_tx, _client_rx) = mpsc::channel(1);
2574        let pending = forwarding
2575            .begin_route_bind_relay_for_test(
2576                client_connection,
2577                FrameSink::new(client_tx),
2578                800,
2579                "credit-provider",
2580            )
2581            .unwrap();
2582        forwarding
2583            .complete_pending_relay(
2584                module_connection,
2585                pending.corr,
2586                RouteBindRelayOutcome::Accepted,
2587            )
2588            .unwrap();
2589        let DataRoute::Client(DataRouteState::Bound(route)) = forwarding
2590            .lookup_data_route(
2591                client_connection,
2592                pending.client_channel,
2593                pending.client_epoch,
2594            )
2595            .unwrap()
2596        else {
2597            panic!("expected a bound client route");
2598        };
2599        route.flow.acquire_tagged(801, false).await.unwrap();
2600        assert_eq!(route.flow.drain_in_flight(), 1);
2601
2602        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2603        let module_ctx = RouteCtx {
2604            connection_id: module_connection,
2605            egress: FrameSink::new(module_egress_tx),
2606        };
2607        let terminal = Frame::build(
2608            FrameType::Response,
2609            Flags::new(false, Priority::Interactive, true),
2610            pending.module_channel,
2611            pending.module_epoch,
2612            801,
2613            b"terminal".to_vec(),
2614        )
2615        .unwrap();
2616        router
2617            .route_for_connection(&module_ctx, terminal)
2618            .await
2619            .unwrap();
2620
2621        assert_eq!(
2622            router.counters.snapshot()["client_egress_close_delivery_failed"],
2623            1,
2624            "the client delivery must have failed for this test to mean anything"
2625        );
2626        assert_eq!(
2627            route.flow.drain_in_flight(),
2628            0,
2629            "the module's terminal frame must release its credit even though the client could not take it"
2630        );
2631    }
2632
2633    #[tokio::test]
2634    async fn full_route_goodbye_sink_requests_target_close_without_erroring_module() {
2635        let forwarding = Arc::new(ForwardingTable::default());
2636        let control = Arc::new(ControlHandler::with_forwarding(
2637            Arc::new(crate::Registry::default()),
2638            Arc::clone(&forwarding),
2639        ));
2640        let router = Router::with_control_handler(control);
2641        let module_connection = ConnectionId::new(30);
2642        let client_connection = ConnectionId::new(40);
2643        let mut close_receiver = forwarding.register_connection_close(client_connection);
2644        let (module_tx, _module_rx) = mpsc::channel(1);
2645        forwarding
2646            .register_module_connection(
2647                module_connection,
2648                "goodbye-full-provider".to_string(),
2649                1,
2650                Concurrency::ModuleManaged,
2651                FrameSink::new(module_tx),
2652            )
2653            .unwrap();
2654        let (client_tx, mut client_rx) = mpsc::channel(1);
2655        let pending = forwarding
2656            .begin_route_bind_relay_for_test(
2657                client_connection,
2658                FrameSink::new(client_tx),
2659                800,
2660                "goodbye-full-provider",
2661            )
2662            .unwrap();
2663        forwarding
2664            .complete_pending_relay(
2665                module_connection,
2666                pending.corr,
2667                RouteBindRelayOutcome::Accepted,
2668            )
2669            .unwrap();
2670
2671        let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2672        let module_ctx = RouteCtx {
2673            connection_id: module_connection,
2674            egress: FrameSink::new(module_egress_tx),
2675        };
2676        let goodbye = Frame::build(
2677            FrameType::Goodbye,
2678            Flags::new(false, Priority::Passive, true),
2679            pending.module_channel,
2680            pending.module_epoch,
2681            801,
2682            Vec::new(),
2683        )
2684        .unwrap();
2685
2686        router
2687            .route_for_connection(&module_ctx, goodbye)
2688            .await
2689            .unwrap();
2690        let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2691            .await
2692            .expect("close request should be sent for the full GOODBYE sink")
2693            .expect("close sender should include a reason");
2694        assert!(
2695            reason.to_string().contains("route_goodbye_delivery_failed"),
2696            "unexpected close reason: {reason}"
2697        );
2698        assert_eq!(client_rx.try_recv().unwrap().header.corr, 800);
2699        assert!(client_rx.try_recv().is_err());
2700        assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 1);
2701        assert_eq!(router.counters.snapshot()["route_released_epoch_fenced"], 1);
2702    }
2703
2704    /// One module and one client connection with `routes` routes bound between
2705    /// them. The client uses a real connection egress queue (the same byte
2706    /// budget and frame-count backstop as a live connection), and its route.open
2707    /// responses are drained so the queue starts empty.
2708    async fn multi_route_client(
2709        module_id: &str,
2710        module_connection: ConnectionId,
2711        client_connection: ConnectionId,
2712        routes: usize,
2713    ) -> (
2714        Router,
2715        FrameSink,
2716        mpsc::Receiver<OutboundFrame>,
2717        RouteCtx,
2718        Vec<crate::forwarding::PendingRouteBindRelay>,
2719        ConnectionCloseReceiver,
2720    ) {
2721        let forwarding = Arc::new(ForwardingTable::default());
2722        let control = Arc::new(ControlHandler::with_forwarding(
2723            Arc::new(crate::Registry::default()),
2724            Arc::clone(&forwarding),
2725        ));
2726        let router = Router::with_control_handler(control);
2727        let close_receiver = forwarding.register_connection_close(client_connection);
2728        let (module_tx, _module_rx) = mpsc::channel(8);
2729        forwarding
2730            .register_module_connection(
2731                module_connection,
2732                module_id.to_string(),
2733                1,
2734                Concurrency::ModuleManaged,
2735                FrameSink::new(module_tx),
2736            )
2737            .unwrap();
2738        let (client_sink, mut client_rx) = crate::server::connection_egress();
2739        let mut bound = Vec::with_capacity(routes);
2740        for index in 0..routes {
2741            let pending = forwarding
2742                .begin_route_bind_relay_for_test(
2743                    client_connection,
2744                    client_sink.clone(),
2745                    900 + index as u64,
2746                    module_id,
2747                )
2748                .unwrap();
2749            forwarding
2750                .complete_pending_relay(
2751                    module_connection,
2752                    pending.corr,
2753                    RouteBindRelayOutcome::Accepted,
2754                )
2755                .unwrap();
2756            assert_eq!(
2757                client_rx.recv().await.unwrap().header.corr,
2758                900 + index as u64
2759            );
2760            bound.push(pending);
2761        }
2762        let (module_egress_tx, _module_egress_rx) = mpsc::channel(8);
2763        let module_ctx = RouteCtx {
2764            connection_id: module_connection,
2765            egress: FrameSink::new(module_egress_tx),
2766        };
2767        (
2768            router,
2769            client_sink,
2770            client_rx,
2771            module_ctx,
2772            bound,
2773            close_receiver,
2774        )
2775    }
2776
2777    /// Per-frame cost of the egress sink's admission and release accounting:
2778    /// one million 200-byte frames enqueued with `try_send` and taken off the
2779    /// queue the way the connection writer does, in batches of 1,000 so the
2780    /// queue stays well inside its budget. Prints nanoseconds per frame; it is
2781    /// a measurement, not a gate. Run with
2782    /// `cargo test --release -p subc-daemon --lib egress_sink_per_frame_cost -- --ignored --nocapture`.
2783    #[test]
2784    #[ignore = "timing measurement, run on demand"]
2785    fn egress_sink_per_frame_cost() {
2786        const FRAMES: usize = 1_000_000;
2787        const BATCH: usize = 1_000;
2788        let (sink, mut rx) = crate::server::connection_egress();
2789        let template = stream_frame(9, 1, 0, vec![b't'; 200]);
2790        let started = Instant::now();
2791        for _ in 0..FRAMES / BATCH {
2792            for _ in 0..BATCH {
2793                sink.try_send(template.clone()).unwrap();
2794            }
2795            for _ in 0..BATCH {
2796                let outbound = rx.try_recv().unwrap();
2797                if let Some(charge) = &outbound.charge {
2798                    charge.taken_by_writer();
2799                }
2800                drop(std::hint::black_box(outbound));
2801            }
2802        }
2803        let elapsed = started.elapsed();
2804        assert_eq!(sink.backlog().queued_bytes, 0);
2805        println!(
2806            "egress sink: {FRAMES} frames in {elapsed:?}, {:.1} ns/frame",
2807            elapsed.as_nanos() as f64 / FRAMES as f64
2808        );
2809    }
2810
2811    fn stream_frame(channel: u16, epoch: u32, corr: u64, body: Vec<u8>) -> Frame {
2812        Frame::build(
2813            FrameType::StreamData,
2814            Flags::new(false, Priority::Interactive, false),
2815            channel,
2816            epoch,
2817            corr,
2818            body,
2819        )
2820        .unwrap()
2821    }
2822
2823    fn module_route_for_test(
2824        router: &Router,
2825        ctx: &RouteCtx,
2826        route: &crate::forwarding::PendingRouteBindRelay,
2827    ) -> Arc<RouteBinding> {
2828        match router
2829            .forwarding
2830            .lookup_data_route(ctx.connection_id, route.module_channel, route.module_epoch)
2831            .unwrap()
2832        {
2833            DataRoute::Module(DataRouteState::Bound(binding)) => binding,
2834            other => panic!("expected a bound module route, got {other:?}"),
2835        }
2836    }
2837
2838    fn assert_frame_size_error(frame: &Frame, channel: u16, epoch: u32, corr: u64, bytes: usize) {
2839        assert_eq!(frame.header.ty, FrameType::Error);
2840        assert_eq!(frame.header.channel, channel);
2841        assert_eq!(frame.header.epoch, epoch);
2842        assert_eq!(frame.header.corr, corr);
2843        assert!(frame.header.flags.is_daemon_origin());
2844        let body: ErrorBody = serde_json::from_slice(&frame.body).unwrap();
2845        assert_eq!(body.code, error_codes::FRAME_TOO_LARGE);
2846        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2847        assert!(body.message.contains(&bytes.to_string()));
2848        assert!(body.message.contains(&limit.to_string()));
2849        assert_eq!(
2850            body.detail,
2851            Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit }))
2852        );
2853    }
2854
2855    #[tokio::test]
2856    async fn module_egress_response_oversized_reply_keeps_connection_and_other_route() {
2857        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
2858        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2859            "oversize-provider",
2860            ConnectionId::new(80),
2861            ConnectionId::new(81),
2862            2,
2863        )
2864        .await;
2865        let binding = module_route_for_test(&router, &ctx, &routes[0]);
2866        binding.flow.acquire_tagged(101, false).await.unwrap();
2867        module_route_for_test(&router, &ctx, &routes[1])
2868            .flow
2869            .acquire_tagged(102, false)
2870            .await
2871            .unwrap();
2872        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2873        let oversized = Frame::build(
2874            FrameType::Response,
2875            Flags::new(false, Priority::Interactive, true),
2876            routes[0].module_channel,
2877            routes[0].module_epoch,
2878            101,
2879            vec![b'x'; limit - subc_protocol::HEADER_LEN + 1],
2880        )
2881        .unwrap();
2882        assert_eq!(sink.backlog().queued_bytes, 0);
2883        router.route_for_connection(&ctx, oversized).await.unwrap();
2884
2885        // Reproduce a large reply followed by ordinary traffic on another route
2886        // before the client reads anything. The large reply must not monopolize
2887        // the queue and make that second delivery close the shared connection.
2888        router
2889            .route_for_connection(
2890                &ctx,
2891                route_frame(
2892                    FrameType::Response,
2893                    routes[1].module_channel,
2894                    routes[1].module_epoch,
2895                    102,
2896                ),
2897            )
2898            .await
2899            .unwrap();
2900        assert!(
2901            close.try_recv().is_err(),
2902            "an oversized reply must not close the multiplexed client"
2903        );
2904        let error = rx.try_recv().expect("the waiting client gets an ERROR");
2905        assert_frame_size_error(
2906            &error,
2907            routes[0].client_channel,
2908            routes[0].client_epoch,
2909            101,
2910            limit + 1,
2911        );
2912        let second = rx.try_recv().expect("the other route's reply arrives");
2913        assert_eq!(second.header.ty, FrameType::Response);
2914        assert_eq!(second.header.channel, routes[1].client_channel);
2915        assert_eq!(second.header.epoch, routes[1].client_epoch);
2916        assert_eq!(second.header.corr, 102);
2917        assert_eq!(second.body, b"route-body");
2918        assert_eq!(binding.flow.drain_in_flight(), 0);
2919        for route in &routes {
2920            module_route_for_test(&router, &ctx, route);
2921        }
2922        assert!(rx.try_recv().is_err());
2923        drop((error, second));
2924        assert_eq!(sink.backlog().queued_bytes, 0);
2925        assert_eq!(
2926            router.counters.snapshot()["client_egress_close_delivery_failed"],
2927            0
2928        );
2929
2930        let captured = test_log::captured_logs(&logs);
2931        let warnings: Vec<_> = captured
2932            .lines()
2933            .filter(|line| line.contains("WARN"))
2934            .collect();
2935        assert_eq!(warnings.len(), 1, "{captured}");
2936        let warning = warnings[0];
2937        for expected in [
2938            "module_id=oversize-provider".to_string(),
2939            "client_principal=direct".to_string(),
2940            format!("client_channel={}", routes[0].client_channel),
2941            format!("frame_bytes={}", limit + 1),
2942            format!("byte_limit={limit}"),
2943        ] {
2944            assert!(warning.contains(&expected), "missing {expected}: {warning}");
2945        }
2946    }
2947
2948    async fn poll_once<F: std::future::Future>(
2949        mut future: std::pin::Pin<&mut F>,
2950    ) -> std::task::Poll<F::Output> {
2951        std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
2952    }
2953
2954    fn fill_client_budget(sink: &FrameSink, route: &crate::forwarding::PendingRouteBindRelay) {
2955        sink.try_send(
2956            Frame::build(
2957                FrameType::Response,
2958                Flags::new(false, Priority::Interactive, true),
2959                route.client_channel,
2960                route.client_epoch,
2961                100,
2962                vec![
2963                    b'x';
2964                    crate::server::CONNECTION_EGRESS_BYTE_BUDGET - subc_protocol::HEADER_LEN
2965                ],
2966            )
2967            .unwrap(),
2968        )
2969        .unwrap();
2970        assert_eq!(
2971            sink.backlog().queued_bytes,
2972            crate::server::CONNECTION_EGRESS_BYTE_BUDGET
2973        );
2974    }
2975
2976    async fn assert_full_client_burst_survives(ty: FrameType, oversized: bool) {
2977        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2978            "draining-burst",
2979            ConnectionId::new(92),
2980            ConnectionId::new(93),
2981            2,
2982        )
2983        .await;
2984        fill_client_budget(&sink, &routes[0]);
2985        let frame = if oversized {
2986            Frame::build(
2987                ty,
2988                Flags::new(false, Priority::Interactive, false),
2989                routes[1].module_channel,
2990                routes[1].module_epoch,
2991                101,
2992                vec![b's'; crate::server::CONNECTION_EGRESS_BYTE_BUDGET],
2993            )
2994            .unwrap()
2995        } else {
2996            route_frame(ty, routes[1].module_channel, routes[1].module_epoch, 101)
2997        };
2998        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2999        assert!(poll_once(delivery.as_mut()).await.is_pending());
3000        assert!(
3001            close.try_recv().is_err(),
3002            "fresh bursts must not close: {ty:?}"
3003        );
3004        tokio::time::advance(Duration::from_millis(50)).await;
3005        let queued = rx.try_recv().unwrap();
3006        queued.charge.as_ref().unwrap().taken_by_writer();
3007        drop(queued);
3008        assert!(matches!(
3009            poll_once(delivery.as_mut()).await,
3010            std::task::Poll::Ready(Ok(()))
3011        ));
3012        let delivered = rx
3013            .try_recv()
3014            .expect("the burst frame must be delivered after draining");
3015        assert_eq!(delivered.header.channel, routes[1].client_channel);
3016        assert_eq!(delivered.header.epoch, routes[1].client_epoch);
3017        assert_eq!(delivered.header.corr, 101);
3018        if oversized {
3019            assert_frame_size_error(
3020                &delivered,
3021                routes[1].client_channel,
3022                routes[1].client_epoch,
3023                101,
3024                crate::server::CONNECTION_EGRESS_BYTE_BUDGET + subc_protocol::HEADER_LEN,
3025            );
3026        } else {
3027            assert_eq!(delivered.header.ty, ty);
3028        }
3029        assert!(rx.try_recv().is_err());
3030        assert!(close.try_recv().is_err());
3031        assert_eq!(
3032            router.counters.snapshot()["client_egress_close_delivery_failed"],
3033            0
3034        );
3035        assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 0);
3036    }
3037
3038    #[tokio::test(start_paused = true)]
3039    async fn module_egress_fresh_full_queue_burst_survives() {
3040        for ty in [
3041            FrameType::Response,
3042            FrameType::StreamData,
3043            FrameType::Request,
3044            FrameType::Push,
3045        ] {
3046            assert_full_client_burst_survives(ty, false).await;
3047        }
3048    }
3049
3050    #[tokio::test(start_paused = true)]
3051    async fn module_goodbye_fresh_full_queue_burst_survives() {
3052        assert_full_client_burst_survives(FrameType::Goodbye, false).await;
3053    }
3054
3055    #[tokio::test(start_paused = true)]
3056    async fn frame_count_backstop_also_allows_a_draining_burst() {
3057        let (tx, mut rx) = mpsc::channel(1);
3058        let sink = FrameSink::with_byte_budget(tx, 2_000);
3059        sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
3060            .unwrap();
3061        let mut delivery = Box::pin(sink.send_to_client(stream_frame(1, 1, 2, vec![b'y'; 100])));
3062        assert!(poll_once(delivery.as_mut()).await.is_pending());
3063        assert!(
3064            sink.backlog().queued_bytes < 2_000,
3065            "only frame capacity is exhausted"
3066        );
3067        tokio::time::advance(Duration::from_millis(50)).await;
3068        drop(rx.try_recv().unwrap());
3069        assert!(matches!(
3070            poll_once(delivery.as_mut()).await,
3071            std::task::Poll::Ready(Ok(()))
3072        ));
3073        assert_eq!(rx.try_recv().unwrap().header.corr, 2);
3074    }
3075
3076    #[tokio::test(start_paused = true)]
3077    async fn oversized_reply_replacement_fresh_full_queue_burst_survives() {
3078        for ty in [FrameType::Response, FrameType::StreamData] {
3079            assert_full_client_burst_survives(ty, true).await;
3080        }
3081    }
3082
3083    #[tokio::test(start_paused = true)]
3084    async fn module_egress_stopped_reader_closes_without_waiting() {
3085        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3086        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3087            "oldest-stopped-reader",
3088            ConnectionId::new(94),
3089            ConnectionId::new(95),
3090            2,
3091        )
3092        .await;
3093        fill_client_budget(&sink, &routes[0]);
3094        // The writer has taken the head but cannot finish writing it. Its
3095        // bytes stay charged, and the atomic age still diagnoses that stall.
3096        let writing = rx.try_recv().unwrap();
3097        writing.charge.as_ref().unwrap().taken_by_writer();
3098        tokio::time::advance(Duration::from_millis(251)).await;
3099        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(251)));
3100        assert_eq!(sink.backlog().queued_bytes, queued_frame_bytes(&writing));
3101        let started = tokio::time::Instant::now();
3102        let frame = stream_frame(
3103            routes[0].module_channel,
3104            routes[0].module_epoch,
3105            101,
3106            b"secret-body-never-log".to_vec(),
3107        );
3108        let flags = frame.header.flags.0;
3109        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3110        assert!(
3111            matches!(
3112                poll_once(delivery.as_mut()).await,
3113                std::task::Poll::Ready(Ok(()))
3114            ),
3115            "an oldest frame older than 250 ms must close immediately, without a new wait"
3116        );
3117        assert_eq!(started.elapsed(), Duration::ZERO);
3118        assert!(close
3119            .try_recv()
3120            .unwrap()
3121            .to_string()
3122            .contains("module_to_client_delivery_failed"));
3123        let captured = test_log::captured_logs(&logs);
3124        for expected in [
3125            "frame_type=Some(StreamData)".to_string(),
3126            format!("frame_flags=Some({flags})"),
3127            "oldest_queued_ms=251".to_string(),
3128            "close_cause=\"oldest_frame_older_than_250_ms\"".to_string(),
3129        ] {
3130            assert!(
3131                captured.contains(&expected),
3132                "missing {expected}: {captured}"
3133            );
3134        }
3135        assert!(!captured.contains("secret-body-never-log"));
3136    }
3137
3138    #[tokio::test(start_paused = true)]
3139    async fn client_gone_mid_wait_resumes_module_reader_immediately() {
3140        for writer_gone in [false, true] {
3141            let (router, sink, mut rx, ctx, routes, _close) = multi_route_client(
3142                "gone-during-burst",
3143                ConnectionId::new(96),
3144                ConnectionId::new(97),
3145                1,
3146            )
3147            .await;
3148            fill_client_budget(&sink, &routes[0]);
3149            let frame = route_frame(
3150                FrameType::Response,
3151                routes[0].module_channel,
3152                routes[0].module_epoch,
3153                101,
3154            );
3155            let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3156            assert!(poll_once(delivery.as_mut()).await.is_pending());
3157            tokio::time::advance(Duration::from_millis(50)).await;
3158            let started = tokio::time::Instant::now();
3159            if writer_gone {
3160                rx.close();
3161            } else {
3162                sink.close_client_delivery();
3163            }
3164            assert!(
3165                matches!(
3166                    poll_once(delivery.as_mut()).await,
3167                    std::task::Poll::Ready(Ok(()))
3168                ),
3169                "client teardown must wake the module reader before the writer drains"
3170            );
3171            assert_eq!(started.elapsed(), Duration::ZERO);
3172            assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
3173            assert_eq!(rx.try_recv().unwrap().header.corr, 100);
3174            assert!(rx.try_recv().is_err());
3175        }
3176    }
3177
3178    #[tokio::test(start_paused = true)]
3179    async fn cancelled_module_delivery_releases_waiter_without_closing_client() {
3180        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3181            "cancelled-burst",
3182            ConnectionId::new(98),
3183            ConnectionId::new(99),
3184            1,
3185        )
3186        .await;
3187        fill_client_budget(&sink, &routes[0]);
3188        let frame = route_frame(
3189            FrameType::Response,
3190            routes[0].module_channel,
3191            routes[0].module_epoch,
3192            101,
3193        );
3194        let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3195        assert!(poll_once(delivery.as_mut()).await.is_pending());
3196        assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 1);
3197        drop(delivery);
3198        assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
3199        tokio::time::advance(Duration::from_secs(1)).await;
3200        assert!(close.try_recv().is_err());
3201        drop(rx.try_recv().unwrap());
3202        assert_eq!(sink.backlog().queued_bytes, 0);
3203        assert!(sink.backlog().oldest_age.is_none());
3204    }
3205
3206    #[tokio::test(start_paused = true)]
3207    async fn egress_atomic_age_follows_writer_and_clears_when_drained() {
3208        let (sink, mut rx) = crate::server::connection_egress();
3209        sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
3210            .unwrap();
3211        tokio::time::advance(Duration::from_millis(200)).await;
3212        sink.try_send(stream_frame(2, 1, 2, vec![b'y'; 100]))
3213            .unwrap();
3214        let writing = rx.try_recv().unwrap();
3215        writing.charge.as_ref().unwrap().taken_by_writer();
3216        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3217        drop(writing);
3218        // Between frames the lock-free marker can still name the previous
3219        // head. The next writer take advances it without scanning the queue.
3220        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3221        tokio::time::advance(Duration::from_millis(50)).await;
3222        let next = rx.try_recv().unwrap();
3223        next.charge.as_ref().unwrap().taken_by_writer();
3224        assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(50)));
3225        assert!(
3226            sink.backlog().queued_bytes > 0,
3227            "in-flight bytes remain charged until written"
3228        );
3229        drop(next);
3230        assert_eq!(sink.backlog().queued_bytes, 0);
3231        assert_eq!(sink.backlog().oldest_age, None);
3232    }
3233
3234    #[tokio::test(start_paused = true)]
3235    async fn module_egress_response_full_client_waits_only_remaining_grace() {
3236        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3237            "backpressure-provider",
3238            ConnectionId::new(82),
3239            ConnectionId::new(83),
3240            2,
3241        )
3242        .await;
3243        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3244        // Exactly the limit, including its header, must be admitted to an empty
3245        // queue. Leave it unread to deterministically exhaust only byte capacity.
3246        router
3247            .route_for_connection(
3248                &ctx,
3249                Frame::build(
3250                    FrameType::Response,
3251                    Flags::new(false, Priority::Interactive, true),
3252                    routes[0].module_channel,
3253                    routes[0].module_epoch,
3254                    201,
3255                    vec![b'x'; limit - subc_protocol::HEADER_LEN],
3256                )
3257                .unwrap(),
3258            )
3259            .await
3260            .unwrap();
3261        assert_eq!(sink.backlog().queued_bytes, limit);
3262        assert_eq!(sink.backlog().queued_frames, 1);
3263        let ordinary = route_frame(
3264            FrameType::Response,
3265            routes[1].module_channel,
3266            routes[1].module_epoch,
3267            202,
3268        );
3269        assert!(queued_frame_bytes(&ordinary) < limit);
3270        // A fitting reply gets only the unused portion of the oldest frame's
3271        // 250 ms grace, not an immediate close or a fresh 250 ms wait.
3272        tokio::time::advance(Duration::from_millis(100)).await;
3273        let mut delivery = Box::pin(router.route_for_connection(&ctx, ordinary));
3274        assert!(poll_once(delivery.as_mut()).await.is_pending());
3275        assert!(close.try_recv().is_err());
3276        tokio::time::advance(Duration::from_millis(149)).await;
3277        assert!(poll_once(delivery.as_mut()).await.is_pending());
3278        assert!(close.try_recv().is_err());
3279        tokio::time::advance(Duration::from_millis(1)).await;
3280        assert!(matches!(
3281            poll_once(delivery.as_mut()).await,
3282            std::task::Poll::Ready(Ok(()))
3283        ));
3284        drop(delivery);
3285        let reason = close.try_recv().expect("the full client must close");
3286        assert!(reason
3287            .to_string()
3288            .contains("module_to_client_delivery_failed"));
3289        assert!(!reason.to_string().contains("frame_too_large"));
3290        assert_eq!(
3291            router.counters.snapshot()["client_egress_close_delivery_failed"],
3292            1
3293        );
3294        let admitted = rx.try_recv().expect("the budget-sized frame was admitted");
3295        assert_eq!(admitted.header.ty, FrameType::Response);
3296        assert_eq!(admitted.header.corr, 201);
3297        assert!(
3298            rx.try_recv().is_err(),
3299            "no oversized-frame ERROR is substituted for backpressure"
3300        );
3301    }
3302
3303    #[tokio::test(start_paused = true)]
3304    async fn oversized_reply_to_a_full_client_waits_only_remaining_grace() {
3305        let (router, sink, _rx, ctx, routes, mut close) = multi_route_client(
3306            "full-oversize-provider",
3307            ConnectionId::new(90),
3308            ConnectionId::new(91),
3309            2,
3310        )
3311        .await;
3312        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3313        // Fill the client's queue to its budget and leave it unread.
3314        router
3315            .route_for_connection(
3316                &ctx,
3317                Frame::build(
3318                    FrameType::Response,
3319                    Flags::new(false, Priority::Interactive, true),
3320                    routes[0].module_channel,
3321                    routes[0].module_epoch,
3322                    501,
3323                    vec![b'x'; limit - subc_protocol::HEADER_LEN],
3324                )
3325                .unwrap(),
3326            )
3327            .await
3328            .unwrap();
3329        assert_eq!(sink.backlog().queued_bytes, limit);
3330        let oversized = Frame::build(
3331            FrameType::Response,
3332            Flags::new(false, Priority::Interactive, true),
3333            routes[1].module_channel,
3334            routes[1].module_epoch,
3335            502,
3336            vec![b'y'; limit - subc_protocol::HEADER_LEN + 1],
3337        )
3338        .unwrap();
3339        // The replacement ERROR gets the same bounded burst grace as the
3340        // ordinary reply; oversized payloads themselves are never queued.
3341        tokio::time::advance(Duration::from_millis(80)).await;
3342        let mut delivery = Box::pin(router.route_for_connection(&ctx, oversized));
3343        assert!(poll_once(delivery.as_mut()).await.is_pending());
3344        assert!(close.try_recv().is_err());
3345        tokio::time::advance(Duration::from_millis(169)).await;
3346        assert!(poll_once(delivery.as_mut()).await.is_pending());
3347        assert!(close.try_recv().is_err());
3348        tokio::time::advance(Duration::from_millis(1)).await;
3349        assert!(matches!(
3350            poll_once(delivery.as_mut()).await,
3351            std::task::Poll::Ready(Ok(()))
3352        ));
3353        drop(delivery);
3354        let reason = close.try_recv().expect("the full client must close");
3355        assert!(reason
3356            .to_string()
3357            .contains("module_to_client_delivery_failed"));
3358    }
3359
3360    #[tokio::test]
3361    async fn oversized_reverse_request_errors_module_without_reaching_client() {
3362        let (router, _sink, mut rx, mut ctx, routes, mut close) = multi_route_client(
3363            "reverse-oversize-provider",
3364            ConnectionId::new(84),
3365            ConnectionId::new(85),
3366            2,
3367        )
3368        .await;
3369        let (module_tx, mut module_rx) = mpsc::channel(8);
3370        ctx.egress = FrameSink::new(module_tx);
3371        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3372        router
3373            .route_for_connection(
3374                &ctx,
3375                Frame::build(
3376                    FrameType::Request,
3377                    Flags::new(false, Priority::Interactive, false),
3378                    routes[0].module_channel,
3379                    routes[0].module_epoch,
3380                    301,
3381                    vec![b'r'; limit - subc_protocol::HEADER_LEN + 1],
3382                )
3383                .unwrap(),
3384            )
3385            .await
3386            .unwrap();
3387        assert!(
3388            rx.try_recv().is_err(),
3389            "the reverse request never reaches the client"
3390        );
3391        let error = module_rx.try_recv().expect("the module gets the refusal");
3392        assert_frame_size_error(
3393            &error,
3394            routes[0].module_channel,
3395            routes[0].module_epoch,
3396            301,
3397            limit + 1,
3398        );
3399        assert!(module_rx.try_recv().is_err());
3400        for (index, route) in routes.iter().enumerate() {
3401            module_route_for_test(&router, &ctx, route);
3402            router
3403                .route_for_connection(
3404                    &ctx,
3405                    route_frame(
3406                        FrameType::Response,
3407                        route.module_channel,
3408                        route.module_epoch,
3409                        302 + index as u64,
3410                    ),
3411                )
3412                .await
3413                .unwrap();
3414            let reply = rx.try_recv().expect("both routes still deliver");
3415            assert_eq!(reply.header.ty, FrameType::Response);
3416            assert_eq!(reply.header.channel, route.client_channel);
3417        }
3418        assert!(close.try_recv().is_err());
3419        assert_eq!(
3420            router.counters.snapshot()["client_egress_close_delivery_failed"],
3421            0
3422        );
3423    }
3424
3425    #[tokio::test]
3426    async fn oversized_push_closes_client_with_precise_warn() {
3427        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3428        let (router, _sink, mut rx, ctx, routes, mut close) = multi_route_client(
3429            "push-oversize-provider",
3430            ConnectionId::new(86),
3431            ConnectionId::new(87),
3432            2,
3433        )
3434        .await;
3435        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3436        router
3437            .route_for_connection(
3438                &ctx,
3439                Frame::build(
3440                    FrameType::Push,
3441                    Flags::new(false, Priority::Interactive, false),
3442                    routes[0].module_channel,
3443                    routes[0].module_epoch,
3444                    0,
3445                    vec![b'p'; limit - subc_protocol::HEADER_LEN + 1],
3446                )
3447                .unwrap(),
3448            )
3449            .await
3450            .unwrap();
3451        let reason = close
3452            .try_recv()
3453            .expect("unsolicited oversize still closes the client");
3454        assert!(reason
3455            .to_string()
3456            .contains("frame_too_large: oversized push"));
3457        assert!(
3458            rx.try_recv().is_err(),
3459            "no part of the oversized push is forwarded"
3460        );
3461        let captured = test_log::captured_logs(&logs);
3462        let warnings: Vec<_> = captured
3463            .lines()
3464            .filter(|line| line.contains("WARN"))
3465            .collect();
3466        assert_eq!(warnings.len(), 1, "{captured}");
3467        let warning = warnings[0];
3468        for expected in [
3469            "oversized push".to_string(),
3470            "module_id=\"push-oversize-provider\"".to_string(),
3471            "principals=direct".to_string(),
3472            format!("client_channel={}", routes[0].client_channel),
3473            format!("size={}", limit + 1),
3474            format!("limit={limit}"),
3475        ] {
3476            assert!(warning.contains(&expected), "missing {expected}: {warning}");
3477        }
3478        assert_eq!(
3479            router.counters.snapshot()["client_egress_close_delivery_failed"],
3480            1
3481        );
3482    }
3483
3484    #[tokio::test]
3485    async fn oversized_stream_and_error_frames_end_request_even_with_queued_traffic() {
3486        let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3487            "stream-oversize-provider",
3488            ConnectionId::new(88),
3489            ConnectionId::new(89),
3490            2,
3491        )
3492        .await;
3493        let binding = module_route_for_test(&router, &ctx, &routes[0]);
3494        let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3495        for ty in [
3496            FrameType::StreamData,
3497            FrameType::StreamEnd,
3498            FrameType::Error,
3499        ] {
3500            binding.flow.acquire_tagged(401, false).await.unwrap();
3501            sink.try_send(route_frame(
3502                FrameType::Response,
3503                routes[1].client_channel,
3504                routes[1].client_epoch,
3505                402,
3506            ))
3507            .unwrap();
3508            router
3509                .route_for_connection(
3510                    &ctx,
3511                    Frame::build(
3512                        ty,
3513                        Flags::new(false, Priority::Interactive, false),
3514                        routes[0].module_channel,
3515                        routes[0].module_epoch,
3516                        401,
3517                        vec![b's'; limit - subc_protocol::HEADER_LEN + 1],
3518                    )
3519                    .unwrap(),
3520                )
3521                .await
3522                .unwrap();
3523            assert!(close.try_recv().is_err());
3524            assert_eq!(rx.try_recv().unwrap().header.corr, 402);
3525            let error = rx
3526                .try_recv()
3527                .expect("every correlated oversize frame becomes ERROR");
3528            assert_frame_size_error(
3529                &error,
3530                routes[0].client_channel,
3531                routes[0].client_epoch,
3532                401,
3533                limit + 1,
3534            );
3535            assert_eq!(binding.flow.drain_in_flight(), 0, "{ty:?} ends the request");
3536            assert!(rx.try_recv().is_err());
3537        }
3538    }
3539
3540    /// An awaited send (a control or error reply) is never refused by the byte
3541    /// budget: it parks until the writer frees bytes. If a release could miss a
3542    /// parked sender, that reply would hang for the connection's lifetime, so
3543    /// this pins the wakeup: the send stays parked while the queue is full and
3544    /// completes as soon as one frame leaves it.
3545    #[tokio::test]
3546    async fn awaited_send_parked_behind_a_full_byte_budget_wakes_when_bytes_free() {
3547        let (tx, mut rx) = mpsc::channel(1024);
3548        let sink = FrameSink::with_byte_budget(tx, 2_000);
3549        let mut queued = 0u64;
3550        while sink
3551            .try_send(stream_frame(7, 1, queued, vec![b'x'; 200]))
3552            .is_ok()
3553        {
3554            queued += 1;
3555        }
3556        assert!(queued > 0, "the budget admitted nothing");
3557
3558        let parked = tokio::spawn({
3559            let sink = sink.clone();
3560            async move { sink.send(stream_frame(0, 0, 999, vec![b'r'; 200])).await }
3561        });
3562        tokio::time::sleep(Duration::from_millis(100)).await;
3563        assert!(
3564            !parked.is_finished(),
3565            "the awaited send must wait while the byte budget is full"
3566        );
3567
3568        // The writer takes one frame and drops it, releasing its bytes.
3569        drop(rx.recv().await.expect("a queued frame"));
3570        tokio::time::timeout(Duration::from_secs(2), parked)
3571            .await
3572            .expect("the parked send was never woken after bytes were freed")
3573            .unwrap()
3574            .unwrap();
3575    }
3576
3577    /// A client multiplexing several token streams pauses its reader while the
3578    /// module keeps producing small frames. Far more frames than the old
3579    /// 64-frame queue allowed, but far fewer bytes than the budget, must all be
3580    /// held without closing the connection and delivered in order afterwards.
3581    #[tokio::test]
3582    async fn paused_client_reader_keeps_connection_through_small_frame_burst() {
3583        const ROUTES: usize = 4;
3584        const FRAMES_PER_ROUTE: usize = 250;
3585        let (router, client_sink, mut client_rx, module_ctx, routes, mut close_receiver) =
3586            multi_route_client(
3587                "burst-provider",
3588                ConnectionId::new(60),
3589                ConnectionId::new(61),
3590                ROUTES,
3591            )
3592            .await;
3593
3594        // The reader is paused: nothing is received until every frame is sent.
3595        for seq in 0..FRAMES_PER_ROUTE as u64 {
3596            for (index, route) in routes.iter().enumerate() {
3597                let body = format!("route-{index}-token-{seq:05}-{}", "t".repeat(170));
3598                router
3599                    .route_for_connection(
3600                        &module_ctx,
3601                        stream_frame(
3602                            route.module_channel,
3603                            route.module_epoch,
3604                            seq,
3605                            body.into_bytes(),
3606                        ),
3607                    )
3608                    .await
3609                    .unwrap();
3610            }
3611        }
3612
3613        let backlog = client_sink.backlog();
3614        assert_eq!(backlog.queued_frames, ROUTES * FRAMES_PER_ROUTE);
3615        assert!(backlog.queued_bytes < crate::server::CONNECTION_EGRESS_BYTE_BUDGET);
3616        assert!(
3617            close_receiver.try_recv().is_err(),
3618            "a paused reader under the byte budget must not be closed"
3619        );
3620        assert_eq!(
3621            router.counters.snapshot()["client_egress_close_delivery_failed"],
3622            0
3623        );
3624
3625        // The reader resumes: every frame arrives, in order within each route.
3626        let mut next_seq = vec![0u64; ROUTES];
3627        for _ in 0..ROUTES * FRAMES_PER_ROUTE {
3628            let frame = client_rx.try_recv().expect("every queued frame arrives");
3629            let index = routes
3630                .iter()
3631                .position(|route| route.client_channel == frame.header.channel)
3632                .expect("frame arrives on one of the bound client channels");
3633            assert_eq!(frame.header.corr, next_seq[index], "route {index} order");
3634            let expected_prefix = format!("route-{index}-token-{:05}-", next_seq[index]);
3635            assert!(frame.body.starts_with(expected_prefix.as_bytes()));
3636            next_seq[index] += 1;
3637        }
3638        assert!(client_rx.try_recv().is_err());
3639        assert_eq!(next_seq, vec![FRAMES_PER_ROUTE as u64; ROUTES]);
3640        assert_eq!(client_sink.backlog().queued_bytes, 0);
3641    }
3642
3643    /// A client that never reads exhausts its byte budget and the oldest
3644    /// frame's 250 ms grace, then closes with one header-only diagnosis.
3645    #[tokio::test(start_paused = true)]
3646    async fn never_reading_client_is_closed_at_byte_budget_with_warn_diagnosis() {
3647        let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3648        const BODY: usize = 16 * 1024;
3649        let (router, client_sink, _client_rx, module_ctx, routes, mut close_receiver) =
3650            multi_route_client(
3651                "stuck-reader-provider",
3652                ConnectionId::new(70),
3653                ConnectionId::new(71),
3654                2,
3655            )
3656            .await;
3657
3658        let mut admitted = 0usize;
3659        let mut sent = 0u64;
3660        let started = tokio::time::Instant::now();
3661        while router.counters.snapshot()["client_egress_close_delivery_failed"] == 0 {
3662            assert!(sent < 1_000, "the byte budget never refused a frame");
3663            let route = &routes[(sent % 2) as usize];
3664            router
3665                .route_for_connection(
3666                    &module_ctx,
3667                    stream_frame(
3668                        route.module_channel,
3669                        route.module_epoch,
3670                        sent,
3671                        vec![b'z'; BODY],
3672                    ),
3673                )
3674                .await
3675                .unwrap();
3676            sent += 1;
3677            admitted = client_sink.backlog().queued_frames;
3678        }
3679        assert_eq!(
3680            started.elapsed(),
3681            crate::server::CONNECTION_EGRESS_STALL_GRACE
3682        );
3683        // The budget, not the frame-count backstop, did the refusing.
3684        let frame_bytes = subc_protocol::HEADER_LEN + BODY;
3685        assert_eq!(
3686            admitted,
3687            crate::server::CONNECTION_EGRESS_BYTE_BUDGET / frame_bytes
3688        );
3689        let reason = close_receiver
3690            .try_recv()
3691            .expect("the client connection must be asked to close");
3692        assert!(reason
3693            .to_string()
3694            .contains("module_to_client_delivery_failed"));
3695
3696        // A second refused frame for the same connection adds no second WARN.
3697        router
3698            .route_for_connection(
3699                &module_ctx,
3700                stream_frame(
3701                    routes[0].module_channel,
3702                    routes[0].module_epoch,
3703                    sent,
3704                    vec![b'z'; BODY],
3705                ),
3706            )
3707            .await
3708            .unwrap();
3709
3710        let captured = test_log::captured_logs(&logs);
3711        let warn_lines = captured
3712            .lines()
3713            .filter(|line| {
3714                line.contains("closing client connection: its egress queue could not take a frame")
3715            })
3716            .collect::<Vec<_>>();
3717        assert_eq!(warn_lines.len(), 1, "exactly one WARN, got: {captured}");
3718        let line = warn_lines[0];
3719        assert!(line.contains("WARN"), "{line}");
3720        assert!(line.contains("connection_id=71"), "{line}");
3721        assert!(
3722            line.contains("module_id=\"stuck-reader-provider\""),
3723            "{line}"
3724        );
3725        assert!(line.contains("client_channel="), "{line}");
3726        assert!(line.contains("principals=direct"), "{line}");
3727        let queued_bytes: usize = line
3728            .split("queued_bytes=")
3729            .nth(1)
3730            .and_then(|rest| rest.split_whitespace().next())
3731            .and_then(|value| value.parse().ok())
3732            .expect("queued_bytes is logged");
3733        assert_eq!(queued_bytes, admitted * frame_bytes);
3734        assert!(
3735            line.contains(&format!("queued_frames={admitted}")),
3736            "{line}"
3737        );
3738        assert!(line.contains("oldest_queued_ms="), "{line}");
3739        assert!(line.contains("oldest_queued_ms=250"), "{line}");
3740        assert!(line.contains("frame_type=Some(StreamData)"), "{line}");
3741        assert!(
3742            line.contains(&format!(
3743                "frame_flags=Some({})",
3744                Flags::new(false, Priority::Interactive, false).0
3745            )),
3746            "{line}"
3747        );
3748        assert!(
3749            line.contains("close_cause=\"waited_250_ms_without_room\""),
3750            "{line}"
3751        );
3752        assert!(
3753            !line.contains(&"z".repeat(20)),
3754            "body bytes must never be logged: {line}"
3755        );
3756    }
3757
3758    fn route_frame(ty: FrameType, channel: u16, epoch: u32, corr: u64) -> Frame {
3759        Frame::build(
3760            ty,
3761            Flags::new(false, Priority::Interactive, false),
3762            channel,
3763            epoch,
3764            corr,
3765            if ty == FrameType::Request || ty == FrameType::Response {
3766                b"route-body".to_vec()
3767            } else {
3768                Vec::new()
3769            },
3770        )
3771        .unwrap()
3772    }
3773
3774    type DynamicRouteFixture = (
3775        Router,
3776        Arc<ForwardingTable>,
3777        RouteCtx,
3778        mpsc::Receiver<crate::router::OutboundFrame>,
3779        RouteCtx,
3780        mpsc::Receiver<crate::router::OutboundFrame>,
3781        mpsc::Receiver<crate::router::OutboundFrame>,
3782        crate::forwarding::PendingRouteBindRelay,
3783    );
3784
3785    fn dynamic_route_fixture(commit: bool) -> DynamicRouteFixture {
3786        dynamic_route_fixture_with_busy(commit, false)
3787    }
3788
3789    fn dynamic_route_fixture_with_busy(commit: bool, busy_refusals: bool) -> DynamicRouteFixture {
3790        let forwarding = Arc::new(ForwardingTable::default());
3791        let control = Arc::new(crate::ControlHandler::with_forwarding(
3792            Arc::new(crate::Registry::default()),
3793            Arc::clone(&forwarding),
3794        ));
3795        let router = Router::with_control_handler(control);
3796        let module_connection = ConnectionId::new(500);
3797        let client_connection = ConnectionId::new(501);
3798        let (module_tx, module_rx) = mpsc::channel(if busy_refusals { 64 } else { 8 });
3799        forwarding
3800            .register_module_connection(
3801                module_connection,
3802                "epoch-router".into(),
3803                2,
3804                Concurrency::ModuleManaged,
3805                FrameSink::new(module_tx),
3806            )
3807            .unwrap();
3808        let (client_tx, client_rx) = mpsc::channel(if busy_refusals { 64 } else { 8 });
3809        let client_sink = FrameSink::new(client_tx);
3810        let pending = forwarding
3811            .begin_route_bind_relay_with_busy_for_test(
3812                client_connection,
3813                client_sink.clone(),
3814                700,
3815                "epoch-router",
3816                busy_refusals,
3817            )
3818            .unwrap();
3819        if commit {
3820            forwarding
3821                .complete_pending_relay(
3822                    module_connection,
3823                    pending.corr,
3824                    RouteBindRelayOutcome::Accepted,
3825                )
3826                .unwrap();
3827        }
3828        let (module_egress_tx, module_egress_rx) = mpsc::channel(8);
3829        (
3830            router,
3831            forwarding,
3832            RouteCtx {
3833                connection_id: client_connection,
3834                egress: client_sink,
3835            },
3836            client_rx,
3837            RouteCtx {
3838                connection_id: module_connection,
3839                egress: FrameSink::new(module_egress_tx),
3840            },
3841            module_egress_rx,
3842            module_rx,
3843            pending,
3844        )
3845    }
3846
3847    #[tokio::test]
3848    async fn route_epochs_validate_both_directions_and_rewrite_to_peer_handle() {
3849        let (
3850            router,
3851            _forwarding,
3852            client_ctx,
3853            mut client_rx,
3854            module_ctx,
3855            _module_egress_rx,
3856            mut module_rx,
3857            pending,
3858        ) = dynamic_route_fixture(true);
3859        let route_open = client_rx.recv().await.unwrap();
3860        assert_eq!(route_open.header.corr, 700);
3861
3862        router
3863            .route_for_connection(
3864                &client_ctx,
3865                route_frame(
3866                    FrameType::Request,
3867                    pending.client_channel,
3868                    pending.client_epoch,
3869                    701,
3870                ),
3871            )
3872            .await
3873            .unwrap();
3874        let forwarded = module_rx.recv().await.unwrap();
3875        assert_eq!(forwarded.header.channel, pending.module_channel);
3876        assert_eq!(forwarded.header.epoch, pending.module_epoch);
3877
3878        router
3879            .route_for_connection(
3880                &module_ctx,
3881                route_frame(
3882                    FrameType::Response,
3883                    pending.module_channel,
3884                    pending.module_epoch,
3885                    701,
3886                ),
3887            )
3888            .await
3889            .unwrap();
3890        let delivered = client_rx.recv().await.unwrap();
3891        assert_eq!(delivered.header.channel, pending.client_channel);
3892        assert_eq!(delivered.header.epoch, pending.client_epoch);
3893
3894        router
3895            .route_for_connection(
3896                &client_ctx,
3897                route_frame(
3898                    FrameType::Request,
3899                    pending.client_channel,
3900                    pending.client_epoch + 1,
3901                    702,
3902                ),
3903            )
3904            .await
3905            .unwrap();
3906        router
3907            .route_for_connection(
3908                &module_ctx,
3909                route_frame(
3910                    FrameType::Response,
3911                    pending.module_channel,
3912                    pending.module_epoch + 1,
3913                    703,
3914                ),
3915            )
3916            .await
3917            .unwrap();
3918        let stale_error = client_rx.recv().await.unwrap();
3919        assert_eq!(stale_error.header.ty, FrameType::Error);
3920        assert_eq!(stale_error.header.channel, pending.client_channel);
3921        assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
3922        assert_eq!(stale_error.header.corr, 702);
3923        let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
3924        assert_eq!(body.code, "stale_route_epoch");
3925        assert!(module_rx.try_recv().is_err());
3926        assert!(client_rx.try_recv().is_err());
3927        let counters = router.counters.snapshot();
3928        assert_eq!(counters["client_frames_dropped_stale_route"], 1);
3929        assert_eq!(counters["module_frames_dropped_no_route"], 1);
3930    }
3931
3932    #[tokio::test]
3933    async fn accepted_route_publishes_route_open_before_immediate_reverse_request() {
3934        let (
3935            router,
3936            _,
3937            _client_ctx,
3938            mut client_rx,
3939            module_ctx,
3940            _module_egress_rx,
3941            _module_rx,
3942            pending,
3943        ) = dynamic_route_fixture(true);
3944        router
3945            .route_for_connection(
3946                &module_ctx,
3947                route_frame(
3948                    FrameType::Request,
3949                    pending.module_channel,
3950                    pending.module_epoch,
3951                    800,
3952                ),
3953            )
3954            .await
3955            .unwrap();
3956
3957        let first = client_rx.recv().await.unwrap();
3958        let second = client_rx.recv().await.unwrap();
3959        assert_eq!(first.header.channel, 0);
3960        assert_eq!(first.header.corr, 700);
3961        assert_eq!(second.header.channel, pending.client_channel);
3962        assert_eq!(second.header.epoch, pending.client_epoch);
3963        assert_eq!(second.header.corr, 800);
3964    }
3965
3966    #[tokio::test]
3967    async fn reserved_slot_ingress_errors_only_matching_client_requests() {
3968        let (
3969            router,
3970            _forwarding,
3971            client_ctx,
3972            mut client_rx,
3973            _module_ctx,
3974            _module_egress_rx,
3975            mut module_rx,
3976            pending,
3977        ) = dynamic_route_fixture(false);
3978        router
3979            .route_for_connection(
3980                &client_ctx,
3981                route_frame(
3982                    FrameType::Request,
3983                    pending.client_channel,
3984                    pending.client_epoch,
3985                    900,
3986                ),
3987            )
3988            .await
3989            .unwrap();
3990        let error = client_rx.recv().await.unwrap();
3991        assert_eq!(error.header.ty, FrameType::Error);
3992        assert_eq!(error.header.channel, pending.client_channel);
3993        assert_eq!(error.header.epoch, pending.client_epoch);
3994        assert_eq!(error.header.corr, 900);
3995
3996        router
3997            .route_for_connection(
3998                &client_ctx,
3999                route_frame(
4000                    FrameType::Response,
4001                    pending.client_channel,
4002                    pending.client_epoch,
4003                    901,
4004                ),
4005            )
4006            .await
4007            .unwrap();
4008        router
4009            .route_for_connection(
4010                &client_ctx,
4011                route_frame(
4012                    FrameType::Request,
4013                    pending.client_channel,
4014                    pending.client_epoch + 1,
4015                    902,
4016                ),
4017            )
4018            .await
4019            .unwrap();
4020        let stale_error = client_rx.recv().await.unwrap();
4021        assert_eq!(stale_error.header.ty, FrameType::Error);
4022        assert_eq!(stale_error.header.channel, pending.client_channel);
4023        assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
4024        assert_eq!(stale_error.header.corr, 902);
4025        let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
4026        assert_eq!(body.code, "stale_route_epoch");
4027        assert!(module_rx.try_recv().is_err());
4028        let counters = router.counters.snapshot();
4029        assert_eq!(counters["client_frames_dropped_stale_route"], 1);
4030        assert_eq!(counters["module_frames_dropped_no_route"], 0);
4031    }
4032
4033    #[tokio::test]
4034    async fn dropped_module_route_goodbye_increments_counter() {
4035        let (
4036            router,
4037            _forwarding,
4038            client_ctx,
4039            mut client_rx,
4040            _module_ctx,
4041            _module_egress_rx,
4042            mut module_rx,
4043            pending,
4044        ) = dynamic_route_fixture(true);
4045        let _ = client_rx.recv().await;
4046        module_rx.close();
4047
4048        router
4049            .route_for_connection(
4050                &client_ctx,
4051                route_frame(
4052                    FrameType::Goodbye,
4053                    pending.client_channel,
4054                    pending.client_epoch,
4055                    999,
4056                ),
4057            )
4058            .await
4059            .unwrap();
4060
4061        let counters = router.counters.snapshot();
4062        assert_eq!(counters["goodbye_relay_module_dropped"], 1);
4063        assert_eq!(
4064            counters["goodbye_relay_module_dropped_by_module"],
4065            serde_json::json!({ "epoch-router": 1 })
4066        );
4067        assert_eq!(counters["route_released_epoch_fenced"], 1);
4068    }
4069
4070    #[tokio::test]
4071    async fn module_request_on_stale_epoch_receives_stale_route_epoch() {
4072        let (
4073            router,
4074            _forwarding,
4075            _client_ctx,
4076            _client_rx,
4077            module_ctx,
4078            mut module_egress_rx,
4079            mut module_rx,
4080            pending,
4081        ) = dynamic_route_fixture(true);
4082
4083        router
4084            .route_for_connection(
4085                &module_ctx,
4086                route_frame(
4087                    FrameType::Request,
4088                    pending.module_channel,
4089                    pending.module_epoch + 1,
4090                    1_000,
4091                ),
4092            )
4093            .await
4094            .unwrap();
4095
4096        let error = module_egress_rx.try_recv().unwrap();
4097        assert_eq!(error.header.ty, FrameType::Error);
4098        assert_eq!(error.header.channel, pending.module_channel);
4099        assert_eq!(error.header.epoch, pending.module_epoch + 1);
4100        assert_eq!(error.header.corr, 1_000);
4101        let body: ErrorBody = serde_json::from_slice(&error.body).unwrap();
4102        assert_eq!(body.code, "stale_route_epoch");
4103        assert!(module_rx.try_recv().is_err());
4104        let counters = router.counters.snapshot();
4105        assert_eq!(counters["module_requests_dropped_stale_route"], 1);
4106        assert_eq!(counters["module_frames_dropped_no_route"], 0);
4107    }
4108
4109    #[tokio::test(start_paused = true)]
4110    async fn stale_epoch_drop_warns_once_per_minute_with_suppressed_count() {
4111        let (output, _guard) = test_log::log_capture(tracing::Level::WARN);
4112        let (
4113            router,
4114            _forwarding,
4115            client_ctx,
4116            mut client_rx,
4117            _module_ctx,
4118            _module_egress_rx,
4119            mut module_rx,
4120            pending,
4121        ) = dynamic_route_fixture(true);
4122        let _ = client_rx.recv().await;
4123        for corr in 1..=3 {
4124            router
4125                .route_for_connection(
4126                    &client_ctx,
4127                    route_frame(
4128                        FrameType::Request,
4129                        pending.client_channel,
4130                        pending.client_epoch + 1,
4131                        corr,
4132                    ),
4133                )
4134                .await
4135                .unwrap();
4136            assert_eq!(client_rx.recv().await.unwrap().header.ty, FrameType::Error);
4137        }
4138        assert!(module_rx.try_recv().is_err());
4139        let logs = test_log::captured_logs(&output);
4140        assert_eq!(
4141            logs.matches("dropping data-plane frame").count(),
4142            1,
4143            "{logs}"
4144        );
4145        assert!(logs.contains("WARN"), "{logs}");
4146        assert!(
4147            logs.contains(&format!("connection_id={}", client_ctx.connection_id.get())),
4148            "{logs}"
4149        );
4150        assert!(
4151            logs.contains(&format!("channel={}", pending.client_channel)),
4152            "{logs}"
4153        );
4154        assert!(
4155            logs.contains(&format!("epoch={}", pending.client_epoch + 1)),
4156            "{logs}"
4157        );
4158        assert!(
4159            logs.contains("client_to_module") && logs.contains("stale_epoch"),
4160            "{logs}"
4161        );
4162        assert!(logs.contains("suppressed=0"), "{logs}");
4163
4164        tokio::time::advance(Duration::from_secs(60)).await;
4165        router
4166            .route_for_connection(
4167                &client_ctx,
4168                route_frame(
4169                    FrameType::Request,
4170                    pending.client_channel,
4171                    pending.client_epoch + 2,
4172                    4,
4173                ),
4174            )
4175            .await
4176            .unwrap();
4177        let logs = test_log::captured_logs(&output);
4178        assert_eq!(
4179            logs.matches("dropping data-plane frame").count(),
4180            2,
4181            "{logs}"
4182        );
4183        assert!(logs.contains("suppressed=2"), "{logs}");
4184    }
4185
4186    #[tokio::test]
4187    async fn module_request_on_reserved_or_absent_route_receives_unknown_channel() {
4188        let (
4189            reserved_router,
4190            _forwarding,
4191            _client_ctx,
4192            _client_rx,
4193            reserved_module_ctx,
4194            mut reserved_module_egress_rx,
4195            _module_rx,
4196            reserved,
4197        ) = dynamic_route_fixture(false);
4198        reserved_router
4199            .route_for_connection(
4200                &reserved_module_ctx,
4201                route_frame(
4202                    FrameType::Request,
4203                    reserved.module_channel,
4204                    reserved.module_epoch,
4205                    1_001,
4206                ),
4207            )
4208            .await
4209            .unwrap();
4210        let reserved_error = reserved_module_egress_rx.try_recv().unwrap();
4211        let reserved_body: ErrorBody = serde_json::from_slice(&reserved_error.body).unwrap();
4212        assert_eq!(reserved_error.header.ty, FrameType::Error);
4213        assert_eq!(reserved_error.header.channel, reserved.module_channel);
4214        assert_eq!(reserved_error.header.epoch, reserved.module_epoch);
4215        assert_eq!(reserved_error.header.corr, 1_001);
4216        assert_eq!(reserved_body.code, "unknown_channel");
4217        assert_eq!(
4218            reserved_router.counters.snapshot()["module_requests_dropped_stale_route"],
4219            1
4220        );
4221
4222        let (
4223            absent_router,
4224            _forwarding,
4225            _client_ctx,
4226            _client_rx,
4227            absent_module_ctx,
4228            mut absent_module_egress_rx,
4229            _module_rx,
4230            absent,
4231        ) = dynamic_route_fixture(false);
4232        absent_router
4233            .route_for_connection(
4234                &absent_module_ctx,
4235                route_frame(
4236                    FrameType::Request,
4237                    absent.module_channel + 1,
4238                    absent.module_epoch,
4239                    1_002,
4240                ),
4241            )
4242            .await
4243            .unwrap();
4244        let absent_error = absent_module_egress_rx.try_recv().unwrap();
4245        let absent_body: ErrorBody = serde_json::from_slice(&absent_error.body).unwrap();
4246        assert_eq!(absent_error.header.ty, FrameType::Error);
4247        assert_eq!(absent_error.header.channel, absent.module_channel + 1);
4248        assert_eq!(absent_error.header.epoch, absent.module_epoch);
4249        assert_eq!(absent_error.header.corr, 1_002);
4250        assert_eq!(absent_body.code, "unknown_channel");
4251        assert_eq!(
4252            absent_router.counters.snapshot()["module_requests_dropped_stale_route"],
4253            1
4254        );
4255    }
4256
4257    #[tokio::test]
4258    async fn non_request_module_frame_on_dead_route_is_counted_without_error() {
4259        let (
4260            router,
4261            forwarding,
4262            client_ctx,
4263            mut client_rx,
4264            module_ctx,
4265            mut module_egress_rx,
4266            mut module_rx,
4267            pending,
4268        ) = dynamic_route_fixture(true);
4269        let (other_module_tx, _other_module_rx) = mpsc::channel(8);
4270        forwarding
4271            .register_module_connection(
4272                ConnectionId::new(502),
4273                "other-module".into(),
4274                2,
4275                Concurrency::ModuleManaged,
4276                FrameSink::new(other_module_tx),
4277            )
4278            .unwrap();
4279        let _ = client_rx.recv().await.unwrap();
4280
4281        router
4282            .route_for_connection(
4283                &client_ctx,
4284                route_frame(
4285                    FrameType::Goodbye,
4286                    pending.client_channel,
4287                    pending.client_epoch,
4288                    1_003,
4289                ),
4290            )
4291            .await
4292            .unwrap();
4293        let _ = module_rx.recv().await.unwrap();
4294
4295        router
4296            .route_for_connection(
4297                &module_ctx,
4298                route_frame(
4299                    FrameType::StreamData,
4300                    pending.module_channel,
4301                    pending.module_epoch,
4302                    1_004,
4303                ),
4304            )
4305            .await
4306            .unwrap();
4307
4308        // No ERROR goes back for a non-request frame. The one reply is the
4309        // route GOODBYE telling the module to let go of the released route.
4310        let reply = module_egress_rx.try_recv().unwrap();
4311        assert_eq!(reply.header.ty, FrameType::Goodbye);
4312        assert!(module_egress_rx.try_recv().is_err());
4313        let counters = router.counters.snapshot();
4314        assert_eq!(counters["module_frames_dropped_no_route"], 1);
4315        assert_eq!(
4316            counters["module_frames_dropped_no_route_by_module"],
4317            serde_json::json!({ "epoch-router": 1 })
4318        );
4319        assert_eq!(counters["module_requests_dropped_stale_route"], 0);
4320    }
4321
4322    /// Drain whatever the module connection's queue holds right now, the way a
4323    /// module's reader would before it stalls.
4324    fn drain_now(rx: &mut mpsc::Receiver<OutboundFrame>) {
4325        while rx.try_recv().is_ok() {}
4326    }
4327
4328    /// A module that stops reading for a moment when a client closes one of
4329    /// its routes still learns the route is gone: the GOODBYE its full egress
4330    /// queue refused is delivered as soon as it reads again, rather than
4331    /// dropped, which would leave the module holding the route for the rest of
4332    /// its connection.
4333    #[tokio::test]
4334    async fn route_goodbye_refused_by_stalled_module_is_delivered_when_it_resumes_reading() {
4335        const BUDGET: usize = 4_096;
4336        let forwarding = Arc::new(ForwardingTable::default());
4337        let control = Arc::new(ControlHandler::with_forwarding(
4338            Arc::new(Registry::default()),
4339            Arc::clone(&forwarding),
4340        ));
4341        let router = Router::with_control_handler(control);
4342        let module_connection = ConnectionId::new(80);
4343        let client_connection = ConnectionId::new(81);
4344        let (module_tx, mut module_rx) = mpsc::channel(64);
4345        let module_sink = FrameSink::with_byte_budget(module_tx, BUDGET);
4346        forwarding
4347            .register_module_connection(
4348                module_connection,
4349                "stalled-provider".into(),
4350                2,
4351                Concurrency::ModuleManaged,
4352                module_sink.clone(),
4353            )
4354            .unwrap();
4355        let (client_tx, mut client_rx) = mpsc::channel(8);
4356        let client_sink = FrameSink::new(client_tx);
4357        let pending = forwarding
4358            .begin_route_bind_relay_for_test(
4359                client_connection,
4360                client_sink.clone(),
4361                1_100,
4362                "stalled-provider",
4363            )
4364            .unwrap();
4365        forwarding
4366            .complete_pending_relay(
4367                module_connection,
4368                pending.corr,
4369                RouteBindRelayOutcome::Accepted,
4370            )
4371            .unwrap();
4372        let _ = client_rx.recv().await.unwrap();
4373        drain_now(&mut module_rx);
4374
4375        // The module stalls: its queue holds more than the byte budget, so the
4376        // queue refuses anything further.
4377        module_sink
4378            .try_send(stream_frame(9, 1, 0, vec![b'f'; BUDGET]))
4379            .unwrap();
4380        assert!(module_sink
4381            .try_send(stream_frame(9, 1, 1, Vec::new()))
4382            .is_err());
4383
4384        let client_ctx = RouteCtx {
4385            connection_id: client_connection,
4386            egress: client_sink,
4387        };
4388        router
4389            .route_for_connection(
4390                &client_ctx,
4391                route_frame(
4392                    FrameType::Goodbye,
4393                    pending.client_channel,
4394                    pending.client_epoch,
4395                    1_101,
4396                ),
4397            )
4398            .await
4399            .unwrap();
4400        tokio::task::yield_now().await;
4401        assert_eq!(
4402            router.counters.snapshot()["goodbye_relay_module_dropped"],
4403            0,
4404            "a GOODBYE refused by a momentarily full module queue must not be dropped"
4405        );
4406
4407        // The module reads again: the filler comes off, then the GOODBYE.
4408        let filler = module_rx.recv().await.unwrap();
4409        assert_eq!(filler.header.ty, FrameType::StreamData);
4410        drop(filler);
4411        let goodbye = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
4412            .await
4413            .expect("the refused GOODBYE must be delivered once the module frees room")
4414            .unwrap();
4415        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
4416        assert_eq!(goodbye.header.channel, pending.module_channel);
4417        assert_eq!(goodbye.header.epoch, pending.module_epoch);
4418        assert_eq!(
4419            router.counters.snapshot()["goodbye_relay_module_dropped"],
4420            0
4421        );
4422    }
4423
4424    /// A module still sending on a route the daemon released is told, with a
4425    /// route GOODBYE for exactly the (channel, epoch) it sent on. The same
4426    /// happens for a stale epoch on a channel whose route moved on and for a
4427    /// channel that never had a route; only the first is counted as traffic on
4428    /// a released route.
4429    #[tokio::test]
4430    async fn module_frame_on_route_the_daemon_does_not_hold_is_answered_with_goodbye() {
4431        let (
4432            router,
4433            _forwarding,
4434            client_ctx,
4435            mut client_rx,
4436            module_ctx,
4437            mut module_egress_rx,
4438            mut module_rx,
4439            pending,
4440        ) = dynamic_route_fixture(true);
4441        let _ = client_rx.recv().await.unwrap();
4442        router
4443            .route_for_connection(
4444                &client_ctx,
4445                route_frame(
4446                    FrameType::Goodbye,
4447                    pending.client_channel,
4448                    pending.client_epoch,
4449                    1_200,
4450                ),
4451            )
4452            .await
4453            .unwrap();
4454        drain_now(&mut module_rx);
4455
4456        let cases = [
4457            // Released route: the daemon allocated this (channel, epoch).
4458            (pending.module_channel, pending.module_epoch),
4459            // A channel the daemon never allocated on this connection.
4460            (pending.module_channel + 1, 1),
4461        ];
4462        for (channel, epoch) in cases {
4463            router
4464                .route_for_connection(
4465                    &module_ctx,
4466                    route_frame(FrameType::StreamData, channel, epoch, 1_201),
4467                )
4468                .await
4469                .unwrap();
4470            let reply = module_egress_rx
4471                .try_recv()
4472                .expect("a frame on a route the daemon does not hold is answered");
4473            assert_eq!(reply.header.ty, FrameType::Goodbye);
4474            assert_eq!(reply.header.channel, channel);
4475            assert_eq!(reply.header.epoch, epoch);
4476            assert_eq!(reply.header.corr, 0);
4477            assert!(module_egress_rx.try_recv().is_err());
4478        }
4479
4480        // A GOODBYE from the module for a route the daemon already released
4481        // is not answered: the module is letting go already.
4482        router
4483            .route_for_connection(
4484                &module_ctx,
4485                route_frame(FrameType::Goodbye, pending.module_channel + 2, 1, 0),
4486            )
4487            .await
4488            .unwrap();
4489        assert!(module_egress_rx.try_recv().is_err());
4490
4491        let counters = router.counters.snapshot();
4492        assert_eq!(counters["module_frames_dropped_no_route"], 3);
4493        assert_eq!(counters["module_frames_dropped_released_route"], 1);
4494        assert_eq!(
4495            counters["module_frames_dropped_released_route_by_module"],
4496            serde_json::json!({ "epoch-router": 1 })
4497        );
4498        assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4499    }
4500
4501    /// A chatty orphan (a streaming module still producing on a route it
4502    /// missed the GOODBYE for) is answered once per interval, not once per
4503    /// frame, and answered again once the interval has passed.
4504    #[tokio::test(start_paused = true)]
4505    async fn burst_of_orphan_module_frames_is_answered_once_per_interval() {
4506        let (
4507            router,
4508            _forwarding,
4509            client_ctx,
4510            mut client_rx,
4511            module_ctx,
4512            mut module_egress_rx,
4513            mut module_rx,
4514            pending,
4515        ) = dynamic_route_fixture(true);
4516        let _ = client_rx.recv().await.unwrap();
4517        router
4518            .route_for_connection(
4519                &client_ctx,
4520                route_frame(
4521                    FrameType::Goodbye,
4522                    pending.client_channel,
4523                    pending.client_epoch,
4524                    1_300,
4525                ),
4526            )
4527            .await
4528            .unwrap();
4529        drain_now(&mut module_rx);
4530
4531        let send_burst = |corr: u64| {
4532            route_frame(
4533                FrameType::StreamData,
4534                pending.module_channel,
4535                pending.module_epoch,
4536                corr,
4537            )
4538        };
4539        for corr in 0..20 {
4540            router
4541                .route_for_connection(&module_ctx, send_burst(corr))
4542                .await
4543                .unwrap();
4544        }
4545        let mut replies = 0;
4546        while let Ok(reply) = module_egress_rx.try_recv() {
4547            assert_eq!(reply.header.ty, FrameType::Goodbye);
4548            replies += 1;
4549        }
4550        assert_eq!(replies, 1, "a burst within the interval gets one GOODBYE");
4551
4552        tokio::time::advance(ORPHAN_ROUTE_GOODBYE_INTERVAL).await;
4553        router
4554            .route_for_connection(&module_ctx, send_burst(20))
4555            .await
4556            .unwrap();
4557        let retry = module_egress_rx
4558            .try_recv()
4559            .expect("the first orphan frame after the interval is answered again");
4560        assert_eq!(retry.header.channel, pending.module_channel);
4561        assert_eq!(retry.header.epoch, pending.module_epoch);
4562
4563        let counters = router.counters.snapshot();
4564        assert_eq!(counters["module_frames_dropped_no_route"], 21);
4565        assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4566    }
4567
4568    /// The rate-limit memory is per connection and goes away with it.
4569    #[test]
4570    fn orphan_goodbye_rate_limit_state_is_released_with_the_connection() {
4571        let router = Router::with_default_self_handler();
4572        let connection = router.begin_connection();
4573        let id = connection.id();
4574        assert!(router.orphan_goodbyes.claim(id, 7));
4575        assert!(!router.orphan_goodbyes.claim(id, 7));
4576        assert!(router.orphan_goodbyes.claim(id, 8));
4577        drop(connection);
4578        assert!(router
4579            .orphan_goodbyes
4580            .last_sent
4581            .lock()
4582            .unwrap()
4583            .get(&id)
4584            .is_none());
4585    }
4586
4587    #[test]
4588    fn channel_zero_cannot_be_registered_as_backend() {
4589        let mut router = Router::with_default_self_handler();
4590
4591        let err = router.register_backend(0, EchoBackend).unwrap_err();
4592
4593        assert_eq!(err, RouterError::ReservedChannelZero);
4594    }
4595}