1use std::{
2 collections::HashMap,
3 error::Error,
4 fmt,
5 sync::{
6 atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
7 Arc, Mutex,
8 },
9 time::{Duration, Instant},
10};
11
12use subc_protocol::{error_codes, ErrorBody, Flags, FrameType, Priority};
13use tokio::sync::{mpsc, Notify};
14use tracing::{debug, warn};
15
16use crate::{
17 control::ControlHandler,
18 forwarding::{
19 principal_label, CloseReason, ConnectionCloseReceiver, DataRoute, DataRouteState,
20 ForwardingError, ForwardingTable, RouteBinding, RouteRelease, UndeliveredFrame,
21 },
22 registry::ConnectionId,
23 DaemonCounters, Frame, FrameBuildError,
24};
25
26#[derive(Debug)]
35pub struct OutboundFrame {
36 pub frame: Frame,
37 pub enqueued_at: std::time::Instant,
38 pub(crate) flushed: Option<tokio::sync::oneshot::Sender<()>>,
39 pub(crate) charge: Option<EgressCharge>,
44}
45
46impl OutboundFrame {
47 fn charged(frame: Frame, charge: EgressCharge) -> Self {
48 Self {
49 frame,
50 enqueued_at: charge.enqueued_at,
51 flushed: None,
52 charge: Some(charge),
53 }
54 }
55}
56
57fn queued_frame_bytes(frame: &Frame) -> usize {
60 subc_protocol::HEADER_LEN + frame.body.len()
61}
62
63#[derive(Debug, Clone, Copy, PartialEq, Eq)]
66pub(crate) struct EgressBacklog {
67 pub queued_bytes: usize,
68 pub queued_frames: usize,
69 pub oldest_age: Option<Duration>,
76}
77
78#[derive(Debug)]
83struct EgressAccounting {
84 ingress_seen: AtomicBool,
87 ingress_terminal: AtomicBool,
88 egress_terminal: AtomicBool,
89 egress_stopped: Notify,
90 byte_budget: usize,
91 queued_bytes: AtomicUsize,
92 queued_frames: AtomicUsize,
93 time_base: Instant,
95 oldest_enqueued_nanos: AtomicU64,
98 waiters: AtomicUsize,
101 freed: Notify,
104 client_delivery_closed: AtomicBool,
105 client_delivery_stopped: Notify,
106 #[cfg(test)]
107 retire_after_reserve: AtomicBool,
108}
109
110impl EgressAccounting {
111 fn new(byte_budget: usize) -> Self {
112 Self {
113 ingress_seen: AtomicBool::new(false),
114 ingress_terminal: AtomicBool::new(false),
115 egress_terminal: AtomicBool::new(false),
116 egress_stopped: Notify::new(),
117 byte_budget,
118 queued_bytes: AtomicUsize::new(0),
119 queued_frames: AtomicUsize::new(0),
120 time_base: tokio::time::Instant::now().into_std(),
121 oldest_enqueued_nanos: AtomicU64::new(0),
122 waiters: AtomicUsize::new(0),
123 freed: Notify::new(),
124 client_delivery_closed: AtomicBool::new(false),
125 client_delivery_stopped: Notify::new(),
126 #[cfg(test)]
127 retire_after_reserve: AtomicBool::new(false),
128 }
129 }
130
131 fn stamp(&self, at: Instant) -> u64 {
132 (at.saturating_duration_since(self.time_base).as_nanos() as u64).saturating_add(1)
133 }
134
135 fn try_charge(self: &Arc<Self>, bytes: usize) -> Option<EgressCharge> {
140 let mut current = self.queued_bytes.load(Ordering::SeqCst);
143 loop {
144 if current != 0 && current.saturating_add(bytes) > self.byte_budget {
145 return None;
146 }
147 match self.queued_bytes.compare_exchange_weak(
148 current,
149 current + bytes,
150 Ordering::SeqCst,
151 Ordering::SeqCst,
152 ) {
153 Ok(_) => return Some(self.record(bytes)),
154 Err(actual) => current = actual,
155 }
156 }
157 }
158
159 fn charge_unconditionally(self: &Arc<Self>, bytes: usize) -> EgressCharge {
162 self.queued_bytes.fetch_add(bytes, Ordering::SeqCst);
163 self.record(bytes)
164 }
165
166 fn record(self: &Arc<Self>, bytes: usize) -> EgressCharge {
167 let enqueued_at = tokio::time::Instant::now().into_std();
168 let stamp = self.stamp(enqueued_at);
169 if self.queued_frames.fetch_add(1, Ordering::AcqRel) == 0 {
170 self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
173 }
174 EgressCharge {
175 accounting: Arc::clone(self),
176 bytes,
177 stamp,
178 enqueued_at,
179 }
180 }
181
182 fn taken_by_writer(&self, stamp: u64) {
184 self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
185 }
186
187 fn release(&self, bytes: usize, stamp: u64) {
188 self.queued_bytes.fetch_sub(bytes, Ordering::SeqCst);
189 if self.queued_frames.fetch_sub(1, Ordering::AcqRel) == 1 {
190 let _ = self.oldest_enqueued_nanos.compare_exchange(
193 stamp,
194 0,
195 Ordering::AcqRel,
196 Ordering::Acquire,
197 );
198 }
199 if self.waiters.load(Ordering::SeqCst) != 0 {
200 self.freed.notify_waiters();
201 }
202 }
203
204 fn backlog(&self) -> EgressBacklog {
205 let queued_frames = self.queued_frames.load(Ordering::Acquire);
206 let stamp = self.oldest_enqueued_nanos.load(Ordering::Acquire);
207 let oldest_age = (queued_frames != 0 && stamp != 0).then(|| {
208 let enqueued = self.time_base + Duration::from_nanos(stamp - 1);
209 tokio::time::Instant::from_std(enqueued).elapsed()
210 });
211 EgressBacklog {
212 queued_bytes: self.queued_bytes.load(Ordering::Acquire),
213 queued_frames,
214 oldest_age,
215 }
216 }
217}
218
219#[derive(Debug)]
222pub(crate) struct EgressCharge {
223 accounting: Arc<EgressAccounting>,
224 bytes: usize,
225 stamp: u64,
226 enqueued_at: Instant,
227}
228
229impl EgressCharge {
230 pub(crate) fn taken_by_writer(&self) {
233 self.accounting.taken_by_writer(self.stamp);
234 }
235}
236
237impl Drop for EgressCharge {
238 fn drop(&mut self) {
239 self.accounting.release(self.bytes, self.stamp);
240 }
241}
242
243#[derive(Debug)]
249pub(crate) struct EgressPermit {
250 permit: mpsc::OwnedPermit<OutboundFrame>,
251 accounting: Arc<EgressAccounting>,
252}
253
254impl EgressPermit {
255 pub(crate) fn send(self, frame: Frame) -> bool {
259 if self.accounting.egress_terminal.load(Ordering::Acquire) {
260 return true;
261 }
262 let charge = self
263 .accounting
264 .charge_unconditionally(queued_frame_bytes(&frame));
265 let sender = self.permit.send(OutboundFrame::charged(frame, charge));
266 sender.is_closed()
267 }
268}
269
270impl std::ops::Deref for OutboundFrame {
273 type Target = Frame;
274
275 fn deref(&self) -> &Frame {
276 &self.frame
277 }
278}
279
280#[cfg(test)]
283pub(crate) mod test_log {
284 use std::{
285 io::Write,
286 sync::{Arc, Mutex, OnceLock},
287 };
288
289 #[derive(Clone)]
290 struct TestLogWriter(Arc<Mutex<Vec<u8>>>);
291
292 impl Write for TestLogWriter {
293 fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
294 self.0
295 .lock()
296 .expect("test log capture is not poisoned")
297 .extend(buffer);
298 Ok(buffer.len())
299 }
300
301 fn flush(&mut self) -> std::io::Result<()> {
302 Ok(())
303 }
304 }
305
306 pub(crate) fn log_capture(
307 level: tracing::Level,
308 ) -> (Arc<Mutex<Vec<u8>>>, tracing::dispatcher::DefaultGuard) {
309 static INTEREST_ANCHOR: OnceLock<tracing::Dispatch> = OnceLock::new();
316 INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
317 let output = Arc::new(Mutex::new(Vec::new()));
318 let writer = Arc::clone(&output);
319 let subscriber = tracing_subscriber::fmt()
320 .with_max_level(level)
321 .with_ansi(false)
322 .without_time()
323 .with_target(false)
324 .with_writer(move || TestLogWriter(Arc::clone(&writer)))
325 .finish();
326 let guard = tracing::subscriber::set_default(subscriber);
327 (output, guard)
328 }
329
330 pub(crate) fn captured_logs(output: &Arc<Mutex<Vec<u8>>>) -> String {
331 String::from_utf8(
332 output
333 .lock()
334 .expect("test log capture is not poisoned")
335 .clone(),
336 )
337 .expect("tracing output is UTF-8")
338 }
339
340 #[cfg(test)]
341 mod tests {
342 use super::*;
343
344 fn emit_capture_probe(source: &str) {
345 tracing::error!(source, "parallel capture probe");
346 }
347
348 #[test]
349 fn parallel_callsite_registration_keeps_the_test_capture_enabled() {
350 let (output, _guard) = log_capture(tracing::Level::ERROR);
351 std::thread::spawn(|| emit_capture_probe("other-thread"))
354 .join()
355 .expect("unsubscribed thread exits");
356 emit_capture_probe("captured-thread");
357
358 let logs = captured_logs(&output);
359 assert!(
360 logs.contains("captured-thread"),
361 "capture was disabled: {logs}"
362 );
363 assert!(!logs.contains("other-thread"), "capture leaked: {logs}");
364 }
365 }
366}
367
368#[derive(Debug, Clone)]
376pub struct FrameSink {
377 tx: mpsc::Sender<OutboundFrame>,
378 accounting: Arc<EgressAccounting>,
379}
380
381impl FrameSink {
382 pub(crate) fn terminate_ingress(&self) {
383 self.accounting
384 .ingress_terminal
385 .store(true, Ordering::Release);
386 }
387
388 pub(crate) fn ingress_terminal(&self) -> bool {
389 self.accounting.ingress_terminal.load(Ordering::Acquire)
390 }
391
392 pub(crate) fn terminate_egress(&self) {
395 self.accounting
396 .egress_terminal
397 .store(true, Ordering::Release);
398 self.accounting.egress_stopped.notify_waiters();
399 self.accounting.freed.notify_waiters();
400 }
401
402 pub(crate) fn egress_terminal(&self) -> bool {
403 self.accounting.egress_terminal.load(Ordering::Acquire)
404 }
405
406 pub(crate) fn first_ingress_frame(&self) -> bool {
407 !self.accounting.ingress_seen.swap(true, Ordering::Relaxed)
408 }
409 pub fn new(tx: mpsc::Sender<OutboundFrame>) -> Self {
413 Self::with_byte_budget(tx, crate::server::CONNECTION_EGRESS_BYTE_BUDGET)
414 }
415
416 pub(crate) fn with_byte_budget(tx: mpsc::Sender<OutboundFrame>, byte_budget: usize) -> Self {
417 Self {
418 tx,
419 accounting: Arc::new(EgressAccounting::new(byte_budget)),
420 }
421 }
422
423 async fn charge_waiting(&self, bytes: usize) -> Option<EgressCharge> {
430 if self.egress_terminal() {
431 return None;
432 }
433 if let Some(charge) = self.accounting.try_charge(bytes) {
434 return Some(charge);
435 }
436 struct Waiting<'a>(&'a AtomicUsize);
439 impl Drop for Waiting<'_> {
440 fn drop(&mut self) {
441 self.0.fetch_sub(1, Ordering::SeqCst);
442 }
443 }
444 self.accounting.waiters.fetch_add(1, Ordering::SeqCst);
445 let _waiting = Waiting(&self.accounting.waiters);
446 loop {
447 let freed = self.accounting.freed.notified();
448 tokio::pin!(freed);
449 freed.as_mut().enable();
452 if self.egress_terminal() {
453 return None;
454 }
455 if let Some(charge) = self.accounting.try_charge(bytes) {
456 return Some(charge);
457 }
458 tokio::select! {
459 _ = &mut freed => {}
460 _ = self.tx.closed() => return None,
461 }
462 }
463 }
464
465 async fn send_outbound(&self, outbound: OutboundFrame) -> Result<bool, RouterError> {
470 let stopped = self.accounting.egress_stopped.notified();
471 tokio::pin!(stopped);
472 stopped.as_mut().enable();
473 if self.egress_terminal() {
474 return Ok(false);
475 }
476 let header = &outbound.frame.header;
477 let closed = || {
478 RouterError::backend_with_epoch(
479 header.channel,
480 header.epoch,
481 header.corr,
482 "connection writer closed",
483 )
484 };
485 let permit = tokio::select! {
486 biased;
487 _ = &mut stopped => return Ok(false),
488 permit = self.tx.reserve() => permit.map_err(|_| closed())?,
489 };
490 if self.egress_terminal() {
491 return Ok(false);
492 }
493 permit.send(outbound);
494 Ok(true)
495 }
496
497 pub async fn send(&self, frame: Frame) -> Result<(), RouterError> {
498 if self.egress_terminal() {
499 return Ok(());
500 }
501 let channel = frame.header.channel;
502 let epoch = frame.header.epoch;
503 let corr = frame.header.corr;
504 let closed =
505 || RouterError::backend_with_epoch(channel, epoch, corr, "connection writer closed");
506 let Some(charge) = self.charge_waiting(queued_frame_bytes(&frame)).await else {
507 return if self.egress_terminal() {
508 Ok(())
509 } else {
510 Err(closed())
511 };
512 };
513 self.send_outbound(OutboundFrame::charged(frame, charge))
514 .await
515 .map(|_| ())
516 }
517
518 pub async fn send_flushed(&self, frame: Frame) -> Result<(), RouterError> {
525 self.send_flushed_notifying_enqueue(frame, || {}).await
526 }
527
528 pub(crate) async fn send_flushed_notifying_enqueue(
531 &self,
532 frame: Frame,
533 queued: impl FnOnce(),
534 ) -> Result<(), RouterError> {
535 let (tx, rx) = tokio::sync::oneshot::channel();
536 let charge = self
537 .charge_waiting(queued_frame_bytes(&frame))
538 .await
539 .ok_or_else(|| RouterError::backend(0, 0, "connection writer closed"))?;
540 let mut outbound = OutboundFrame::charged(frame, charge);
541 outbound.flushed = Some(tx);
542 if !self.send_outbound(outbound).await? {
543 return Err(RouterError::backend(0, 0, "connection egress retired"));
544 }
545 queued();
546 rx.await
547 .map_err(|_| RouterError::backend(0, 0, "connection flush failed"))
548 }
549
550 pub(crate) async fn reserve_owned(&self) -> Result<EgressPermit, RouterError> {
551 let stopped = self.accounting.egress_stopped.notified();
552 tokio::pin!(stopped);
553 stopped.as_mut().enable();
554 if self.egress_terminal() {
555 return Err(RouterError::backend(0, 0, "connection egress retired"));
556 }
557 let permit = tokio::select! {
558 biased;
559 _ = &mut stopped => return Err(RouterError::backend(0, 0, "connection egress retired")),
560 permit = self.tx.clone().reserve_owned() => permit
561 .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?,
562 };
563 #[cfg(test)]
564 if self
565 .accounting
566 .retire_after_reserve
567 .swap(false, Ordering::Relaxed)
568 {
569 self.terminate_egress();
570 }
571 if self.egress_terminal() {
572 return Err(RouterError::backend(0, 0, "connection egress retired"));
573 }
574 Ok(EgressPermit {
575 permit,
576 accounting: Arc::clone(&self.accounting),
577 })
578 }
579
580 #[cfg(test)]
581 pub(crate) fn try_reserve_owned(&self) -> Result<EgressPermit, RouterError> {
582 if self.egress_terminal() {
583 return Err(RouterError::backend(0, 0, "connection egress retired"));
584 }
585 let permit = self
586 .tx
587 .clone()
588 .try_reserve_owned()
589 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
590 Ok(EgressPermit {
591 permit,
592 accounting: Arc::clone(&self.accounting),
593 })
594 }
595
596 pub(crate) fn is_closed(&self) -> bool {
597 self.tx.is_closed()
598 }
599
600 pub(crate) fn backlog(&self) -> EgressBacklog {
602 self.accounting.backlog()
603 }
604
605 pub(crate) fn close_client_delivery(&self) {
608 self.accounting
609 .client_delivery_closed
610 .store(true, Ordering::Release);
611 self.accounting.client_delivery_stopped.notify_waiters();
612 }
613
614 async fn send_to_client(&self, frame: Frame) -> Result<(), ClientDeliveryFailure> {
617 if self.egress_terminal() {
618 return Ok(());
619 }
620 let frame_type = frame.header.ty;
621 let flags = frame.header.flags;
622 let stopped = self.accounting.client_delivery_stopped.notified();
623 tokio::pin!(stopped);
624 stopped.as_mut().enable();
625 let failure = |cause, error| ClientDeliveryFailure {
626 diagnosis: ClientDeliveryDiagnosis {
627 frame_type,
628 flags,
629 cause,
630 },
631 error,
632 };
633 let closed = || {
634 RouterError::backend_with_epoch(
635 frame.header.channel,
636 frame.header.epoch,
637 frame.header.corr,
638 "client connection closed",
639 )
640 };
641 if self
642 .accounting
643 .client_delivery_closed
644 .load(Ordering::Acquire)
645 {
646 return Err(failure("client_closed", closed()));
647 }
648 let (frame, error) = match self.try_send_owned(frame) {
649 Ok(()) => return Ok(()),
650 Err(refused) => refused,
651 };
652 if self.is_closed() {
653 return Err(failure("client_closed", error));
654 }
655 let grace = crate::server::CONNECTION_EGRESS_STALL_GRACE;
656 let age = self.backlog().oldest_age.unwrap_or_default();
657 if age > grace {
658 return Err(failure("oldest_frame_older_than_250_ms", error));
659 }
660 let result = tokio::select! {
661 biased;
662 _ = &mut stopped => return Err(failure("client_closed", error)),
663 result = tokio::time::timeout(grace.saturating_sub(age), self.send(frame)) => result,
664 };
665 match result {
666 Ok(Ok(())) => Ok(()),
667 Ok(Err(error)) => Err(failure("client_closed", error)),
668 Err(_) => Err(failure("waited_250_ms_without_room", error)),
669 }
670 }
671
672 pub(crate) fn try_send(&self, frame: Frame) -> Result<(), RouterError> {
676 self.try_send_owned(frame).map_err(|(_, error)| error)
677 }
678
679 fn try_send_owned(&self, frame: Frame) -> Result<(), (Frame, RouterError)> {
680 if self.egress_terminal() {
681 return Ok(());
682 }
683 let channel = frame.header.channel;
684 let epoch = frame.header.epoch;
685 let corr = frame.header.corr;
686 let unavailable = |why: String| {
687 RouterError::backend_with_epoch(
688 channel,
689 epoch,
690 corr,
691 format!("connection writer unavailable: {why}"),
692 )
693 };
694 let bytes = queued_frame_bytes(&frame);
695 let Some(charge) = self.accounting.try_charge(bytes) else {
696 if self.tx.is_closed() {
697 return Err((frame, unavailable("channel closed".to_string())));
698 }
699 return Err((
700 frame,
701 unavailable(format!(
702 "egress byte budget exhausted ({} queued bytes, frame of {bytes} bytes, budget {})",
703 self.accounting.queued_bytes.load(Ordering::Acquire),
704 self.accounting.byte_budget
705 )),
706 ));
707 };
708 if self.egress_terminal() {
711 return Ok(());
712 }
713 self.tx
714 .try_send(OutboundFrame::charged(frame, charge))
715 .map_err(|err| {
716 let error = unavailable(err.to_string());
717 (err.into_inner().frame, error)
718 })
719 }
720}
721
722#[derive(Debug)]
723struct ClientDeliveryFailure {
724 diagnosis: ClientDeliveryDiagnosis,
725 error: RouterError,
726}
727
728#[derive(Debug)]
730pub(crate) struct ClientDeliveryDiagnosis {
731 pub frame_type: FrameType,
732 pub flags: Flags,
733 pub cause: &'static str,
734}
735
736impl fmt::Display for ClientDeliveryFailure {
737 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
738 self.error.fmt(f)
739 }
740}
741
742const ORPHAN_ROUTE_GOODBYE_INTERVAL: Duration = Duration::from_secs(5);
754
755const ORPHAN_ROUTE_GOODBYE_PRUNE_AT: usize = 256;
762
763#[derive(Debug, Default)]
767struct OrphanGoodbyeLimiter {
768 last_sent: Mutex<HashMap<ConnectionId, HashMap<u16, tokio::time::Instant>>>,
769}
770
771impl OrphanGoodbyeLimiter {
772 fn claim(&self, connection_id: ConnectionId, channel: u16) -> bool {
775 let now = tokio::time::Instant::now();
776 let mut last_sent = self
777 .last_sent
778 .lock()
779 .unwrap_or_else(std::sync::PoisonError::into_inner);
780 let channels = last_sent.entry(connection_id).or_default();
781 if channels.get(&channel).is_some_and(|sent| {
782 now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
783 }) {
784 return false;
785 }
786 if channels.len() >= ORPHAN_ROUTE_GOODBYE_PRUNE_AT {
787 channels.retain(|_, sent| {
788 now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
789 });
790 }
791 channels.insert(channel, now);
792 true
793 }
794
795 fn forget_connection(&self, connection_id: ConnectionId) {
796 self.last_sent
797 .lock()
798 .unwrap_or_else(std::sync::PoisonError::into_inner)
799 .remove(&connection_id);
800 }
801}
802
803#[derive(Debug, Clone)]
805pub struct RouteCtx {
806 pub connection_id: ConnectionId,
807 pub egress: FrameSink,
808}
809
810#[derive(Debug, Clone)]
816pub enum Backend {
817 Echo(EchoBackend),
818 Forward(ForwardBackend),
819}
820
821impl From<EchoBackend> for Backend {
822 fn from(backend: EchoBackend) -> Self {
823 Self::Echo(backend)
824 }
825}
826
827impl From<ForwardBackend> for Backend {
828 fn from(backend: ForwardBackend) -> Self {
829 Self::Forward(backend)
830 }
831}
832
833impl Backend {
834 pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
835 match self {
836 Self::Echo(backend) => backend.handle(ctx, frame).await,
837 Self::Forward(backend) => backend.handle(ctx, frame).await,
838 }
839 }
840}
841
842pub struct Router {
863 backends: HashMap<u16, Backend>,
864 control: Arc<ControlHandler>,
865 forwarding: Arc<ForwardingTable>,
866 forward_backend: ForwardBackend,
867 counters: DaemonCounters,
868 next_connection_id: AtomicU64,
869 orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
870}
871
872impl Router {
873 pub fn with_control_handler(control: Arc<ControlHandler>) -> Self {
874 control.install_swap_promotion_observer();
877 let forwarding = control.forwarding();
878 let counters = control.counters();
879 Self {
880 backends: HashMap::new(),
881 control,
882 forwarding: Arc::clone(&forwarding),
883 forward_backend: ForwardBackend::new(forwarding),
884 counters,
885 next_connection_id: AtomicU64::new(1),
887 orphan_goodbyes: Arc::default(),
888 }
889 }
890
891 pub(crate) fn install_registration_service(self: &Arc<Self>, auth: crate::server::ServerAuth) {
894 self.control
895 .install_registration_service(Arc::downgrade(self), auth);
896 }
897
898 pub fn with_default_self_handler() -> Self {
899 Self::with_control_handler(Arc::new(ControlHandler::default()))
900 }
901
902 pub fn forwarding(&self) -> Arc<ForwardingTable> {
903 Arc::clone(&self.forwarding)
904 }
905
906 pub fn register_backend(
907 &mut self,
908 channel: u16,
909 backend: impl Into<Backend>,
910 ) -> Result<(), RouterError> {
911 self.register_backend_arc(channel, Arc::new(backend.into()))
912 }
913
914 pub(crate) fn register_backend_arc(
915 &mut self,
916 channel: u16,
917 backend: Arc<Backend>,
918 ) -> Result<(), RouterError> {
919 if channel == 0 {
920 return Err(RouterError::ReservedChannelZero);
921 }
922 if self.backends.contains_key(&channel) {
923 return Err(RouterError::DuplicateChannel { channel });
924 }
925 self.backends.insert(channel, backend.as_ref().clone());
926 Ok(())
927 }
928
929 fn record_module_frame_drop(&self, connection_id: ConnectionId) -> Result<(), RouterError> {
932 let module_id = self
933 .forwarding
934 .module_id_for_connection(connection_id)
935 .map_err(RouterError::Forwarding)?;
936 self.counters
937 .increment_module_frames_dropped_no_route(module_id.as_deref());
938 Ok(())
939 }
940
941 fn handle_orphan_module_frame(&self, ctx: &RouteCtx, frame: &Frame) -> Result<(), RouterError> {
953 let channel = frame.header.channel;
954 let epoch = frame.header.epoch;
955 let module_id = self
956 .forwarding
957 .module_id_for_connection(ctx.connection_id)
958 .map_err(RouterError::Forwarding)?;
959 self.counters
960 .increment_module_frames_dropped_no_route(module_id.as_deref());
961 if self
962 .forwarding
963 .module_route_epoch_was_allocated(ctx.connection_id, channel, epoch)
964 .map_err(RouterError::Forwarding)?
965 {
966 self.counters
967 .increment_module_frames_dropped_released_route(module_id.as_deref());
968 }
969 if frame.header.ty == FrameType::Goodbye
970 || !self.orphan_goodbyes.claim(ctx.connection_id, channel)
971 {
972 return Ok(());
973 }
974 let goodbye = Frame::build_with_version(
975 frame.header.ver,
976 FrameType::Goodbye,
977 Flags::new(false, Priority::Passive, false),
978 channel,
979 epoch,
980 0,
981 Vec::new(),
982 )
983 .map_err(RouterError::FrameBuild)?;
984 match ctx.egress.try_send(goodbye) {
985 Ok(()) => {
986 self.counters.increment_module_orphan_route_goodbyes_sent();
987 debug!(
988 connection_id = ctx.connection_id.get(),
989 module_id = module_id.as_deref().unwrap_or("unknown"),
990 channel,
991 epoch,
992 "answered module frame on a route the daemon does not hold with a route GOODBYE"
993 );
994 }
995 Err(err) => {
996 self.forwarding.warn_frame_drop(
997 ctx.connection_id,
998 channel,
999 epoch,
1000 "daemon_to_module",
1001 if ctx.egress.is_closed() {
1002 "endpoint_gone"
1003 } else {
1004 "egress_budget"
1005 },
1006 );
1007 debug!(
1008 connection_id = ctx.connection_id.get(),
1009 module_id = module_id.as_deref().unwrap_or("unknown"),
1010 channel,
1011 epoch,
1012 error = %err,
1013 "could not enqueue route GOODBYE for module frame on a route the daemon does not hold; the next such frame retries"
1014 );
1015 }
1016 }
1017 Ok(())
1018 }
1019
1020 pub fn begin_connection(&self) -> RouterConnection {
1021 let raw = self.next_connection_id.fetch_add(1, Ordering::Relaxed);
1022 let id = ConnectionId::new(raw);
1023 let close_receiver = self.forwarding.register_connection_close(id);
1024 RouterConnection {
1025 id,
1026 control_handler: Arc::clone(&self.control),
1027 forwarding: Arc::clone(&self.forwarding),
1028 close_receiver: Some(close_receiver),
1029 orphan_goodbyes: Arc::clone(&self.orphan_goodbyes),
1030 }
1031 }
1032
1033 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1034 self.control.route_open_target(frame)
1035 }
1036
1037 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
1038 self.control.record_local_connection(connection);
1039 }
1040
1041 pub(crate) fn record_captured_peer(
1042 &self,
1043 connection: ConnectionId,
1044 peer: subc_os::local_peer::CapturedPeer,
1045 ) {
1046 self.control.record_captured_peer(connection, peer);
1047 }
1048
1049 pub(crate) fn record_caller_endpoints(
1050 &self,
1051 connection: ConnectionId,
1052 endpoints: Option<(std::net::SocketAddr, std::net::SocketAddr)>,
1053 ) {
1054 self.control.record_caller_endpoints(connection, endpoints);
1055 }
1056
1057 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1058 self.control.defers_privileged_control(ctx, frame)
1059 }
1060
1061 pub(crate) fn defers_config_approval(&self, frame: &Frame) -> bool {
1062 self.control.defers_config_approval(frame)
1063 }
1064
1065 pub(crate) fn reserve_config_approval(
1066 &self,
1067 ctx: &RouteCtx,
1068 frame: &Frame,
1069 ) -> Result<
1070 Option<crate::config_gate::runtime::ApprovalRequestGuard>,
1071 crate::config_gate::GateError,
1072 > {
1073 self.control.reserve_config_approval(ctx, frame)
1074 }
1075
1076 pub(crate) fn route_open_capacity_refusal(
1077 &self,
1078 ctx: &RouteCtx,
1079 frame: &Frame,
1080 target_module_id: &str,
1081 in_flight: usize,
1082 limit: usize,
1083 ) -> Result<Frame, RouterError> {
1084 self.control
1085 .route_open_capacity_refusal(ctx, frame, target_module_id, in_flight, limit)
1086 }
1087
1088 pub async fn route_for_connection(
1089 &self,
1090 ctx: &RouteCtx,
1091 frame: Frame,
1092 ) -> Result<(), RouterError> {
1093 self.route_for_connection_started(ctx, frame, None).await
1094 }
1095
1096 pub(crate) async fn observe_connection_wait<F: std::future::Future>(
1097 &self,
1098 ctx: &RouteCtx,
1099 kind: &'static str,
1100 future: F,
1101 ) -> F::Output {
1102 self.counters
1103 .observe_reader_wait(
1104 || {
1105 let module = self
1106 .forwarding
1107 .module_id_for_connection(ctx.connection_id)
1108 .ok()
1109 .flatten();
1110 let principal = if module.is_some() { "module" } else { "direct" };
1111 (
1112 module.unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
1113 principal.into(),
1114 )
1115 },
1116 kind,
1117 future,
1118 )
1119 .await
1120 }
1121
1122 pub(crate) async fn route_for_connection_started(
1123 &self,
1124 ctx: &RouteCtx,
1125 frame: Frame,
1126 dispatch_started_at: Option<Instant>,
1127 ) -> Result<(), RouterError> {
1128 let first = ctx.egress.first_ingress_frame();
1130 self.route_for_connection_ingress(ctx, frame, dispatch_started_at, first)
1131 .await
1132 }
1133
1134 pub(crate) async fn route_for_connection_ingress(
1135 &self,
1136 ctx: &RouteCtx,
1137 frame: Frame,
1138 dispatch_started_at: Option<Instant>,
1139 first: bool,
1140 ) -> Result<(), RouterError> {
1141 if ctx.egress.ingress_terminal() {
1142 return Ok(());
1143 }
1144 let channel = frame.header.channel;
1145 let epoch = frame.header.epoch;
1146 let corr = frame.header.corr;
1147 if channel == 0 {
1148 debug!(
1149 connection_id = ctx.connection_id.get(),
1150 corr,
1151 frame_type = ?frame.header.ty,
1152 "routing control frame"
1153 );
1154 let dispatch_started_at = (frame.header.ty == FrameType::Request)
1161 .then(|| dispatch_started_at.unwrap_or_else(Instant::now));
1162 let review_request = self.control.config_desktop_review_request(&frame);
1165 let responses = self
1166 .control
1167 .handle_control_frame_ingress(ctx, frame, dispatch_started_at, first)
1168 .await?;
1169 for response in responses {
1170 if let Some(view) = review_request
1171 .then(|| {
1172 self.control
1173 .config_desktop_review_response(ctx.connection_id, &response)
1174 })
1175 .flatten()
1176 {
1177 self.observe_connection_wait(
1178 ctx,
1179 "reply_egress",
1180 self.control
1181 .deliver_config_desktop_review(ctx, response, view),
1182 )
1183 .await?;
1184 continue;
1185 }
1186 self.observe_connection_wait(ctx, "reply_egress", ctx.egress.send(response))
1187 .await?;
1188 }
1189 return Ok(());
1190 }
1191
1192 let data_route = self
1193 .forwarding
1194 .lookup_data_route(ctx.connection_id, channel, epoch)
1195 .map_err(RouterError::Forwarding)?;
1196
1197 match data_route {
1198 DataRoute::Module(DataRouteState::EpochMismatch) => {
1199 self.forwarding.warn_frame_drop(
1200 ctx.connection_id,
1201 channel,
1202 epoch,
1203 "module_to_client",
1204 "stale_epoch",
1205 );
1206 if frame.header.ty == FrameType::Request {
1207 self.counters
1208 .increment_module_requests_dropped_stale_route();
1209 let err = RouterError::StaleRouteEpoch {
1210 channel,
1211 epoch,
1212 corr,
1213 };
1214 if let Some(error_frame) = err.to_error_frame() {
1215 self.observe_connection_wait(
1216 ctx,
1217 "reply_egress",
1218 ctx.egress.send(error_frame),
1219 )
1220 .await?;
1221 }
1222 } else {
1223 self.handle_orphan_module_frame(ctx, &frame)?;
1224 }
1225 debug!(
1226 connection_id = ctx.connection_id.get(),
1227 channel, epoch, corr, "dropping module frame for stale route epoch"
1228 );
1229 return Ok(());
1230 }
1231 DataRoute::Module(DataRouteState::Reserved) => {
1232 self.forwarding.warn_frame_drop(
1233 ctx.connection_id,
1234 channel,
1235 epoch,
1236 "module_to_client",
1237 "missing_binding",
1238 );
1239 if frame.header.ty == FrameType::Request {
1240 self.counters
1241 .increment_module_requests_dropped_stale_route();
1242 let err = RouterError::UnknownChannel {
1243 channel,
1244 epoch,
1245 corr,
1246 };
1247 if let Some(error_frame) = err.to_error_frame() {
1248 self.observe_connection_wait(
1249 ctx,
1250 "reply_egress",
1251 ctx.egress.send(error_frame),
1252 )
1253 .await?;
1254 }
1255 } else {
1256 self.record_module_frame_drop(ctx.connection_id)?;
1257 }
1258 debug!(
1259 connection_id = ctx.connection_id.get(),
1260 channel, epoch, corr, "dropping module frame for reserved route handle"
1261 );
1262 return Ok(());
1263 }
1264 DataRoute::Module(DataRouteState::Absent) => {
1265 self.forwarding.warn_frame_drop(
1266 ctx.connection_id,
1267 channel,
1268 epoch,
1269 "module_to_client",
1270 "unknown_channel",
1271 );
1272 if frame.header.ty == FrameType::Request {
1273 self.counters
1274 .increment_module_requests_dropped_stale_route();
1275 let err = RouterError::UnknownChannel {
1276 channel,
1277 epoch,
1278 corr,
1279 };
1280 if let Some(error_frame) = err.to_error_frame() {
1281 self.observe_connection_wait(
1282 ctx,
1283 "reply_egress",
1284 ctx.egress.send(error_frame),
1285 )
1286 .await?;
1287 }
1288 } else {
1289 self.handle_orphan_module_frame(ctx, &frame)?;
1290 }
1291 debug!(
1292 connection_id = ctx.connection_id.get(),
1293 channel, epoch, corr, "dropping module frame for absent route handle"
1294 );
1295 return Ok(());
1296 }
1297 DataRoute::Module(DataRouteState::Bound(route)) => {
1298 if frame.header.ty == FrameType::Goodbye {
1299 if let RouteRelease::Removed(target) = self
1300 .forwarding
1301 .release_module_route(ctx.connection_id, channel, epoch)
1302 .map_err(RouterError::Forwarding)?
1303 {
1304 let mut goodbye = frame;
1305 goodbye.header.channel = target.channel;
1306 goodbye.header.epoch = target.epoch;
1307 goodbye.header.flags = goodbye.header.flags.without_daemon_origin();
1311 if let Err(err) = self
1312 .observe_connection_wait(
1313 ctx,
1314 "module_to_client",
1315 target.sink.send_to_client(goodbye),
1316 )
1317 .await
1318 {
1319 if !target.close_on_delivery_failure() {
1320 self.forwarding.warn_frame_drop(
1321 target.connection_id,
1322 target.channel,
1323 target.epoch,
1324 "module_to_client",
1325 err.diagnosis.cause,
1326 );
1327 }
1328 if target.close_on_delivery_failure()
1329 && self
1330 .forwarding
1331 .escalate_client_delivery_failure_diagnosed(
1332 target.connection_id,
1333 target.channel,
1334 target.epoch,
1335 CloseReason::new(
1336 "route_goodbye_delivery_failed",
1337 format!(
1338 "failed to enqueue route GOODBYE for client channel {}: {err}",
1339 target.channel
1340 ),
1341 ),
1342 UndeliveredFrame {
1343 module_id: Some(&route.module_id),
1344 sink: &target.sink,
1345 },
1346 Some(&err.diagnosis),
1347 )
1348 .map_err(RouterError::Forwarding)?
1349 {
1350 self.counters.increment_goodbye_relay_client_failed();
1351 }
1352 }
1353 } else {
1354 self.forwarding.warn_frame_drop(
1355 ctx.connection_id,
1356 channel,
1357 epoch,
1358 "module_to_client",
1359 "missing_binding",
1360 );
1361 }
1362 return Ok(());
1363 }
1364
1365 let bytes = queued_frame_bytes(&frame);
1369 let limit = route.client_sink.accounting.byte_budget;
1370 if bytes > limit {
1371 let ty = frame.header.ty;
1372 let flags = frame.header.flags;
1373 drop(frame);
1374 return self
1375 .refuse_oversized_module_frame(ctx, &route, (ty, flags), corr, bytes)
1376 .await;
1377 }
1378
1379 let releases_credit = is_terminal_frame(frame.header.ty);
1385 if releases_credit {
1386 route.flow.release_corr(corr);
1387 }
1388 let mut frame = frame;
1389 frame.header.channel = route.client_channel;
1390 frame.header.epoch = route.client_epoch;
1391 frame.header.flags = frame.header.flags.without_daemon_origin();
1397 if let Err(err) = self
1398 .counters
1399 .observe_reader_wait(
1400 || (route.module_id.clone(), principal_label(&route.principal)),
1401 "module_to_client",
1402 route.client_sink.send_to_client(frame),
1403 )
1404 .await
1405 {
1406 if self
1407 .forwarding
1408 .escalate_client_delivery_failure_diagnosed(
1409 route.client_connection_id,
1410 route.client_channel,
1411 route.client_epoch,
1412 CloseReason::new(
1413 "module_to_client_delivery_failed",
1414 format!(
1415 "failed to enqueue module frame for client channel {} corr {corr}: {err}",
1416 route.client_channel
1417 ),
1418 ),
1419 UndeliveredFrame {
1420 module_id: Some(&route.module_id),
1421 sink: &route.client_sink,
1422 },
1423 Some(&err.diagnosis),
1424 )
1425 .map_err(RouterError::Forwarding)?
1426 {
1427 self.counters
1428 .increment_client_egress_close_delivery_failed();
1429 }
1430 return Ok(());
1431 }
1432 return Ok(());
1433 }
1434 DataRoute::Client(DataRouteState::EpochMismatch) => {
1435 self.forwarding.warn_frame_drop(
1436 ctx.connection_id,
1437 channel,
1438 epoch,
1439 "client_to_module",
1440 "stale_epoch",
1441 );
1442 if frame.header.ty == FrameType::Request {
1443 self.counters.increment_client_frames_dropped_stale_route();
1444 let err = RouterError::StaleRouteEpoch {
1446 channel,
1447 epoch,
1448 corr,
1449 };
1450 if let Some(error_frame) = err.to_error_frame() {
1451 self.observe_connection_wait(
1452 ctx,
1453 "reply_egress",
1454 ctx.egress.send(error_frame),
1455 )
1456 .await?;
1457 }
1458 }
1459 debug!(
1460 connection_id = ctx.connection_id.get(),
1461 channel, epoch, corr, "dropping client frame for stale route epoch"
1462 );
1463 return Ok(());
1464 }
1465 DataRoute::Client(DataRouteState::Reserved) => {
1466 self.forwarding.warn_frame_drop(
1467 ctx.connection_id,
1468 channel,
1469 epoch,
1470 "client_to_module",
1471 "missing_binding",
1472 );
1473 if frame.header.ty == FrameType::Request {
1474 let err = RouterError::UnknownChannel {
1475 channel,
1476 epoch,
1477 corr,
1478 };
1479 if let Some(error_frame) = err.to_error_frame() {
1480 self.observe_connection_wait(
1481 ctx,
1482 "reply_egress",
1483 ctx.egress.send(error_frame),
1484 )
1485 .await?;
1486 }
1487 }
1488 return Ok(());
1489 }
1490 DataRoute::Client(DataRouteState::Bound(route)) => {
1491 if frame.header.ty == FrameType::Goodbye {
1492 let _ = self
1493 .control
1494 .handle_route_goodbye(ctx.connection_id, channel, epoch)?;
1495 return Ok(());
1496 }
1497 return self.forward_backend.handle_bound(frame, route).await;
1498 }
1499 DataRoute::Client(DataRouteState::Absent) => {}
1500 }
1501
1502 if let Some(backend) = self.backends.get(&channel) {
1503 return backend.handle(ctx.clone(), frame).await.inspect_err(|_| {
1504 self.forwarding.warn_frame_drop(
1505 ctx.connection_id,
1506 channel,
1507 epoch,
1508 "client_to_backend",
1509 "endpoint_gone",
1510 );
1511 });
1512 }
1513 self.forwarding.warn_frame_drop(
1514 ctx.connection_id,
1515 channel,
1516 epoch,
1517 "client_to_module",
1518 "unknown_channel",
1519 );
1520 if frame.header.ty == FrameType::Request {
1521 let err = RouterError::UnknownChannel {
1522 channel,
1523 epoch,
1524 corr,
1525 };
1526 if let Some(error_frame) = err.to_error_frame() {
1527 self.observe_connection_wait(ctx, "reply_egress", ctx.egress.send(error_frame))
1528 .await?;
1529 }
1530 }
1531 Ok(())
1532 }
1533
1534 async fn refuse_oversized_module_frame(
1535 &self,
1536 ctx: &RouteCtx,
1537 route: &RouteBinding,
1538 header: (FrameType, Flags),
1539 corr: u64,
1540 bytes: usize,
1541 ) -> Result<(), RouterError> {
1542 let (ty, flags) = header;
1543 let limit = route.client_sink.accounting.byte_budget;
1544 if ty == FrameType::Push {
1545 let diagnosis = ClientDeliveryDiagnosis {
1549 frame_type: ty,
1550 flags,
1551 cause: "frame_too_large",
1552 };
1553 if self
1554 .forwarding
1555 .escalate_client_delivery_failure_diagnosed(
1556 route.client_connection_id,
1557 route.client_channel,
1558 route.client_epoch,
1559 CloseReason::new(
1560 error_codes::FRAME_TOO_LARGE,
1561 format!(
1562 "oversized push: size={bytes} limit={limit} (header plus body bytes)"
1563 ),
1564 ),
1565 UndeliveredFrame {
1566 module_id: Some(&route.module_id),
1567 sink: &route.client_sink,
1568 },
1569 Some(&diagnosis),
1570 )
1571 .map_err(RouterError::Forwarding)?
1572 {
1573 self.counters
1574 .increment_client_egress_close_delivery_failed();
1575 }
1576 return Ok(());
1577 }
1578
1579 warn!(
1580 connection_id = route.client_connection_id.get(),
1581 module_id = %route.module_id,
1582 client_principal = %principal_label(&route.principal),
1583 client_channel = route.client_channel,
1584 frame_type = ?ty,
1585 frame_bytes = bytes,
1586 byte_limit = limit,
1587 "refusing oversized module-to-client frame with ERROR frame_too_large"
1588 );
1589 if ty == FrameType::Request {
1590 let error = frame_too_large_error(
1593 route.module_channel,
1594 route.module_epoch,
1595 corr,
1596 bytes,
1597 limit,
1598 )?;
1599 self.observe_connection_wait(ctx, "reply_egress", ctx.egress.send(error))
1600 .await?;
1601 } else {
1602 route.flow.release_corr(corr);
1605 let error = frame_too_large_error(
1606 route.client_channel,
1607 route.client_epoch,
1608 corr,
1609 bytes,
1610 limit,
1611 )?;
1612 if let Err(err) = self
1615 .counters
1616 .observe_reader_wait(
1617 || (route.module_id.clone(), principal_label(&route.principal)),
1618 "module_to_client",
1619 route.client_sink.send_to_client(error),
1620 )
1621 .await
1622 {
1623 if self
1624 .forwarding
1625 .escalate_client_delivery_failure_diagnosed(
1626 route.client_connection_id,
1627 route.client_channel,
1628 route.client_epoch,
1629 CloseReason::new(
1630 "module_to_client_delivery_failed",
1631 format!(
1632 "failed to enqueue frame_too_large for client channel {} corr {corr}: {err}",
1633 route.client_channel
1634 ),
1635 ),
1636 UndeliveredFrame {
1637 module_id: Some(&route.module_id),
1638 sink: &route.client_sink,
1639 },
1640 Some(&err.diagnosis),
1641 )
1642 .map_err(RouterError::Forwarding)?
1643 {
1644 self.counters
1645 .increment_client_egress_close_delivery_failed();
1646 }
1647 }
1648 }
1649 Ok(())
1650 }
1651}
1652
1653impl Default for Router {
1654 fn default() -> Self {
1655 Self::with_default_self_handler()
1656 }
1657}
1658
1659#[must_use]
1661pub struct RouterConnection {
1662 id: ConnectionId,
1663 control_handler: Arc<ControlHandler>,
1664 forwarding: Arc<ForwardingTable>,
1665 close_receiver: Option<ConnectionCloseReceiver>,
1666 orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
1667}
1668
1669impl RouterConnection {
1670 pub fn id(&self) -> ConnectionId {
1671 self.id
1672 }
1673
1674 pub(crate) fn take_close_receiver(&mut self) -> ConnectionCloseReceiver {
1675 self.close_receiver
1676 .take()
1677 .expect("connection close receiver can only be taken once")
1678 }
1679}
1680
1681impl Drop for RouterConnection {
1682 fn drop(&mut self) {
1683 self.orphan_goodbyes.forget_connection(self.id);
1684 let _ = self.control_handler.cleanup_connection(self.id);
1687 self.forwarding.unregister_connection_close(self.id);
1691 }
1692}
1693
1694#[derive(Debug, Default, Clone, Copy)]
1697pub struct EchoBackend;
1698
1699impl EchoBackend {
1700 pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1701 let response = Frame::build_with_version(
1702 frame.header.ver,
1703 FrameType::Response,
1704 frame.header.flags,
1705 frame.header.channel,
1706 frame.header.epoch,
1707 frame.header.corr,
1708 frame.body,
1709 )
1710 .map_err(RouterError::FrameBuild)?;
1711 ctx.egress.send(response).await
1712 }
1713}
1714
1715#[derive(Debug, Clone)]
1717pub struct ForwardBackend {
1718 forwarding: Arc<ForwardingTable>,
1719}
1720
1721impl ForwardBackend {
1722 pub fn new(forwarding: Arc<ForwardingTable>) -> Self {
1723 Self { forwarding }
1724 }
1725
1726 pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1727 let channel = frame.header.channel;
1728 let corr = frame.header.corr;
1729 let route = match self
1730 .forwarding
1731 .lookup_data_route(ctx.connection_id, channel, frame.header.epoch)
1732 .map_err(RouterError::Forwarding)?
1733 {
1734 DataRoute::Client(DataRouteState::Bound(route)) => route,
1735 DataRoute::Client(_) | DataRoute::Module(_) => {
1736 self.forwarding.warn_frame_drop(
1737 ctx.connection_id,
1738 channel,
1739 frame.header.epoch,
1740 "client_to_module",
1741 "missing_binding",
1742 );
1743 return Err(RouterError::UnknownChannel {
1744 channel,
1745 epoch: frame.header.epoch,
1746 corr,
1747 });
1748 }
1749 };
1750 self.handle_bound(frame, route).await
1751 }
1752
1753 pub(crate) async fn handle_bound(
1754 &self,
1755 frame: Frame,
1756 route: Arc<RouteBinding>,
1757 ) -> Result<(), RouterError> {
1758 let channel = frame.header.channel;
1759 let corr = frame.header.corr;
1760 let frame_type = frame.header.ty;
1761
1762 let acquired_credit = frame_type == FrameType::Request;
1765 if acquired_credit && route.flow.busy_refusals() {
1766 use crate::forwarding::RequestAdmission;
1767 match self
1768 .forwarding
1769 .try_admit_request(
1770 &route,
1771 frame.header.epoch,
1772 corr,
1773 frame.header.flags.is_subscription(),
1774 )
1775 .map_err(RouterError::Forwarding)?
1776 {
1777 RequestAdmission::Admitted => {}
1778 RequestAdmission::Busy(detail) => {
1779 let principal = principal_label(&route.principal);
1780 let counters = self.forwarding.counters_ref();
1781 counters.increment_route_busy(&route.module_id, &principal);
1782 if route.flow.should_warn_busy() {
1783 warn!(module_id = %route.module_id, client_principal = %principal, client_channel = channel,
1784 route_epoch = route.client_epoch, ?detail, "request refused: route_busy");
1785 }
1786 let error = RouterError::RouteBusy {
1787 channel,
1788 epoch: frame.header.epoch,
1789 corr,
1790 detail: Box::new(detail),
1791 };
1792 let refusal = error
1793 .to_error_frame()
1794 .expect("route_busy is a serializable wire error");
1795 counters
1796 .observe_reader_wait(
1797 || (route.module_id.clone(), principal),
1798 "route_busy_egress",
1799 route.client_sink.send(refusal),
1800 )
1801 .await?;
1802 return Ok(());
1803 }
1804 RequestAdmission::ModuleReloading => {
1805 return Err(RouterError::route_error_with_epoch(
1806 channel,
1807 frame.header.epoch,
1808 corr,
1809 error_codes::MODULE_RELOADING,
1810 format!("module endpoint for route channel {channel} is reloading"),
1811 ));
1812 }
1813 RequestAdmission::Stale => {
1814 return Err(RouterError::StaleRouteEpoch {
1815 channel,
1816 epoch: frame.header.epoch,
1817 corr,
1818 })
1819 }
1820 }
1821 } else if acquired_credit {
1822 let counters = self.forwarding.counters_ref();
1823 if let Err(err) = counters
1824 .observe_reader_wait(
1825 || (route.module_id.clone(), principal_label(&route.principal)),
1826 "legacy_credit",
1827 route
1828 .flow
1829 .acquire_tagged(corr, frame.header.flags.is_subscription()),
1830 )
1831 .await
1832 {
1833 self.forwarding.warn_frame_drop(
1834 route.client_connection_id,
1835 channel,
1836 frame.header.epoch,
1837 "client_to_module",
1838 "endpoint_gone",
1839 );
1840 if self
1847 .forwarding
1848 .endpoint_is_draining(route.module_endpoint)
1849 .map_err(RouterError::Forwarding)?
1850 {
1851 return Err(RouterError::route_error_with_epoch(
1852 channel,
1853 frame.header.epoch,
1854 corr,
1855 "module_reloading",
1856 format!("module endpoint for route channel {channel} is reloading"),
1857 ));
1858 }
1859 return Err(RouterError::backend_with_epoch(
1860 channel,
1861 frame.header.epoch,
1862 corr,
1863 format!("{err} for route channel {channel}"),
1864 ));
1865 }
1866 }
1867
1868 let mut frame = frame;
1869 frame.header.channel = route.module_channel;
1870 frame.header.epoch = route.module_epoch;
1871 frame.header.flags = frame.header.flags.without_daemon_origin();
1875 let counters = self.forwarding.counters_ref();
1876 let result = counters
1877 .observe_reader_wait(
1878 || (route.module_id.clone(), principal_label(&route.principal)),
1879 "module_queue",
1880 route.module_sink.send(frame),
1881 )
1882 .await
1883 .map_err(|err| {
1884 self.forwarding.warn_frame_drop(
1885 route.client_connection_id,
1886 channel,
1887 route.client_epoch,
1888 "client_to_module",
1889 "endpoint_gone",
1890 );
1891 RouterError::backend_with_epoch(channel, route.client_epoch, corr, err.to_string())
1892 });
1893 if acquired_credit && result.is_err() {
1894 route.flow.release_corr(corr);
1895 }
1896 result
1897 }
1898}
1899
1900fn is_terminal_frame(frame_type: FrameType) -> bool {
1901 matches!(
1902 frame_type,
1903 FrameType::Response | FrameType::Error | FrameType::StreamEnd
1904 )
1905}
1906
1907#[derive(Debug, Clone, PartialEq, Eq)]
1910pub enum RouterError {
1911 ReservedChannelZero,
1912 DuplicateChannel {
1913 channel: u16,
1914 },
1915 UnknownChannel {
1916 channel: u16,
1917 epoch: u32,
1918 corr: u64,
1919 },
1920 StaleRouteEpoch {
1921 channel: u16,
1922 epoch: u32,
1923 corr: u64,
1924 },
1925 RouteBusy {
1926 channel: u16,
1927 epoch: u32,
1928 corr: u64,
1929 detail: Box<subc_protocol::RouteBusyDetail>,
1930 },
1931 Backend {
1932 channel: u16,
1933 epoch: u32,
1934 corr: u64,
1935 message: String,
1936 },
1937 RouteError {
1938 channel: u16,
1939 epoch: u32,
1940 corr: u64,
1941 code: String,
1942 message: String,
1943 },
1944 FrameBuild(FrameBuildError),
1945 Forwarding(ForwardingError),
1946}
1947
1948impl RouterError {
1949 pub fn backend(channel: u16, corr: u64, message: impl Into<String>) -> Self {
1950 Self::backend_with_epoch(channel, 0, corr, message)
1951 }
1952
1953 pub fn backend_with_epoch(
1954 channel: u16,
1955 epoch: u32,
1956 corr: u64,
1957 message: impl Into<String>,
1958 ) -> Self {
1959 Self::Backend {
1960 channel,
1961 epoch,
1962 corr,
1963 message: message.into(),
1964 }
1965 }
1966
1967 pub fn route_error(
1968 channel: u16,
1969 corr: u64,
1970 code: impl Into<String>,
1971 message: impl Into<String>,
1972 ) -> Self {
1973 Self::route_error_with_epoch(channel, 0, corr, code, message)
1974 }
1975
1976 pub fn route_error_with_epoch(
1977 channel: u16,
1978 epoch: u32,
1979 corr: u64,
1980 code: impl Into<String>,
1981 message: impl Into<String>,
1982 ) -> Self {
1983 Self::RouteError {
1984 channel,
1985 epoch,
1986 corr,
1987 code: code.into(),
1988 message: message.into(),
1989 }
1990 }
1991
1992 pub fn to_error_frame(&self) -> Option<Frame> {
1994 match self {
1995 Self::RouteBusy {
1996 channel,
1997 epoch,
1998 corr,
1999 detail,
2000 } => {
2001 let body = serde_json::to_vec(&ErrorBody {
2002 code: error_codes::ROUTE_BUSY.into(),
2003 message: "route has no free request or watch credit; retry on the same route"
2004 .into(),
2005 detail: Some(serde_json::to_value(detail).ok()?),
2006 })
2007 .ok()?;
2008 Frame::build(
2009 FrameType::Error,
2010 daemon_error_flags(),
2011 *channel,
2012 *epoch,
2013 *corr,
2014 body,
2015 )
2016 .ok()
2017 }
2018 Self::UnknownChannel {
2019 channel,
2020 epoch,
2021 corr,
2022 } => error_frame(
2023 *channel,
2024 *epoch,
2025 *corr,
2026 error_codes::UNKNOWN_CHANNEL,
2027 format!("unknown channel {channel}"),
2028 ),
2029 Self::StaleRouteEpoch {
2030 channel,
2031 epoch,
2032 corr,
2033 } => error_frame(
2034 *channel,
2035 *epoch,
2036 *corr,
2037 error_codes::STALE_ROUTE_EPOCH,
2038 format!("stale route epoch for channel {channel}"),
2039 ),
2040 Self::Backend {
2041 channel,
2042 epoch,
2043 corr,
2044 message,
2045 } => error_frame(*channel, *epoch, *corr, "backend_error", message.clone()),
2046 Self::RouteError {
2047 channel,
2048 epoch,
2049 corr,
2050 code,
2051 message,
2052 } => error_frame(*channel, *epoch, *corr, code, message.clone()),
2053 Self::ReservedChannelZero
2054 | Self::DuplicateChannel { .. }
2055 | Self::FrameBuild(_)
2056 | Self::Forwarding(_) => None,
2057 }
2058 }
2059}
2060
2061pub(crate) fn daemon_error_flags() -> Flags {
2073 Flags::new(false, Priority::Passive, false).with_daemon_origin()
2074}
2075
2076fn error_frame(channel: u16, epoch: u32, corr: u64, code: &str, message: String) -> Option<Frame> {
2077 let body = serde_json::to_vec(&ErrorBody {
2078 code: code.to_string(),
2079 message,
2080 detail: None,
2081 })
2082 .ok()?;
2083
2084 Frame::build(
2085 FrameType::Error,
2086 daemon_error_flags(),
2087 channel,
2088 epoch,
2089 corr,
2090 body,
2091 )
2092 .ok()
2093}
2094
2095fn frame_too_large_error(
2096 channel: u16,
2097 epoch: u32,
2098 corr: u64,
2099 bytes: usize,
2100 limit: usize,
2101) -> Result<Frame, RouterError> {
2102 let body = serde_json::to_vec(&ErrorBody {
2103 code: error_codes::FRAME_TOO_LARGE.to_string(),
2104 message: format!(
2105 "module frame of {bytes} bytes exceeds client egress limit of {limit} bytes"
2106 ),
2107 detail: Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit })),
2108 })
2109 .expect("serializing a frame size error");
2110 Frame::build(
2111 FrameType::Error,
2112 daemon_error_flags(),
2113 channel,
2114 epoch,
2115 corr,
2116 body,
2117 )
2118 .map_err(RouterError::FrameBuild)
2119}
2120
2121impl fmt::Display for RouterError {
2122 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2123 match self {
2124 Self::ReservedChannelZero => write!(f, "channel 0 is reserved for subc"),
2125 Self::DuplicateChannel { channel } => {
2126 write!(f, "backend already registered for channel {channel}")
2127 }
2128 Self::UnknownChannel { channel, corr, .. } => {
2129 write!(f, "unknown channel {channel} for corr {corr}")
2130 }
2131 Self::StaleRouteEpoch { channel, corr, .. } => {
2132 write!(f, "stale route epoch for channel {channel} corr {corr}")
2133 }
2134 Self::RouteBusy { channel, corr, .. } => {
2135 write!(f, "route busy on channel {channel} corr {corr}")
2136 }
2137 Self::Backend {
2138 channel,
2139 corr,
2140 message,
2141 ..
2142 } => write!(
2143 f,
2144 "backend error on channel {channel} corr {corr}: {message}"
2145 ),
2146 Self::RouteError {
2147 channel,
2148 corr,
2149 code,
2150 message,
2151 ..
2152 } => write!(
2153 f,
2154 "route error {code} on channel {channel} corr {corr}: {message}"
2155 ),
2156 Self::FrameBuild(err) => write!(f, "failed to build routed frame: {err}"),
2157 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
2158 }
2159 }
2160}
2161
2162impl Error for RouterError {
2163 fn source(&self) -> Option<&(dyn Error + 'static)> {
2164 match self {
2165 Self::FrameBuild(err) => Some(err),
2166 Self::Forwarding(err) => Some(err),
2167 Self::ReservedChannelZero
2168 | Self::DuplicateChannel { .. }
2169 | Self::UnknownChannel { .. }
2170 | Self::StaleRouteEpoch { .. }
2171 | Self::RouteBusy { .. }
2172 | Self::Backend { .. }
2173 | Self::RouteError { .. } => None,
2174 }
2175 }
2176}
2177
2178#[cfg(test)]
2179mod tests {
2180 include!("route_busy_router_tests.rs");
2181 use super::*;
2182 use crate::{
2183 forwarding::RouteBindRelayOutcome,
2184 supervise::{ModuleSpec, RestartPolicy, Supervisor, SupervisorHandle},
2185 ControlHandler, Registry,
2186 };
2187 use std::{
2188 sync::{mpsc as std_mpsc, Arc},
2189 time::Duration,
2190 };
2191 use subc_control::ModuleProtocol;
2192 use subc_protocol::{manifest::Concurrency, ErrorBody, Flags, FrameType, Priority};
2193 use tokio::sync::mpsc;
2194
2195 pub(crate) use crate::router::test_log::{captured_logs, log_capture};
2196
2197 fn logged_millis(logs: &str, field: &str) -> u64 {
2198 logs.split_whitespace()
2199 .find_map(|part| part.strip_prefix(field))
2200 .and_then(|value| value.parse().ok())
2201 .unwrap_or_else(|| panic!("missing numeric {field} in logs: {logs}"))
2202 }
2203
2204 fn request(channel: u16, corr: u64, body: &[u8]) -> Frame {
2205 Frame::build(
2206 FrameType::Request,
2207 Flags::new(true, Priority::Interactive, false),
2208 channel,
2209 0,
2210 corr,
2211 body.to_vec(),
2212 )
2213 .unwrap()
2214 }
2215
2216 fn ping(corr: u64) -> Frame {
2217 Frame::build(
2218 FrameType::Ping,
2219 Flags::new(false, Priority::Passive, false),
2220 0,
2221 0,
2222 corr,
2223 Vec::new(),
2224 )
2225 .unwrap()
2226 }
2227
2228 fn route_ctx() -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
2229 let (tx, rx) = mpsc::channel(8);
2230 (
2231 RouteCtx {
2232 connection_id: ConnectionId::LOCAL,
2233 egress: FrameSink::new(tx),
2234 },
2235 rx,
2236 )
2237 }
2238
2239 #[tokio::test]
2240 async fn retirement_egress_all_send_paths_drop_after_fence() {
2241 let (tx, mut rx) = mpsc::channel(8);
2242 let sink = FrameSink::new(tx);
2243 let reserved = sink.reserve_owned().await.unwrap();
2244 let immediate = sink.try_reserve_owned().unwrap();
2245 sink.terminate_egress();
2246 assert!(reserved.send(ping(1)));
2247 assert!(immediate.send(ping(2)));
2248 sink.send(ping(3)).await.unwrap();
2249 sink.try_send(ping(4)).unwrap();
2250 sink.send_to_client(ping(5)).await.unwrap();
2251 let queued = AtomicBool::new(false);
2252 assert!(sink
2253 .send_flushed_notifying_enqueue(ping(6), || queued.store(true, Ordering::Relaxed))
2254 .await
2255 .is_err());
2256 assert!(!queued.load(Ordering::Relaxed));
2257 assert!(sink.send_flushed(ping(7)).await.is_err());
2258 assert!(sink.reserve_owned().await.is_err());
2259 assert!(sink.try_reserve_owned().is_err());
2260 assert!(
2261 rx.try_recv().is_err(),
2262 "retired frames must not enter any writer queue"
2263 );
2264 assert_eq!(sink.backlog().queued_frames, 0);
2265 assert_eq!(sink.backlog().queued_bytes, 0);
2266 }
2267
2268 #[tokio::test]
2269 async fn retirement_egress_reservation_rechecks_after_slot_acquisition() {
2270 let (tx, mut rx) = mpsc::channel(1);
2271 let sink = FrameSink::new(tx);
2272 sink.accounting
2275 .retire_after_reserve
2276 .store(true, Ordering::Relaxed);
2277 assert!(
2278 sink.reserve_owned().await.is_err(),
2279 "a slot acquired before retirement must not escape as a live reservation"
2280 );
2281 assert_eq!(
2282 sink.tx.capacity(),
2283 1,
2284 "refusal must release the reserved slot"
2285 );
2286 assert!(rx.try_recv().is_err());
2287 }
2288
2289 #[tokio::test]
2290 async fn retirement_egress_pending_budget_and_slot_sends_stop() {
2291 for budget in [1, usize::MAX] {
2292 let (tx, mut rx) = mpsc::channel(1);
2293 let sink = FrameSink::with_byte_budget(tx, budget);
2294 sink.try_send(ping(1)).unwrap();
2295 let sending = sink.send(ping(2));
2296 tokio::pin!(sending);
2297 assert!(poll_once(sending.as_mut()).await.is_pending());
2298 sink.terminate_egress();
2299 tokio::time::timeout(Duration::from_secs(2), sending)
2300 .await
2301 .unwrap()
2302 .unwrap();
2303 drop(rx.try_recv().unwrap());
2304 assert!(
2305 rx.try_recv().is_err(),
2306 "a send parked before retirement must not enqueue afterward"
2307 );
2308 assert_eq!(sink.backlog().queued_bytes, 0);
2309 }
2310 }
2311
2312 #[tokio::test]
2313 async fn retirement_egress_pending_flush_and_reservation_stop() {
2314 let (tx, mut rx) = mpsc::channel(1);
2315 let sink = FrameSink::new(tx);
2316 sink.try_send(ping(1)).unwrap();
2317 let queued = AtomicBool::new(false);
2318 let sending =
2319 sink.send_flushed_notifying_enqueue(ping(2), || queued.store(true, Ordering::Relaxed));
2320 let reserving = sink.reserve_owned();
2321 tokio::pin!(sending, reserving);
2322 assert!(poll_once(sending.as_mut()).await.is_pending());
2323 assert!(poll_once(reserving.as_mut()).await.is_pending());
2324 sink.terminate_egress();
2325 assert!(tokio::time::timeout(Duration::from_secs(2), sending)
2326 .await
2327 .unwrap()
2328 .is_err());
2329 assert!(tokio::time::timeout(Duration::from_secs(2), reserving)
2330 .await
2331 .unwrap()
2332 .is_err());
2333 assert!(!queued.load(Ordering::Relaxed));
2334 drop(rx.try_recv().unwrap());
2335 assert!(rx.try_recv().is_err());
2336 assert_eq!(sink.backlog().queued_bytes, 0);
2337 }
2338
2339 #[tokio::test]
2340 async fn echo_backend_returns_response_with_byte_identical_body() {
2341 let mut router = Router::with_default_self_handler();
2342 router.register_backend(7, EchoBackend).unwrap();
2343 let (ctx, mut rx) = route_ctx();
2344 let body = b"{not parsed}\0\xff";
2345
2346 router
2347 .route_for_connection(&ctx, request(7, 123, body))
2348 .await
2349 .unwrap();
2350 let response = rx.recv().await.unwrap();
2351
2352 assert_eq!(response.header.ty, FrameType::Response);
2353 assert_eq!(response.header.channel, 7);
2354 assert_eq!(response.header.corr, 123);
2355 assert_eq!(response.body, body);
2356 assert!(rx.try_recv().is_err());
2357 }
2358
2359 #[tokio::test]
2360 async fn unknown_channel_emits_canonical_error_frame() {
2361 let router = Router::with_default_self_handler();
2362 let (ctx, mut rx) = route_ctx();
2363
2364 router
2365 .route_for_connection(&ctx, request(99, 5, b"payload"))
2366 .await
2367 .unwrap();
2368 let error_frame = rx.recv().await.unwrap();
2369
2370 assert_eq!(error_frame.header.ty, FrameType::Error);
2371 assert_eq!(error_frame.header.channel, 99);
2372 assert_eq!(error_frame.header.corr, 5);
2373 let body: ErrorBody = serde_json::from_slice(&error_frame.body).unwrap();
2374 assert_eq!(body.code, "unknown_channel");
2375 assert_eq!(body.message, "unknown channel 99");
2376 }
2377
2378 #[tokio::test]
2383 async fn daemon_authored_data_plane_errors_carry_daemon_origin() {
2384 let router = Router::with_default_self_handler();
2385 let (ctx, mut rx) = route_ctx();
2386 router
2387 .route_for_connection(&ctx, request(99, 5, b"payload"))
2388 .await
2389 .unwrap();
2390 let unknown = rx.recv().await.unwrap();
2391 assert_eq!(unknown.header.ty, FrameType::Error);
2392 let body: ErrorBody = serde_json::from_slice(&unknown.body).unwrap();
2393 assert_eq!(body.code, "unknown_channel");
2394 assert!(
2395 unknown.header.flags.is_daemon_origin(),
2396 "unknown_channel must carry daemon origin: {:?}",
2397 unknown.header.flags
2398 );
2399
2400 let (
2401 router,
2402 _forwarding,
2403 client_ctx,
2404 mut client_rx,
2405 _module_ctx,
2406 _module_egress_rx,
2407 _module_rx,
2408 pending,
2409 ) = dynamic_route_fixture(true);
2410 let _ = client_rx.recv().await;
2411 router
2412 .route_for_connection(
2413 &client_ctx,
2414 route_frame(
2415 FrameType::Request,
2416 pending.client_channel,
2417 pending.client_epoch + 1,
2418 702,
2419 ),
2420 )
2421 .await
2422 .unwrap();
2423 let stale = client_rx.recv().await.unwrap();
2424 assert_eq!(stale.header.ty, FrameType::Error);
2425 let body: ErrorBody = serde_json::from_slice(&stale.body).unwrap();
2426 assert_eq!(body.code, "stale_route_epoch");
2427 assert!(
2428 stale.header.flags.is_daemon_origin(),
2429 "stale_route_epoch must carry daemon origin: {:?}",
2430 stale.header.flags
2431 );
2432 }
2433
2434 #[tokio::test]
2439 async fn module_error_with_daemon_bit_is_stripped_before_client_delivery() {
2440 let (
2441 router,
2442 _forwarding,
2443 client_ctx,
2444 mut client_rx,
2445 module_ctx,
2446 _module_egress_rx,
2447 mut module_rx,
2448 pending,
2449 ) = dynamic_route_fixture(true);
2450 let _ = client_rx.recv().await;
2451 router
2454 .route_for_connection(
2455 &client_ctx,
2456 route_frame(
2457 FrameType::Request,
2458 pending.client_channel,
2459 pending.client_epoch,
2460 701,
2461 ),
2462 )
2463 .await
2464 .unwrap();
2465 let _ = module_rx.recv().await;
2466
2467 let body = serde_json::to_vec(&ErrorBody::new("module_reloading", "try again")).unwrap();
2468 let spoof = Frame::build(
2469 FrameType::Error,
2470 Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2471 pending.module_channel,
2472 pending.module_epoch,
2473 701,
2474 body.clone(),
2475 )
2476 .unwrap();
2477 router
2478 .route_for_connection(&module_ctx, spoof)
2479 .await
2480 .unwrap();
2481 let delivered = client_rx.recv().await.unwrap();
2482 assert_eq!(delivered.header.ty, FrameType::Error);
2483 assert_eq!(delivered.header.channel, pending.client_channel);
2484 assert_eq!(delivered.header.epoch, pending.client_epoch);
2485 assert_eq!(delivered.header.corr, 701);
2486 assert!(
2487 !delivered.header.flags.is_daemon_origin(),
2488 "module-set daemon origin must be cleared: {:?}",
2489 delivered.header.flags
2490 );
2491 assert_eq!(delivered.body, body, "the body travels byte-identical");
2492 }
2493
2494 #[tokio::test]
2497 async fn module_response_with_daemon_bit_is_stripped_preserving_other_flags() {
2498 use subc_protocol::FLAG_SUBSCRIPTION;
2499 let (
2500 router,
2501 _forwarding,
2502 client_ctx,
2503 mut client_rx,
2504 module_ctx,
2505 _module_egress_rx,
2506 mut module_rx,
2507 pending,
2508 ) = dynamic_route_fixture(true);
2509 let _ = client_rx.recv().await;
2510 router
2511 .route_for_connection(
2512 &client_ctx,
2513 route_frame(
2514 FrameType::Request,
2515 pending.client_channel,
2516 pending.client_epoch,
2517 701,
2518 ),
2519 )
2520 .await
2521 .unwrap();
2522 let _ = module_rx.recv().await;
2523
2524 let sent_flags = Flags(
2525 Flags::new(true, Priority::Background, true).0
2526 | FLAG_SUBSCRIPTION
2527 | subc_protocol::FLAG_DAEMON_ORIGIN,
2528 );
2529 assert!(sent_flags.is_daemon_origin());
2530 let response = Frame::build(
2531 FrameType::Response,
2532 sent_flags,
2533 pending.module_channel,
2534 pending.module_epoch,
2535 701,
2536 b"route-body".to_vec(),
2537 )
2538 .unwrap();
2539 router
2540 .route_for_connection(&module_ctx, response)
2541 .await
2542 .unwrap();
2543 let delivered = client_rx.recv().await.unwrap();
2544 assert_eq!(delivered.header.ty, FrameType::Response);
2545 assert!(
2546 !delivered.header.flags.is_daemon_origin(),
2547 "module-set daemon origin must be cleared: {:?}",
2548 delivered.header.flags
2549 );
2550 assert!(
2551 delivered.header.flags.is_binary(),
2552 "binary flag must survive: {:?}",
2553 delivered.header.flags
2554 );
2555 assert!(
2556 delivered.header.flags.is_last(),
2557 "LAST flag must survive: {:?}",
2558 delivered.header.flags
2559 );
2560 assert!(
2561 delivered.header.flags.is_subscription(),
2562 "subscription flag must survive: {:?}",
2563 delivered.header.flags
2564 );
2565 assert_eq!(
2566 delivered.header.flags.priority(),
2567 Some(Priority::Background),
2568 "priority must survive"
2569 );
2570 }
2571
2572 #[tokio::test]
2575 async fn client_frame_with_daemon_bit_is_stripped_before_module_delivery() {
2576 let (
2577 router,
2578 _forwarding,
2579 client_ctx,
2580 mut client_rx,
2581 _module_ctx,
2582 _module_egress_rx,
2583 mut module_rx,
2584 pending,
2585 ) = dynamic_route_fixture(true);
2586 let _ = client_rx.recv().await;
2587 let spoof = Frame::build(
2588 FrameType::Request,
2589 Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2590 pending.client_channel,
2591 pending.client_epoch,
2592 701,
2593 b"route-body".to_vec(),
2594 )
2595 .unwrap();
2596 router
2597 .route_for_connection(&client_ctx, spoof)
2598 .await
2599 .unwrap();
2600 let delivered = module_rx.recv().await.unwrap();
2601 assert_eq!(delivered.header.ty, FrameType::Request);
2602 assert!(
2603 !delivered.header.flags.is_daemon_origin(),
2604 "client-set daemon origin must be cleared: {:?}",
2605 delivered.header.flags
2606 );
2607 }
2608
2609 #[tokio::test]
2610 async fn channel_zero_uses_control_handler_not_backend_registry() {
2611 let mut router = Router::with_default_self_handler();
2612 router.register_backend(1, EchoBackend).unwrap();
2613 let (ctx, mut rx) = route_ctx();
2614
2615 router.route_for_connection(&ctx, ping(77)).await.unwrap();
2616 let response = rx.recv().await.unwrap();
2617
2618 assert_eq!(response.header.ty, FrameType::Pong);
2619 assert_eq!(response.header.channel, 0);
2620 assert_eq!(response.header.corr, 77);
2621 assert!(response.body.is_empty());
2622 }
2623
2624 #[tokio::test]
2625 async fn slow_control_dispatch_logs_decoded_op_and_elapsed_time() {
2626 let control = Arc::new(
2627 ControlHandler::new(Arc::new(Registry::default()))
2628 .with_control_dispatch_delay(Duration::from_millis(1050)),
2629 );
2630 let router = Router::with_control_handler(control);
2631 let (ctx, mut rx) = route_ctx();
2632 let (output, guard) = log_capture(tracing::Level::WARN);
2633
2634 router
2635 .route_for_connection(&ctx, request(0, 41, br#"{"op":"server.describe"}"#))
2636 .await
2637 .expect("slow request routes");
2638 assert!(rx.recv().await.is_some(), "request receives a response");
2639 drop(guard);
2640
2641 let logs = captured_logs(&output);
2642 assert!(logs.contains("slow control dispatch"));
2643 assert!(logs.contains("op=server.describe"));
2644 assert!(logs.contains("connection_id=0"));
2645 assert!(logs.contains("corr=41"));
2646 assert!(
2647 logged_millis(&logs, "elapsed_ms=") >= 1050,
2648 "elapsed must include the injected handler delay: {logs}"
2649 );
2650 }
2651
2652 #[tokio::test]
2653 async fn fast_control_dispatch_emits_arrival_without_slow_warning() {
2654 let router = Router::with_default_self_handler();
2655 let (ctx, mut rx) = route_ctx();
2656 let (output, guard) = log_capture(tracing::Level::DEBUG);
2657
2658 router
2659 .route_for_connection(&ctx, request(0, 42, br#"{"op":"server.describe"}"#))
2660 .await
2661 .expect("fast request routes");
2662 assert!(rx.recv().await.is_some(), "request receives a response");
2663 drop(guard);
2664
2665 let logs = captured_logs(&output);
2666 assert!(logs.contains("control dispatch op=server.describe connection_id=0 corr=42"));
2667 assert!(!logs.contains("slow control dispatch"));
2668 }
2669
2670 #[tokio::test]
2671 async fn control_dispatch_arrival_is_hidden_at_info() {
2672 let router = Router::with_default_self_handler();
2673 let (ctx, mut rx) = route_ctx();
2674 let (output, guard) = log_capture(tracing::Level::INFO);
2675
2676 router
2677 .route_for_connection(&ctx, request(0, 43, br#"{"op":"server.describe"}"#))
2678 .await
2679 .expect("fast request routes");
2680 assert!(rx.recv().await.is_some(), "request receives a response");
2681 drop(guard);
2682
2683 assert!(
2684 !captured_logs(&output).contains("control dispatch"),
2685 "arrival logging must stay hidden at INFO"
2686 );
2687 }
2688
2689 #[tokio::test]
2690 async fn supervisor_list_logs_contended_snapshot_lock_only() {
2691 let registry = Arc::new(Registry::default());
2692 let handle = SupervisorHandle::new();
2693 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
2694 .with_handle(handle.clone());
2695 let module = supervisor
2696 .supervise_configured(
2697 ModuleSpec {
2698 script: None,
2699 carry_containment: false,
2700 module_id: "held-module".to_string(),
2701 program: "test-module".into(),
2702 args: Vec::new(),
2703 env: Vec::new(),
2704 reserved: false,
2705 reserved_prefixes: Vec::new(),
2706 protocol: ModuleProtocol::Subc,
2707 overlap: Default::default(),
2708 },
2709 false,
2710 )
2711 .expect("disabled test module is supervised");
2712 let router = Router::with_control_handler(Arc::new(
2713 ControlHandler::new(Arc::clone(®istry)).with_supervisor(handle),
2714 ));
2715 let (ctx, mut rx) = route_ctx();
2716 let (acquired, ready) = std_mpsc::channel();
2717 tokio::task::yield_now().await;
2720 let holder = module.hold_snapshot_for_test(acquired, Duration::from_millis(400));
2721 ready.recv().expect("holder acquired snapshot lock");
2722 let (output, guard) = log_capture(tracing::Level::WARN);
2723
2724 router
2725 .route_for_connection(&ctx, request(0, 44, br#"{"op":"supervisor.list"}"#))
2726 .await
2727 .expect("list request routes after the lock releases");
2728 assert!(
2729 rx.recv().await.is_some(),
2730 "list request receives a response"
2731 );
2732 holder.join().expect("snapshot holder exits cleanly");
2733 drop(guard);
2734
2735 let logs = captured_logs(&output);
2736 assert!(logs.contains("slow snapshot lock"));
2737 assert!(logs.contains("module_id=held-module"));
2738 assert!(logs.contains("caller=list"));
2739 assert!(
2740 logged_millis(&logs, "waited_ms=") >= 250,
2741 "wait must exceed the slow-lock threshold: {logs}"
2742 );
2743
2744 let (output, guard) = log_capture(tracing::Level::WARN);
2745 router
2746 .route_for_connection(&ctx, request(0, 45, br#"{"op":"supervisor.list"}"#))
2747 .await
2748 .expect("uncontended list request routes");
2749 assert!(
2750 rx.recv().await.is_some(),
2751 "uncontended list receives a response"
2752 );
2753 drop(guard);
2754 assert!(
2755 !captured_logs(&output).contains("slow snapshot lock"),
2756 "uncontended list acquisition must not warn"
2757 );
2758 }
2759
2760 #[tokio::test]
2761 async fn full_module_to_client_sink_requests_client_close_without_erroring_module() {
2762 let forwarding = Arc::new(ForwardingTable::default());
2763 let control = Arc::new(ControlHandler::with_forwarding(
2764 Arc::new(crate::Registry::default()),
2765 Arc::clone(&forwarding),
2766 ));
2767 let router = Router::with_control_handler(control);
2768 let module_connection = ConnectionId::new(10);
2769 let client_connection = ConnectionId::new(20);
2770 let mut close_receiver = forwarding.register_connection_close(client_connection);
2771 let (module_tx, _module_rx) = mpsc::channel(1);
2772 forwarding
2773 .register_module_connection(
2774 module_connection,
2775 "full-sink-provider".to_string(),
2776 1,
2777 Concurrency::ModuleManaged,
2778 FrameSink::new(module_tx),
2779 )
2780 .unwrap();
2781 let (client_tx, mut client_rx) = mpsc::channel(1);
2782 let pending = forwarding
2783 .begin_route_bind_relay_for_test(
2784 client_connection,
2785 FrameSink::new(client_tx),
2786 700,
2787 "full-sink-provider",
2788 )
2789 .unwrap();
2790 forwarding
2791 .complete_pending_relay(
2792 module_connection,
2793 pending.corr,
2794 RouteBindRelayOutcome::Accepted,
2795 )
2796 .unwrap();
2797
2798 let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2799 let module_ctx = RouteCtx {
2800 connection_id: module_connection,
2801 egress: FrameSink::new(module_egress_tx),
2802 };
2803 let terminal = Frame::build(
2804 FrameType::Response,
2805 Flags::new(false, Priority::Interactive, true),
2806 pending.module_channel,
2807 pending.module_epoch,
2808 701,
2809 b"terminal".to_vec(),
2810 )
2811 .unwrap();
2812
2813 router
2814 .route_for_connection(&module_ctx, terminal)
2815 .await
2816 .unwrap();
2817 let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2818 .await
2819 .expect("close request should be sent for the full client sink")
2820 .expect("close sender should include a reason");
2821 assert!(
2822 reason
2823 .to_string()
2824 .contains("module_to_client_delivery_failed"),
2825 "unexpected close reason: {reason}"
2826 );
2827 assert_eq!(client_rx.try_recv().unwrap().header.corr, 700);
2828 assert!(client_rx.try_recv().is_err());
2829 assert_eq!(
2830 router.counters.snapshot()["client_egress_close_delivery_failed"],
2831 1
2832 );
2833 }
2834
2835 #[tokio::test]
2839 async fn terminal_frame_releases_its_credit_even_when_client_delivery_fails() {
2840 let forwarding = Arc::new(ForwardingTable::default());
2841 let control = Arc::new(ControlHandler::with_forwarding(
2842 Arc::new(crate::Registry::default()),
2843 Arc::clone(&forwarding),
2844 ));
2845 let router = Router::with_control_handler(control);
2846 let module_connection = ConnectionId::new(11);
2847 let client_connection = ConnectionId::new(21);
2848 let _close_receiver = forwarding.register_connection_close(client_connection);
2849 let (module_tx, _module_rx) = mpsc::channel(1);
2850 forwarding
2851 .register_module_connection(
2852 module_connection,
2853 "credit-provider".to_string(),
2854 1,
2855 Concurrency::ModuleManaged,
2856 FrameSink::new(module_tx),
2857 )
2858 .unwrap();
2859 let (client_tx, _client_rx) = mpsc::channel(1);
2862 let pending = forwarding
2863 .begin_route_bind_relay_for_test(
2864 client_connection,
2865 FrameSink::new(client_tx),
2866 800,
2867 "credit-provider",
2868 )
2869 .unwrap();
2870 forwarding
2871 .complete_pending_relay(
2872 module_connection,
2873 pending.corr,
2874 RouteBindRelayOutcome::Accepted,
2875 )
2876 .unwrap();
2877 let DataRoute::Client(DataRouteState::Bound(route)) = forwarding
2878 .lookup_data_route(
2879 client_connection,
2880 pending.client_channel,
2881 pending.client_epoch,
2882 )
2883 .unwrap()
2884 else {
2885 panic!("expected a bound client route");
2886 };
2887 route.flow.acquire_tagged(801, false).await.unwrap();
2888 assert_eq!(route.flow.drain_in_flight(), 1);
2889
2890 let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2891 let module_ctx = RouteCtx {
2892 connection_id: module_connection,
2893 egress: FrameSink::new(module_egress_tx),
2894 };
2895 let terminal = Frame::build(
2896 FrameType::Response,
2897 Flags::new(false, Priority::Interactive, true),
2898 pending.module_channel,
2899 pending.module_epoch,
2900 801,
2901 b"terminal".to_vec(),
2902 )
2903 .unwrap();
2904 router
2905 .route_for_connection(&module_ctx, terminal)
2906 .await
2907 .unwrap();
2908
2909 assert_eq!(
2910 router.counters.snapshot()["client_egress_close_delivery_failed"],
2911 1,
2912 "the client delivery must have failed for this test to mean anything"
2913 );
2914 assert_eq!(
2915 route.flow.drain_in_flight(),
2916 0,
2917 "the module's terminal frame must release its credit even though the client could not take it"
2918 );
2919 }
2920
2921 #[tokio::test]
2922 async fn full_route_goodbye_sink_requests_target_close_without_erroring_module() {
2923 let forwarding = Arc::new(ForwardingTable::default());
2924 let control = Arc::new(ControlHandler::with_forwarding(
2925 Arc::new(crate::Registry::default()),
2926 Arc::clone(&forwarding),
2927 ));
2928 let router = Router::with_control_handler(control);
2929 let module_connection = ConnectionId::new(30);
2930 let client_connection = ConnectionId::new(40);
2931 let mut close_receiver = forwarding.register_connection_close(client_connection);
2932 let (module_tx, _module_rx) = mpsc::channel(1);
2933 forwarding
2934 .register_module_connection(
2935 module_connection,
2936 "goodbye-full-provider".to_string(),
2937 1,
2938 Concurrency::ModuleManaged,
2939 FrameSink::new(module_tx),
2940 )
2941 .unwrap();
2942 let (client_tx, mut client_rx) = mpsc::channel(1);
2943 let pending = forwarding
2944 .begin_route_bind_relay_for_test(
2945 client_connection,
2946 FrameSink::new(client_tx),
2947 800,
2948 "goodbye-full-provider",
2949 )
2950 .unwrap();
2951 forwarding
2952 .complete_pending_relay(
2953 module_connection,
2954 pending.corr,
2955 RouteBindRelayOutcome::Accepted,
2956 )
2957 .unwrap();
2958
2959 let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2960 let module_ctx = RouteCtx {
2961 connection_id: module_connection,
2962 egress: FrameSink::new(module_egress_tx),
2963 };
2964 let goodbye = Frame::build(
2965 FrameType::Goodbye,
2966 Flags::new(false, Priority::Passive, true),
2967 pending.module_channel,
2968 pending.module_epoch,
2969 801,
2970 Vec::new(),
2971 )
2972 .unwrap();
2973
2974 router
2975 .route_for_connection(&module_ctx, goodbye)
2976 .await
2977 .unwrap();
2978 let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2979 .await
2980 .expect("close request should be sent for the full GOODBYE sink")
2981 .expect("close sender should include a reason");
2982 assert!(
2983 reason.to_string().contains("route_goodbye_delivery_failed"),
2984 "unexpected close reason: {reason}"
2985 );
2986 assert_eq!(client_rx.try_recv().unwrap().header.corr, 800);
2987 assert!(client_rx.try_recv().is_err());
2988 assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 1);
2989 assert_eq!(router.counters.snapshot()["route_released_epoch_fenced"], 1);
2990 }
2991
2992 async fn multi_route_client(
2997 module_id: &str,
2998 module_connection: ConnectionId,
2999 client_connection: ConnectionId,
3000 routes: usize,
3001 ) -> (
3002 Router,
3003 FrameSink,
3004 mpsc::Receiver<OutboundFrame>,
3005 RouteCtx,
3006 Vec<crate::forwarding::PendingRouteBindRelay>,
3007 ConnectionCloseReceiver,
3008 ) {
3009 let forwarding = Arc::new(ForwardingTable::default());
3010 let control = Arc::new(ControlHandler::with_forwarding(
3011 Arc::new(crate::Registry::default()),
3012 Arc::clone(&forwarding),
3013 ));
3014 let router = Router::with_control_handler(control);
3015 let close_receiver = forwarding.register_connection_close(client_connection);
3016 let (module_tx, _module_rx) = mpsc::channel(8);
3017 forwarding
3018 .register_module_connection(
3019 module_connection,
3020 module_id.to_string(),
3021 1,
3022 Concurrency::ModuleManaged,
3023 FrameSink::new(module_tx),
3024 )
3025 .unwrap();
3026 let (client_sink, mut client_rx) = crate::server::connection_egress();
3027 let mut bound = Vec::with_capacity(routes);
3028 for index in 0..routes {
3029 let pending = forwarding
3030 .begin_route_bind_relay_for_test(
3031 client_connection,
3032 client_sink.clone(),
3033 900 + index as u64,
3034 module_id,
3035 )
3036 .unwrap();
3037 forwarding
3038 .complete_pending_relay(
3039 module_connection,
3040 pending.corr,
3041 RouteBindRelayOutcome::Accepted,
3042 )
3043 .unwrap();
3044 assert_eq!(
3045 client_rx.recv().await.unwrap().header.corr,
3046 900 + index as u64
3047 );
3048 bound.push(pending);
3049 }
3050 let (module_egress_tx, _module_egress_rx) = mpsc::channel(8);
3051 let module_ctx = RouteCtx {
3052 connection_id: module_connection,
3053 egress: FrameSink::new(module_egress_tx),
3054 };
3055 (
3056 router,
3057 client_sink,
3058 client_rx,
3059 module_ctx,
3060 bound,
3061 close_receiver,
3062 )
3063 }
3064
3065 #[test]
3072 #[ignore = "timing measurement, run on demand"]
3073 fn egress_sink_per_frame_cost() {
3074 const FRAMES: usize = 1_000_000;
3075 const BATCH: usize = 1_000;
3076 let (sink, mut rx) = crate::server::connection_egress();
3077 let template = stream_frame(9, 1, 0, vec![b't'; 200]);
3078 let started = Instant::now();
3079 for _ in 0..FRAMES / BATCH {
3080 for _ in 0..BATCH {
3081 sink.try_send(template.clone()).unwrap();
3082 }
3083 for _ in 0..BATCH {
3084 let outbound = rx.try_recv().unwrap();
3085 if let Some(charge) = &outbound.charge {
3086 charge.taken_by_writer();
3087 }
3088 drop(std::hint::black_box(outbound));
3089 }
3090 }
3091 let elapsed = started.elapsed();
3092 assert_eq!(sink.backlog().queued_bytes, 0);
3093 println!(
3094 "egress sink: {FRAMES} frames in {elapsed:?}, {:.1} ns/frame",
3095 elapsed.as_nanos() as f64 / FRAMES as f64
3096 );
3097 }
3098
3099 fn stream_frame(channel: u16, epoch: u32, corr: u64, body: Vec<u8>) -> Frame {
3100 Frame::build(
3101 FrameType::StreamData,
3102 Flags::new(false, Priority::Interactive, false),
3103 channel,
3104 epoch,
3105 corr,
3106 body,
3107 )
3108 .unwrap()
3109 }
3110
3111 fn module_route_for_test(
3112 router: &Router,
3113 ctx: &RouteCtx,
3114 route: &crate::forwarding::PendingRouteBindRelay,
3115 ) -> Arc<RouteBinding> {
3116 match router
3117 .forwarding
3118 .lookup_data_route(ctx.connection_id, route.module_channel, route.module_epoch)
3119 .unwrap()
3120 {
3121 DataRoute::Module(DataRouteState::Bound(binding)) => binding,
3122 other => panic!("expected a bound module route, got {other:?}"),
3123 }
3124 }
3125
3126 fn assert_frame_size_error(frame: &Frame, channel: u16, epoch: u32, corr: u64, bytes: usize) {
3127 assert_eq!(frame.header.ty, FrameType::Error);
3128 assert_eq!(frame.header.channel, channel);
3129 assert_eq!(frame.header.epoch, epoch);
3130 assert_eq!(frame.header.corr, corr);
3131 assert!(frame.header.flags.is_daemon_origin());
3132 let body: ErrorBody = serde_json::from_slice(&frame.body).unwrap();
3133 assert_eq!(body.code, error_codes::FRAME_TOO_LARGE);
3134 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3135 assert!(body.message.contains(&bytes.to_string()));
3136 assert!(body.message.contains(&limit.to_string()));
3137 assert_eq!(
3138 body.detail,
3139 Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit }))
3140 );
3141 }
3142
3143 #[tokio::test]
3144 async fn module_egress_response_oversized_reply_keeps_connection_and_other_route() {
3145 let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3146 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3147 "oversize-provider",
3148 ConnectionId::new(80),
3149 ConnectionId::new(81),
3150 2,
3151 )
3152 .await;
3153 let binding = module_route_for_test(&router, &ctx, &routes[0]);
3154 binding.flow.acquire_tagged(101, false).await.unwrap();
3155 module_route_for_test(&router, &ctx, &routes[1])
3156 .flow
3157 .acquire_tagged(102, false)
3158 .await
3159 .unwrap();
3160 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3161 let oversized = Frame::build(
3162 FrameType::Response,
3163 Flags::new(false, Priority::Interactive, true),
3164 routes[0].module_channel,
3165 routes[0].module_epoch,
3166 101,
3167 vec![b'x'; limit - subc_protocol::HEADER_LEN + 1],
3168 )
3169 .unwrap();
3170 assert_eq!(sink.backlog().queued_bytes, 0);
3171 router.route_for_connection(&ctx, oversized).await.unwrap();
3172
3173 router
3177 .route_for_connection(
3178 &ctx,
3179 route_frame(
3180 FrameType::Response,
3181 routes[1].module_channel,
3182 routes[1].module_epoch,
3183 102,
3184 ),
3185 )
3186 .await
3187 .unwrap();
3188 assert!(
3189 close.try_recv().is_err(),
3190 "an oversized reply must not close the multiplexed client"
3191 );
3192 let error = rx.try_recv().expect("the waiting client gets an ERROR");
3193 assert_frame_size_error(
3194 &error,
3195 routes[0].client_channel,
3196 routes[0].client_epoch,
3197 101,
3198 limit + 1,
3199 );
3200 let second = rx.try_recv().expect("the other route's reply arrives");
3201 assert_eq!(second.header.ty, FrameType::Response);
3202 assert_eq!(second.header.channel, routes[1].client_channel);
3203 assert_eq!(second.header.epoch, routes[1].client_epoch);
3204 assert_eq!(second.header.corr, 102);
3205 assert_eq!(second.body, b"route-body");
3206 assert_eq!(binding.flow.drain_in_flight(), 0);
3207 for route in &routes {
3208 module_route_for_test(&router, &ctx, route);
3209 }
3210 assert!(rx.try_recv().is_err());
3211 drop((error, second));
3212 assert_eq!(sink.backlog().queued_bytes, 0);
3213 assert_eq!(
3214 router.counters.snapshot()["client_egress_close_delivery_failed"],
3215 0
3216 );
3217
3218 let captured = test_log::captured_logs(&logs);
3219 let warnings: Vec<_> = captured
3220 .lines()
3221 .filter(|line| line.contains("WARN"))
3222 .collect();
3223 assert_eq!(warnings.len(), 1, "{captured}");
3224 let warning = warnings[0];
3225 for expected in [
3226 "module_id=oversize-provider".to_string(),
3227 "client_principal=direct".to_string(),
3228 format!("client_channel={}", routes[0].client_channel),
3229 format!("frame_bytes={}", limit + 1),
3230 format!("byte_limit={limit}"),
3231 ] {
3232 assert!(warning.contains(&expected), "missing {expected}: {warning}");
3233 }
3234 }
3235
3236 async fn poll_once<F: std::future::Future>(
3237 mut future: std::pin::Pin<&mut F>,
3238 ) -> std::task::Poll<F::Output> {
3239 std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
3240 }
3241
3242 fn fill_client_budget(sink: &FrameSink, route: &crate::forwarding::PendingRouteBindRelay) {
3243 sink.try_send(
3244 Frame::build(
3245 FrameType::Response,
3246 Flags::new(false, Priority::Interactive, true),
3247 route.client_channel,
3248 route.client_epoch,
3249 100,
3250 vec![
3251 b'x';
3252 crate::server::CONNECTION_EGRESS_BYTE_BUDGET - subc_protocol::HEADER_LEN
3253 ],
3254 )
3255 .unwrap(),
3256 )
3257 .unwrap();
3258 assert_eq!(
3259 sink.backlog().queued_bytes,
3260 crate::server::CONNECTION_EGRESS_BYTE_BUDGET
3261 );
3262 }
3263
3264 async fn assert_full_client_burst_survives(ty: FrameType, oversized: bool) {
3265 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3266 "draining-burst",
3267 ConnectionId::new(92),
3268 ConnectionId::new(93),
3269 2,
3270 )
3271 .await;
3272 fill_client_budget(&sink, &routes[0]);
3273 let frame = if oversized {
3274 Frame::build(
3275 ty,
3276 Flags::new(false, Priority::Interactive, false),
3277 routes[1].module_channel,
3278 routes[1].module_epoch,
3279 101,
3280 vec![b's'; crate::server::CONNECTION_EGRESS_BYTE_BUDGET],
3281 )
3282 .unwrap()
3283 } else {
3284 route_frame(ty, routes[1].module_channel, routes[1].module_epoch, 101)
3285 };
3286 let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3287 assert!(poll_once(delivery.as_mut()).await.is_pending());
3288 assert!(
3289 close.try_recv().is_err(),
3290 "fresh bursts must not close: {ty:?}"
3291 );
3292 tokio::time::advance(Duration::from_millis(50)).await;
3293 let queued = rx.try_recv().unwrap();
3294 queued.charge.as_ref().unwrap().taken_by_writer();
3295 drop(queued);
3296 assert!(matches!(
3297 poll_once(delivery.as_mut()).await,
3298 std::task::Poll::Ready(Ok(()))
3299 ));
3300 let delivered = rx
3301 .try_recv()
3302 .expect("the burst frame must be delivered after draining");
3303 assert_eq!(delivered.header.channel, routes[1].client_channel);
3304 assert_eq!(delivered.header.epoch, routes[1].client_epoch);
3305 assert_eq!(delivered.header.corr, 101);
3306 if oversized {
3307 assert_frame_size_error(
3308 &delivered,
3309 routes[1].client_channel,
3310 routes[1].client_epoch,
3311 101,
3312 crate::server::CONNECTION_EGRESS_BYTE_BUDGET + subc_protocol::HEADER_LEN,
3313 );
3314 } else {
3315 assert_eq!(delivered.header.ty, ty);
3316 }
3317 assert!(rx.try_recv().is_err());
3318 assert!(close.try_recv().is_err());
3319 assert_eq!(
3320 router.counters.snapshot()["client_egress_close_delivery_failed"],
3321 0
3322 );
3323 assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 0);
3324 }
3325
3326 #[tokio::test(start_paused = true)]
3327 async fn module_egress_fresh_full_queue_burst_survives() {
3328 for ty in [
3329 FrameType::Response,
3330 FrameType::StreamData,
3331 FrameType::Request,
3332 FrameType::Push,
3333 ] {
3334 assert_full_client_burst_survives(ty, false).await;
3335 }
3336 }
3337
3338 #[tokio::test(start_paused = true)]
3339 async fn module_goodbye_fresh_full_queue_burst_survives() {
3340 assert_full_client_burst_survives(FrameType::Goodbye, false).await;
3341 }
3342
3343 #[tokio::test(start_paused = true)]
3344 async fn frame_count_backstop_also_allows_a_draining_burst() {
3345 let (tx, mut rx) = mpsc::channel(1);
3346 let sink = FrameSink::with_byte_budget(tx, 2_000);
3347 sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
3348 .unwrap();
3349 let mut delivery = Box::pin(sink.send_to_client(stream_frame(1, 1, 2, vec![b'y'; 100])));
3350 assert!(poll_once(delivery.as_mut()).await.is_pending());
3351 assert!(
3352 sink.backlog().queued_bytes < 2_000,
3353 "only frame capacity is exhausted"
3354 );
3355 tokio::time::advance(Duration::from_millis(50)).await;
3356 drop(rx.try_recv().unwrap());
3357 assert!(matches!(
3358 poll_once(delivery.as_mut()).await,
3359 std::task::Poll::Ready(Ok(()))
3360 ));
3361 assert_eq!(rx.try_recv().unwrap().header.corr, 2);
3362 }
3363
3364 #[tokio::test(start_paused = true)]
3365 async fn oversized_reply_replacement_fresh_full_queue_burst_survives() {
3366 for ty in [FrameType::Response, FrameType::StreamData] {
3367 assert_full_client_burst_survives(ty, true).await;
3368 }
3369 }
3370
3371 #[tokio::test(start_paused = true)]
3372 async fn module_egress_stopped_reader_closes_without_waiting() {
3373 let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3374 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3375 "oldest-stopped-reader",
3376 ConnectionId::new(94),
3377 ConnectionId::new(95),
3378 2,
3379 )
3380 .await;
3381 fill_client_budget(&sink, &routes[0]);
3382 let writing = rx.try_recv().unwrap();
3385 writing.charge.as_ref().unwrap().taken_by_writer();
3386 tokio::time::advance(Duration::from_millis(251)).await;
3387 assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(251)));
3388 assert_eq!(sink.backlog().queued_bytes, queued_frame_bytes(&writing));
3389 let started = tokio::time::Instant::now();
3390 let frame = stream_frame(
3391 routes[0].module_channel,
3392 routes[0].module_epoch,
3393 101,
3394 b"secret-body-never-log".to_vec(),
3395 );
3396 let flags = frame.header.flags.0;
3397 let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3398 assert!(
3399 matches!(
3400 poll_once(delivery.as_mut()).await,
3401 std::task::Poll::Ready(Ok(()))
3402 ),
3403 "an oldest frame older than 250 ms must close immediately, without a new wait"
3404 );
3405 assert_eq!(started.elapsed(), Duration::ZERO);
3406 assert!(close
3407 .try_recv()
3408 .unwrap()
3409 .to_string()
3410 .contains("module_to_client_delivery_failed"));
3411 let captured = test_log::captured_logs(&logs);
3412 for expected in [
3413 "frame_type=Some(StreamData)".to_string(),
3414 format!("frame_flags=Some({flags})"),
3415 "oldest_queued_ms=251".to_string(),
3416 "close_cause=\"oldest_frame_older_than_250_ms\"".to_string(),
3417 ] {
3418 assert!(
3419 captured.contains(&expected),
3420 "missing {expected}: {captured}"
3421 );
3422 }
3423 assert!(!captured.contains("secret-body-never-log"));
3424 }
3425
3426 #[tokio::test(start_paused = true)]
3427 async fn client_gone_mid_wait_resumes_module_reader_immediately() {
3428 for writer_gone in [false, true] {
3429 let (router, sink, mut rx, ctx, routes, _close) = multi_route_client(
3430 "gone-during-burst",
3431 ConnectionId::new(96),
3432 ConnectionId::new(97),
3433 1,
3434 )
3435 .await;
3436 fill_client_budget(&sink, &routes[0]);
3437 let frame = route_frame(
3438 FrameType::Response,
3439 routes[0].module_channel,
3440 routes[0].module_epoch,
3441 101,
3442 );
3443 let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3444 assert!(poll_once(delivery.as_mut()).await.is_pending());
3445 tokio::time::advance(Duration::from_millis(50)).await;
3446 let started = tokio::time::Instant::now();
3447 if writer_gone {
3448 rx.close();
3449 } else {
3450 sink.close_client_delivery();
3451 }
3452 assert!(
3453 matches!(
3454 poll_once(delivery.as_mut()).await,
3455 std::task::Poll::Ready(Ok(()))
3456 ),
3457 "client teardown must wake the module reader before the writer drains"
3458 );
3459 assert_eq!(started.elapsed(), Duration::ZERO);
3460 assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
3461 assert_eq!(rx.try_recv().unwrap().header.corr, 100);
3462 assert!(rx.try_recv().is_err());
3463 }
3464 }
3465
3466 #[tokio::test(start_paused = true)]
3467 async fn cancelled_module_delivery_releases_waiter_without_closing_client() {
3468 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3469 "cancelled-burst",
3470 ConnectionId::new(98),
3471 ConnectionId::new(99),
3472 1,
3473 )
3474 .await;
3475 fill_client_budget(&sink, &routes[0]);
3476 let frame = route_frame(
3477 FrameType::Response,
3478 routes[0].module_channel,
3479 routes[0].module_epoch,
3480 101,
3481 );
3482 let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3483 assert!(poll_once(delivery.as_mut()).await.is_pending());
3484 assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 1);
3485 drop(delivery);
3486 assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
3487 tokio::time::advance(Duration::from_secs(1)).await;
3488 assert!(close.try_recv().is_err());
3489 drop(rx.try_recv().unwrap());
3490 assert_eq!(sink.backlog().queued_bytes, 0);
3491 assert!(sink.backlog().oldest_age.is_none());
3492 }
3493
3494 #[tokio::test(start_paused = true)]
3495 async fn egress_atomic_age_follows_writer_and_clears_when_drained() {
3496 let (sink, mut rx) = crate::server::connection_egress();
3497 sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
3498 .unwrap();
3499 tokio::time::advance(Duration::from_millis(200)).await;
3500 sink.try_send(stream_frame(2, 1, 2, vec![b'y'; 100]))
3501 .unwrap();
3502 let writing = rx.try_recv().unwrap();
3503 writing.charge.as_ref().unwrap().taken_by_writer();
3504 assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3505 drop(writing);
3506 assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3509 tokio::time::advance(Duration::from_millis(50)).await;
3510 let next = rx.try_recv().unwrap();
3511 next.charge.as_ref().unwrap().taken_by_writer();
3512 assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(50)));
3513 assert!(
3514 sink.backlog().queued_bytes > 0,
3515 "in-flight bytes remain charged until written"
3516 );
3517 drop(next);
3518 assert_eq!(sink.backlog().queued_bytes, 0);
3519 assert_eq!(sink.backlog().oldest_age, None);
3520 }
3521
3522 #[tokio::test(start_paused = true)]
3523 async fn module_egress_response_full_client_waits_only_remaining_grace() {
3524 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3525 "backpressure-provider",
3526 ConnectionId::new(82),
3527 ConnectionId::new(83),
3528 2,
3529 )
3530 .await;
3531 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3532 router
3535 .route_for_connection(
3536 &ctx,
3537 Frame::build(
3538 FrameType::Response,
3539 Flags::new(false, Priority::Interactive, true),
3540 routes[0].module_channel,
3541 routes[0].module_epoch,
3542 201,
3543 vec![b'x'; limit - subc_protocol::HEADER_LEN],
3544 )
3545 .unwrap(),
3546 )
3547 .await
3548 .unwrap();
3549 assert_eq!(sink.backlog().queued_bytes, limit);
3550 assert_eq!(sink.backlog().queued_frames, 1);
3551 let ordinary = route_frame(
3552 FrameType::Response,
3553 routes[1].module_channel,
3554 routes[1].module_epoch,
3555 202,
3556 );
3557 assert!(queued_frame_bytes(&ordinary) < limit);
3558 tokio::time::advance(Duration::from_millis(100)).await;
3561 let mut delivery = Box::pin(router.route_for_connection(&ctx, ordinary));
3562 assert!(poll_once(delivery.as_mut()).await.is_pending());
3563 assert!(close.try_recv().is_err());
3564 tokio::time::advance(Duration::from_millis(149)).await;
3565 assert!(poll_once(delivery.as_mut()).await.is_pending());
3566 assert!(close.try_recv().is_err());
3567 tokio::time::advance(Duration::from_millis(1)).await;
3568 assert!(matches!(
3569 poll_once(delivery.as_mut()).await,
3570 std::task::Poll::Ready(Ok(()))
3571 ));
3572 drop(delivery);
3573 let reason = close.try_recv().expect("the full client must close");
3574 assert!(reason
3575 .to_string()
3576 .contains("module_to_client_delivery_failed"));
3577 assert!(!reason.to_string().contains("frame_too_large"));
3578 assert_eq!(
3579 router.counters.snapshot()["client_egress_close_delivery_failed"],
3580 1
3581 );
3582 let admitted = rx.try_recv().expect("the budget-sized frame was admitted");
3583 assert_eq!(admitted.header.ty, FrameType::Response);
3584 assert_eq!(admitted.header.corr, 201);
3585 assert!(
3586 rx.try_recv().is_err(),
3587 "no oversized-frame ERROR is substituted for backpressure"
3588 );
3589 }
3590
3591 #[tokio::test(start_paused = true)]
3592 async fn oversized_reply_to_a_full_client_waits_only_remaining_grace() {
3593 let (router, sink, _rx, ctx, routes, mut close) = multi_route_client(
3594 "full-oversize-provider",
3595 ConnectionId::new(90),
3596 ConnectionId::new(91),
3597 2,
3598 )
3599 .await;
3600 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3601 router
3603 .route_for_connection(
3604 &ctx,
3605 Frame::build(
3606 FrameType::Response,
3607 Flags::new(false, Priority::Interactive, true),
3608 routes[0].module_channel,
3609 routes[0].module_epoch,
3610 501,
3611 vec![b'x'; limit - subc_protocol::HEADER_LEN],
3612 )
3613 .unwrap(),
3614 )
3615 .await
3616 .unwrap();
3617 assert_eq!(sink.backlog().queued_bytes, limit);
3618 let oversized = Frame::build(
3619 FrameType::Response,
3620 Flags::new(false, Priority::Interactive, true),
3621 routes[1].module_channel,
3622 routes[1].module_epoch,
3623 502,
3624 vec![b'y'; limit - subc_protocol::HEADER_LEN + 1],
3625 )
3626 .unwrap();
3627 tokio::time::advance(Duration::from_millis(80)).await;
3630 let mut delivery = Box::pin(router.route_for_connection(&ctx, oversized));
3631 assert!(poll_once(delivery.as_mut()).await.is_pending());
3632 assert!(close.try_recv().is_err());
3633 tokio::time::advance(Duration::from_millis(169)).await;
3634 assert!(poll_once(delivery.as_mut()).await.is_pending());
3635 assert!(close.try_recv().is_err());
3636 tokio::time::advance(Duration::from_millis(1)).await;
3637 assert!(matches!(
3638 poll_once(delivery.as_mut()).await,
3639 std::task::Poll::Ready(Ok(()))
3640 ));
3641 drop(delivery);
3642 let reason = close.try_recv().expect("the full client must close");
3643 assert!(reason
3644 .to_string()
3645 .contains("module_to_client_delivery_failed"));
3646 }
3647
3648 #[tokio::test]
3649 async fn oversized_reverse_request_errors_module_without_reaching_client() {
3650 let (router, _sink, mut rx, mut ctx, routes, mut close) = multi_route_client(
3651 "reverse-oversize-provider",
3652 ConnectionId::new(84),
3653 ConnectionId::new(85),
3654 2,
3655 )
3656 .await;
3657 let (module_tx, mut module_rx) = mpsc::channel(8);
3658 ctx.egress = FrameSink::new(module_tx);
3659 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3660 router
3661 .route_for_connection(
3662 &ctx,
3663 Frame::build(
3664 FrameType::Request,
3665 Flags::new(false, Priority::Interactive, false),
3666 routes[0].module_channel,
3667 routes[0].module_epoch,
3668 301,
3669 vec![b'r'; limit - subc_protocol::HEADER_LEN + 1],
3670 )
3671 .unwrap(),
3672 )
3673 .await
3674 .unwrap();
3675 assert!(
3676 rx.try_recv().is_err(),
3677 "the reverse request never reaches the client"
3678 );
3679 let error = module_rx.try_recv().expect("the module gets the refusal");
3680 assert_frame_size_error(
3681 &error,
3682 routes[0].module_channel,
3683 routes[0].module_epoch,
3684 301,
3685 limit + 1,
3686 );
3687 assert!(module_rx.try_recv().is_err());
3688 for (index, route) in routes.iter().enumerate() {
3689 module_route_for_test(&router, &ctx, route);
3690 router
3691 .route_for_connection(
3692 &ctx,
3693 route_frame(
3694 FrameType::Response,
3695 route.module_channel,
3696 route.module_epoch,
3697 302 + index as u64,
3698 ),
3699 )
3700 .await
3701 .unwrap();
3702 let reply = rx.try_recv().expect("both routes still deliver");
3703 assert_eq!(reply.header.ty, FrameType::Response);
3704 assert_eq!(reply.header.channel, route.client_channel);
3705 }
3706 assert!(close.try_recv().is_err());
3707 assert_eq!(
3708 router.counters.snapshot()["client_egress_close_delivery_failed"],
3709 0
3710 );
3711 }
3712
3713 #[tokio::test]
3714 async fn oversized_push_closes_client_with_precise_warn() {
3715 let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3716 let (router, _sink, mut rx, ctx, routes, mut close) = multi_route_client(
3717 "push-oversize-provider",
3718 ConnectionId::new(86),
3719 ConnectionId::new(87),
3720 2,
3721 )
3722 .await;
3723 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3724 router
3725 .route_for_connection(
3726 &ctx,
3727 Frame::build(
3728 FrameType::Push,
3729 Flags::new(false, Priority::Interactive, false),
3730 routes[0].module_channel,
3731 routes[0].module_epoch,
3732 0,
3733 vec![b'p'; limit - subc_protocol::HEADER_LEN + 1],
3734 )
3735 .unwrap(),
3736 )
3737 .await
3738 .unwrap();
3739 let reason = close
3740 .try_recv()
3741 .expect("unsolicited oversize still closes the client");
3742 assert!(reason
3743 .to_string()
3744 .contains("frame_too_large: oversized push"));
3745 assert!(
3746 rx.try_recv().is_err(),
3747 "no part of the oversized push is forwarded"
3748 );
3749 let captured = test_log::captured_logs(&logs);
3750 let warnings: Vec<_> = captured
3751 .lines()
3752 .filter(|line| line.contains("WARN"))
3753 .collect();
3754 assert_eq!(warnings.len(), 1, "{captured}");
3755 let warning = warnings[0];
3756 for expected in [
3757 "oversized push".to_string(),
3758 "module_id=\"push-oversize-provider\"".to_string(),
3759 "principals=direct".to_string(),
3760 format!("client_channel={}", routes[0].client_channel),
3761 format!("size={}", limit + 1),
3762 format!("limit={limit}"),
3763 ] {
3764 assert!(warning.contains(&expected), "missing {expected}: {warning}");
3765 }
3766 assert_eq!(
3767 router.counters.snapshot()["client_egress_close_delivery_failed"],
3768 1
3769 );
3770 }
3771
3772 #[tokio::test]
3773 async fn oversized_stream_and_error_frames_end_request_even_with_queued_traffic() {
3774 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3775 "stream-oversize-provider",
3776 ConnectionId::new(88),
3777 ConnectionId::new(89),
3778 2,
3779 )
3780 .await;
3781 let binding = module_route_for_test(&router, &ctx, &routes[0]);
3782 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3783 for ty in [
3784 FrameType::StreamData,
3785 FrameType::StreamEnd,
3786 FrameType::Error,
3787 ] {
3788 binding.flow.acquire_tagged(401, false).await.unwrap();
3789 sink.try_send(route_frame(
3790 FrameType::Response,
3791 routes[1].client_channel,
3792 routes[1].client_epoch,
3793 402,
3794 ))
3795 .unwrap();
3796 router
3797 .route_for_connection(
3798 &ctx,
3799 Frame::build(
3800 ty,
3801 Flags::new(false, Priority::Interactive, false),
3802 routes[0].module_channel,
3803 routes[0].module_epoch,
3804 401,
3805 vec![b's'; limit - subc_protocol::HEADER_LEN + 1],
3806 )
3807 .unwrap(),
3808 )
3809 .await
3810 .unwrap();
3811 assert!(close.try_recv().is_err());
3812 assert_eq!(rx.try_recv().unwrap().header.corr, 402);
3813 let error = rx
3814 .try_recv()
3815 .expect("every correlated oversize frame becomes ERROR");
3816 assert_frame_size_error(
3817 &error,
3818 routes[0].client_channel,
3819 routes[0].client_epoch,
3820 401,
3821 limit + 1,
3822 );
3823 assert_eq!(binding.flow.drain_in_flight(), 0, "{ty:?} ends the request");
3824 assert!(rx.try_recv().is_err());
3825 }
3826 }
3827
3828 #[tokio::test]
3834 async fn awaited_send_parked_behind_a_full_byte_budget_wakes_when_bytes_free() {
3835 let (tx, mut rx) = mpsc::channel(1024);
3836 let sink = FrameSink::with_byte_budget(tx, 2_000);
3837 let mut queued = 0u64;
3838 while sink
3839 .try_send(stream_frame(7, 1, queued, vec![b'x'; 200]))
3840 .is_ok()
3841 {
3842 queued += 1;
3843 }
3844 assert!(queued > 0, "the budget admitted nothing");
3845
3846 let parked = tokio::spawn({
3847 let sink = sink.clone();
3848 async move { sink.send(stream_frame(0, 0, 999, vec![b'r'; 200])).await }
3849 });
3850 tokio::time::sleep(Duration::from_millis(100)).await;
3851 assert!(
3852 !parked.is_finished(),
3853 "the awaited send must wait while the byte budget is full"
3854 );
3855
3856 drop(rx.recv().await.expect("a queued frame"));
3858 tokio::time::timeout(Duration::from_secs(2), parked)
3859 .await
3860 .expect("the parked send was never woken after bytes were freed")
3861 .unwrap()
3862 .unwrap();
3863 }
3864
3865 #[tokio::test]
3870 async fn paused_client_reader_keeps_connection_through_small_frame_burst() {
3871 const ROUTES: usize = 4;
3872 const FRAMES_PER_ROUTE: usize = 250;
3873 let (router, client_sink, mut client_rx, module_ctx, routes, mut close_receiver) =
3874 multi_route_client(
3875 "burst-provider",
3876 ConnectionId::new(60),
3877 ConnectionId::new(61),
3878 ROUTES,
3879 )
3880 .await;
3881
3882 for seq in 0..FRAMES_PER_ROUTE as u64 {
3884 for (index, route) in routes.iter().enumerate() {
3885 let body = format!("route-{index}-token-{seq:05}-{}", "t".repeat(170));
3886 router
3887 .route_for_connection(
3888 &module_ctx,
3889 stream_frame(
3890 route.module_channel,
3891 route.module_epoch,
3892 seq,
3893 body.into_bytes(),
3894 ),
3895 )
3896 .await
3897 .unwrap();
3898 }
3899 }
3900
3901 let backlog = client_sink.backlog();
3902 assert_eq!(backlog.queued_frames, ROUTES * FRAMES_PER_ROUTE);
3903 assert!(backlog.queued_bytes < crate::server::CONNECTION_EGRESS_BYTE_BUDGET);
3904 assert!(
3905 close_receiver.try_recv().is_err(),
3906 "a paused reader under the byte budget must not be closed"
3907 );
3908 assert_eq!(
3909 router.counters.snapshot()["client_egress_close_delivery_failed"],
3910 0
3911 );
3912
3913 let mut next_seq = vec![0u64; ROUTES];
3915 for _ in 0..ROUTES * FRAMES_PER_ROUTE {
3916 let frame = client_rx.try_recv().expect("every queued frame arrives");
3917 let index = routes
3918 .iter()
3919 .position(|route| route.client_channel == frame.header.channel)
3920 .expect("frame arrives on one of the bound client channels");
3921 assert_eq!(frame.header.corr, next_seq[index], "route {index} order");
3922 let expected_prefix = format!("route-{index}-token-{:05}-", next_seq[index]);
3923 assert!(frame.body.starts_with(expected_prefix.as_bytes()));
3924 next_seq[index] += 1;
3925 }
3926 assert!(client_rx.try_recv().is_err());
3927 assert_eq!(next_seq, vec![FRAMES_PER_ROUTE as u64; ROUTES]);
3928 assert_eq!(client_sink.backlog().queued_bytes, 0);
3929 }
3930
3931 #[tokio::test(start_paused = true)]
3934 async fn never_reading_client_is_closed_at_byte_budget_with_warn_diagnosis() {
3935 let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3936 const BODY: usize = 16 * 1024;
3937 let (router, client_sink, _client_rx, module_ctx, routes, mut close_receiver) =
3938 multi_route_client(
3939 "stuck-reader-provider",
3940 ConnectionId::new(70),
3941 ConnectionId::new(71),
3942 2,
3943 )
3944 .await;
3945
3946 let mut admitted = 0usize;
3947 let mut sent = 0u64;
3948 let started = tokio::time::Instant::now();
3949 while router.counters.snapshot()["client_egress_close_delivery_failed"] == 0 {
3950 assert!(sent < 1_000, "the byte budget never refused a frame");
3951 let route = &routes[(sent % 2) as usize];
3952 router
3953 .route_for_connection(
3954 &module_ctx,
3955 stream_frame(
3956 route.module_channel,
3957 route.module_epoch,
3958 sent,
3959 vec![b'z'; BODY],
3960 ),
3961 )
3962 .await
3963 .unwrap();
3964 sent += 1;
3965 admitted = client_sink.backlog().queued_frames;
3966 }
3967 assert_eq!(
3968 started.elapsed(),
3969 crate::server::CONNECTION_EGRESS_STALL_GRACE
3970 );
3971 let frame_bytes = subc_protocol::HEADER_LEN + BODY;
3973 assert_eq!(
3974 admitted,
3975 crate::server::CONNECTION_EGRESS_BYTE_BUDGET / frame_bytes
3976 );
3977 let reason = close_receiver
3978 .try_recv()
3979 .expect("the client connection must be asked to close");
3980 assert!(reason
3981 .to_string()
3982 .contains("module_to_client_delivery_failed"));
3983
3984 router
3986 .route_for_connection(
3987 &module_ctx,
3988 stream_frame(
3989 routes[0].module_channel,
3990 routes[0].module_epoch,
3991 sent,
3992 vec![b'z'; BODY],
3993 ),
3994 )
3995 .await
3996 .unwrap();
3997
3998 let captured = test_log::captured_logs(&logs);
3999 let warn_lines = captured
4000 .lines()
4001 .filter(|line| {
4002 line.contains("closing client connection: its egress queue could not take a frame")
4003 })
4004 .collect::<Vec<_>>();
4005 assert_eq!(warn_lines.len(), 1, "exactly one WARN, got: {captured}");
4006 let line = warn_lines[0];
4007 assert!(line.contains("WARN"), "{line}");
4008 assert!(line.contains("connection_id=71"), "{line}");
4009 assert!(
4010 line.contains("module_id=\"stuck-reader-provider\""),
4011 "{line}"
4012 );
4013 assert!(line.contains("client_channel="), "{line}");
4014 assert!(line.contains("principals=direct"), "{line}");
4015 let queued_bytes: usize = line
4016 .split("queued_bytes=")
4017 .nth(1)
4018 .and_then(|rest| rest.split_whitespace().next())
4019 .and_then(|value| value.parse().ok())
4020 .expect("queued_bytes is logged");
4021 assert_eq!(queued_bytes, admitted * frame_bytes);
4022 assert!(
4023 line.contains(&format!("queued_frames={admitted}")),
4024 "{line}"
4025 );
4026 assert!(line.contains("oldest_queued_ms="), "{line}");
4027 assert!(line.contains("oldest_queued_ms=250"), "{line}");
4028 assert!(line.contains("frame_type=Some(StreamData)"), "{line}");
4029 assert!(
4030 line.contains(&format!(
4031 "frame_flags=Some({})",
4032 Flags::new(false, Priority::Interactive, false).0
4033 )),
4034 "{line}"
4035 );
4036 assert!(
4037 line.contains("close_cause=\"waited_250_ms_without_room\""),
4038 "{line}"
4039 );
4040 assert!(
4041 !line.contains(&"z".repeat(20)),
4042 "body bytes must never be logged: {line}"
4043 );
4044 }
4045
4046 fn route_frame(ty: FrameType, channel: u16, epoch: u32, corr: u64) -> Frame {
4047 Frame::build(
4048 ty,
4049 Flags::new(false, Priority::Interactive, false),
4050 channel,
4051 epoch,
4052 corr,
4053 if ty == FrameType::Request || ty == FrameType::Response {
4054 b"route-body".to_vec()
4055 } else {
4056 Vec::new()
4057 },
4058 )
4059 .unwrap()
4060 }
4061
4062 type DynamicRouteFixture = (
4063 Router,
4064 Arc<ForwardingTable>,
4065 RouteCtx,
4066 mpsc::Receiver<crate::router::OutboundFrame>,
4067 RouteCtx,
4068 mpsc::Receiver<crate::router::OutboundFrame>,
4069 mpsc::Receiver<crate::router::OutboundFrame>,
4070 crate::forwarding::PendingRouteBindRelay,
4071 );
4072
4073 fn dynamic_route_fixture(commit: bool) -> DynamicRouteFixture {
4074 dynamic_route_fixture_with_busy(commit, false)
4075 }
4076
4077 fn dynamic_route_fixture_with_busy(commit: bool, busy_refusals: bool) -> DynamicRouteFixture {
4078 let forwarding = Arc::new(ForwardingTable::default());
4079 let control = Arc::new(crate::ControlHandler::with_forwarding(
4080 Arc::new(crate::Registry::default()),
4081 Arc::clone(&forwarding),
4082 ));
4083 let router = Router::with_control_handler(control);
4084 let module_connection = ConnectionId::new(500);
4085 let client_connection = ConnectionId::new(501);
4086 let (module_tx, module_rx) = mpsc::channel(if busy_refusals { 64 } else { 8 });
4087 forwarding
4088 .register_module_connection(
4089 module_connection,
4090 "epoch-router".into(),
4091 2,
4092 Concurrency::ModuleManaged,
4093 FrameSink::new(module_tx),
4094 )
4095 .unwrap();
4096 let (client_tx, client_rx) = mpsc::channel(if busy_refusals { 64 } else { 8 });
4097 let client_sink = FrameSink::new(client_tx);
4098 let pending = forwarding
4099 .begin_route_bind_relay_with_busy_for_test(
4100 client_connection,
4101 client_sink.clone(),
4102 700,
4103 "epoch-router",
4104 busy_refusals,
4105 )
4106 .unwrap();
4107 if commit {
4108 forwarding
4109 .complete_pending_relay(
4110 module_connection,
4111 pending.corr,
4112 RouteBindRelayOutcome::Accepted,
4113 )
4114 .unwrap();
4115 }
4116 let (module_egress_tx, module_egress_rx) = mpsc::channel(8);
4117 (
4118 router,
4119 forwarding,
4120 RouteCtx {
4121 connection_id: client_connection,
4122 egress: client_sink,
4123 },
4124 client_rx,
4125 RouteCtx {
4126 connection_id: module_connection,
4127 egress: FrameSink::new(module_egress_tx),
4128 },
4129 module_egress_rx,
4130 module_rx,
4131 pending,
4132 )
4133 }
4134
4135 #[tokio::test]
4136 async fn route_epochs_validate_both_directions_and_rewrite_to_peer_handle() {
4137 let (
4138 router,
4139 _forwarding,
4140 client_ctx,
4141 mut client_rx,
4142 module_ctx,
4143 _module_egress_rx,
4144 mut module_rx,
4145 pending,
4146 ) = dynamic_route_fixture(true);
4147 let route_open = client_rx.recv().await.unwrap();
4148 assert_eq!(route_open.header.corr, 700);
4149
4150 router
4151 .route_for_connection(
4152 &client_ctx,
4153 route_frame(
4154 FrameType::Request,
4155 pending.client_channel,
4156 pending.client_epoch,
4157 701,
4158 ),
4159 )
4160 .await
4161 .unwrap();
4162 let forwarded = module_rx.recv().await.unwrap();
4163 assert_eq!(forwarded.header.channel, pending.module_channel);
4164 assert_eq!(forwarded.header.epoch, pending.module_epoch);
4165
4166 router
4167 .route_for_connection(
4168 &module_ctx,
4169 route_frame(
4170 FrameType::Response,
4171 pending.module_channel,
4172 pending.module_epoch,
4173 701,
4174 ),
4175 )
4176 .await
4177 .unwrap();
4178 let delivered = client_rx.recv().await.unwrap();
4179 assert_eq!(delivered.header.channel, pending.client_channel);
4180 assert_eq!(delivered.header.epoch, pending.client_epoch);
4181
4182 router
4183 .route_for_connection(
4184 &client_ctx,
4185 route_frame(
4186 FrameType::Request,
4187 pending.client_channel,
4188 pending.client_epoch + 1,
4189 702,
4190 ),
4191 )
4192 .await
4193 .unwrap();
4194 router
4195 .route_for_connection(
4196 &module_ctx,
4197 route_frame(
4198 FrameType::Response,
4199 pending.module_channel,
4200 pending.module_epoch + 1,
4201 703,
4202 ),
4203 )
4204 .await
4205 .unwrap();
4206 let stale_error = client_rx.recv().await.unwrap();
4207 assert_eq!(stale_error.header.ty, FrameType::Error);
4208 assert_eq!(stale_error.header.channel, pending.client_channel);
4209 assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
4210 assert_eq!(stale_error.header.corr, 702);
4211 let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
4212 assert_eq!(body.code, "stale_route_epoch");
4213 assert!(module_rx.try_recv().is_err());
4214 assert!(client_rx.try_recv().is_err());
4215 let counters = router.counters.snapshot();
4216 assert_eq!(counters["client_frames_dropped_stale_route"], 1);
4217 assert_eq!(counters["module_frames_dropped_no_route"], 1);
4218 }
4219
4220 #[tokio::test]
4221 async fn accepted_route_publishes_route_open_before_immediate_reverse_request() {
4222 let (
4223 router,
4224 _,
4225 _client_ctx,
4226 mut client_rx,
4227 module_ctx,
4228 _module_egress_rx,
4229 _module_rx,
4230 pending,
4231 ) = dynamic_route_fixture(true);
4232 router
4233 .route_for_connection(
4234 &module_ctx,
4235 route_frame(
4236 FrameType::Request,
4237 pending.module_channel,
4238 pending.module_epoch,
4239 800,
4240 ),
4241 )
4242 .await
4243 .unwrap();
4244
4245 let first = client_rx.recv().await.unwrap();
4246 let second = client_rx.recv().await.unwrap();
4247 assert_eq!(first.header.channel, 0);
4248 assert_eq!(first.header.corr, 700);
4249 assert_eq!(second.header.channel, pending.client_channel);
4250 assert_eq!(second.header.epoch, pending.client_epoch);
4251 assert_eq!(second.header.corr, 800);
4252 }
4253
4254 #[tokio::test]
4255 async fn reserved_slot_ingress_errors_only_matching_client_requests() {
4256 let (
4257 router,
4258 _forwarding,
4259 client_ctx,
4260 mut client_rx,
4261 _module_ctx,
4262 _module_egress_rx,
4263 mut module_rx,
4264 pending,
4265 ) = dynamic_route_fixture(false);
4266 router
4267 .route_for_connection(
4268 &client_ctx,
4269 route_frame(
4270 FrameType::Request,
4271 pending.client_channel,
4272 pending.client_epoch,
4273 900,
4274 ),
4275 )
4276 .await
4277 .unwrap();
4278 let error = client_rx.recv().await.unwrap();
4279 assert_eq!(error.header.ty, FrameType::Error);
4280 assert_eq!(error.header.channel, pending.client_channel);
4281 assert_eq!(error.header.epoch, pending.client_epoch);
4282 assert_eq!(error.header.corr, 900);
4283
4284 router
4285 .route_for_connection(
4286 &client_ctx,
4287 route_frame(
4288 FrameType::Response,
4289 pending.client_channel,
4290 pending.client_epoch,
4291 901,
4292 ),
4293 )
4294 .await
4295 .unwrap();
4296 router
4297 .route_for_connection(
4298 &client_ctx,
4299 route_frame(
4300 FrameType::Request,
4301 pending.client_channel,
4302 pending.client_epoch + 1,
4303 902,
4304 ),
4305 )
4306 .await
4307 .unwrap();
4308 let stale_error = client_rx.recv().await.unwrap();
4309 assert_eq!(stale_error.header.ty, FrameType::Error);
4310 assert_eq!(stale_error.header.channel, pending.client_channel);
4311 assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
4312 assert_eq!(stale_error.header.corr, 902);
4313 let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
4314 assert_eq!(body.code, "stale_route_epoch");
4315 assert!(module_rx.try_recv().is_err());
4316 let counters = router.counters.snapshot();
4317 assert_eq!(counters["client_frames_dropped_stale_route"], 1);
4318 assert_eq!(counters["module_frames_dropped_no_route"], 0);
4319 }
4320
4321 #[tokio::test]
4322 async fn dropped_module_route_goodbye_increments_counter() {
4323 let (
4324 router,
4325 _forwarding,
4326 client_ctx,
4327 mut client_rx,
4328 _module_ctx,
4329 _module_egress_rx,
4330 mut module_rx,
4331 pending,
4332 ) = dynamic_route_fixture(true);
4333 let _ = client_rx.recv().await;
4334 module_rx.close();
4335
4336 router
4337 .route_for_connection(
4338 &client_ctx,
4339 route_frame(
4340 FrameType::Goodbye,
4341 pending.client_channel,
4342 pending.client_epoch,
4343 999,
4344 ),
4345 )
4346 .await
4347 .unwrap();
4348
4349 let counters = router.counters.snapshot();
4350 assert_eq!(counters["goodbye_relay_module_dropped"], 1);
4351 assert_eq!(
4352 counters["goodbye_relay_module_dropped_by_module"],
4353 serde_json::json!({ "epoch-router": 1 })
4354 );
4355 assert_eq!(counters["route_released_epoch_fenced"], 1);
4356 }
4357
4358 #[tokio::test]
4359 async fn module_request_on_stale_epoch_receives_stale_route_epoch() {
4360 let (
4361 router,
4362 _forwarding,
4363 _client_ctx,
4364 _client_rx,
4365 module_ctx,
4366 mut module_egress_rx,
4367 mut module_rx,
4368 pending,
4369 ) = dynamic_route_fixture(true);
4370
4371 router
4372 .route_for_connection(
4373 &module_ctx,
4374 route_frame(
4375 FrameType::Request,
4376 pending.module_channel,
4377 pending.module_epoch + 1,
4378 1_000,
4379 ),
4380 )
4381 .await
4382 .unwrap();
4383
4384 let error = module_egress_rx.try_recv().unwrap();
4385 assert_eq!(error.header.ty, FrameType::Error);
4386 assert_eq!(error.header.channel, pending.module_channel);
4387 assert_eq!(error.header.epoch, pending.module_epoch + 1);
4388 assert_eq!(error.header.corr, 1_000);
4389 let body: ErrorBody = serde_json::from_slice(&error.body).unwrap();
4390 assert_eq!(body.code, "stale_route_epoch");
4391 assert!(module_rx.try_recv().is_err());
4392 let counters = router.counters.snapshot();
4393 assert_eq!(counters["module_requests_dropped_stale_route"], 1);
4394 assert_eq!(counters["module_frames_dropped_no_route"], 0);
4395 }
4396
4397 #[tokio::test(start_paused = true)]
4398 async fn stale_epoch_drop_warns_once_per_minute_with_suppressed_count() {
4399 let (output, _guard) = test_log::log_capture(tracing::Level::WARN);
4400 let (
4401 router,
4402 _forwarding,
4403 client_ctx,
4404 mut client_rx,
4405 _module_ctx,
4406 _module_egress_rx,
4407 mut module_rx,
4408 pending,
4409 ) = dynamic_route_fixture(true);
4410 let _ = client_rx.recv().await;
4411 for corr in 1..=3 {
4412 router
4413 .route_for_connection(
4414 &client_ctx,
4415 route_frame(
4416 FrameType::Request,
4417 pending.client_channel,
4418 pending.client_epoch + 1,
4419 corr,
4420 ),
4421 )
4422 .await
4423 .unwrap();
4424 assert_eq!(client_rx.recv().await.unwrap().header.ty, FrameType::Error);
4425 }
4426 assert!(module_rx.try_recv().is_err());
4427 let logs = test_log::captured_logs(&output);
4428 assert_eq!(
4429 logs.matches("dropping data-plane frame").count(),
4430 1,
4431 "{logs}"
4432 );
4433 assert!(logs.contains("WARN"), "{logs}");
4434 assert!(
4435 logs.contains(&format!("connection_id={}", client_ctx.connection_id.get())),
4436 "{logs}"
4437 );
4438 assert!(
4439 logs.contains(&format!("channel={}", pending.client_channel)),
4440 "{logs}"
4441 );
4442 assert!(
4443 logs.contains(&format!("epoch={}", pending.client_epoch + 1)),
4444 "{logs}"
4445 );
4446 assert!(
4447 logs.contains("client_to_module") && logs.contains("stale_epoch"),
4448 "{logs}"
4449 );
4450 assert!(logs.contains("suppressed=0"), "{logs}");
4451
4452 tokio::time::advance(Duration::from_secs(60)).await;
4453 router
4454 .route_for_connection(
4455 &client_ctx,
4456 route_frame(
4457 FrameType::Request,
4458 pending.client_channel,
4459 pending.client_epoch + 2,
4460 4,
4461 ),
4462 )
4463 .await
4464 .unwrap();
4465 let logs = test_log::captured_logs(&output);
4466 assert_eq!(
4467 logs.matches("dropping data-plane frame").count(),
4468 2,
4469 "{logs}"
4470 );
4471 assert!(logs.contains("suppressed=2"), "{logs}");
4472 }
4473
4474 #[tokio::test]
4475 async fn module_request_on_reserved_or_absent_route_receives_unknown_channel() {
4476 let (
4477 reserved_router,
4478 _forwarding,
4479 _client_ctx,
4480 _client_rx,
4481 reserved_module_ctx,
4482 mut reserved_module_egress_rx,
4483 _module_rx,
4484 reserved,
4485 ) = dynamic_route_fixture(false);
4486 reserved_router
4487 .route_for_connection(
4488 &reserved_module_ctx,
4489 route_frame(
4490 FrameType::Request,
4491 reserved.module_channel,
4492 reserved.module_epoch,
4493 1_001,
4494 ),
4495 )
4496 .await
4497 .unwrap();
4498 let reserved_error = reserved_module_egress_rx.try_recv().unwrap();
4499 let reserved_body: ErrorBody = serde_json::from_slice(&reserved_error.body).unwrap();
4500 assert_eq!(reserved_error.header.ty, FrameType::Error);
4501 assert_eq!(reserved_error.header.channel, reserved.module_channel);
4502 assert_eq!(reserved_error.header.epoch, reserved.module_epoch);
4503 assert_eq!(reserved_error.header.corr, 1_001);
4504 assert_eq!(reserved_body.code, "unknown_channel");
4505 assert_eq!(
4506 reserved_router.counters.snapshot()["module_requests_dropped_stale_route"],
4507 1
4508 );
4509
4510 let (
4511 absent_router,
4512 _forwarding,
4513 _client_ctx,
4514 _client_rx,
4515 absent_module_ctx,
4516 mut absent_module_egress_rx,
4517 _module_rx,
4518 absent,
4519 ) = dynamic_route_fixture(false);
4520 absent_router
4521 .route_for_connection(
4522 &absent_module_ctx,
4523 route_frame(
4524 FrameType::Request,
4525 absent.module_channel + 1,
4526 absent.module_epoch,
4527 1_002,
4528 ),
4529 )
4530 .await
4531 .unwrap();
4532 let absent_error = absent_module_egress_rx.try_recv().unwrap();
4533 let absent_body: ErrorBody = serde_json::from_slice(&absent_error.body).unwrap();
4534 assert_eq!(absent_error.header.ty, FrameType::Error);
4535 assert_eq!(absent_error.header.channel, absent.module_channel + 1);
4536 assert_eq!(absent_error.header.epoch, absent.module_epoch);
4537 assert_eq!(absent_error.header.corr, 1_002);
4538 assert_eq!(absent_body.code, "unknown_channel");
4539 assert_eq!(
4540 absent_router.counters.snapshot()["module_requests_dropped_stale_route"],
4541 1
4542 );
4543 }
4544
4545 #[tokio::test]
4546 async fn non_request_module_frame_on_dead_route_is_counted_without_error() {
4547 let (
4548 router,
4549 forwarding,
4550 client_ctx,
4551 mut client_rx,
4552 module_ctx,
4553 mut module_egress_rx,
4554 mut module_rx,
4555 pending,
4556 ) = dynamic_route_fixture(true);
4557 let (other_module_tx, _other_module_rx) = mpsc::channel(8);
4558 forwarding
4559 .register_module_connection(
4560 ConnectionId::new(502),
4561 "other-module".into(),
4562 2,
4563 Concurrency::ModuleManaged,
4564 FrameSink::new(other_module_tx),
4565 )
4566 .unwrap();
4567 let _ = client_rx.recv().await.unwrap();
4568
4569 router
4570 .route_for_connection(
4571 &client_ctx,
4572 route_frame(
4573 FrameType::Goodbye,
4574 pending.client_channel,
4575 pending.client_epoch,
4576 1_003,
4577 ),
4578 )
4579 .await
4580 .unwrap();
4581 let _ = module_rx.recv().await.unwrap();
4582
4583 router
4584 .route_for_connection(
4585 &module_ctx,
4586 route_frame(
4587 FrameType::StreamData,
4588 pending.module_channel,
4589 pending.module_epoch,
4590 1_004,
4591 ),
4592 )
4593 .await
4594 .unwrap();
4595
4596 let reply = module_egress_rx.try_recv().unwrap();
4599 assert_eq!(reply.header.ty, FrameType::Goodbye);
4600 assert!(module_egress_rx.try_recv().is_err());
4601 let counters = router.counters.snapshot();
4602 assert_eq!(counters["module_frames_dropped_no_route"], 1);
4603 assert_eq!(
4604 counters["module_frames_dropped_no_route_by_module"],
4605 serde_json::json!({ "epoch-router": 1 })
4606 );
4607 assert_eq!(counters["module_requests_dropped_stale_route"], 0);
4608 }
4609
4610 fn drain_now(rx: &mut mpsc::Receiver<OutboundFrame>) {
4613 while rx.try_recv().is_ok() {}
4614 }
4615
4616 #[tokio::test]
4622 async fn route_goodbye_refused_by_stalled_module_is_delivered_when_it_resumes_reading() {
4623 const BUDGET: usize = 4_096;
4624 let forwarding = Arc::new(ForwardingTable::default());
4625 let control = Arc::new(ControlHandler::with_forwarding(
4626 Arc::new(Registry::default()),
4627 Arc::clone(&forwarding),
4628 ));
4629 let router = Router::with_control_handler(control);
4630 let module_connection = ConnectionId::new(80);
4631 let client_connection = ConnectionId::new(81);
4632 let (module_tx, mut module_rx) = mpsc::channel(64);
4633 let module_sink = FrameSink::with_byte_budget(module_tx, BUDGET);
4634 forwarding
4635 .register_module_connection(
4636 module_connection,
4637 "stalled-provider".into(),
4638 2,
4639 Concurrency::ModuleManaged,
4640 module_sink.clone(),
4641 )
4642 .unwrap();
4643 let (client_tx, mut client_rx) = mpsc::channel(8);
4644 let client_sink = FrameSink::new(client_tx);
4645 let pending = forwarding
4646 .begin_route_bind_relay_for_test(
4647 client_connection,
4648 client_sink.clone(),
4649 1_100,
4650 "stalled-provider",
4651 )
4652 .unwrap();
4653 forwarding
4654 .complete_pending_relay(
4655 module_connection,
4656 pending.corr,
4657 RouteBindRelayOutcome::Accepted,
4658 )
4659 .unwrap();
4660 let _ = client_rx.recv().await.unwrap();
4661 drain_now(&mut module_rx);
4662
4663 module_sink
4666 .try_send(stream_frame(9, 1, 0, vec![b'f'; BUDGET]))
4667 .unwrap();
4668 assert!(module_sink
4669 .try_send(stream_frame(9, 1, 1, Vec::new()))
4670 .is_err());
4671
4672 let client_ctx = RouteCtx {
4673 connection_id: client_connection,
4674 egress: client_sink,
4675 };
4676 router
4677 .route_for_connection(
4678 &client_ctx,
4679 route_frame(
4680 FrameType::Goodbye,
4681 pending.client_channel,
4682 pending.client_epoch,
4683 1_101,
4684 ),
4685 )
4686 .await
4687 .unwrap();
4688 tokio::task::yield_now().await;
4689 assert_eq!(
4690 router.counters.snapshot()["goodbye_relay_module_dropped"],
4691 0,
4692 "a GOODBYE refused by a momentarily full module queue must not be dropped"
4693 );
4694
4695 let filler = module_rx.recv().await.unwrap();
4697 assert_eq!(filler.header.ty, FrameType::StreamData);
4698 drop(filler);
4699 let goodbye = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
4700 .await
4701 .expect("the refused GOODBYE must be delivered once the module frees room")
4702 .unwrap();
4703 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
4704 assert_eq!(goodbye.header.channel, pending.module_channel);
4705 assert_eq!(goodbye.header.epoch, pending.module_epoch);
4706 assert_eq!(
4707 router.counters.snapshot()["goodbye_relay_module_dropped"],
4708 0
4709 );
4710 }
4711
4712 #[tokio::test]
4718 async fn module_frame_on_route_the_daemon_does_not_hold_is_answered_with_goodbye() {
4719 let (
4720 router,
4721 _forwarding,
4722 client_ctx,
4723 mut client_rx,
4724 module_ctx,
4725 mut module_egress_rx,
4726 mut module_rx,
4727 pending,
4728 ) = dynamic_route_fixture(true);
4729 let _ = client_rx.recv().await.unwrap();
4730 router
4731 .route_for_connection(
4732 &client_ctx,
4733 route_frame(
4734 FrameType::Goodbye,
4735 pending.client_channel,
4736 pending.client_epoch,
4737 1_200,
4738 ),
4739 )
4740 .await
4741 .unwrap();
4742 drain_now(&mut module_rx);
4743
4744 let cases = [
4745 (pending.module_channel, pending.module_epoch),
4747 (pending.module_channel + 1, 1),
4749 ];
4750 for (channel, epoch) in cases {
4751 router
4752 .route_for_connection(
4753 &module_ctx,
4754 route_frame(FrameType::StreamData, channel, epoch, 1_201),
4755 )
4756 .await
4757 .unwrap();
4758 let reply = module_egress_rx
4759 .try_recv()
4760 .expect("a frame on a route the daemon does not hold is answered");
4761 assert_eq!(reply.header.ty, FrameType::Goodbye);
4762 assert_eq!(reply.header.channel, channel);
4763 assert_eq!(reply.header.epoch, epoch);
4764 assert_eq!(reply.header.corr, 0);
4765 assert!(module_egress_rx.try_recv().is_err());
4766 }
4767
4768 router
4771 .route_for_connection(
4772 &module_ctx,
4773 route_frame(FrameType::Goodbye, pending.module_channel + 2, 1, 0),
4774 )
4775 .await
4776 .unwrap();
4777 assert!(module_egress_rx.try_recv().is_err());
4778
4779 let counters = router.counters.snapshot();
4780 assert_eq!(counters["module_frames_dropped_no_route"], 3);
4781 assert_eq!(counters["module_frames_dropped_released_route"], 1);
4782 assert_eq!(
4783 counters["module_frames_dropped_released_route_by_module"],
4784 serde_json::json!({ "epoch-router": 1 })
4785 );
4786 assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4787 }
4788
4789 #[tokio::test(start_paused = true)]
4793 async fn burst_of_orphan_module_frames_is_answered_once_per_interval() {
4794 let (
4795 router,
4796 _forwarding,
4797 client_ctx,
4798 mut client_rx,
4799 module_ctx,
4800 mut module_egress_rx,
4801 mut module_rx,
4802 pending,
4803 ) = dynamic_route_fixture(true);
4804 let _ = client_rx.recv().await.unwrap();
4805 router
4806 .route_for_connection(
4807 &client_ctx,
4808 route_frame(
4809 FrameType::Goodbye,
4810 pending.client_channel,
4811 pending.client_epoch,
4812 1_300,
4813 ),
4814 )
4815 .await
4816 .unwrap();
4817 drain_now(&mut module_rx);
4818
4819 let send_burst = |corr: u64| {
4820 route_frame(
4821 FrameType::StreamData,
4822 pending.module_channel,
4823 pending.module_epoch,
4824 corr,
4825 )
4826 };
4827 for corr in 0..20 {
4828 router
4829 .route_for_connection(&module_ctx, send_burst(corr))
4830 .await
4831 .unwrap();
4832 }
4833 let mut replies = 0;
4834 while let Ok(reply) = module_egress_rx.try_recv() {
4835 assert_eq!(reply.header.ty, FrameType::Goodbye);
4836 replies += 1;
4837 }
4838 assert_eq!(replies, 1, "a burst within the interval gets one GOODBYE");
4839
4840 tokio::time::advance(ORPHAN_ROUTE_GOODBYE_INTERVAL).await;
4841 router
4842 .route_for_connection(&module_ctx, send_burst(20))
4843 .await
4844 .unwrap();
4845 let retry = module_egress_rx
4846 .try_recv()
4847 .expect("the first orphan frame after the interval is answered again");
4848 assert_eq!(retry.header.channel, pending.module_channel);
4849 assert_eq!(retry.header.epoch, pending.module_epoch);
4850
4851 let counters = router.counters.snapshot();
4852 assert_eq!(counters["module_frames_dropped_no_route"], 21);
4853 assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4854 }
4855
4856 #[test]
4858 fn orphan_goodbye_rate_limit_state_is_released_with_the_connection() {
4859 let router = Router::with_default_self_handler();
4860 let connection = router.begin_connection();
4861 let id = connection.id();
4862 assert!(router.orphan_goodbyes.claim(id, 7));
4863 assert!(!router.orphan_goodbyes.claim(id, 7));
4864 assert!(router.orphan_goodbyes.claim(id, 8));
4865 drop(connection);
4866 assert!(router
4867 .orphan_goodbyes
4868 .last_sent
4869 .lock()
4870 .unwrap()
4871 .get(&id)
4872 .is_none());
4873 }
4874
4875 #[test]
4876 fn channel_zero_cannot_be_registered_as_backend() {
4877 let mut router = Router::with_default_self_handler();
4878
4879 let err = router.register_backend(0, EchoBackend).unwrap_err();
4880
4881 assert_eq!(err, RouterError::ReservedChannelZero);
4882 }
4883}