1use std::{
2 collections::HashMap,
3 error::Error,
4 fmt,
5 sync::{
6 atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
7 Arc, Mutex,
8 },
9 time::{Duration, Instant},
10};
11
12use subc_protocol::{error_codes, ErrorBody, Flags, FrameType, Priority};
13use tokio::sync::{mpsc, Notify};
14use tracing::{debug, warn};
15
16use crate::{
17 control::ControlHandler,
18 forwarding::{
19 principal_label, CloseReason, ConnectionCloseReceiver, DataRoute, DataRouteState,
20 ForwardingError, ForwardingTable, RouteBinding, RouteRelease, UndeliveredFrame,
21 },
22 registry::ConnectionId,
23 DaemonCounters, Frame, FrameBuildError,
24};
25
26#[derive(Debug)]
35pub struct OutboundFrame {
36 pub frame: Frame,
37 pub enqueued_at: std::time::Instant,
38 pub(crate) flushed: Option<tokio::sync::oneshot::Sender<()>>,
39 pub(crate) charge: Option<EgressCharge>,
44}
45
46impl OutboundFrame {
47 fn charged(frame: Frame, charge: EgressCharge) -> Self {
48 Self {
49 frame,
50 enqueued_at: charge.enqueued_at,
51 flushed: None,
52 charge: Some(charge),
53 }
54 }
55}
56
57fn queued_frame_bytes(frame: &Frame) -> usize {
60 subc_protocol::HEADER_LEN + frame.body.len()
61}
62
63#[derive(Debug, Clone, Copy, PartialEq, Eq)]
66pub(crate) struct EgressBacklog {
67 pub queued_bytes: usize,
68 pub queued_frames: usize,
69 pub oldest_age: Option<Duration>,
76}
77
78#[derive(Debug)]
83struct EgressAccounting {
84 byte_budget: usize,
85 queued_bytes: AtomicUsize,
86 queued_frames: AtomicUsize,
87 time_base: Instant,
89 oldest_enqueued_nanos: AtomicU64,
92 waiters: AtomicUsize,
95 freed: Notify,
98 client_delivery_closed: AtomicBool,
99 client_delivery_stopped: Notify,
100}
101
102impl EgressAccounting {
103 fn new(byte_budget: usize) -> Self {
104 Self {
105 byte_budget,
106 queued_bytes: AtomicUsize::new(0),
107 queued_frames: AtomicUsize::new(0),
108 time_base: tokio::time::Instant::now().into_std(),
109 oldest_enqueued_nanos: AtomicU64::new(0),
110 waiters: AtomicUsize::new(0),
111 freed: Notify::new(),
112 client_delivery_closed: AtomicBool::new(false),
113 client_delivery_stopped: Notify::new(),
114 }
115 }
116
117 fn stamp(&self, at: Instant) -> u64 {
118 (at.saturating_duration_since(self.time_base).as_nanos() as u64).saturating_add(1)
119 }
120
121 fn try_charge(self: &Arc<Self>, bytes: usize) -> Option<EgressCharge> {
126 let mut current = self.queued_bytes.load(Ordering::SeqCst);
129 loop {
130 if current != 0 && current.saturating_add(bytes) > self.byte_budget {
131 return None;
132 }
133 match self.queued_bytes.compare_exchange_weak(
134 current,
135 current + bytes,
136 Ordering::SeqCst,
137 Ordering::SeqCst,
138 ) {
139 Ok(_) => return Some(self.record(bytes)),
140 Err(actual) => current = actual,
141 }
142 }
143 }
144
145 fn charge_unconditionally(self: &Arc<Self>, bytes: usize) -> EgressCharge {
148 self.queued_bytes.fetch_add(bytes, Ordering::SeqCst);
149 self.record(bytes)
150 }
151
152 fn record(self: &Arc<Self>, bytes: usize) -> EgressCharge {
153 let enqueued_at = tokio::time::Instant::now().into_std();
154 let stamp = self.stamp(enqueued_at);
155 if self.queued_frames.fetch_add(1, Ordering::AcqRel) == 0 {
156 self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
159 }
160 EgressCharge {
161 accounting: Arc::clone(self),
162 bytes,
163 stamp,
164 enqueued_at,
165 }
166 }
167
168 fn taken_by_writer(&self, stamp: u64) {
170 self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
171 }
172
173 fn release(&self, bytes: usize, stamp: u64) {
174 self.queued_bytes.fetch_sub(bytes, Ordering::SeqCst);
175 if self.queued_frames.fetch_sub(1, Ordering::AcqRel) == 1 {
176 let _ = self.oldest_enqueued_nanos.compare_exchange(
179 stamp,
180 0,
181 Ordering::AcqRel,
182 Ordering::Acquire,
183 );
184 }
185 if self.waiters.load(Ordering::SeqCst) != 0 {
186 self.freed.notify_waiters();
187 }
188 }
189
190 fn backlog(&self) -> EgressBacklog {
191 let queued_frames = self.queued_frames.load(Ordering::Acquire);
192 let stamp = self.oldest_enqueued_nanos.load(Ordering::Acquire);
193 let oldest_age = (queued_frames != 0 && stamp != 0).then(|| {
194 let enqueued = self.time_base + Duration::from_nanos(stamp - 1);
195 tokio::time::Instant::from_std(enqueued).elapsed()
196 });
197 EgressBacklog {
198 queued_bytes: self.queued_bytes.load(Ordering::Acquire),
199 queued_frames,
200 oldest_age,
201 }
202 }
203}
204
205#[derive(Debug)]
208pub(crate) struct EgressCharge {
209 accounting: Arc<EgressAccounting>,
210 bytes: usize,
211 stamp: u64,
212 enqueued_at: Instant,
213}
214
215impl EgressCharge {
216 pub(crate) fn taken_by_writer(&self) {
219 self.accounting.taken_by_writer(self.stamp);
220 }
221}
222
223impl Drop for EgressCharge {
224 fn drop(&mut self) {
225 self.accounting.release(self.bytes, self.stamp);
226 }
227}
228
229#[derive(Debug)]
235pub(crate) struct EgressPermit {
236 permit: mpsc::OwnedPermit<OutboundFrame>,
237 accounting: Arc<EgressAccounting>,
238}
239
240impl EgressPermit {
241 pub(crate) fn send(self, frame: Frame) -> bool {
244 let charge = self
245 .accounting
246 .charge_unconditionally(queued_frame_bytes(&frame));
247 let sender = self.permit.send(OutboundFrame::charged(frame, charge));
248 sender.is_closed()
249 }
250}
251
252impl std::ops::Deref for OutboundFrame {
255 type Target = Frame;
256
257 fn deref(&self) -> &Frame {
258 &self.frame
259 }
260}
261
262#[cfg(test)]
265pub(crate) mod test_log {
266 use std::{
267 io::Write,
268 sync::{Arc, Mutex, OnceLock},
269 };
270
271 #[derive(Clone)]
272 struct TestLogWriter(Arc<Mutex<Vec<u8>>>);
273
274 impl Write for TestLogWriter {
275 fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
276 self.0
277 .lock()
278 .expect("test log capture is not poisoned")
279 .extend(buffer);
280 Ok(buffer.len())
281 }
282
283 fn flush(&mut self) -> std::io::Result<()> {
284 Ok(())
285 }
286 }
287
288 pub(crate) fn log_capture(
289 level: tracing::Level,
290 ) -> (Arc<Mutex<Vec<u8>>>, tracing::dispatcher::DefaultGuard) {
291 static INTEREST_ANCHOR: OnceLock<tracing::Dispatch> = OnceLock::new();
298 INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
299 let output = Arc::new(Mutex::new(Vec::new()));
300 let writer = Arc::clone(&output);
301 let subscriber = tracing_subscriber::fmt()
302 .with_max_level(level)
303 .with_ansi(false)
304 .without_time()
305 .with_target(false)
306 .with_writer(move || TestLogWriter(Arc::clone(&writer)))
307 .finish();
308 let guard = tracing::subscriber::set_default(subscriber);
309 (output, guard)
310 }
311
312 pub(crate) fn captured_logs(output: &Arc<Mutex<Vec<u8>>>) -> String {
313 String::from_utf8(
314 output
315 .lock()
316 .expect("test log capture is not poisoned")
317 .clone(),
318 )
319 .expect("tracing output is UTF-8")
320 }
321
322 #[cfg(test)]
323 mod tests {
324 use super::*;
325
326 fn emit_capture_probe(source: &str) {
327 tracing::error!(source, "parallel capture probe");
328 }
329
330 #[test]
331 fn parallel_callsite_registration_keeps_the_test_capture_enabled() {
332 let (output, _guard) = log_capture(tracing::Level::ERROR);
333 std::thread::spawn(|| emit_capture_probe("other-thread"))
336 .join()
337 .expect("unsubscribed thread exits");
338 emit_capture_probe("captured-thread");
339
340 let logs = captured_logs(&output);
341 assert!(
342 logs.contains("captured-thread"),
343 "capture was disabled: {logs}"
344 );
345 assert!(!logs.contains("other-thread"), "capture leaked: {logs}");
346 }
347 }
348}
349
350#[derive(Debug, Clone)]
358pub struct FrameSink {
359 tx: mpsc::Sender<OutboundFrame>,
360 accounting: Arc<EgressAccounting>,
361}
362
363impl FrameSink {
364 pub fn new(tx: mpsc::Sender<OutboundFrame>) -> Self {
368 Self::with_byte_budget(tx, crate::server::CONNECTION_EGRESS_BYTE_BUDGET)
369 }
370
371 pub(crate) fn with_byte_budget(tx: mpsc::Sender<OutboundFrame>, byte_budget: usize) -> Self {
372 Self {
373 tx,
374 accounting: Arc::new(EgressAccounting::new(byte_budget)),
375 }
376 }
377
378 async fn charge_waiting(&self, bytes: usize) -> Option<EgressCharge> {
385 if let Some(charge) = self.accounting.try_charge(bytes) {
386 return Some(charge);
387 }
388 struct Waiting<'a>(&'a AtomicUsize);
391 impl Drop for Waiting<'_> {
392 fn drop(&mut self) {
393 self.0.fetch_sub(1, Ordering::SeqCst);
394 }
395 }
396 self.accounting.waiters.fetch_add(1, Ordering::SeqCst);
397 let _waiting = Waiting(&self.accounting.waiters);
398 loop {
399 let freed = self.accounting.freed.notified();
400 tokio::pin!(freed);
401 freed.as_mut().enable();
404 if let Some(charge) = self.accounting.try_charge(bytes) {
405 return Some(charge);
406 }
407 tokio::select! {
408 _ = &mut freed => {}
409 _ = self.tx.closed() => return None,
410 }
411 }
412 }
413
414 pub async fn send(&self, frame: Frame) -> Result<(), RouterError> {
415 let channel = frame.header.channel;
416 let epoch = frame.header.epoch;
417 let corr = frame.header.corr;
418 let closed =
419 || RouterError::backend_with_epoch(channel, epoch, corr, "connection writer closed");
420 let charge = self
421 .charge_waiting(queued_frame_bytes(&frame))
422 .await
423 .ok_or_else(closed)?;
424 self.tx
425 .send(OutboundFrame::charged(frame, charge))
426 .await
427 .map_err(|_| closed())
428 }
429
430 #[cfg(unix)]
437 pub async fn send_flushed(&self, frame: Frame) -> Result<(), RouterError> {
438 let (tx, rx) = tokio::sync::oneshot::channel();
439 let charge = self
440 .charge_waiting(queued_frame_bytes(&frame))
441 .await
442 .ok_or_else(|| RouterError::backend(0, 0, "connection writer closed"))?;
443 let mut outbound = OutboundFrame::charged(frame, charge);
444 outbound.flushed = Some(tx);
445 self.tx
446 .send(outbound)
447 .await
448 .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
449 rx.await
450 .map_err(|_| RouterError::backend(0, 0, "connection flush failed"))
451 }
452
453 pub(crate) async fn reserve_owned(&self) -> Result<EgressPermit, RouterError> {
454 let permit = self
455 .tx
456 .clone()
457 .reserve_owned()
458 .await
459 .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
460 Ok(EgressPermit {
461 permit,
462 accounting: Arc::clone(&self.accounting),
463 })
464 }
465
466 #[cfg(test)]
467 pub(crate) fn try_reserve_owned(&self) -> Result<EgressPermit, RouterError> {
468 let permit = self
469 .tx
470 .clone()
471 .try_reserve_owned()
472 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
473 Ok(EgressPermit {
474 permit,
475 accounting: Arc::clone(&self.accounting),
476 })
477 }
478
479 pub(crate) fn is_closed(&self) -> bool {
480 self.tx.is_closed()
481 }
482
483 pub(crate) fn backlog(&self) -> EgressBacklog {
485 self.accounting.backlog()
486 }
487
488 pub(crate) fn close_client_delivery(&self) {
491 self.accounting
492 .client_delivery_closed
493 .store(true, Ordering::Release);
494 self.accounting.client_delivery_stopped.notify_waiters();
495 }
496
497 async fn send_to_client(&self, frame: Frame) -> Result<(), ClientDeliveryFailure> {
500 let frame_type = frame.header.ty;
501 let flags = frame.header.flags;
502 let stopped = self.accounting.client_delivery_stopped.notified();
503 tokio::pin!(stopped);
504 stopped.as_mut().enable();
505 let failure = |cause, error| ClientDeliveryFailure {
506 diagnosis: ClientDeliveryDiagnosis {
507 frame_type,
508 flags,
509 cause,
510 },
511 error,
512 };
513 let closed = || {
514 RouterError::backend_with_epoch(
515 frame.header.channel,
516 frame.header.epoch,
517 frame.header.corr,
518 "client connection closed",
519 )
520 };
521 if self
522 .accounting
523 .client_delivery_closed
524 .load(Ordering::Acquire)
525 {
526 return Err(failure("client_closed", closed()));
527 }
528 let (frame, error) = match self.try_send_owned(frame) {
529 Ok(()) => return Ok(()),
530 Err(refused) => refused,
531 };
532 if self.is_closed() {
533 return Err(failure("client_closed", error));
534 }
535 let grace = crate::server::CONNECTION_EGRESS_STALL_GRACE;
536 let age = self.backlog().oldest_age.unwrap_or_default();
537 if age > grace {
538 return Err(failure("oldest_frame_older_than_250_ms", error));
539 }
540 let result = tokio::select! {
541 biased;
542 _ = &mut stopped => return Err(failure("client_closed", error)),
543 result = tokio::time::timeout(grace.saturating_sub(age), self.send(frame)) => result,
544 };
545 match result {
546 Ok(Ok(())) => Ok(()),
547 Ok(Err(error)) => Err(failure("client_closed", error)),
548 Err(_) => Err(failure("waited_250_ms_without_room", error)),
549 }
550 }
551
552 pub(crate) fn try_send(&self, frame: Frame) -> Result<(), RouterError> {
556 self.try_send_owned(frame).map_err(|(_, error)| error)
557 }
558
559 fn try_send_owned(&self, frame: Frame) -> Result<(), (Frame, RouterError)> {
560 let channel = frame.header.channel;
561 let epoch = frame.header.epoch;
562 let corr = frame.header.corr;
563 let unavailable = |why: String| {
564 RouterError::backend_with_epoch(
565 channel,
566 epoch,
567 corr,
568 format!("connection writer unavailable: {why}"),
569 )
570 };
571 let bytes = queued_frame_bytes(&frame);
572 let Some(charge) = self.accounting.try_charge(bytes) else {
573 if self.tx.is_closed() {
574 return Err((frame, unavailable("channel closed".to_string())));
575 }
576 return Err((
577 frame,
578 unavailable(format!(
579 "egress byte budget exhausted ({} queued bytes, frame of {bytes} bytes, budget {})",
580 self.accounting.queued_bytes.load(Ordering::Acquire),
581 self.accounting.byte_budget
582 )),
583 ));
584 };
585 self.tx
588 .try_send(OutboundFrame::charged(frame, charge))
589 .map_err(|err| {
590 let error = unavailable(err.to_string());
591 (err.into_inner().frame, error)
592 })
593 }
594}
595
596#[derive(Debug)]
597struct ClientDeliveryFailure {
598 diagnosis: ClientDeliveryDiagnosis,
599 error: RouterError,
600}
601
602#[derive(Debug)]
604pub(crate) struct ClientDeliveryDiagnosis {
605 pub frame_type: FrameType,
606 pub flags: Flags,
607 pub cause: &'static str,
608}
609
610impl fmt::Display for ClientDeliveryFailure {
611 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
612 self.error.fmt(f)
613 }
614}
615
616const ORPHAN_ROUTE_GOODBYE_INTERVAL: Duration = Duration::from_secs(5);
628
629const ORPHAN_ROUTE_GOODBYE_PRUNE_AT: usize = 256;
636
637#[derive(Debug, Default)]
641struct OrphanGoodbyeLimiter {
642 last_sent: Mutex<HashMap<ConnectionId, HashMap<u16, tokio::time::Instant>>>,
643}
644
645impl OrphanGoodbyeLimiter {
646 fn claim(&self, connection_id: ConnectionId, channel: u16) -> bool {
649 let now = tokio::time::Instant::now();
650 let mut last_sent = self
651 .last_sent
652 .lock()
653 .unwrap_or_else(std::sync::PoisonError::into_inner);
654 let channels = last_sent.entry(connection_id).or_default();
655 if channels.get(&channel).is_some_and(|sent| {
656 now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
657 }) {
658 return false;
659 }
660 if channels.len() >= ORPHAN_ROUTE_GOODBYE_PRUNE_AT {
661 channels.retain(|_, sent| {
662 now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
663 });
664 }
665 channels.insert(channel, now);
666 true
667 }
668
669 fn forget_connection(&self, connection_id: ConnectionId) {
670 self.last_sent
671 .lock()
672 .unwrap_or_else(std::sync::PoisonError::into_inner)
673 .remove(&connection_id);
674 }
675}
676
677#[derive(Debug, Clone)]
679pub struct RouteCtx {
680 pub connection_id: ConnectionId,
681 pub egress: FrameSink,
682}
683
684#[derive(Debug, Clone)]
690pub enum Backend {
691 Echo(EchoBackend),
692 Forward(ForwardBackend),
693}
694
695impl From<EchoBackend> for Backend {
696 fn from(backend: EchoBackend) -> Self {
697 Self::Echo(backend)
698 }
699}
700
701impl From<ForwardBackend> for Backend {
702 fn from(backend: ForwardBackend) -> Self {
703 Self::Forward(backend)
704 }
705}
706
707impl Backend {
708 pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
709 match self {
710 Self::Echo(backend) => backend.handle(ctx, frame).await,
711 Self::Forward(backend) => backend.handle(ctx, frame).await,
712 }
713 }
714}
715
716pub struct Router {
737 backends: HashMap<u16, Backend>,
738 control: Arc<ControlHandler>,
739 forwarding: Arc<ForwardingTable>,
740 forward_backend: ForwardBackend,
741 counters: DaemonCounters,
742 next_connection_id: AtomicU64,
743 orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
744}
745
746impl Router {
747 pub fn with_control_handler(control: Arc<ControlHandler>) -> Self {
748 control.install_swap_promotion_observer();
751 let forwarding = control.forwarding();
752 let counters = control.counters();
753 Self {
754 backends: HashMap::new(),
755 control,
756 forwarding: Arc::clone(&forwarding),
757 forward_backend: ForwardBackend::new(forwarding),
758 counters,
759 next_connection_id: AtomicU64::new(1),
761 orphan_goodbyes: Arc::default(),
762 }
763 }
764
765 pub fn with_default_self_handler() -> Self {
766 Self::with_control_handler(Arc::new(ControlHandler::default()))
767 }
768
769 pub fn forwarding(&self) -> Arc<ForwardingTable> {
770 Arc::clone(&self.forwarding)
771 }
772
773 pub fn register_backend(
774 &mut self,
775 channel: u16,
776 backend: impl Into<Backend>,
777 ) -> Result<(), RouterError> {
778 self.register_backend_arc(channel, Arc::new(backend.into()))
779 }
780
781 pub(crate) fn register_backend_arc(
782 &mut self,
783 channel: u16,
784 backend: Arc<Backend>,
785 ) -> Result<(), RouterError> {
786 if channel == 0 {
787 return Err(RouterError::ReservedChannelZero);
788 }
789 if self.backends.contains_key(&channel) {
790 return Err(RouterError::DuplicateChannel { channel });
791 }
792 self.backends.insert(channel, backend.as_ref().clone());
793 Ok(())
794 }
795
796 fn record_module_frame_drop(&self, connection_id: ConnectionId) -> Result<(), RouterError> {
799 let module_id = self
800 .forwarding
801 .module_id_for_connection(connection_id)
802 .map_err(RouterError::Forwarding)?;
803 self.counters
804 .increment_module_frames_dropped_no_route(module_id.as_deref());
805 Ok(())
806 }
807
808 fn handle_orphan_module_frame(&self, ctx: &RouteCtx, frame: &Frame) -> Result<(), RouterError> {
820 let channel = frame.header.channel;
821 let epoch = frame.header.epoch;
822 let module_id = self
823 .forwarding
824 .module_id_for_connection(ctx.connection_id)
825 .map_err(RouterError::Forwarding)?;
826 self.counters
827 .increment_module_frames_dropped_no_route(module_id.as_deref());
828 if self
829 .forwarding
830 .module_route_epoch_was_allocated(ctx.connection_id, channel, epoch)
831 .map_err(RouterError::Forwarding)?
832 {
833 self.counters
834 .increment_module_frames_dropped_released_route(module_id.as_deref());
835 }
836 if frame.header.ty == FrameType::Goodbye
837 || !self.orphan_goodbyes.claim(ctx.connection_id, channel)
838 {
839 return Ok(());
840 }
841 let goodbye = Frame::build_with_version(
842 frame.header.ver,
843 FrameType::Goodbye,
844 Flags::new(false, Priority::Passive, false),
845 channel,
846 epoch,
847 0,
848 Vec::new(),
849 )
850 .map_err(RouterError::FrameBuild)?;
851 match ctx.egress.try_send(goodbye) {
852 Ok(()) => {
853 self.counters.increment_module_orphan_route_goodbyes_sent();
854 debug!(
855 connection_id = ctx.connection_id.get(),
856 module_id = module_id.as_deref().unwrap_or("unknown"),
857 channel,
858 epoch,
859 "answered module frame on a route the daemon does not hold with a route GOODBYE"
860 );
861 }
862 Err(err) => {
863 self.forwarding.warn_frame_drop(
864 ctx.connection_id,
865 channel,
866 epoch,
867 "daemon_to_module",
868 if ctx.egress.is_closed() {
869 "endpoint_gone"
870 } else {
871 "egress_budget"
872 },
873 );
874 debug!(
875 connection_id = ctx.connection_id.get(),
876 module_id = module_id.as_deref().unwrap_or("unknown"),
877 channel,
878 epoch,
879 error = %err,
880 "could not enqueue route GOODBYE for module frame on a route the daemon does not hold; the next such frame retries"
881 );
882 }
883 }
884 Ok(())
885 }
886
887 pub fn begin_connection(&self) -> RouterConnection {
888 let raw = self.next_connection_id.fetch_add(1, Ordering::Relaxed);
889 let id = ConnectionId::new(raw);
890 let close_receiver = self.forwarding.register_connection_close(id);
891 RouterConnection {
892 id,
893 control_handler: Arc::clone(&self.control),
894 forwarding: Arc::clone(&self.forwarding),
895 close_receiver: Some(close_receiver),
896 orphan_goodbyes: Arc::clone(&self.orphan_goodbyes),
897 }
898 }
899
900 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
901 self.control.route_open_target(frame)
902 }
903
904 pub(crate) fn record_captured_peer(
905 &self,
906 connection: ConnectionId,
907 peer: subc_os::local_peer::CapturedPeer,
908 ) {
909 self.control.record_captured_peer(connection, peer);
910 }
911
912 pub(crate) fn record_caller_endpoints(
913 &self,
914 connection: ConnectionId,
915 endpoints: Option<(std::net::SocketAddr, std::net::SocketAddr)>,
916 ) {
917 self.control.record_caller_endpoints(connection, endpoints);
918 }
919
920 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
921 self.control.defers_privileged_control(ctx, frame)
922 }
923
924 pub(crate) fn route_open_capacity_refusal(
925 &self,
926 ctx: &RouteCtx,
927 frame: &Frame,
928 target_module_id: &str,
929 in_flight: usize,
930 limit: usize,
931 ) -> Result<Frame, RouterError> {
932 self.control
933 .route_open_capacity_refusal(ctx, frame, target_module_id, in_flight, limit)
934 }
935
936 pub async fn route_for_connection(
937 &self,
938 ctx: &RouteCtx,
939 frame: Frame,
940 ) -> Result<(), RouterError> {
941 self.route_for_connection_started(ctx, frame, None).await
942 }
943
944 pub(crate) async fn route_for_connection_started(
945 &self,
946 ctx: &RouteCtx,
947 frame: Frame,
948 dispatch_started_at: Option<Instant>,
949 ) -> Result<(), RouterError> {
950 let channel = frame.header.channel;
951 let epoch = frame.header.epoch;
952 let corr = frame.header.corr;
953 if channel == 0 {
954 debug!(
955 connection_id = ctx.connection_id.get(),
956 corr,
957 frame_type = ?frame.header.ty,
958 "routing control frame"
959 );
960 let dispatch_started_at = (frame.header.ty == FrameType::Request)
966 .then(|| dispatch_started_at.unwrap_or_else(Instant::now));
967 let responses = self
968 .control
969 .handle_control_frame_timed(ctx, frame, dispatch_started_at)
970 .await?;
971 for response in responses {
972 ctx.egress.send(response).await?;
973 }
974 return Ok(());
975 }
976
977 let data_route = self
978 .forwarding
979 .lookup_data_route(ctx.connection_id, channel, epoch)
980 .map_err(RouterError::Forwarding)?;
981
982 match data_route {
983 DataRoute::Module(DataRouteState::EpochMismatch) => {
984 self.forwarding.warn_frame_drop(
985 ctx.connection_id,
986 channel,
987 epoch,
988 "module_to_client",
989 "stale_epoch",
990 );
991 if frame.header.ty == FrameType::Request {
992 self.counters
993 .increment_module_requests_dropped_stale_route();
994 let err = RouterError::StaleRouteEpoch {
995 channel,
996 epoch,
997 corr,
998 };
999 if let Some(error_frame) = err.to_error_frame() {
1000 ctx.egress.send(error_frame).await?;
1001 }
1002 } else {
1003 self.handle_orphan_module_frame(ctx, &frame)?;
1004 }
1005 debug!(
1006 connection_id = ctx.connection_id.get(),
1007 channel, epoch, corr, "dropping module frame for stale route epoch"
1008 );
1009 return Ok(());
1010 }
1011 DataRoute::Module(DataRouteState::Reserved) => {
1012 self.forwarding.warn_frame_drop(
1013 ctx.connection_id,
1014 channel,
1015 epoch,
1016 "module_to_client",
1017 "missing_binding",
1018 );
1019 if frame.header.ty == FrameType::Request {
1020 self.counters
1021 .increment_module_requests_dropped_stale_route();
1022 let err = RouterError::UnknownChannel {
1023 channel,
1024 epoch,
1025 corr,
1026 };
1027 if let Some(error_frame) = err.to_error_frame() {
1028 ctx.egress.send(error_frame).await?;
1029 }
1030 } else {
1031 self.record_module_frame_drop(ctx.connection_id)?;
1032 }
1033 debug!(
1034 connection_id = ctx.connection_id.get(),
1035 channel, epoch, corr, "dropping module frame for reserved route handle"
1036 );
1037 return Ok(());
1038 }
1039 DataRoute::Module(DataRouteState::Absent) => {
1040 self.forwarding.warn_frame_drop(
1041 ctx.connection_id,
1042 channel,
1043 epoch,
1044 "module_to_client",
1045 "unknown_channel",
1046 );
1047 if frame.header.ty == FrameType::Request {
1048 self.counters
1049 .increment_module_requests_dropped_stale_route();
1050 let err = RouterError::UnknownChannel {
1051 channel,
1052 epoch,
1053 corr,
1054 };
1055 if let Some(error_frame) = err.to_error_frame() {
1056 ctx.egress.send(error_frame).await?;
1057 }
1058 } else {
1059 self.handle_orphan_module_frame(ctx, &frame)?;
1060 }
1061 debug!(
1062 connection_id = ctx.connection_id.get(),
1063 channel, epoch, corr, "dropping module frame for absent route handle"
1064 );
1065 return Ok(());
1066 }
1067 DataRoute::Module(DataRouteState::Bound(route)) => {
1068 if frame.header.ty == FrameType::Goodbye {
1069 if let RouteRelease::Removed(target) = self
1070 .forwarding
1071 .release_module_route(ctx.connection_id, channel, epoch)
1072 .map_err(RouterError::Forwarding)?
1073 {
1074 let mut goodbye = frame;
1075 goodbye.header.channel = target.channel;
1076 goodbye.header.epoch = target.epoch;
1077 goodbye.header.flags = goodbye.header.flags.without_daemon_origin();
1081 if let Err(err) = target.sink.send_to_client(goodbye).await {
1082 if !target.close_on_delivery_failure() {
1083 self.forwarding.warn_frame_drop(
1084 target.connection_id,
1085 target.channel,
1086 target.epoch,
1087 "module_to_client",
1088 err.diagnosis.cause,
1089 );
1090 }
1091 if target.close_on_delivery_failure()
1092 && self
1093 .forwarding
1094 .escalate_client_delivery_failure_diagnosed(
1095 target.connection_id,
1096 target.channel,
1097 target.epoch,
1098 CloseReason::new(
1099 "route_goodbye_delivery_failed",
1100 format!(
1101 "failed to enqueue route GOODBYE for client channel {}: {err}",
1102 target.channel
1103 ),
1104 ),
1105 UndeliveredFrame {
1106 module_id: Some(&route.module_id),
1107 sink: &target.sink,
1108 },
1109 Some(&err.diagnosis),
1110 )
1111 .map_err(RouterError::Forwarding)?
1112 {
1113 self.counters.increment_goodbye_relay_client_failed();
1114 }
1115 }
1116 } else {
1117 self.forwarding.warn_frame_drop(
1118 ctx.connection_id,
1119 channel,
1120 epoch,
1121 "module_to_client",
1122 "missing_binding",
1123 );
1124 }
1125 return Ok(());
1126 }
1127
1128 let bytes = queued_frame_bytes(&frame);
1132 let limit = route.client_sink.accounting.byte_budget;
1133 if bytes > limit {
1134 let ty = frame.header.ty;
1135 let flags = frame.header.flags;
1136 drop(frame);
1137 return self
1138 .refuse_oversized_module_frame(ctx, &route, (ty, flags), corr, bytes)
1139 .await;
1140 }
1141
1142 let releases_credit = is_terminal_frame(frame.header.ty);
1148 if releases_credit {
1149 route.flow.release_corr(corr);
1150 }
1151 let mut frame = frame;
1152 frame.header.channel = route.client_channel;
1153 frame.header.epoch = route.client_epoch;
1154 frame.header.flags = frame.header.flags.without_daemon_origin();
1160 if let Err(err) = route.client_sink.send_to_client(frame).await {
1161 if self
1162 .forwarding
1163 .escalate_client_delivery_failure_diagnosed(
1164 route.client_connection_id,
1165 route.client_channel,
1166 route.client_epoch,
1167 CloseReason::new(
1168 "module_to_client_delivery_failed",
1169 format!(
1170 "failed to enqueue module frame for client channel {} corr {corr}: {err}",
1171 route.client_channel
1172 ),
1173 ),
1174 UndeliveredFrame {
1175 module_id: Some(&route.module_id),
1176 sink: &route.client_sink,
1177 },
1178 Some(&err.diagnosis),
1179 )
1180 .map_err(RouterError::Forwarding)?
1181 {
1182 self.counters
1183 .increment_client_egress_close_delivery_failed();
1184 }
1185 return Ok(());
1186 }
1187 return Ok(());
1188 }
1189 DataRoute::Client(DataRouteState::EpochMismatch) => {
1190 self.forwarding.warn_frame_drop(
1191 ctx.connection_id,
1192 channel,
1193 epoch,
1194 "client_to_module",
1195 "stale_epoch",
1196 );
1197 if frame.header.ty == FrameType::Request {
1198 self.counters.increment_client_frames_dropped_stale_route();
1199 let err = RouterError::StaleRouteEpoch {
1201 channel,
1202 epoch,
1203 corr,
1204 };
1205 if let Some(error_frame) = err.to_error_frame() {
1206 ctx.egress.send(error_frame).await?;
1207 }
1208 }
1209 debug!(
1210 connection_id = ctx.connection_id.get(),
1211 channel, epoch, corr, "dropping client frame for stale route epoch"
1212 );
1213 return Ok(());
1214 }
1215 DataRoute::Client(DataRouteState::Reserved) => {
1216 self.forwarding.warn_frame_drop(
1217 ctx.connection_id,
1218 channel,
1219 epoch,
1220 "client_to_module",
1221 "missing_binding",
1222 );
1223 if frame.header.ty == FrameType::Request {
1224 let err = RouterError::UnknownChannel {
1225 channel,
1226 epoch,
1227 corr,
1228 };
1229 if let Some(error_frame) = err.to_error_frame() {
1230 ctx.egress.send(error_frame).await?;
1231 }
1232 }
1233 return Ok(());
1234 }
1235 DataRoute::Client(DataRouteState::Bound(route)) => {
1236 if frame.header.ty == FrameType::Goodbye {
1237 let _ = self
1238 .control
1239 .handle_route_goodbye(ctx.connection_id, channel, epoch)?;
1240 return Ok(());
1241 }
1242 return self.forward_backend.handle_bound(frame, route).await;
1243 }
1244 DataRoute::Client(DataRouteState::Absent) => {}
1245 }
1246
1247 if let Some(backend) = self.backends.get(&channel) {
1248 return backend.handle(ctx.clone(), frame).await.inspect_err(|_| {
1249 self.forwarding.warn_frame_drop(
1250 ctx.connection_id,
1251 channel,
1252 epoch,
1253 "client_to_backend",
1254 "endpoint_gone",
1255 );
1256 });
1257 }
1258 self.forwarding.warn_frame_drop(
1259 ctx.connection_id,
1260 channel,
1261 epoch,
1262 "client_to_module",
1263 "unknown_channel",
1264 );
1265 if frame.header.ty == FrameType::Request {
1266 let err = RouterError::UnknownChannel {
1267 channel,
1268 epoch,
1269 corr,
1270 };
1271 if let Some(error_frame) = err.to_error_frame() {
1272 ctx.egress.send(error_frame).await?;
1273 }
1274 }
1275 Ok(())
1276 }
1277
1278 async fn refuse_oversized_module_frame(
1279 &self,
1280 ctx: &RouteCtx,
1281 route: &RouteBinding,
1282 header: (FrameType, Flags),
1283 corr: u64,
1284 bytes: usize,
1285 ) -> Result<(), RouterError> {
1286 let (ty, flags) = header;
1287 let limit = route.client_sink.accounting.byte_budget;
1288 if ty == FrameType::Push {
1289 let diagnosis = ClientDeliveryDiagnosis {
1293 frame_type: ty,
1294 flags,
1295 cause: "frame_too_large",
1296 };
1297 if self
1298 .forwarding
1299 .escalate_client_delivery_failure_diagnosed(
1300 route.client_connection_id,
1301 route.client_channel,
1302 route.client_epoch,
1303 CloseReason::new(
1304 error_codes::FRAME_TOO_LARGE,
1305 format!(
1306 "oversized push: size={bytes} limit={limit} (header plus body bytes)"
1307 ),
1308 ),
1309 UndeliveredFrame {
1310 module_id: Some(&route.module_id),
1311 sink: &route.client_sink,
1312 },
1313 Some(&diagnosis),
1314 )
1315 .map_err(RouterError::Forwarding)?
1316 {
1317 self.counters
1318 .increment_client_egress_close_delivery_failed();
1319 }
1320 return Ok(());
1321 }
1322
1323 warn!(
1324 connection_id = route.client_connection_id.get(),
1325 module_id = %route.module_id,
1326 client_principal = %principal_label(&route.principal),
1327 client_channel = route.client_channel,
1328 frame_type = ?ty,
1329 frame_bytes = bytes,
1330 byte_limit = limit,
1331 "refusing oversized module-to-client frame with ERROR frame_too_large"
1332 );
1333 if ty == FrameType::Request {
1334 ctx.egress
1337 .send(frame_too_large_error(
1338 route.module_channel,
1339 route.module_epoch,
1340 corr,
1341 bytes,
1342 limit,
1343 )?)
1344 .await?;
1345 } else {
1346 route.flow.release_corr(corr);
1349 let error = frame_too_large_error(
1350 route.client_channel,
1351 route.client_epoch,
1352 corr,
1353 bytes,
1354 limit,
1355 )?;
1356 if let Err(err) = route.client_sink.send_to_client(error).await {
1359 if self
1360 .forwarding
1361 .escalate_client_delivery_failure_diagnosed(
1362 route.client_connection_id,
1363 route.client_channel,
1364 route.client_epoch,
1365 CloseReason::new(
1366 "module_to_client_delivery_failed",
1367 format!(
1368 "failed to enqueue frame_too_large for client channel {} corr {corr}: {err}",
1369 route.client_channel
1370 ),
1371 ),
1372 UndeliveredFrame {
1373 module_id: Some(&route.module_id),
1374 sink: &route.client_sink,
1375 },
1376 Some(&err.diagnosis),
1377 )
1378 .map_err(RouterError::Forwarding)?
1379 {
1380 self.counters
1381 .increment_client_egress_close_delivery_failed();
1382 }
1383 }
1384 }
1385 Ok(())
1386 }
1387}
1388
1389impl Default for Router {
1390 fn default() -> Self {
1391 Self::with_default_self_handler()
1392 }
1393}
1394
1395#[must_use]
1397pub struct RouterConnection {
1398 id: ConnectionId,
1399 control_handler: Arc<ControlHandler>,
1400 forwarding: Arc<ForwardingTable>,
1401 close_receiver: Option<ConnectionCloseReceiver>,
1402 orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
1403}
1404
1405impl RouterConnection {
1406 pub fn id(&self) -> ConnectionId {
1407 self.id
1408 }
1409
1410 pub(crate) fn take_close_receiver(&mut self) -> ConnectionCloseReceiver {
1411 self.close_receiver
1412 .take()
1413 .expect("connection close receiver can only be taken once")
1414 }
1415}
1416
1417impl Drop for RouterConnection {
1418 fn drop(&mut self) {
1419 self.forwarding.unregister_connection_close(self.id);
1420 self.orphan_goodbyes.forget_connection(self.id);
1421 let _ = self.control_handler.cleanup_connection(self.id);
1424 }
1425}
1426
1427#[derive(Debug, Default, Clone, Copy)]
1430pub struct EchoBackend;
1431
1432impl EchoBackend {
1433 pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1434 let response = Frame::build_with_version(
1435 frame.header.ver,
1436 FrameType::Response,
1437 frame.header.flags,
1438 frame.header.channel,
1439 frame.header.epoch,
1440 frame.header.corr,
1441 frame.body,
1442 )
1443 .map_err(RouterError::FrameBuild)?;
1444 ctx.egress.send(response).await
1445 }
1446}
1447
1448#[derive(Debug, Clone)]
1450pub struct ForwardBackend {
1451 forwarding: Arc<ForwardingTable>,
1452}
1453
1454impl ForwardBackend {
1455 pub fn new(forwarding: Arc<ForwardingTable>) -> Self {
1456 Self { forwarding }
1457 }
1458
1459 pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1460 let channel = frame.header.channel;
1461 let corr = frame.header.corr;
1462 let route = match self
1463 .forwarding
1464 .lookup_data_route(ctx.connection_id, channel, frame.header.epoch)
1465 .map_err(RouterError::Forwarding)?
1466 {
1467 DataRoute::Client(DataRouteState::Bound(route)) => route,
1468 DataRoute::Client(_) | DataRoute::Module(_) => {
1469 self.forwarding.warn_frame_drop(
1470 ctx.connection_id,
1471 channel,
1472 frame.header.epoch,
1473 "client_to_module",
1474 "missing_binding",
1475 );
1476 return Err(RouterError::UnknownChannel {
1477 channel,
1478 epoch: frame.header.epoch,
1479 corr,
1480 });
1481 }
1482 };
1483 self.handle_bound(frame, route).await
1484 }
1485
1486 pub(crate) async fn handle_bound(
1487 &self,
1488 frame: Frame,
1489 route: Arc<RouteBinding>,
1490 ) -> Result<(), RouterError> {
1491 let channel = frame.header.channel;
1492 let corr = frame.header.corr;
1493 let frame_type = frame.header.ty;
1494
1495 let acquired_credit = frame_type == FrameType::Request;
1498 if acquired_credit {
1499 if let Err(err) = route
1500 .flow
1501 .acquire_tagged(corr, frame.header.flags.is_subscription())
1502 .await
1503 {
1504 self.forwarding.warn_frame_drop(
1505 route.client_connection_id,
1506 channel,
1507 frame.header.epoch,
1508 "client_to_module",
1509 "endpoint_gone",
1510 );
1511 if self
1518 .forwarding
1519 .endpoint_is_draining(route.module_endpoint)
1520 .map_err(RouterError::Forwarding)?
1521 {
1522 return Err(RouterError::route_error_with_epoch(
1523 channel,
1524 frame.header.epoch,
1525 corr,
1526 "module_reloading",
1527 format!("module endpoint for route channel {channel} is reloading"),
1528 ));
1529 }
1530 return Err(RouterError::backend_with_epoch(
1531 channel,
1532 frame.header.epoch,
1533 corr,
1534 format!("{err} for route channel {channel}"),
1535 ));
1536 }
1537 }
1538
1539 let mut frame = frame;
1540 frame.header.channel = route.module_channel;
1541 frame.header.epoch = route.module_epoch;
1542 frame.header.flags = frame.header.flags.without_daemon_origin();
1546 let result = route.module_sink.send(frame).await.map_err(|err| {
1547 self.forwarding.warn_frame_drop(
1548 route.client_connection_id,
1549 channel,
1550 route.client_epoch,
1551 "client_to_module",
1552 "endpoint_gone",
1553 );
1554 RouterError::backend_with_epoch(channel, route.client_epoch, corr, err.to_string())
1555 });
1556 if acquired_credit && result.is_err() {
1557 route.flow.release_corr(corr);
1558 }
1559 result
1560 }
1561}
1562
1563fn is_terminal_frame(frame_type: FrameType) -> bool {
1564 matches!(
1565 frame_type,
1566 FrameType::Response | FrameType::Error | FrameType::StreamEnd
1567 )
1568}
1569
1570#[derive(Debug, Clone, PartialEq, Eq)]
1573pub enum RouterError {
1574 ReservedChannelZero,
1575 DuplicateChannel {
1576 channel: u16,
1577 },
1578 UnknownChannel {
1579 channel: u16,
1580 epoch: u32,
1581 corr: u64,
1582 },
1583 StaleRouteEpoch {
1584 channel: u16,
1585 epoch: u32,
1586 corr: u64,
1587 },
1588 Backend {
1589 channel: u16,
1590 epoch: u32,
1591 corr: u64,
1592 message: String,
1593 },
1594 RouteError {
1595 channel: u16,
1596 epoch: u32,
1597 corr: u64,
1598 code: String,
1599 message: String,
1600 },
1601 FrameBuild(FrameBuildError),
1602 Forwarding(ForwardingError),
1603}
1604
1605impl RouterError {
1606 pub fn backend(channel: u16, corr: u64, message: impl Into<String>) -> Self {
1607 Self::backend_with_epoch(channel, 0, corr, message)
1608 }
1609
1610 pub fn backend_with_epoch(
1611 channel: u16,
1612 epoch: u32,
1613 corr: u64,
1614 message: impl Into<String>,
1615 ) -> Self {
1616 Self::Backend {
1617 channel,
1618 epoch,
1619 corr,
1620 message: message.into(),
1621 }
1622 }
1623
1624 pub fn route_error(
1625 channel: u16,
1626 corr: u64,
1627 code: impl Into<String>,
1628 message: impl Into<String>,
1629 ) -> Self {
1630 Self::route_error_with_epoch(channel, 0, corr, code, message)
1631 }
1632
1633 pub fn route_error_with_epoch(
1634 channel: u16,
1635 epoch: u32,
1636 corr: u64,
1637 code: impl Into<String>,
1638 message: impl Into<String>,
1639 ) -> Self {
1640 Self::RouteError {
1641 channel,
1642 epoch,
1643 corr,
1644 code: code.into(),
1645 message: message.into(),
1646 }
1647 }
1648
1649 pub fn to_error_frame(&self) -> Option<Frame> {
1651 match self {
1652 Self::UnknownChannel {
1653 channel,
1654 epoch,
1655 corr,
1656 } => error_frame(
1657 *channel,
1658 *epoch,
1659 *corr,
1660 error_codes::UNKNOWN_CHANNEL,
1661 format!("unknown channel {channel}"),
1662 ),
1663 Self::StaleRouteEpoch {
1664 channel,
1665 epoch,
1666 corr,
1667 } => error_frame(
1668 *channel,
1669 *epoch,
1670 *corr,
1671 error_codes::STALE_ROUTE_EPOCH,
1672 format!("stale route epoch for channel {channel}"),
1673 ),
1674 Self::Backend {
1675 channel,
1676 epoch,
1677 corr,
1678 message,
1679 } => error_frame(*channel, *epoch, *corr, "backend_error", message.clone()),
1680 Self::RouteError {
1681 channel,
1682 epoch,
1683 corr,
1684 code,
1685 message,
1686 } => error_frame(*channel, *epoch, *corr, code, message.clone()),
1687 Self::ReservedChannelZero
1688 | Self::DuplicateChannel { .. }
1689 | Self::FrameBuild(_)
1690 | Self::Forwarding(_) => None,
1691 }
1692 }
1693}
1694
1695pub(crate) fn daemon_error_flags() -> Flags {
1707 Flags::new(false, Priority::Passive, false).with_daemon_origin()
1708}
1709
1710fn error_frame(channel: u16, epoch: u32, corr: u64, code: &str, message: String) -> Option<Frame> {
1711 let body = serde_json::to_vec(&ErrorBody {
1712 code: code.to_string(),
1713 message,
1714 detail: None,
1715 })
1716 .ok()?;
1717
1718 Frame::build(
1719 FrameType::Error,
1720 daemon_error_flags(),
1721 channel,
1722 epoch,
1723 corr,
1724 body,
1725 )
1726 .ok()
1727}
1728
1729fn frame_too_large_error(
1730 channel: u16,
1731 epoch: u32,
1732 corr: u64,
1733 bytes: usize,
1734 limit: usize,
1735) -> Result<Frame, RouterError> {
1736 let body = serde_json::to_vec(&ErrorBody {
1737 code: error_codes::FRAME_TOO_LARGE.to_string(),
1738 message: format!(
1739 "module frame of {bytes} bytes exceeds client egress limit of {limit} bytes"
1740 ),
1741 detail: Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit })),
1742 })
1743 .expect("serializing a frame size error");
1744 Frame::build(
1745 FrameType::Error,
1746 daemon_error_flags(),
1747 channel,
1748 epoch,
1749 corr,
1750 body,
1751 )
1752 .map_err(RouterError::FrameBuild)
1753}
1754
1755impl fmt::Display for RouterError {
1756 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1757 match self {
1758 Self::ReservedChannelZero => write!(f, "channel 0 is reserved for subc"),
1759 Self::DuplicateChannel { channel } => {
1760 write!(f, "backend already registered for channel {channel}")
1761 }
1762 Self::UnknownChannel { channel, corr, .. } => {
1763 write!(f, "unknown channel {channel} for corr {corr}")
1764 }
1765 Self::StaleRouteEpoch { channel, corr, .. } => {
1766 write!(f, "stale route epoch for channel {channel} corr {corr}")
1767 }
1768 Self::Backend {
1769 channel,
1770 corr,
1771 message,
1772 ..
1773 } => write!(
1774 f,
1775 "backend error on channel {channel} corr {corr}: {message}"
1776 ),
1777 Self::RouteError {
1778 channel,
1779 corr,
1780 code,
1781 message,
1782 ..
1783 } => write!(
1784 f,
1785 "route error {code} on channel {channel} corr {corr}: {message}"
1786 ),
1787 Self::FrameBuild(err) => write!(f, "failed to build routed frame: {err}"),
1788 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
1789 }
1790 }
1791}
1792
1793impl Error for RouterError {
1794 fn source(&self) -> Option<&(dyn Error + 'static)> {
1795 match self {
1796 Self::FrameBuild(err) => Some(err),
1797 Self::Forwarding(err) => Some(err),
1798 Self::ReservedChannelZero
1799 | Self::DuplicateChannel { .. }
1800 | Self::UnknownChannel { .. }
1801 | Self::StaleRouteEpoch { .. }
1802 | Self::Backend { .. }
1803 | Self::RouteError { .. } => None,
1804 }
1805 }
1806}
1807
1808#[cfg(test)]
1809mod tests {
1810 use super::*;
1811 use crate::{
1812 forwarding::RouteBindRelayOutcome,
1813 supervise::{ModuleSpec, RestartPolicy, Supervisor, SupervisorHandle},
1814 ControlHandler, Registry,
1815 };
1816 use std::{
1817 sync::{mpsc as std_mpsc, Arc},
1818 time::Duration,
1819 };
1820 use subc_control::ModuleProtocol;
1821 use subc_protocol::{manifest::Concurrency, ErrorBody, Flags, FrameType, Priority};
1822 use tokio::sync::mpsc;
1823
1824 pub(crate) use crate::router::test_log::{captured_logs, log_capture};
1825
1826 fn logged_millis(logs: &str, field: &str) -> u64 {
1827 logs.split_whitespace()
1828 .find_map(|part| part.strip_prefix(field))
1829 .and_then(|value| value.parse().ok())
1830 .unwrap_or_else(|| panic!("missing numeric {field} in logs: {logs}"))
1831 }
1832
1833 fn request(channel: u16, corr: u64, body: &[u8]) -> Frame {
1834 Frame::build(
1835 FrameType::Request,
1836 Flags::new(true, Priority::Interactive, false),
1837 channel,
1838 0,
1839 corr,
1840 body.to_vec(),
1841 )
1842 .unwrap()
1843 }
1844
1845 fn ping(corr: u64) -> Frame {
1846 Frame::build(
1847 FrameType::Ping,
1848 Flags::new(false, Priority::Passive, false),
1849 0,
1850 0,
1851 corr,
1852 Vec::new(),
1853 )
1854 .unwrap()
1855 }
1856
1857 fn route_ctx() -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
1858 let (tx, rx) = mpsc::channel(8);
1859 (
1860 RouteCtx {
1861 connection_id: ConnectionId::LOCAL,
1862 egress: FrameSink::new(tx),
1863 },
1864 rx,
1865 )
1866 }
1867
1868 #[tokio::test]
1869 async fn echo_backend_returns_response_with_byte_identical_body() {
1870 let mut router = Router::with_default_self_handler();
1871 router.register_backend(7, EchoBackend).unwrap();
1872 let (ctx, mut rx) = route_ctx();
1873 let body = b"{not parsed}\0\xff";
1874
1875 router
1876 .route_for_connection(&ctx, request(7, 123, body))
1877 .await
1878 .unwrap();
1879 let response = rx.recv().await.unwrap();
1880
1881 assert_eq!(response.header.ty, FrameType::Response);
1882 assert_eq!(response.header.channel, 7);
1883 assert_eq!(response.header.corr, 123);
1884 assert_eq!(response.body, body);
1885 assert!(rx.try_recv().is_err());
1886 }
1887
1888 #[tokio::test]
1889 async fn unknown_channel_emits_canonical_error_frame() {
1890 let router = Router::with_default_self_handler();
1891 let (ctx, mut rx) = route_ctx();
1892
1893 router
1894 .route_for_connection(&ctx, request(99, 5, b"payload"))
1895 .await
1896 .unwrap();
1897 let error_frame = rx.recv().await.unwrap();
1898
1899 assert_eq!(error_frame.header.ty, FrameType::Error);
1900 assert_eq!(error_frame.header.channel, 99);
1901 assert_eq!(error_frame.header.corr, 5);
1902 let body: ErrorBody = serde_json::from_slice(&error_frame.body).unwrap();
1903 assert_eq!(body.code, "unknown_channel");
1904 assert_eq!(body.message, "unknown channel 99");
1905 }
1906
1907 #[tokio::test]
1912 async fn daemon_authored_data_plane_errors_carry_daemon_origin() {
1913 let router = Router::with_default_self_handler();
1914 let (ctx, mut rx) = route_ctx();
1915 router
1916 .route_for_connection(&ctx, request(99, 5, b"payload"))
1917 .await
1918 .unwrap();
1919 let unknown = rx.recv().await.unwrap();
1920 assert_eq!(unknown.header.ty, FrameType::Error);
1921 let body: ErrorBody = serde_json::from_slice(&unknown.body).unwrap();
1922 assert_eq!(body.code, "unknown_channel");
1923 assert!(
1924 unknown.header.flags.is_daemon_origin(),
1925 "unknown_channel must carry daemon origin: {:?}",
1926 unknown.header.flags
1927 );
1928
1929 let (
1930 router,
1931 _forwarding,
1932 client_ctx,
1933 mut client_rx,
1934 _module_ctx,
1935 _module_egress_rx,
1936 _module_rx,
1937 pending,
1938 ) = dynamic_route_fixture(true);
1939 let _ = client_rx.recv().await;
1940 router
1941 .route_for_connection(
1942 &client_ctx,
1943 route_frame(
1944 FrameType::Request,
1945 pending.client_channel,
1946 pending.client_epoch + 1,
1947 702,
1948 ),
1949 )
1950 .await
1951 .unwrap();
1952 let stale = client_rx.recv().await.unwrap();
1953 assert_eq!(stale.header.ty, FrameType::Error);
1954 let body: ErrorBody = serde_json::from_slice(&stale.body).unwrap();
1955 assert_eq!(body.code, "stale_route_epoch");
1956 assert!(
1957 stale.header.flags.is_daemon_origin(),
1958 "stale_route_epoch must carry daemon origin: {:?}",
1959 stale.header.flags
1960 );
1961 }
1962
1963 #[tokio::test]
1968 async fn module_error_with_daemon_bit_is_stripped_before_client_delivery() {
1969 let (
1970 router,
1971 _forwarding,
1972 client_ctx,
1973 mut client_rx,
1974 module_ctx,
1975 _module_egress_rx,
1976 mut module_rx,
1977 pending,
1978 ) = dynamic_route_fixture(true);
1979 let _ = client_rx.recv().await;
1980 router
1983 .route_for_connection(
1984 &client_ctx,
1985 route_frame(
1986 FrameType::Request,
1987 pending.client_channel,
1988 pending.client_epoch,
1989 701,
1990 ),
1991 )
1992 .await
1993 .unwrap();
1994 let _ = module_rx.recv().await;
1995
1996 let body = serde_json::to_vec(&ErrorBody::new("module_reloading", "try again")).unwrap();
1997 let spoof = Frame::build(
1998 FrameType::Error,
1999 Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2000 pending.module_channel,
2001 pending.module_epoch,
2002 701,
2003 body.clone(),
2004 )
2005 .unwrap();
2006 router
2007 .route_for_connection(&module_ctx, spoof)
2008 .await
2009 .unwrap();
2010 let delivered = client_rx.recv().await.unwrap();
2011 assert_eq!(delivered.header.ty, FrameType::Error);
2012 assert_eq!(delivered.header.channel, pending.client_channel);
2013 assert_eq!(delivered.header.epoch, pending.client_epoch);
2014 assert_eq!(delivered.header.corr, 701);
2015 assert!(
2016 !delivered.header.flags.is_daemon_origin(),
2017 "module-set daemon origin must be cleared: {:?}",
2018 delivered.header.flags
2019 );
2020 assert_eq!(delivered.body, body, "the body travels byte-identical");
2021 }
2022
2023 #[tokio::test]
2026 async fn module_response_with_daemon_bit_is_stripped_preserving_other_flags() {
2027 use subc_protocol::FLAG_SUBSCRIPTION;
2028 let (
2029 router,
2030 _forwarding,
2031 client_ctx,
2032 mut client_rx,
2033 module_ctx,
2034 _module_egress_rx,
2035 mut module_rx,
2036 pending,
2037 ) = dynamic_route_fixture(true);
2038 let _ = client_rx.recv().await;
2039 router
2040 .route_for_connection(
2041 &client_ctx,
2042 route_frame(
2043 FrameType::Request,
2044 pending.client_channel,
2045 pending.client_epoch,
2046 701,
2047 ),
2048 )
2049 .await
2050 .unwrap();
2051 let _ = module_rx.recv().await;
2052
2053 let sent_flags = Flags(
2054 Flags::new(true, Priority::Background, true).0
2055 | FLAG_SUBSCRIPTION
2056 | subc_protocol::FLAG_DAEMON_ORIGIN,
2057 );
2058 assert!(sent_flags.is_daemon_origin());
2059 let response = Frame::build(
2060 FrameType::Response,
2061 sent_flags,
2062 pending.module_channel,
2063 pending.module_epoch,
2064 701,
2065 b"route-body".to_vec(),
2066 )
2067 .unwrap();
2068 router
2069 .route_for_connection(&module_ctx, response)
2070 .await
2071 .unwrap();
2072 let delivered = client_rx.recv().await.unwrap();
2073 assert_eq!(delivered.header.ty, FrameType::Response);
2074 assert!(
2075 !delivered.header.flags.is_daemon_origin(),
2076 "module-set daemon origin must be cleared: {:?}",
2077 delivered.header.flags
2078 );
2079 assert!(
2080 delivered.header.flags.is_binary(),
2081 "binary flag must survive: {:?}",
2082 delivered.header.flags
2083 );
2084 assert!(
2085 delivered.header.flags.is_last(),
2086 "LAST flag must survive: {:?}",
2087 delivered.header.flags
2088 );
2089 assert!(
2090 delivered.header.flags.is_subscription(),
2091 "subscription flag must survive: {:?}",
2092 delivered.header.flags
2093 );
2094 assert_eq!(
2095 delivered.header.flags.priority(),
2096 Some(Priority::Background),
2097 "priority must survive"
2098 );
2099 }
2100
2101 #[tokio::test]
2104 async fn client_frame_with_daemon_bit_is_stripped_before_module_delivery() {
2105 let (
2106 router,
2107 _forwarding,
2108 client_ctx,
2109 mut client_rx,
2110 _module_ctx,
2111 _module_egress_rx,
2112 mut module_rx,
2113 pending,
2114 ) = dynamic_route_fixture(true);
2115 let _ = client_rx.recv().await;
2116 let spoof = Frame::build(
2117 FrameType::Request,
2118 Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2119 pending.client_channel,
2120 pending.client_epoch,
2121 701,
2122 b"route-body".to_vec(),
2123 )
2124 .unwrap();
2125 router
2126 .route_for_connection(&client_ctx, spoof)
2127 .await
2128 .unwrap();
2129 let delivered = module_rx.recv().await.unwrap();
2130 assert_eq!(delivered.header.ty, FrameType::Request);
2131 assert!(
2132 !delivered.header.flags.is_daemon_origin(),
2133 "client-set daemon origin must be cleared: {:?}",
2134 delivered.header.flags
2135 );
2136 }
2137
2138 #[tokio::test]
2139 async fn channel_zero_uses_control_handler_not_backend_registry() {
2140 let mut router = Router::with_default_self_handler();
2141 router.register_backend(1, EchoBackend).unwrap();
2142 let (ctx, mut rx) = route_ctx();
2143
2144 router.route_for_connection(&ctx, ping(77)).await.unwrap();
2145 let response = rx.recv().await.unwrap();
2146
2147 assert_eq!(response.header.ty, FrameType::Pong);
2148 assert_eq!(response.header.channel, 0);
2149 assert_eq!(response.header.corr, 77);
2150 assert!(response.body.is_empty());
2151 }
2152
2153 #[tokio::test]
2154 async fn slow_control_dispatch_logs_decoded_op_and_elapsed_time() {
2155 let control = Arc::new(
2156 ControlHandler::new(Arc::new(Registry::default()))
2157 .with_control_dispatch_delay(Duration::from_millis(1050)),
2158 );
2159 let router = Router::with_control_handler(control);
2160 let (ctx, mut rx) = route_ctx();
2161 let (output, guard) = log_capture(tracing::Level::WARN);
2162
2163 router
2164 .route_for_connection(&ctx, request(0, 41, br#"{"op":"server.describe"}"#))
2165 .await
2166 .expect("slow request routes");
2167 assert!(rx.recv().await.is_some(), "request receives a response");
2168 drop(guard);
2169
2170 let logs = captured_logs(&output);
2171 assert!(logs.contains("slow control dispatch"));
2172 assert!(logs.contains("op=server.describe"));
2173 assert!(logs.contains("connection_id=0"));
2174 assert!(logs.contains("corr=41"));
2175 assert!(
2176 logged_millis(&logs, "elapsed_ms=") >= 1050,
2177 "elapsed must include the injected handler delay: {logs}"
2178 );
2179 }
2180
2181 #[tokio::test]
2182 async fn fast_control_dispatch_emits_arrival_without_slow_warning() {
2183 let router = Router::with_default_self_handler();
2184 let (ctx, mut rx) = route_ctx();
2185 let (output, guard) = log_capture(tracing::Level::DEBUG);
2186
2187 router
2188 .route_for_connection(&ctx, request(0, 42, br#"{"op":"server.describe"}"#))
2189 .await
2190 .expect("fast request routes");
2191 assert!(rx.recv().await.is_some(), "request receives a response");
2192 drop(guard);
2193
2194 let logs = captured_logs(&output);
2195 assert!(logs.contains("control dispatch op=server.describe connection_id=0 corr=42"));
2196 assert!(!logs.contains("slow control dispatch"));
2197 }
2198
2199 #[tokio::test]
2200 async fn control_dispatch_arrival_is_hidden_at_info() {
2201 let router = Router::with_default_self_handler();
2202 let (ctx, mut rx) = route_ctx();
2203 let (output, guard) = log_capture(tracing::Level::INFO);
2204
2205 router
2206 .route_for_connection(&ctx, request(0, 43, br#"{"op":"server.describe"}"#))
2207 .await
2208 .expect("fast request routes");
2209 assert!(rx.recv().await.is_some(), "request receives a response");
2210 drop(guard);
2211
2212 assert!(
2213 !captured_logs(&output).contains("control dispatch"),
2214 "arrival logging must stay hidden at INFO"
2215 );
2216 }
2217
2218 #[tokio::test]
2219 async fn supervisor_list_logs_contended_snapshot_lock_only() {
2220 let registry = Arc::new(Registry::default());
2221 let handle = SupervisorHandle::new();
2222 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
2223 .with_handle(handle.clone());
2224 let module = supervisor
2225 .supervise_configured(
2226 ModuleSpec {
2227 module_id: "held-module".to_string(),
2228 program: "test-module".into(),
2229 args: Vec::new(),
2230 env: Vec::new(),
2231 reserved: false,
2232 reserved_prefixes: Vec::new(),
2233 protocol: ModuleProtocol::Subc,
2234 overlap: Default::default(),
2235 },
2236 false,
2237 )
2238 .expect("disabled test module is supervised");
2239 let router = Router::with_control_handler(Arc::new(
2240 ControlHandler::new(Arc::clone(®istry)).with_supervisor(handle),
2241 ));
2242 let (ctx, mut rx) = route_ctx();
2243 let (acquired, ready) = std_mpsc::channel();
2244 let holder = module.hold_snapshot_for_test(acquired, Duration::from_millis(400));
2245 ready.recv().expect("holder acquired snapshot lock");
2246 let (output, guard) = log_capture(tracing::Level::WARN);
2247
2248 router
2249 .route_for_connection(&ctx, request(0, 44, br#"{"op":"supervisor.list"}"#))
2250 .await
2251 .expect("list request routes after the lock releases");
2252 assert!(
2253 rx.recv().await.is_some(),
2254 "list request receives a response"
2255 );
2256 holder.join().expect("snapshot holder exits cleanly");
2257 drop(guard);
2258
2259 let logs = captured_logs(&output);
2260 assert!(logs.contains("slow snapshot lock"));
2261 assert!(logs.contains("module_id=held-module"));
2262 assert!(logs.contains("caller=list"));
2263 assert!(
2264 logged_millis(&logs, "waited_ms=") >= 250,
2265 "wait must exceed the slow-lock threshold: {logs}"
2266 );
2267
2268 let (output, guard) = log_capture(tracing::Level::WARN);
2269 router
2270 .route_for_connection(&ctx, request(0, 45, br#"{"op":"supervisor.list"}"#))
2271 .await
2272 .expect("uncontended list request routes");
2273 assert!(
2274 rx.recv().await.is_some(),
2275 "uncontended list receives a response"
2276 );
2277 drop(guard);
2278 assert!(
2279 !captured_logs(&output).contains("slow snapshot lock"),
2280 "uncontended list acquisition must not warn"
2281 );
2282 }
2283
2284 #[tokio::test]
2285 async fn full_module_to_client_sink_requests_client_close_without_erroring_module() {
2286 let forwarding = Arc::new(ForwardingTable::default());
2287 let control = Arc::new(ControlHandler::with_forwarding(
2288 Arc::new(crate::Registry::default()),
2289 Arc::clone(&forwarding),
2290 ));
2291 let router = Router::with_control_handler(control);
2292 let module_connection = ConnectionId::new(10);
2293 let client_connection = ConnectionId::new(20);
2294 let mut close_receiver = forwarding.register_connection_close(client_connection);
2295 let (module_tx, _module_rx) = mpsc::channel(1);
2296 forwarding
2297 .register_module_connection(
2298 module_connection,
2299 "full-sink-provider".to_string(),
2300 1,
2301 Concurrency::ModuleManaged,
2302 FrameSink::new(module_tx),
2303 )
2304 .unwrap();
2305 let (client_tx, mut client_rx) = mpsc::channel(1);
2306 let pending = forwarding
2307 .begin_route_bind_relay_for_test(
2308 client_connection,
2309 FrameSink::new(client_tx),
2310 700,
2311 "full-sink-provider",
2312 )
2313 .unwrap();
2314 forwarding
2315 .complete_pending_relay(
2316 module_connection,
2317 pending.corr,
2318 RouteBindRelayOutcome::Accepted,
2319 )
2320 .unwrap();
2321
2322 let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2323 let module_ctx = RouteCtx {
2324 connection_id: module_connection,
2325 egress: FrameSink::new(module_egress_tx),
2326 };
2327 let terminal = Frame::build(
2328 FrameType::Response,
2329 Flags::new(false, Priority::Interactive, true),
2330 pending.module_channel,
2331 pending.module_epoch,
2332 701,
2333 b"terminal".to_vec(),
2334 )
2335 .unwrap();
2336
2337 router
2338 .route_for_connection(&module_ctx, terminal)
2339 .await
2340 .unwrap();
2341 let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2342 .await
2343 .expect("close request should be sent for the full client sink")
2344 .expect("close sender should include a reason");
2345 assert!(
2346 reason
2347 .to_string()
2348 .contains("module_to_client_delivery_failed"),
2349 "unexpected close reason: {reason}"
2350 );
2351 assert_eq!(client_rx.try_recv().unwrap().header.corr, 700);
2352 assert!(client_rx.try_recv().is_err());
2353 assert_eq!(
2354 router.counters.snapshot()["client_egress_close_delivery_failed"],
2355 1
2356 );
2357 }
2358
2359 #[tokio::test]
2363 async fn terminal_frame_releases_its_credit_even_when_client_delivery_fails() {
2364 let forwarding = Arc::new(ForwardingTable::default());
2365 let control = Arc::new(ControlHandler::with_forwarding(
2366 Arc::new(crate::Registry::default()),
2367 Arc::clone(&forwarding),
2368 ));
2369 let router = Router::with_control_handler(control);
2370 let module_connection = ConnectionId::new(11);
2371 let client_connection = ConnectionId::new(21);
2372 let _close_receiver = forwarding.register_connection_close(client_connection);
2373 let (module_tx, _module_rx) = mpsc::channel(1);
2374 forwarding
2375 .register_module_connection(
2376 module_connection,
2377 "credit-provider".to_string(),
2378 1,
2379 Concurrency::ModuleManaged,
2380 FrameSink::new(module_tx),
2381 )
2382 .unwrap();
2383 let (client_tx, _client_rx) = mpsc::channel(1);
2386 let pending = forwarding
2387 .begin_route_bind_relay_for_test(
2388 client_connection,
2389 FrameSink::new(client_tx),
2390 800,
2391 "credit-provider",
2392 )
2393 .unwrap();
2394 forwarding
2395 .complete_pending_relay(
2396 module_connection,
2397 pending.corr,
2398 RouteBindRelayOutcome::Accepted,
2399 )
2400 .unwrap();
2401 let DataRoute::Client(DataRouteState::Bound(route)) = forwarding
2402 .lookup_data_route(
2403 client_connection,
2404 pending.client_channel,
2405 pending.client_epoch,
2406 )
2407 .unwrap()
2408 else {
2409 panic!("expected a bound client route");
2410 };
2411 route.flow.acquire_tagged(801, false).await.unwrap();
2412 assert_eq!(route.flow.drain_in_flight(), 1);
2413
2414 let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2415 let module_ctx = RouteCtx {
2416 connection_id: module_connection,
2417 egress: FrameSink::new(module_egress_tx),
2418 };
2419 let terminal = Frame::build(
2420 FrameType::Response,
2421 Flags::new(false, Priority::Interactive, true),
2422 pending.module_channel,
2423 pending.module_epoch,
2424 801,
2425 b"terminal".to_vec(),
2426 )
2427 .unwrap();
2428 router
2429 .route_for_connection(&module_ctx, terminal)
2430 .await
2431 .unwrap();
2432
2433 assert_eq!(
2434 router.counters.snapshot()["client_egress_close_delivery_failed"],
2435 1,
2436 "the client delivery must have failed for this test to mean anything"
2437 );
2438 assert_eq!(
2439 route.flow.drain_in_flight(),
2440 0,
2441 "the module's terminal frame must release its credit even though the client could not take it"
2442 );
2443 }
2444
2445 #[tokio::test]
2446 async fn full_route_goodbye_sink_requests_target_close_without_erroring_module() {
2447 let forwarding = Arc::new(ForwardingTable::default());
2448 let control = Arc::new(ControlHandler::with_forwarding(
2449 Arc::new(crate::Registry::default()),
2450 Arc::clone(&forwarding),
2451 ));
2452 let router = Router::with_control_handler(control);
2453 let module_connection = ConnectionId::new(30);
2454 let client_connection = ConnectionId::new(40);
2455 let mut close_receiver = forwarding.register_connection_close(client_connection);
2456 let (module_tx, _module_rx) = mpsc::channel(1);
2457 forwarding
2458 .register_module_connection(
2459 module_connection,
2460 "goodbye-full-provider".to_string(),
2461 1,
2462 Concurrency::ModuleManaged,
2463 FrameSink::new(module_tx),
2464 )
2465 .unwrap();
2466 let (client_tx, mut client_rx) = mpsc::channel(1);
2467 let pending = forwarding
2468 .begin_route_bind_relay_for_test(
2469 client_connection,
2470 FrameSink::new(client_tx),
2471 800,
2472 "goodbye-full-provider",
2473 )
2474 .unwrap();
2475 forwarding
2476 .complete_pending_relay(
2477 module_connection,
2478 pending.corr,
2479 RouteBindRelayOutcome::Accepted,
2480 )
2481 .unwrap();
2482
2483 let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2484 let module_ctx = RouteCtx {
2485 connection_id: module_connection,
2486 egress: FrameSink::new(module_egress_tx),
2487 };
2488 let goodbye = Frame::build(
2489 FrameType::Goodbye,
2490 Flags::new(false, Priority::Passive, true),
2491 pending.module_channel,
2492 pending.module_epoch,
2493 801,
2494 Vec::new(),
2495 )
2496 .unwrap();
2497
2498 router
2499 .route_for_connection(&module_ctx, goodbye)
2500 .await
2501 .unwrap();
2502 let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2503 .await
2504 .expect("close request should be sent for the full GOODBYE sink")
2505 .expect("close sender should include a reason");
2506 assert!(
2507 reason.to_string().contains("route_goodbye_delivery_failed"),
2508 "unexpected close reason: {reason}"
2509 );
2510 assert_eq!(client_rx.try_recv().unwrap().header.corr, 800);
2511 assert!(client_rx.try_recv().is_err());
2512 assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 1);
2513 assert_eq!(router.counters.snapshot()["route_released_epoch_fenced"], 1);
2514 }
2515
2516 async fn multi_route_client(
2521 module_id: &str,
2522 module_connection: ConnectionId,
2523 client_connection: ConnectionId,
2524 routes: usize,
2525 ) -> (
2526 Router,
2527 FrameSink,
2528 mpsc::Receiver<OutboundFrame>,
2529 RouteCtx,
2530 Vec<crate::forwarding::PendingRouteBindRelay>,
2531 ConnectionCloseReceiver,
2532 ) {
2533 let forwarding = Arc::new(ForwardingTable::default());
2534 let control = Arc::new(ControlHandler::with_forwarding(
2535 Arc::new(crate::Registry::default()),
2536 Arc::clone(&forwarding),
2537 ));
2538 let router = Router::with_control_handler(control);
2539 let close_receiver = forwarding.register_connection_close(client_connection);
2540 let (module_tx, _module_rx) = mpsc::channel(8);
2541 forwarding
2542 .register_module_connection(
2543 module_connection,
2544 module_id.to_string(),
2545 1,
2546 Concurrency::ModuleManaged,
2547 FrameSink::new(module_tx),
2548 )
2549 .unwrap();
2550 let (client_sink, mut client_rx) = crate::server::connection_egress();
2551 let mut bound = Vec::with_capacity(routes);
2552 for index in 0..routes {
2553 let pending = forwarding
2554 .begin_route_bind_relay_for_test(
2555 client_connection,
2556 client_sink.clone(),
2557 900 + index as u64,
2558 module_id,
2559 )
2560 .unwrap();
2561 forwarding
2562 .complete_pending_relay(
2563 module_connection,
2564 pending.corr,
2565 RouteBindRelayOutcome::Accepted,
2566 )
2567 .unwrap();
2568 assert_eq!(
2569 client_rx.recv().await.unwrap().header.corr,
2570 900 + index as u64
2571 );
2572 bound.push(pending);
2573 }
2574 let (module_egress_tx, _module_egress_rx) = mpsc::channel(8);
2575 let module_ctx = RouteCtx {
2576 connection_id: module_connection,
2577 egress: FrameSink::new(module_egress_tx),
2578 };
2579 (
2580 router,
2581 client_sink,
2582 client_rx,
2583 module_ctx,
2584 bound,
2585 close_receiver,
2586 )
2587 }
2588
2589 #[test]
2596 #[ignore = "timing measurement, run on demand"]
2597 fn egress_sink_per_frame_cost() {
2598 const FRAMES: usize = 1_000_000;
2599 const BATCH: usize = 1_000;
2600 let (sink, mut rx) = crate::server::connection_egress();
2601 let template = stream_frame(9, 1, 0, vec![b't'; 200]);
2602 let started = Instant::now();
2603 for _ in 0..FRAMES / BATCH {
2604 for _ in 0..BATCH {
2605 sink.try_send(template.clone()).unwrap();
2606 }
2607 for _ in 0..BATCH {
2608 let outbound = rx.try_recv().unwrap();
2609 if let Some(charge) = &outbound.charge {
2610 charge.taken_by_writer();
2611 }
2612 drop(std::hint::black_box(outbound));
2613 }
2614 }
2615 let elapsed = started.elapsed();
2616 assert_eq!(sink.backlog().queued_bytes, 0);
2617 println!(
2618 "egress sink: {FRAMES} frames in {elapsed:?}, {:.1} ns/frame",
2619 elapsed.as_nanos() as f64 / FRAMES as f64
2620 );
2621 }
2622
2623 fn stream_frame(channel: u16, epoch: u32, corr: u64, body: Vec<u8>) -> Frame {
2624 Frame::build(
2625 FrameType::StreamData,
2626 Flags::new(false, Priority::Interactive, false),
2627 channel,
2628 epoch,
2629 corr,
2630 body,
2631 )
2632 .unwrap()
2633 }
2634
2635 fn module_route_for_test(
2636 router: &Router,
2637 ctx: &RouteCtx,
2638 route: &crate::forwarding::PendingRouteBindRelay,
2639 ) -> Arc<RouteBinding> {
2640 match router
2641 .forwarding
2642 .lookup_data_route(ctx.connection_id, route.module_channel, route.module_epoch)
2643 .unwrap()
2644 {
2645 DataRoute::Module(DataRouteState::Bound(binding)) => binding,
2646 other => panic!("expected a bound module route, got {other:?}"),
2647 }
2648 }
2649
2650 fn assert_frame_size_error(frame: &Frame, channel: u16, epoch: u32, corr: u64, bytes: usize) {
2651 assert_eq!(frame.header.ty, FrameType::Error);
2652 assert_eq!(frame.header.channel, channel);
2653 assert_eq!(frame.header.epoch, epoch);
2654 assert_eq!(frame.header.corr, corr);
2655 assert!(frame.header.flags.is_daemon_origin());
2656 let body: ErrorBody = serde_json::from_slice(&frame.body).unwrap();
2657 assert_eq!(body.code, error_codes::FRAME_TOO_LARGE);
2658 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2659 assert!(body.message.contains(&bytes.to_string()));
2660 assert!(body.message.contains(&limit.to_string()));
2661 assert_eq!(
2662 body.detail,
2663 Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit }))
2664 );
2665 }
2666
2667 #[tokio::test]
2668 async fn module_egress_response_oversized_reply_keeps_connection_and_other_route() {
2669 let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
2670 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2671 "oversize-provider",
2672 ConnectionId::new(80),
2673 ConnectionId::new(81),
2674 2,
2675 )
2676 .await;
2677 let binding = module_route_for_test(&router, &ctx, &routes[0]);
2678 binding.flow.acquire_tagged(101, false).await.unwrap();
2679 module_route_for_test(&router, &ctx, &routes[1])
2680 .flow
2681 .acquire_tagged(102, false)
2682 .await
2683 .unwrap();
2684 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2685 let oversized = Frame::build(
2686 FrameType::Response,
2687 Flags::new(false, Priority::Interactive, true),
2688 routes[0].module_channel,
2689 routes[0].module_epoch,
2690 101,
2691 vec![b'x'; limit - subc_protocol::HEADER_LEN + 1],
2692 )
2693 .unwrap();
2694 assert_eq!(sink.backlog().queued_bytes, 0);
2695 router.route_for_connection(&ctx, oversized).await.unwrap();
2696
2697 router
2701 .route_for_connection(
2702 &ctx,
2703 route_frame(
2704 FrameType::Response,
2705 routes[1].module_channel,
2706 routes[1].module_epoch,
2707 102,
2708 ),
2709 )
2710 .await
2711 .unwrap();
2712 assert!(
2713 close.try_recv().is_err(),
2714 "an oversized reply must not close the multiplexed client"
2715 );
2716 let error = rx.try_recv().expect("the waiting client gets an ERROR");
2717 assert_frame_size_error(
2718 &error,
2719 routes[0].client_channel,
2720 routes[0].client_epoch,
2721 101,
2722 limit + 1,
2723 );
2724 let second = rx.try_recv().expect("the other route's reply arrives");
2725 assert_eq!(second.header.ty, FrameType::Response);
2726 assert_eq!(second.header.channel, routes[1].client_channel);
2727 assert_eq!(second.header.epoch, routes[1].client_epoch);
2728 assert_eq!(second.header.corr, 102);
2729 assert_eq!(second.body, b"route-body");
2730 assert_eq!(binding.flow.drain_in_flight(), 0);
2731 for route in &routes {
2732 module_route_for_test(&router, &ctx, route);
2733 }
2734 assert!(rx.try_recv().is_err());
2735 drop((error, second));
2736 assert_eq!(sink.backlog().queued_bytes, 0);
2737 assert_eq!(
2738 router.counters.snapshot()["client_egress_close_delivery_failed"],
2739 0
2740 );
2741
2742 let captured = test_log::captured_logs(&logs);
2743 let warnings: Vec<_> = captured
2744 .lines()
2745 .filter(|line| line.contains("WARN"))
2746 .collect();
2747 assert_eq!(warnings.len(), 1, "{captured}");
2748 let warning = warnings[0];
2749 for expected in [
2750 "module_id=oversize-provider".to_string(),
2751 "client_principal=direct".to_string(),
2752 format!("client_channel={}", routes[0].client_channel),
2753 format!("frame_bytes={}", limit + 1),
2754 format!("byte_limit={limit}"),
2755 ] {
2756 assert!(warning.contains(&expected), "missing {expected}: {warning}");
2757 }
2758 }
2759
2760 async fn poll_once<F: std::future::Future>(
2761 mut future: std::pin::Pin<&mut F>,
2762 ) -> std::task::Poll<F::Output> {
2763 std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
2764 }
2765
2766 fn fill_client_budget(sink: &FrameSink, route: &crate::forwarding::PendingRouteBindRelay) {
2767 sink.try_send(
2768 Frame::build(
2769 FrameType::Response,
2770 Flags::new(false, Priority::Interactive, true),
2771 route.client_channel,
2772 route.client_epoch,
2773 100,
2774 vec![
2775 b'x';
2776 crate::server::CONNECTION_EGRESS_BYTE_BUDGET - subc_protocol::HEADER_LEN
2777 ],
2778 )
2779 .unwrap(),
2780 )
2781 .unwrap();
2782 assert_eq!(
2783 sink.backlog().queued_bytes,
2784 crate::server::CONNECTION_EGRESS_BYTE_BUDGET
2785 );
2786 }
2787
2788 async fn assert_full_client_burst_survives(ty: FrameType, oversized: bool) {
2789 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2790 "draining-burst",
2791 ConnectionId::new(92),
2792 ConnectionId::new(93),
2793 2,
2794 )
2795 .await;
2796 fill_client_budget(&sink, &routes[0]);
2797 let frame = if oversized {
2798 Frame::build(
2799 ty,
2800 Flags::new(false, Priority::Interactive, false),
2801 routes[1].module_channel,
2802 routes[1].module_epoch,
2803 101,
2804 vec![b's'; crate::server::CONNECTION_EGRESS_BYTE_BUDGET],
2805 )
2806 .unwrap()
2807 } else {
2808 route_frame(ty, routes[1].module_channel, routes[1].module_epoch, 101)
2809 };
2810 let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2811 assert!(poll_once(delivery.as_mut()).await.is_pending());
2812 assert!(
2813 close.try_recv().is_err(),
2814 "fresh bursts must not close: {ty:?}"
2815 );
2816 tokio::time::advance(Duration::from_millis(50)).await;
2817 let queued = rx.try_recv().unwrap();
2818 queued.charge.as_ref().unwrap().taken_by_writer();
2819 drop(queued);
2820 assert!(matches!(
2821 poll_once(delivery.as_mut()).await,
2822 std::task::Poll::Ready(Ok(()))
2823 ));
2824 let delivered = rx
2825 .try_recv()
2826 .expect("the burst frame must be delivered after draining");
2827 assert_eq!(delivered.header.channel, routes[1].client_channel);
2828 assert_eq!(delivered.header.epoch, routes[1].client_epoch);
2829 assert_eq!(delivered.header.corr, 101);
2830 if oversized {
2831 assert_frame_size_error(
2832 &delivered,
2833 routes[1].client_channel,
2834 routes[1].client_epoch,
2835 101,
2836 crate::server::CONNECTION_EGRESS_BYTE_BUDGET + subc_protocol::HEADER_LEN,
2837 );
2838 } else {
2839 assert_eq!(delivered.header.ty, ty);
2840 }
2841 assert!(rx.try_recv().is_err());
2842 assert!(close.try_recv().is_err());
2843 assert_eq!(
2844 router.counters.snapshot()["client_egress_close_delivery_failed"],
2845 0
2846 );
2847 assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 0);
2848 }
2849
2850 #[tokio::test(start_paused = true)]
2851 async fn module_egress_fresh_full_queue_burst_survives() {
2852 for ty in [
2853 FrameType::Response,
2854 FrameType::StreamData,
2855 FrameType::Request,
2856 FrameType::Push,
2857 ] {
2858 assert_full_client_burst_survives(ty, false).await;
2859 }
2860 }
2861
2862 #[tokio::test(start_paused = true)]
2863 async fn module_goodbye_fresh_full_queue_burst_survives() {
2864 assert_full_client_burst_survives(FrameType::Goodbye, false).await;
2865 }
2866
2867 #[tokio::test(start_paused = true)]
2868 async fn frame_count_backstop_also_allows_a_draining_burst() {
2869 let (tx, mut rx) = mpsc::channel(1);
2870 let sink = FrameSink::with_byte_budget(tx, 2_000);
2871 sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
2872 .unwrap();
2873 let mut delivery = Box::pin(sink.send_to_client(stream_frame(1, 1, 2, vec![b'y'; 100])));
2874 assert!(poll_once(delivery.as_mut()).await.is_pending());
2875 assert!(
2876 sink.backlog().queued_bytes < 2_000,
2877 "only frame capacity is exhausted"
2878 );
2879 tokio::time::advance(Duration::from_millis(50)).await;
2880 drop(rx.try_recv().unwrap());
2881 assert!(matches!(
2882 poll_once(delivery.as_mut()).await,
2883 std::task::Poll::Ready(Ok(()))
2884 ));
2885 assert_eq!(rx.try_recv().unwrap().header.corr, 2);
2886 }
2887
2888 #[tokio::test(start_paused = true)]
2889 async fn oversized_reply_replacement_fresh_full_queue_burst_survives() {
2890 for ty in [FrameType::Response, FrameType::StreamData] {
2891 assert_full_client_burst_survives(ty, true).await;
2892 }
2893 }
2894
2895 #[tokio::test(start_paused = true)]
2896 async fn module_egress_stopped_reader_closes_without_waiting() {
2897 let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
2898 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2899 "oldest-stopped-reader",
2900 ConnectionId::new(94),
2901 ConnectionId::new(95),
2902 2,
2903 )
2904 .await;
2905 fill_client_budget(&sink, &routes[0]);
2906 let writing = rx.try_recv().unwrap();
2909 writing.charge.as_ref().unwrap().taken_by_writer();
2910 tokio::time::advance(Duration::from_millis(251)).await;
2911 assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(251)));
2912 assert_eq!(sink.backlog().queued_bytes, queued_frame_bytes(&writing));
2913 let started = tokio::time::Instant::now();
2914 let frame = stream_frame(
2915 routes[0].module_channel,
2916 routes[0].module_epoch,
2917 101,
2918 b"secret-body-never-log".to_vec(),
2919 );
2920 let flags = frame.header.flags.0;
2921 let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2922 assert!(
2923 matches!(
2924 poll_once(delivery.as_mut()).await,
2925 std::task::Poll::Ready(Ok(()))
2926 ),
2927 "an oldest frame older than 250 ms must close immediately, without a new wait"
2928 );
2929 assert_eq!(started.elapsed(), Duration::ZERO);
2930 assert!(close
2931 .try_recv()
2932 .unwrap()
2933 .to_string()
2934 .contains("module_to_client_delivery_failed"));
2935 let captured = test_log::captured_logs(&logs);
2936 for expected in [
2937 "frame_type=Some(StreamData)".to_string(),
2938 format!("frame_flags=Some({flags})"),
2939 "oldest_queued_ms=251".to_string(),
2940 "close_cause=\"oldest_frame_older_than_250_ms\"".to_string(),
2941 ] {
2942 assert!(
2943 captured.contains(&expected),
2944 "missing {expected}: {captured}"
2945 );
2946 }
2947 assert!(!captured.contains("secret-body-never-log"));
2948 }
2949
2950 #[tokio::test(start_paused = true)]
2951 async fn client_gone_mid_wait_resumes_module_reader_immediately() {
2952 for writer_gone in [false, true] {
2953 let (router, sink, mut rx, ctx, routes, _close) = multi_route_client(
2954 "gone-during-burst",
2955 ConnectionId::new(96),
2956 ConnectionId::new(97),
2957 1,
2958 )
2959 .await;
2960 fill_client_budget(&sink, &routes[0]);
2961 let frame = route_frame(
2962 FrameType::Response,
2963 routes[0].module_channel,
2964 routes[0].module_epoch,
2965 101,
2966 );
2967 let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2968 assert!(poll_once(delivery.as_mut()).await.is_pending());
2969 tokio::time::advance(Duration::from_millis(50)).await;
2970 let started = tokio::time::Instant::now();
2971 if writer_gone {
2972 rx.close();
2973 } else {
2974 sink.close_client_delivery();
2975 }
2976 assert!(
2977 matches!(
2978 poll_once(delivery.as_mut()).await,
2979 std::task::Poll::Ready(Ok(()))
2980 ),
2981 "client teardown must wake the module reader before the writer drains"
2982 );
2983 assert_eq!(started.elapsed(), Duration::ZERO);
2984 assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
2985 assert_eq!(rx.try_recv().unwrap().header.corr, 100);
2986 assert!(rx.try_recv().is_err());
2987 }
2988 }
2989
2990 #[tokio::test(start_paused = true)]
2991 async fn cancelled_module_delivery_releases_waiter_without_closing_client() {
2992 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2993 "cancelled-burst",
2994 ConnectionId::new(98),
2995 ConnectionId::new(99),
2996 1,
2997 )
2998 .await;
2999 fill_client_budget(&sink, &routes[0]);
3000 let frame = route_frame(
3001 FrameType::Response,
3002 routes[0].module_channel,
3003 routes[0].module_epoch,
3004 101,
3005 );
3006 let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3007 assert!(poll_once(delivery.as_mut()).await.is_pending());
3008 assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 1);
3009 drop(delivery);
3010 assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
3011 tokio::time::advance(Duration::from_secs(1)).await;
3012 assert!(close.try_recv().is_err());
3013 drop(rx.try_recv().unwrap());
3014 assert_eq!(sink.backlog().queued_bytes, 0);
3015 assert!(sink.backlog().oldest_age.is_none());
3016 }
3017
3018 #[tokio::test(start_paused = true)]
3019 async fn egress_atomic_age_follows_writer_and_clears_when_drained() {
3020 let (sink, mut rx) = crate::server::connection_egress();
3021 sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
3022 .unwrap();
3023 tokio::time::advance(Duration::from_millis(200)).await;
3024 sink.try_send(stream_frame(2, 1, 2, vec![b'y'; 100]))
3025 .unwrap();
3026 let writing = rx.try_recv().unwrap();
3027 writing.charge.as_ref().unwrap().taken_by_writer();
3028 assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3029 drop(writing);
3030 assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3033 tokio::time::advance(Duration::from_millis(50)).await;
3034 let next = rx.try_recv().unwrap();
3035 next.charge.as_ref().unwrap().taken_by_writer();
3036 assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(50)));
3037 assert!(
3038 sink.backlog().queued_bytes > 0,
3039 "in-flight bytes remain charged until written"
3040 );
3041 drop(next);
3042 assert_eq!(sink.backlog().queued_bytes, 0);
3043 assert_eq!(sink.backlog().oldest_age, None);
3044 }
3045
3046 #[tokio::test(start_paused = true)]
3047 async fn module_egress_response_full_client_waits_only_remaining_grace() {
3048 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3049 "backpressure-provider",
3050 ConnectionId::new(82),
3051 ConnectionId::new(83),
3052 2,
3053 )
3054 .await;
3055 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3056 router
3059 .route_for_connection(
3060 &ctx,
3061 Frame::build(
3062 FrameType::Response,
3063 Flags::new(false, Priority::Interactive, true),
3064 routes[0].module_channel,
3065 routes[0].module_epoch,
3066 201,
3067 vec![b'x'; limit - subc_protocol::HEADER_LEN],
3068 )
3069 .unwrap(),
3070 )
3071 .await
3072 .unwrap();
3073 assert_eq!(sink.backlog().queued_bytes, limit);
3074 assert_eq!(sink.backlog().queued_frames, 1);
3075 let ordinary = route_frame(
3076 FrameType::Response,
3077 routes[1].module_channel,
3078 routes[1].module_epoch,
3079 202,
3080 );
3081 assert!(queued_frame_bytes(&ordinary) < limit);
3082 tokio::time::advance(Duration::from_millis(100)).await;
3085 let mut delivery = Box::pin(router.route_for_connection(&ctx, ordinary));
3086 assert!(poll_once(delivery.as_mut()).await.is_pending());
3087 assert!(close.try_recv().is_err());
3088 tokio::time::advance(Duration::from_millis(149)).await;
3089 assert!(poll_once(delivery.as_mut()).await.is_pending());
3090 assert!(close.try_recv().is_err());
3091 tokio::time::advance(Duration::from_millis(1)).await;
3092 assert!(matches!(
3093 poll_once(delivery.as_mut()).await,
3094 std::task::Poll::Ready(Ok(()))
3095 ));
3096 drop(delivery);
3097 let reason = close.try_recv().expect("the full client must close");
3098 assert!(reason
3099 .to_string()
3100 .contains("module_to_client_delivery_failed"));
3101 assert!(!reason.to_string().contains("frame_too_large"));
3102 assert_eq!(
3103 router.counters.snapshot()["client_egress_close_delivery_failed"],
3104 1
3105 );
3106 let admitted = rx.try_recv().expect("the budget-sized frame was admitted");
3107 assert_eq!(admitted.header.ty, FrameType::Response);
3108 assert_eq!(admitted.header.corr, 201);
3109 assert!(
3110 rx.try_recv().is_err(),
3111 "no oversized-frame ERROR is substituted for backpressure"
3112 );
3113 }
3114
3115 #[tokio::test(start_paused = true)]
3116 async fn oversized_reply_to_a_full_client_waits_only_remaining_grace() {
3117 let (router, sink, _rx, ctx, routes, mut close) = multi_route_client(
3118 "full-oversize-provider",
3119 ConnectionId::new(90),
3120 ConnectionId::new(91),
3121 2,
3122 )
3123 .await;
3124 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3125 router
3127 .route_for_connection(
3128 &ctx,
3129 Frame::build(
3130 FrameType::Response,
3131 Flags::new(false, Priority::Interactive, true),
3132 routes[0].module_channel,
3133 routes[0].module_epoch,
3134 501,
3135 vec![b'x'; limit - subc_protocol::HEADER_LEN],
3136 )
3137 .unwrap(),
3138 )
3139 .await
3140 .unwrap();
3141 assert_eq!(sink.backlog().queued_bytes, limit);
3142 let oversized = Frame::build(
3143 FrameType::Response,
3144 Flags::new(false, Priority::Interactive, true),
3145 routes[1].module_channel,
3146 routes[1].module_epoch,
3147 502,
3148 vec![b'y'; limit - subc_protocol::HEADER_LEN + 1],
3149 )
3150 .unwrap();
3151 tokio::time::advance(Duration::from_millis(80)).await;
3154 let mut delivery = Box::pin(router.route_for_connection(&ctx, oversized));
3155 assert!(poll_once(delivery.as_mut()).await.is_pending());
3156 assert!(close.try_recv().is_err());
3157 tokio::time::advance(Duration::from_millis(169)).await;
3158 assert!(poll_once(delivery.as_mut()).await.is_pending());
3159 assert!(close.try_recv().is_err());
3160 tokio::time::advance(Duration::from_millis(1)).await;
3161 assert!(matches!(
3162 poll_once(delivery.as_mut()).await,
3163 std::task::Poll::Ready(Ok(()))
3164 ));
3165 drop(delivery);
3166 let reason = close.try_recv().expect("the full client must close");
3167 assert!(reason
3168 .to_string()
3169 .contains("module_to_client_delivery_failed"));
3170 }
3171
3172 #[tokio::test]
3173 async fn oversized_reverse_request_errors_module_without_reaching_client() {
3174 let (router, _sink, mut rx, mut ctx, routes, mut close) = multi_route_client(
3175 "reverse-oversize-provider",
3176 ConnectionId::new(84),
3177 ConnectionId::new(85),
3178 2,
3179 )
3180 .await;
3181 let (module_tx, mut module_rx) = mpsc::channel(8);
3182 ctx.egress = FrameSink::new(module_tx);
3183 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3184 router
3185 .route_for_connection(
3186 &ctx,
3187 Frame::build(
3188 FrameType::Request,
3189 Flags::new(false, Priority::Interactive, false),
3190 routes[0].module_channel,
3191 routes[0].module_epoch,
3192 301,
3193 vec![b'r'; limit - subc_protocol::HEADER_LEN + 1],
3194 )
3195 .unwrap(),
3196 )
3197 .await
3198 .unwrap();
3199 assert!(
3200 rx.try_recv().is_err(),
3201 "the reverse request never reaches the client"
3202 );
3203 let error = module_rx.try_recv().expect("the module gets the refusal");
3204 assert_frame_size_error(
3205 &error,
3206 routes[0].module_channel,
3207 routes[0].module_epoch,
3208 301,
3209 limit + 1,
3210 );
3211 assert!(module_rx.try_recv().is_err());
3212 for (index, route) in routes.iter().enumerate() {
3213 module_route_for_test(&router, &ctx, route);
3214 router
3215 .route_for_connection(
3216 &ctx,
3217 route_frame(
3218 FrameType::Response,
3219 route.module_channel,
3220 route.module_epoch,
3221 302 + index as u64,
3222 ),
3223 )
3224 .await
3225 .unwrap();
3226 let reply = rx.try_recv().expect("both routes still deliver");
3227 assert_eq!(reply.header.ty, FrameType::Response);
3228 assert_eq!(reply.header.channel, route.client_channel);
3229 }
3230 assert!(close.try_recv().is_err());
3231 assert_eq!(
3232 router.counters.snapshot()["client_egress_close_delivery_failed"],
3233 0
3234 );
3235 }
3236
3237 #[tokio::test]
3238 async fn oversized_push_closes_client_with_precise_warn() {
3239 let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3240 let (router, _sink, mut rx, ctx, routes, mut close) = multi_route_client(
3241 "push-oversize-provider",
3242 ConnectionId::new(86),
3243 ConnectionId::new(87),
3244 2,
3245 )
3246 .await;
3247 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3248 router
3249 .route_for_connection(
3250 &ctx,
3251 Frame::build(
3252 FrameType::Push,
3253 Flags::new(false, Priority::Interactive, false),
3254 routes[0].module_channel,
3255 routes[0].module_epoch,
3256 0,
3257 vec![b'p'; limit - subc_protocol::HEADER_LEN + 1],
3258 )
3259 .unwrap(),
3260 )
3261 .await
3262 .unwrap();
3263 let reason = close
3264 .try_recv()
3265 .expect("unsolicited oversize still closes the client");
3266 assert!(reason
3267 .to_string()
3268 .contains("frame_too_large: oversized push"));
3269 assert!(
3270 rx.try_recv().is_err(),
3271 "no part of the oversized push is forwarded"
3272 );
3273 let captured = test_log::captured_logs(&logs);
3274 let warnings: Vec<_> = captured
3275 .lines()
3276 .filter(|line| line.contains("WARN"))
3277 .collect();
3278 assert_eq!(warnings.len(), 1, "{captured}");
3279 let warning = warnings[0];
3280 for expected in [
3281 "oversized push".to_string(),
3282 "module_id=\"push-oversize-provider\"".to_string(),
3283 "principals=direct".to_string(),
3284 format!("client_channel={}", routes[0].client_channel),
3285 format!("size={}", limit + 1),
3286 format!("limit={limit}"),
3287 ] {
3288 assert!(warning.contains(&expected), "missing {expected}: {warning}");
3289 }
3290 assert_eq!(
3291 router.counters.snapshot()["client_egress_close_delivery_failed"],
3292 1
3293 );
3294 }
3295
3296 #[tokio::test]
3297 async fn oversized_stream_and_error_frames_end_request_even_with_queued_traffic() {
3298 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3299 "stream-oversize-provider",
3300 ConnectionId::new(88),
3301 ConnectionId::new(89),
3302 2,
3303 )
3304 .await;
3305 let binding = module_route_for_test(&router, &ctx, &routes[0]);
3306 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3307 for ty in [
3308 FrameType::StreamData,
3309 FrameType::StreamEnd,
3310 FrameType::Error,
3311 ] {
3312 binding.flow.acquire_tagged(401, false).await.unwrap();
3313 sink.try_send(route_frame(
3314 FrameType::Response,
3315 routes[1].client_channel,
3316 routes[1].client_epoch,
3317 402,
3318 ))
3319 .unwrap();
3320 router
3321 .route_for_connection(
3322 &ctx,
3323 Frame::build(
3324 ty,
3325 Flags::new(false, Priority::Interactive, false),
3326 routes[0].module_channel,
3327 routes[0].module_epoch,
3328 401,
3329 vec![b's'; limit - subc_protocol::HEADER_LEN + 1],
3330 )
3331 .unwrap(),
3332 )
3333 .await
3334 .unwrap();
3335 assert!(close.try_recv().is_err());
3336 assert_eq!(rx.try_recv().unwrap().header.corr, 402);
3337 let error = rx
3338 .try_recv()
3339 .expect("every correlated oversize frame becomes ERROR");
3340 assert_frame_size_error(
3341 &error,
3342 routes[0].client_channel,
3343 routes[0].client_epoch,
3344 401,
3345 limit + 1,
3346 );
3347 assert_eq!(binding.flow.drain_in_flight(), 0, "{ty:?} ends the request");
3348 assert!(rx.try_recv().is_err());
3349 }
3350 }
3351
3352 #[tokio::test]
3358 async fn awaited_send_parked_behind_a_full_byte_budget_wakes_when_bytes_free() {
3359 let (tx, mut rx) = mpsc::channel(1024);
3360 let sink = FrameSink::with_byte_budget(tx, 2_000);
3361 let mut queued = 0u64;
3362 while sink
3363 .try_send(stream_frame(7, 1, queued, vec![b'x'; 200]))
3364 .is_ok()
3365 {
3366 queued += 1;
3367 }
3368 assert!(queued > 0, "the budget admitted nothing");
3369
3370 let parked = tokio::spawn({
3371 let sink = sink.clone();
3372 async move { sink.send(stream_frame(0, 0, 999, vec![b'r'; 200])).await }
3373 });
3374 tokio::time::sleep(Duration::from_millis(100)).await;
3375 assert!(
3376 !parked.is_finished(),
3377 "the awaited send must wait while the byte budget is full"
3378 );
3379
3380 drop(rx.recv().await.expect("a queued frame"));
3382 tokio::time::timeout(Duration::from_secs(2), parked)
3383 .await
3384 .expect("the parked send was never woken after bytes were freed")
3385 .unwrap()
3386 .unwrap();
3387 }
3388
3389 #[tokio::test]
3394 async fn paused_client_reader_keeps_connection_through_small_frame_burst() {
3395 const ROUTES: usize = 4;
3396 const FRAMES_PER_ROUTE: usize = 250;
3397 let (router, client_sink, mut client_rx, module_ctx, routes, mut close_receiver) =
3398 multi_route_client(
3399 "burst-provider",
3400 ConnectionId::new(60),
3401 ConnectionId::new(61),
3402 ROUTES,
3403 )
3404 .await;
3405
3406 for seq in 0..FRAMES_PER_ROUTE as u64 {
3408 for (index, route) in routes.iter().enumerate() {
3409 let body = format!("route-{index}-token-{seq:05}-{}", "t".repeat(170));
3410 router
3411 .route_for_connection(
3412 &module_ctx,
3413 stream_frame(
3414 route.module_channel,
3415 route.module_epoch,
3416 seq,
3417 body.into_bytes(),
3418 ),
3419 )
3420 .await
3421 .unwrap();
3422 }
3423 }
3424
3425 let backlog = client_sink.backlog();
3426 assert_eq!(backlog.queued_frames, ROUTES * FRAMES_PER_ROUTE);
3427 assert!(backlog.queued_bytes < crate::server::CONNECTION_EGRESS_BYTE_BUDGET);
3428 assert!(
3429 close_receiver.try_recv().is_err(),
3430 "a paused reader under the byte budget must not be closed"
3431 );
3432 assert_eq!(
3433 router.counters.snapshot()["client_egress_close_delivery_failed"],
3434 0
3435 );
3436
3437 let mut next_seq = vec![0u64; ROUTES];
3439 for _ in 0..ROUTES * FRAMES_PER_ROUTE {
3440 let frame = client_rx.try_recv().expect("every queued frame arrives");
3441 let index = routes
3442 .iter()
3443 .position(|route| route.client_channel == frame.header.channel)
3444 .expect("frame arrives on one of the bound client channels");
3445 assert_eq!(frame.header.corr, next_seq[index], "route {index} order");
3446 let expected_prefix = format!("route-{index}-token-{:05}-", next_seq[index]);
3447 assert!(frame.body.starts_with(expected_prefix.as_bytes()));
3448 next_seq[index] += 1;
3449 }
3450 assert!(client_rx.try_recv().is_err());
3451 assert_eq!(next_seq, vec![FRAMES_PER_ROUTE as u64; ROUTES]);
3452 assert_eq!(client_sink.backlog().queued_bytes, 0);
3453 }
3454
3455 #[tokio::test(start_paused = true)]
3458 async fn never_reading_client_is_closed_at_byte_budget_with_warn_diagnosis() {
3459 let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3460 const BODY: usize = 16 * 1024;
3461 let (router, client_sink, _client_rx, module_ctx, routes, mut close_receiver) =
3462 multi_route_client(
3463 "stuck-reader-provider",
3464 ConnectionId::new(70),
3465 ConnectionId::new(71),
3466 2,
3467 )
3468 .await;
3469
3470 let mut admitted = 0usize;
3471 let mut sent = 0u64;
3472 let started = tokio::time::Instant::now();
3473 while router.counters.snapshot()["client_egress_close_delivery_failed"] == 0 {
3474 assert!(sent < 1_000, "the byte budget never refused a frame");
3475 let route = &routes[(sent % 2) as usize];
3476 router
3477 .route_for_connection(
3478 &module_ctx,
3479 stream_frame(
3480 route.module_channel,
3481 route.module_epoch,
3482 sent,
3483 vec![b'z'; BODY],
3484 ),
3485 )
3486 .await
3487 .unwrap();
3488 sent += 1;
3489 admitted = client_sink.backlog().queued_frames;
3490 }
3491 assert_eq!(
3492 started.elapsed(),
3493 crate::server::CONNECTION_EGRESS_STALL_GRACE
3494 );
3495 let frame_bytes = subc_protocol::HEADER_LEN + BODY;
3497 assert_eq!(
3498 admitted,
3499 crate::server::CONNECTION_EGRESS_BYTE_BUDGET / frame_bytes
3500 );
3501 let reason = close_receiver
3502 .try_recv()
3503 .expect("the client connection must be asked to close");
3504 assert!(reason
3505 .to_string()
3506 .contains("module_to_client_delivery_failed"));
3507
3508 router
3510 .route_for_connection(
3511 &module_ctx,
3512 stream_frame(
3513 routes[0].module_channel,
3514 routes[0].module_epoch,
3515 sent,
3516 vec![b'z'; BODY],
3517 ),
3518 )
3519 .await
3520 .unwrap();
3521
3522 let captured = test_log::captured_logs(&logs);
3523 let warn_lines = captured
3524 .lines()
3525 .filter(|line| {
3526 line.contains("closing client connection: its egress queue could not take a frame")
3527 })
3528 .collect::<Vec<_>>();
3529 assert_eq!(warn_lines.len(), 1, "exactly one WARN, got: {captured}");
3530 let line = warn_lines[0];
3531 assert!(line.contains("WARN"), "{line}");
3532 assert!(line.contains("connection_id=71"), "{line}");
3533 assert!(
3534 line.contains("module_id=\"stuck-reader-provider\""),
3535 "{line}"
3536 );
3537 assert!(line.contains("client_channel="), "{line}");
3538 assert!(line.contains("principals=direct"), "{line}");
3539 let queued_bytes: usize = line
3540 .split("queued_bytes=")
3541 .nth(1)
3542 .and_then(|rest| rest.split_whitespace().next())
3543 .and_then(|value| value.parse().ok())
3544 .expect("queued_bytes is logged");
3545 assert_eq!(queued_bytes, admitted * frame_bytes);
3546 assert!(
3547 line.contains(&format!("queued_frames={admitted}")),
3548 "{line}"
3549 );
3550 assert!(line.contains("oldest_queued_ms="), "{line}");
3551 assert!(line.contains("oldest_queued_ms=250"), "{line}");
3552 assert!(line.contains("frame_type=Some(StreamData)"), "{line}");
3553 assert!(
3554 line.contains(&format!(
3555 "frame_flags=Some({})",
3556 Flags::new(false, Priority::Interactive, false).0
3557 )),
3558 "{line}"
3559 );
3560 assert!(
3561 line.contains("close_cause=\"waited_250_ms_without_room\""),
3562 "{line}"
3563 );
3564 assert!(
3565 !line.contains(&"z".repeat(20)),
3566 "body bytes must never be logged: {line}"
3567 );
3568 }
3569
3570 fn route_frame(ty: FrameType, channel: u16, epoch: u32, corr: u64) -> Frame {
3571 Frame::build(
3572 ty,
3573 Flags::new(false, Priority::Interactive, false),
3574 channel,
3575 epoch,
3576 corr,
3577 if ty == FrameType::Request || ty == FrameType::Response {
3578 b"route-body".to_vec()
3579 } else {
3580 Vec::new()
3581 },
3582 )
3583 .unwrap()
3584 }
3585
3586 type DynamicRouteFixture = (
3587 Router,
3588 Arc<ForwardingTable>,
3589 RouteCtx,
3590 mpsc::Receiver<crate::router::OutboundFrame>,
3591 RouteCtx,
3592 mpsc::Receiver<crate::router::OutboundFrame>,
3593 mpsc::Receiver<crate::router::OutboundFrame>,
3594 crate::forwarding::PendingRouteBindRelay,
3595 );
3596
3597 fn dynamic_route_fixture(commit: bool) -> DynamicRouteFixture {
3598 let forwarding = Arc::new(ForwardingTable::default());
3599 let control = Arc::new(crate::ControlHandler::with_forwarding(
3600 Arc::new(crate::Registry::default()),
3601 Arc::clone(&forwarding),
3602 ));
3603 let router = Router::with_control_handler(control);
3604 let module_connection = ConnectionId::new(500);
3605 let client_connection = ConnectionId::new(501);
3606 let (module_tx, module_rx) = mpsc::channel(8);
3607 forwarding
3608 .register_module_connection(
3609 module_connection,
3610 "epoch-router".into(),
3611 2,
3612 Concurrency::ModuleManaged,
3613 FrameSink::new(module_tx),
3614 )
3615 .unwrap();
3616 let (client_tx, client_rx) = mpsc::channel(8);
3617 let client_sink = FrameSink::new(client_tx);
3618 let pending = forwarding
3619 .begin_route_bind_relay_for_test(
3620 client_connection,
3621 client_sink.clone(),
3622 700,
3623 "epoch-router",
3624 )
3625 .unwrap();
3626 if commit {
3627 forwarding
3628 .complete_pending_relay(
3629 module_connection,
3630 pending.corr,
3631 RouteBindRelayOutcome::Accepted,
3632 )
3633 .unwrap();
3634 }
3635 let (module_egress_tx, module_egress_rx) = mpsc::channel(8);
3636 (
3637 router,
3638 forwarding,
3639 RouteCtx {
3640 connection_id: client_connection,
3641 egress: client_sink,
3642 },
3643 client_rx,
3644 RouteCtx {
3645 connection_id: module_connection,
3646 egress: FrameSink::new(module_egress_tx),
3647 },
3648 module_egress_rx,
3649 module_rx,
3650 pending,
3651 )
3652 }
3653
3654 #[tokio::test]
3655 async fn route_epochs_validate_both_directions_and_rewrite_to_peer_handle() {
3656 let (
3657 router,
3658 _forwarding,
3659 client_ctx,
3660 mut client_rx,
3661 module_ctx,
3662 _module_egress_rx,
3663 mut module_rx,
3664 pending,
3665 ) = dynamic_route_fixture(true);
3666 let route_open = client_rx.recv().await.unwrap();
3667 assert_eq!(route_open.header.corr, 700);
3668
3669 router
3670 .route_for_connection(
3671 &client_ctx,
3672 route_frame(
3673 FrameType::Request,
3674 pending.client_channel,
3675 pending.client_epoch,
3676 701,
3677 ),
3678 )
3679 .await
3680 .unwrap();
3681 let forwarded = module_rx.recv().await.unwrap();
3682 assert_eq!(forwarded.header.channel, pending.module_channel);
3683 assert_eq!(forwarded.header.epoch, pending.module_epoch);
3684
3685 router
3686 .route_for_connection(
3687 &module_ctx,
3688 route_frame(
3689 FrameType::Response,
3690 pending.module_channel,
3691 pending.module_epoch,
3692 701,
3693 ),
3694 )
3695 .await
3696 .unwrap();
3697 let delivered = client_rx.recv().await.unwrap();
3698 assert_eq!(delivered.header.channel, pending.client_channel);
3699 assert_eq!(delivered.header.epoch, pending.client_epoch);
3700
3701 router
3702 .route_for_connection(
3703 &client_ctx,
3704 route_frame(
3705 FrameType::Request,
3706 pending.client_channel,
3707 pending.client_epoch + 1,
3708 702,
3709 ),
3710 )
3711 .await
3712 .unwrap();
3713 router
3714 .route_for_connection(
3715 &module_ctx,
3716 route_frame(
3717 FrameType::Response,
3718 pending.module_channel,
3719 pending.module_epoch + 1,
3720 703,
3721 ),
3722 )
3723 .await
3724 .unwrap();
3725 let stale_error = client_rx.recv().await.unwrap();
3726 assert_eq!(stale_error.header.ty, FrameType::Error);
3727 assert_eq!(stale_error.header.channel, pending.client_channel);
3728 assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
3729 assert_eq!(stale_error.header.corr, 702);
3730 let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
3731 assert_eq!(body.code, "stale_route_epoch");
3732 assert!(module_rx.try_recv().is_err());
3733 assert!(client_rx.try_recv().is_err());
3734 let counters = router.counters.snapshot();
3735 assert_eq!(counters["client_frames_dropped_stale_route"], 1);
3736 assert_eq!(counters["module_frames_dropped_no_route"], 1);
3737 }
3738
3739 #[tokio::test]
3740 async fn accepted_route_publishes_route_open_before_immediate_reverse_request() {
3741 let (
3742 router,
3743 _,
3744 _client_ctx,
3745 mut client_rx,
3746 module_ctx,
3747 _module_egress_rx,
3748 _module_rx,
3749 pending,
3750 ) = dynamic_route_fixture(true);
3751 router
3752 .route_for_connection(
3753 &module_ctx,
3754 route_frame(
3755 FrameType::Request,
3756 pending.module_channel,
3757 pending.module_epoch,
3758 800,
3759 ),
3760 )
3761 .await
3762 .unwrap();
3763
3764 let first = client_rx.recv().await.unwrap();
3765 let second = client_rx.recv().await.unwrap();
3766 assert_eq!(first.header.channel, 0);
3767 assert_eq!(first.header.corr, 700);
3768 assert_eq!(second.header.channel, pending.client_channel);
3769 assert_eq!(second.header.epoch, pending.client_epoch);
3770 assert_eq!(second.header.corr, 800);
3771 }
3772
3773 #[tokio::test]
3774 async fn reserved_slot_ingress_errors_only_matching_client_requests() {
3775 let (
3776 router,
3777 _forwarding,
3778 client_ctx,
3779 mut client_rx,
3780 _module_ctx,
3781 _module_egress_rx,
3782 mut module_rx,
3783 pending,
3784 ) = dynamic_route_fixture(false);
3785 router
3786 .route_for_connection(
3787 &client_ctx,
3788 route_frame(
3789 FrameType::Request,
3790 pending.client_channel,
3791 pending.client_epoch,
3792 900,
3793 ),
3794 )
3795 .await
3796 .unwrap();
3797 let error = client_rx.recv().await.unwrap();
3798 assert_eq!(error.header.ty, FrameType::Error);
3799 assert_eq!(error.header.channel, pending.client_channel);
3800 assert_eq!(error.header.epoch, pending.client_epoch);
3801 assert_eq!(error.header.corr, 900);
3802
3803 router
3804 .route_for_connection(
3805 &client_ctx,
3806 route_frame(
3807 FrameType::Response,
3808 pending.client_channel,
3809 pending.client_epoch,
3810 901,
3811 ),
3812 )
3813 .await
3814 .unwrap();
3815 router
3816 .route_for_connection(
3817 &client_ctx,
3818 route_frame(
3819 FrameType::Request,
3820 pending.client_channel,
3821 pending.client_epoch + 1,
3822 902,
3823 ),
3824 )
3825 .await
3826 .unwrap();
3827 let stale_error = client_rx.recv().await.unwrap();
3828 assert_eq!(stale_error.header.ty, FrameType::Error);
3829 assert_eq!(stale_error.header.channel, pending.client_channel);
3830 assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
3831 assert_eq!(stale_error.header.corr, 902);
3832 let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
3833 assert_eq!(body.code, "stale_route_epoch");
3834 assert!(module_rx.try_recv().is_err());
3835 let counters = router.counters.snapshot();
3836 assert_eq!(counters["client_frames_dropped_stale_route"], 1);
3837 assert_eq!(counters["module_frames_dropped_no_route"], 0);
3838 }
3839
3840 #[tokio::test]
3841 async fn dropped_module_route_goodbye_increments_counter() {
3842 let (
3843 router,
3844 _forwarding,
3845 client_ctx,
3846 mut client_rx,
3847 _module_ctx,
3848 _module_egress_rx,
3849 mut module_rx,
3850 pending,
3851 ) = dynamic_route_fixture(true);
3852 let _ = client_rx.recv().await;
3853 module_rx.close();
3854
3855 router
3856 .route_for_connection(
3857 &client_ctx,
3858 route_frame(
3859 FrameType::Goodbye,
3860 pending.client_channel,
3861 pending.client_epoch,
3862 999,
3863 ),
3864 )
3865 .await
3866 .unwrap();
3867
3868 let counters = router.counters.snapshot();
3869 assert_eq!(counters["goodbye_relay_module_dropped"], 1);
3870 assert_eq!(
3871 counters["goodbye_relay_module_dropped_by_module"],
3872 serde_json::json!({ "epoch-router": 1 })
3873 );
3874 assert_eq!(counters["route_released_epoch_fenced"], 1);
3875 }
3876
3877 #[tokio::test]
3878 async fn module_request_on_stale_epoch_receives_stale_route_epoch() {
3879 let (
3880 router,
3881 _forwarding,
3882 _client_ctx,
3883 _client_rx,
3884 module_ctx,
3885 mut module_egress_rx,
3886 mut module_rx,
3887 pending,
3888 ) = dynamic_route_fixture(true);
3889
3890 router
3891 .route_for_connection(
3892 &module_ctx,
3893 route_frame(
3894 FrameType::Request,
3895 pending.module_channel,
3896 pending.module_epoch + 1,
3897 1_000,
3898 ),
3899 )
3900 .await
3901 .unwrap();
3902
3903 let error = module_egress_rx.try_recv().unwrap();
3904 assert_eq!(error.header.ty, FrameType::Error);
3905 assert_eq!(error.header.channel, pending.module_channel);
3906 assert_eq!(error.header.epoch, pending.module_epoch + 1);
3907 assert_eq!(error.header.corr, 1_000);
3908 let body: ErrorBody = serde_json::from_slice(&error.body).unwrap();
3909 assert_eq!(body.code, "stale_route_epoch");
3910 assert!(module_rx.try_recv().is_err());
3911 let counters = router.counters.snapshot();
3912 assert_eq!(counters["module_requests_dropped_stale_route"], 1);
3913 assert_eq!(counters["module_frames_dropped_no_route"], 0);
3914 }
3915
3916 #[tokio::test(start_paused = true)]
3917 async fn stale_epoch_drop_warns_once_per_minute_with_suppressed_count() {
3918 let (output, _guard) = test_log::log_capture(tracing::Level::WARN);
3919 let (
3920 router,
3921 _forwarding,
3922 client_ctx,
3923 mut client_rx,
3924 _module_ctx,
3925 _module_egress_rx,
3926 mut module_rx,
3927 pending,
3928 ) = dynamic_route_fixture(true);
3929 let _ = client_rx.recv().await;
3930 for corr in 1..=3 {
3931 router
3932 .route_for_connection(
3933 &client_ctx,
3934 route_frame(
3935 FrameType::Request,
3936 pending.client_channel,
3937 pending.client_epoch + 1,
3938 corr,
3939 ),
3940 )
3941 .await
3942 .unwrap();
3943 assert_eq!(client_rx.recv().await.unwrap().header.ty, FrameType::Error);
3944 }
3945 assert!(module_rx.try_recv().is_err());
3946 let logs = test_log::captured_logs(&output);
3947 assert_eq!(
3948 logs.matches("dropping data-plane frame").count(),
3949 1,
3950 "{logs}"
3951 );
3952 assert!(logs.contains("WARN"), "{logs}");
3953 assert!(
3954 logs.contains(&format!("connection_id={}", client_ctx.connection_id.get())),
3955 "{logs}"
3956 );
3957 assert!(
3958 logs.contains(&format!("channel={}", pending.client_channel)),
3959 "{logs}"
3960 );
3961 assert!(
3962 logs.contains(&format!("epoch={}", pending.client_epoch + 1)),
3963 "{logs}"
3964 );
3965 assert!(
3966 logs.contains("client_to_module") && logs.contains("stale_epoch"),
3967 "{logs}"
3968 );
3969 assert!(logs.contains("suppressed=0"), "{logs}");
3970
3971 tokio::time::advance(Duration::from_secs(60)).await;
3972 router
3973 .route_for_connection(
3974 &client_ctx,
3975 route_frame(
3976 FrameType::Request,
3977 pending.client_channel,
3978 pending.client_epoch + 2,
3979 4,
3980 ),
3981 )
3982 .await
3983 .unwrap();
3984 let logs = test_log::captured_logs(&output);
3985 assert_eq!(
3986 logs.matches("dropping data-plane frame").count(),
3987 2,
3988 "{logs}"
3989 );
3990 assert!(logs.contains("suppressed=2"), "{logs}");
3991 }
3992
3993 #[tokio::test]
3994 async fn module_request_on_reserved_or_absent_route_receives_unknown_channel() {
3995 let (
3996 reserved_router,
3997 _forwarding,
3998 _client_ctx,
3999 _client_rx,
4000 reserved_module_ctx,
4001 mut reserved_module_egress_rx,
4002 _module_rx,
4003 reserved,
4004 ) = dynamic_route_fixture(false);
4005 reserved_router
4006 .route_for_connection(
4007 &reserved_module_ctx,
4008 route_frame(
4009 FrameType::Request,
4010 reserved.module_channel,
4011 reserved.module_epoch,
4012 1_001,
4013 ),
4014 )
4015 .await
4016 .unwrap();
4017 let reserved_error = reserved_module_egress_rx.try_recv().unwrap();
4018 let reserved_body: ErrorBody = serde_json::from_slice(&reserved_error.body).unwrap();
4019 assert_eq!(reserved_error.header.ty, FrameType::Error);
4020 assert_eq!(reserved_error.header.channel, reserved.module_channel);
4021 assert_eq!(reserved_error.header.epoch, reserved.module_epoch);
4022 assert_eq!(reserved_error.header.corr, 1_001);
4023 assert_eq!(reserved_body.code, "unknown_channel");
4024 assert_eq!(
4025 reserved_router.counters.snapshot()["module_requests_dropped_stale_route"],
4026 1
4027 );
4028
4029 let (
4030 absent_router,
4031 _forwarding,
4032 _client_ctx,
4033 _client_rx,
4034 absent_module_ctx,
4035 mut absent_module_egress_rx,
4036 _module_rx,
4037 absent,
4038 ) = dynamic_route_fixture(false);
4039 absent_router
4040 .route_for_connection(
4041 &absent_module_ctx,
4042 route_frame(
4043 FrameType::Request,
4044 absent.module_channel + 1,
4045 absent.module_epoch,
4046 1_002,
4047 ),
4048 )
4049 .await
4050 .unwrap();
4051 let absent_error = absent_module_egress_rx.try_recv().unwrap();
4052 let absent_body: ErrorBody = serde_json::from_slice(&absent_error.body).unwrap();
4053 assert_eq!(absent_error.header.ty, FrameType::Error);
4054 assert_eq!(absent_error.header.channel, absent.module_channel + 1);
4055 assert_eq!(absent_error.header.epoch, absent.module_epoch);
4056 assert_eq!(absent_error.header.corr, 1_002);
4057 assert_eq!(absent_body.code, "unknown_channel");
4058 assert_eq!(
4059 absent_router.counters.snapshot()["module_requests_dropped_stale_route"],
4060 1
4061 );
4062 }
4063
4064 #[tokio::test]
4065 async fn non_request_module_frame_on_dead_route_is_counted_without_error() {
4066 let (
4067 router,
4068 forwarding,
4069 client_ctx,
4070 mut client_rx,
4071 module_ctx,
4072 mut module_egress_rx,
4073 mut module_rx,
4074 pending,
4075 ) = dynamic_route_fixture(true);
4076 let (other_module_tx, _other_module_rx) = mpsc::channel(8);
4077 forwarding
4078 .register_module_connection(
4079 ConnectionId::new(502),
4080 "other-module".into(),
4081 2,
4082 Concurrency::ModuleManaged,
4083 FrameSink::new(other_module_tx),
4084 )
4085 .unwrap();
4086 let _ = client_rx.recv().await.unwrap();
4087
4088 router
4089 .route_for_connection(
4090 &client_ctx,
4091 route_frame(
4092 FrameType::Goodbye,
4093 pending.client_channel,
4094 pending.client_epoch,
4095 1_003,
4096 ),
4097 )
4098 .await
4099 .unwrap();
4100 let _ = module_rx.recv().await.unwrap();
4101
4102 router
4103 .route_for_connection(
4104 &module_ctx,
4105 route_frame(
4106 FrameType::StreamData,
4107 pending.module_channel,
4108 pending.module_epoch,
4109 1_004,
4110 ),
4111 )
4112 .await
4113 .unwrap();
4114
4115 let reply = module_egress_rx.try_recv().unwrap();
4118 assert_eq!(reply.header.ty, FrameType::Goodbye);
4119 assert!(module_egress_rx.try_recv().is_err());
4120 let counters = router.counters.snapshot();
4121 assert_eq!(counters["module_frames_dropped_no_route"], 1);
4122 assert_eq!(
4123 counters["module_frames_dropped_no_route_by_module"],
4124 serde_json::json!({ "epoch-router": 1 })
4125 );
4126 assert_eq!(counters["module_requests_dropped_stale_route"], 0);
4127 }
4128
4129 fn drain_now(rx: &mut mpsc::Receiver<OutboundFrame>) {
4132 while rx.try_recv().is_ok() {}
4133 }
4134
4135 #[tokio::test]
4141 async fn route_goodbye_refused_by_stalled_module_is_delivered_when_it_resumes_reading() {
4142 const BUDGET: usize = 4_096;
4143 let forwarding = Arc::new(ForwardingTable::default());
4144 let control = Arc::new(ControlHandler::with_forwarding(
4145 Arc::new(Registry::default()),
4146 Arc::clone(&forwarding),
4147 ));
4148 let router = Router::with_control_handler(control);
4149 let module_connection = ConnectionId::new(80);
4150 let client_connection = ConnectionId::new(81);
4151 let (module_tx, mut module_rx) = mpsc::channel(64);
4152 let module_sink = FrameSink::with_byte_budget(module_tx, BUDGET);
4153 forwarding
4154 .register_module_connection(
4155 module_connection,
4156 "stalled-provider".into(),
4157 2,
4158 Concurrency::ModuleManaged,
4159 module_sink.clone(),
4160 )
4161 .unwrap();
4162 let (client_tx, mut client_rx) = mpsc::channel(8);
4163 let client_sink = FrameSink::new(client_tx);
4164 let pending = forwarding
4165 .begin_route_bind_relay_for_test(
4166 client_connection,
4167 client_sink.clone(),
4168 1_100,
4169 "stalled-provider",
4170 )
4171 .unwrap();
4172 forwarding
4173 .complete_pending_relay(
4174 module_connection,
4175 pending.corr,
4176 RouteBindRelayOutcome::Accepted,
4177 )
4178 .unwrap();
4179 let _ = client_rx.recv().await.unwrap();
4180 drain_now(&mut module_rx);
4181
4182 module_sink
4185 .try_send(stream_frame(9, 1, 0, vec![b'f'; BUDGET]))
4186 .unwrap();
4187 assert!(module_sink
4188 .try_send(stream_frame(9, 1, 1, Vec::new()))
4189 .is_err());
4190
4191 let client_ctx = RouteCtx {
4192 connection_id: client_connection,
4193 egress: client_sink,
4194 };
4195 router
4196 .route_for_connection(
4197 &client_ctx,
4198 route_frame(
4199 FrameType::Goodbye,
4200 pending.client_channel,
4201 pending.client_epoch,
4202 1_101,
4203 ),
4204 )
4205 .await
4206 .unwrap();
4207 tokio::task::yield_now().await;
4208 assert_eq!(
4209 router.counters.snapshot()["goodbye_relay_module_dropped"],
4210 0,
4211 "a GOODBYE refused by a momentarily full module queue must not be dropped"
4212 );
4213
4214 let filler = module_rx.recv().await.unwrap();
4216 assert_eq!(filler.header.ty, FrameType::StreamData);
4217 drop(filler);
4218 let goodbye = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
4219 .await
4220 .expect("the refused GOODBYE must be delivered once the module frees room")
4221 .unwrap();
4222 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
4223 assert_eq!(goodbye.header.channel, pending.module_channel);
4224 assert_eq!(goodbye.header.epoch, pending.module_epoch);
4225 assert_eq!(
4226 router.counters.snapshot()["goodbye_relay_module_dropped"],
4227 0
4228 );
4229 }
4230
4231 #[tokio::test]
4237 async fn module_frame_on_route_the_daemon_does_not_hold_is_answered_with_goodbye() {
4238 let (
4239 router,
4240 _forwarding,
4241 client_ctx,
4242 mut client_rx,
4243 module_ctx,
4244 mut module_egress_rx,
4245 mut module_rx,
4246 pending,
4247 ) = dynamic_route_fixture(true);
4248 let _ = client_rx.recv().await.unwrap();
4249 router
4250 .route_for_connection(
4251 &client_ctx,
4252 route_frame(
4253 FrameType::Goodbye,
4254 pending.client_channel,
4255 pending.client_epoch,
4256 1_200,
4257 ),
4258 )
4259 .await
4260 .unwrap();
4261 drain_now(&mut module_rx);
4262
4263 let cases = [
4264 (pending.module_channel, pending.module_epoch),
4266 (pending.module_channel + 1, 1),
4268 ];
4269 for (channel, epoch) in cases {
4270 router
4271 .route_for_connection(
4272 &module_ctx,
4273 route_frame(FrameType::StreamData, channel, epoch, 1_201),
4274 )
4275 .await
4276 .unwrap();
4277 let reply = module_egress_rx
4278 .try_recv()
4279 .expect("a frame on a route the daemon does not hold is answered");
4280 assert_eq!(reply.header.ty, FrameType::Goodbye);
4281 assert_eq!(reply.header.channel, channel);
4282 assert_eq!(reply.header.epoch, epoch);
4283 assert_eq!(reply.header.corr, 0);
4284 assert!(module_egress_rx.try_recv().is_err());
4285 }
4286
4287 router
4290 .route_for_connection(
4291 &module_ctx,
4292 route_frame(FrameType::Goodbye, pending.module_channel + 2, 1, 0),
4293 )
4294 .await
4295 .unwrap();
4296 assert!(module_egress_rx.try_recv().is_err());
4297
4298 let counters = router.counters.snapshot();
4299 assert_eq!(counters["module_frames_dropped_no_route"], 3);
4300 assert_eq!(counters["module_frames_dropped_released_route"], 1);
4301 assert_eq!(
4302 counters["module_frames_dropped_released_route_by_module"],
4303 serde_json::json!({ "epoch-router": 1 })
4304 );
4305 assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4306 }
4307
4308 #[tokio::test(start_paused = true)]
4312 async fn burst_of_orphan_module_frames_is_answered_once_per_interval() {
4313 let (
4314 router,
4315 _forwarding,
4316 client_ctx,
4317 mut client_rx,
4318 module_ctx,
4319 mut module_egress_rx,
4320 mut module_rx,
4321 pending,
4322 ) = dynamic_route_fixture(true);
4323 let _ = client_rx.recv().await.unwrap();
4324 router
4325 .route_for_connection(
4326 &client_ctx,
4327 route_frame(
4328 FrameType::Goodbye,
4329 pending.client_channel,
4330 pending.client_epoch,
4331 1_300,
4332 ),
4333 )
4334 .await
4335 .unwrap();
4336 drain_now(&mut module_rx);
4337
4338 let send_burst = |corr: u64| {
4339 route_frame(
4340 FrameType::StreamData,
4341 pending.module_channel,
4342 pending.module_epoch,
4343 corr,
4344 )
4345 };
4346 for corr in 0..20 {
4347 router
4348 .route_for_connection(&module_ctx, send_burst(corr))
4349 .await
4350 .unwrap();
4351 }
4352 let mut replies = 0;
4353 while let Ok(reply) = module_egress_rx.try_recv() {
4354 assert_eq!(reply.header.ty, FrameType::Goodbye);
4355 replies += 1;
4356 }
4357 assert_eq!(replies, 1, "a burst within the interval gets one GOODBYE");
4358
4359 tokio::time::advance(ORPHAN_ROUTE_GOODBYE_INTERVAL).await;
4360 router
4361 .route_for_connection(&module_ctx, send_burst(20))
4362 .await
4363 .unwrap();
4364 let retry = module_egress_rx
4365 .try_recv()
4366 .expect("the first orphan frame after the interval is answered again");
4367 assert_eq!(retry.header.channel, pending.module_channel);
4368 assert_eq!(retry.header.epoch, pending.module_epoch);
4369
4370 let counters = router.counters.snapshot();
4371 assert_eq!(counters["module_frames_dropped_no_route"], 21);
4372 assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4373 }
4374
4375 #[test]
4377 fn orphan_goodbye_rate_limit_state_is_released_with_the_connection() {
4378 let router = Router::with_default_self_handler();
4379 let connection = router.begin_connection();
4380 let id = connection.id();
4381 assert!(router.orphan_goodbyes.claim(id, 7));
4382 assert!(!router.orphan_goodbyes.claim(id, 7));
4383 assert!(router.orphan_goodbyes.claim(id, 8));
4384 drop(connection);
4385 assert!(router
4386 .orphan_goodbyes
4387 .last_sent
4388 .lock()
4389 .unwrap()
4390 .get(&id)
4391 .is_none());
4392 }
4393
4394 #[test]
4395 fn channel_zero_cannot_be_registered_as_backend() {
4396 let mut router = Router::with_default_self_handler();
4397
4398 let err = router.register_backend(0, EchoBackend).unwrap_err();
4399
4400 assert_eq!(err, RouterError::ReservedChannelZero);
4401 }
4402}