1use std::{
2 collections::HashMap,
3 error::Error,
4 fmt,
5 sync::{
6 atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
7 Arc, Mutex,
8 },
9 time::{Duration, Instant},
10};
11
12use subc_protocol::{error_codes, ErrorBody, Flags, FrameType, Priority};
13use tokio::sync::{mpsc, Notify};
14use tracing::{debug, warn};
15
16use crate::{
17 control::ControlHandler,
18 forwarding::{
19 principal_label, CloseReason, ConnectionCloseReceiver, DataRoute, DataRouteState,
20 ForwardingError, ForwardingTable, RouteBinding, RouteRelease, UndeliveredFrame,
21 },
22 registry::ConnectionId,
23 DaemonCounters, Frame, FrameBuildError,
24};
25
26#[derive(Debug)]
35pub struct OutboundFrame {
36 pub frame: Frame,
37 pub enqueued_at: std::time::Instant,
38 pub(crate) flushed: Option<tokio::sync::oneshot::Sender<()>>,
39 pub(crate) charge: Option<EgressCharge>,
44}
45
46impl OutboundFrame {
47 fn charged(frame: Frame, charge: EgressCharge) -> Self {
48 Self {
49 frame,
50 enqueued_at: charge.enqueued_at,
51 flushed: None,
52 charge: Some(charge),
53 }
54 }
55}
56
57fn queued_frame_bytes(frame: &Frame) -> usize {
60 subc_protocol::HEADER_LEN + frame.body.len()
61}
62
63#[derive(Debug, Clone, Copy, PartialEq, Eq)]
66pub(crate) struct EgressBacklog {
67 pub queued_bytes: usize,
68 pub queued_frames: usize,
69 pub oldest_age: Option<Duration>,
76}
77
78#[derive(Debug)]
83struct EgressAccounting {
84 byte_budget: usize,
85 queued_bytes: AtomicUsize,
86 queued_frames: AtomicUsize,
87 time_base: Instant,
89 oldest_enqueued_nanos: AtomicU64,
92 waiters: AtomicUsize,
95 freed: Notify,
98 client_delivery_closed: AtomicBool,
99 client_delivery_stopped: Notify,
100}
101
102impl EgressAccounting {
103 fn new(byte_budget: usize) -> Self {
104 Self {
105 byte_budget,
106 queued_bytes: AtomicUsize::new(0),
107 queued_frames: AtomicUsize::new(0),
108 time_base: tokio::time::Instant::now().into_std(),
109 oldest_enqueued_nanos: AtomicU64::new(0),
110 waiters: AtomicUsize::new(0),
111 freed: Notify::new(),
112 client_delivery_closed: AtomicBool::new(false),
113 client_delivery_stopped: Notify::new(),
114 }
115 }
116
117 fn stamp(&self, at: Instant) -> u64 {
118 (at.saturating_duration_since(self.time_base).as_nanos() as u64).saturating_add(1)
119 }
120
121 fn try_charge(self: &Arc<Self>, bytes: usize) -> Option<EgressCharge> {
126 let mut current = self.queued_bytes.load(Ordering::SeqCst);
129 loop {
130 if current != 0 && current.saturating_add(bytes) > self.byte_budget {
131 return None;
132 }
133 match self.queued_bytes.compare_exchange_weak(
134 current,
135 current + bytes,
136 Ordering::SeqCst,
137 Ordering::SeqCst,
138 ) {
139 Ok(_) => return Some(self.record(bytes)),
140 Err(actual) => current = actual,
141 }
142 }
143 }
144
145 fn charge_unconditionally(self: &Arc<Self>, bytes: usize) -> EgressCharge {
148 self.queued_bytes.fetch_add(bytes, Ordering::SeqCst);
149 self.record(bytes)
150 }
151
152 fn record(self: &Arc<Self>, bytes: usize) -> EgressCharge {
153 let enqueued_at = tokio::time::Instant::now().into_std();
154 let stamp = self.stamp(enqueued_at);
155 if self.queued_frames.fetch_add(1, Ordering::AcqRel) == 0 {
156 self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
159 }
160 EgressCharge {
161 accounting: Arc::clone(self),
162 bytes,
163 stamp,
164 enqueued_at,
165 }
166 }
167
168 fn taken_by_writer(&self, stamp: u64) {
170 self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
171 }
172
173 fn release(&self, bytes: usize, stamp: u64) {
174 self.queued_bytes.fetch_sub(bytes, Ordering::SeqCst);
175 if self.queued_frames.fetch_sub(1, Ordering::AcqRel) == 1 {
176 let _ = self.oldest_enqueued_nanos.compare_exchange(
179 stamp,
180 0,
181 Ordering::AcqRel,
182 Ordering::Acquire,
183 );
184 }
185 if self.waiters.load(Ordering::SeqCst) != 0 {
186 self.freed.notify_waiters();
187 }
188 }
189
190 fn backlog(&self) -> EgressBacklog {
191 let queued_frames = self.queued_frames.load(Ordering::Acquire);
192 let stamp = self.oldest_enqueued_nanos.load(Ordering::Acquire);
193 let oldest_age = (queued_frames != 0 && stamp != 0).then(|| {
194 let enqueued = self.time_base + Duration::from_nanos(stamp - 1);
195 tokio::time::Instant::from_std(enqueued).elapsed()
196 });
197 EgressBacklog {
198 queued_bytes: self.queued_bytes.load(Ordering::Acquire),
199 queued_frames,
200 oldest_age,
201 }
202 }
203}
204
205#[derive(Debug)]
208pub(crate) struct EgressCharge {
209 accounting: Arc<EgressAccounting>,
210 bytes: usize,
211 stamp: u64,
212 enqueued_at: Instant,
213}
214
215impl EgressCharge {
216 pub(crate) fn taken_by_writer(&self) {
219 self.accounting.taken_by_writer(self.stamp);
220 }
221}
222
223impl Drop for EgressCharge {
224 fn drop(&mut self) {
225 self.accounting.release(self.bytes, self.stamp);
226 }
227}
228
229#[derive(Debug)]
235pub(crate) struct EgressPermit {
236 permit: mpsc::OwnedPermit<OutboundFrame>,
237 accounting: Arc<EgressAccounting>,
238}
239
240impl EgressPermit {
241 pub(crate) fn send(self, frame: Frame) -> bool {
244 let charge = self
245 .accounting
246 .charge_unconditionally(queued_frame_bytes(&frame));
247 let sender = self.permit.send(OutboundFrame::charged(frame, charge));
248 sender.is_closed()
249 }
250}
251
252impl std::ops::Deref for OutboundFrame {
255 type Target = Frame;
256
257 fn deref(&self) -> &Frame {
258 &self.frame
259 }
260}
261
262#[cfg(test)]
265pub(crate) mod test_log {
266 use std::{
267 io::Write,
268 sync::{Arc, Mutex, OnceLock},
269 };
270
271 #[derive(Clone)]
272 struct TestLogWriter(Arc<Mutex<Vec<u8>>>);
273
274 impl Write for TestLogWriter {
275 fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
276 self.0
277 .lock()
278 .expect("test log capture is not poisoned")
279 .extend(buffer);
280 Ok(buffer.len())
281 }
282
283 fn flush(&mut self) -> std::io::Result<()> {
284 Ok(())
285 }
286 }
287
288 pub(crate) fn log_capture(
289 level: tracing::Level,
290 ) -> (Arc<Mutex<Vec<u8>>>, tracing::dispatcher::DefaultGuard) {
291 static INTEREST_ANCHOR: OnceLock<tracing::Dispatch> = OnceLock::new();
298 INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
299 let output = Arc::new(Mutex::new(Vec::new()));
300 let writer = Arc::clone(&output);
301 let subscriber = tracing_subscriber::fmt()
302 .with_max_level(level)
303 .with_ansi(false)
304 .without_time()
305 .with_target(false)
306 .with_writer(move || TestLogWriter(Arc::clone(&writer)))
307 .finish();
308 let guard = tracing::subscriber::set_default(subscriber);
309 (output, guard)
310 }
311
312 pub(crate) fn captured_logs(output: &Arc<Mutex<Vec<u8>>>) -> String {
313 String::from_utf8(
314 output
315 .lock()
316 .expect("test log capture is not poisoned")
317 .clone(),
318 )
319 .expect("tracing output is UTF-8")
320 }
321
322 #[cfg(test)]
323 mod tests {
324 use super::*;
325
326 fn emit_capture_probe(source: &str) {
327 tracing::error!(source, "parallel capture probe");
328 }
329
330 #[test]
331 fn parallel_callsite_registration_keeps_the_test_capture_enabled() {
332 let (output, _guard) = log_capture(tracing::Level::ERROR);
333 std::thread::spawn(|| emit_capture_probe("other-thread"))
336 .join()
337 .expect("unsubscribed thread exits");
338 emit_capture_probe("captured-thread");
339
340 let logs = captured_logs(&output);
341 assert!(
342 logs.contains("captured-thread"),
343 "capture was disabled: {logs}"
344 );
345 assert!(!logs.contains("other-thread"), "capture leaked: {logs}");
346 }
347 }
348}
349
350#[derive(Debug, Clone)]
358pub struct FrameSink {
359 tx: mpsc::Sender<OutboundFrame>,
360 accounting: Arc<EgressAccounting>,
361}
362
363impl FrameSink {
364 pub fn new(tx: mpsc::Sender<OutboundFrame>) -> Self {
368 Self::with_byte_budget(tx, crate::server::CONNECTION_EGRESS_BYTE_BUDGET)
369 }
370
371 pub(crate) fn with_byte_budget(tx: mpsc::Sender<OutboundFrame>, byte_budget: usize) -> Self {
372 Self {
373 tx,
374 accounting: Arc::new(EgressAccounting::new(byte_budget)),
375 }
376 }
377
378 async fn charge_waiting(&self, bytes: usize) -> Option<EgressCharge> {
385 if let Some(charge) = self.accounting.try_charge(bytes) {
386 return Some(charge);
387 }
388 struct Waiting<'a>(&'a AtomicUsize);
391 impl Drop for Waiting<'_> {
392 fn drop(&mut self) {
393 self.0.fetch_sub(1, Ordering::SeqCst);
394 }
395 }
396 self.accounting.waiters.fetch_add(1, Ordering::SeqCst);
397 let _waiting = Waiting(&self.accounting.waiters);
398 loop {
399 let freed = self.accounting.freed.notified();
400 tokio::pin!(freed);
401 freed.as_mut().enable();
404 if let Some(charge) = self.accounting.try_charge(bytes) {
405 return Some(charge);
406 }
407 tokio::select! {
408 _ = &mut freed => {}
409 _ = self.tx.closed() => return None,
410 }
411 }
412 }
413
414 pub async fn send(&self, frame: Frame) -> Result<(), RouterError> {
415 let channel = frame.header.channel;
416 let epoch = frame.header.epoch;
417 let corr = frame.header.corr;
418 let closed =
419 || RouterError::backend_with_epoch(channel, epoch, corr, "connection writer closed");
420 let charge = self
421 .charge_waiting(queued_frame_bytes(&frame))
422 .await
423 .ok_or_else(closed)?;
424 self.tx
425 .send(OutboundFrame::charged(frame, charge))
426 .await
427 .map_err(|_| closed())
428 }
429
430 #[cfg(unix)]
437 pub async fn send_flushed(&self, frame: Frame) -> Result<(), RouterError> {
438 let (tx, rx) = tokio::sync::oneshot::channel();
439 let charge = self
440 .charge_waiting(queued_frame_bytes(&frame))
441 .await
442 .ok_or_else(|| RouterError::backend(0, 0, "connection writer closed"))?;
443 let mut outbound = OutboundFrame::charged(frame, charge);
444 outbound.flushed = Some(tx);
445 self.tx
446 .send(outbound)
447 .await
448 .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
449 rx.await
450 .map_err(|_| RouterError::backend(0, 0, "connection flush failed"))
451 }
452
453 pub(crate) async fn reserve_owned(&self) -> Result<EgressPermit, RouterError> {
454 let permit = self
455 .tx
456 .clone()
457 .reserve_owned()
458 .await
459 .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
460 Ok(EgressPermit {
461 permit,
462 accounting: Arc::clone(&self.accounting),
463 })
464 }
465
466 #[cfg(test)]
467 pub(crate) fn try_reserve_owned(&self) -> Result<EgressPermit, RouterError> {
468 let permit = self
469 .tx
470 .clone()
471 .try_reserve_owned()
472 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
473 Ok(EgressPermit {
474 permit,
475 accounting: Arc::clone(&self.accounting),
476 })
477 }
478
479 pub(crate) fn is_closed(&self) -> bool {
480 self.tx.is_closed()
481 }
482
483 pub(crate) fn backlog(&self) -> EgressBacklog {
485 self.accounting.backlog()
486 }
487
488 pub(crate) fn close_client_delivery(&self) {
491 self.accounting
492 .client_delivery_closed
493 .store(true, Ordering::Release);
494 self.accounting.client_delivery_stopped.notify_waiters();
495 }
496
497 async fn send_to_client(&self, frame: Frame) -> Result<(), ClientDeliveryFailure> {
500 let frame_type = frame.header.ty;
501 let flags = frame.header.flags;
502 let stopped = self.accounting.client_delivery_stopped.notified();
503 tokio::pin!(stopped);
504 stopped.as_mut().enable();
505 let failure = |cause, error| ClientDeliveryFailure {
506 diagnosis: ClientDeliveryDiagnosis {
507 frame_type,
508 flags,
509 cause,
510 },
511 error,
512 };
513 let closed = || {
514 RouterError::backend_with_epoch(
515 frame.header.channel,
516 frame.header.epoch,
517 frame.header.corr,
518 "client connection closed",
519 )
520 };
521 if self
522 .accounting
523 .client_delivery_closed
524 .load(Ordering::Acquire)
525 {
526 return Err(failure("client_closed", closed()));
527 }
528 let (frame, error) = match self.try_send_owned(frame) {
529 Ok(()) => return Ok(()),
530 Err(refused) => refused,
531 };
532 if self.is_closed() {
533 return Err(failure("client_closed", error));
534 }
535 let grace = crate::server::CONNECTION_EGRESS_STALL_GRACE;
536 let age = self.backlog().oldest_age.unwrap_or_default();
537 if age > grace {
538 return Err(failure("oldest_frame_older_than_250_ms", error));
539 }
540 let result = tokio::select! {
541 biased;
542 _ = &mut stopped => return Err(failure("client_closed", error)),
543 result = tokio::time::timeout(grace.saturating_sub(age), self.send(frame)) => result,
544 };
545 match result {
546 Ok(Ok(())) => Ok(()),
547 Ok(Err(error)) => Err(failure("client_closed", error)),
548 Err(_) => Err(failure("waited_250_ms_without_room", error)),
549 }
550 }
551
552 pub(crate) fn try_send(&self, frame: Frame) -> Result<(), RouterError> {
556 self.try_send_owned(frame).map_err(|(_, error)| error)
557 }
558
559 fn try_send_owned(&self, frame: Frame) -> Result<(), (Frame, RouterError)> {
560 let channel = frame.header.channel;
561 let epoch = frame.header.epoch;
562 let corr = frame.header.corr;
563 let unavailable = |why: String| {
564 RouterError::backend_with_epoch(
565 channel,
566 epoch,
567 corr,
568 format!("connection writer unavailable: {why}"),
569 )
570 };
571 let bytes = queued_frame_bytes(&frame);
572 let Some(charge) = self.accounting.try_charge(bytes) else {
573 if self.tx.is_closed() {
574 return Err((frame, unavailable("channel closed".to_string())));
575 }
576 return Err((
577 frame,
578 unavailable(format!(
579 "egress byte budget exhausted ({} queued bytes, frame of {bytes} bytes, budget {})",
580 self.accounting.queued_bytes.load(Ordering::Acquire),
581 self.accounting.byte_budget
582 )),
583 ));
584 };
585 self.tx
588 .try_send(OutboundFrame::charged(frame, charge))
589 .map_err(|err| {
590 let error = unavailable(err.to_string());
591 (err.into_inner().frame, error)
592 })
593 }
594}
595
596#[derive(Debug)]
597struct ClientDeliveryFailure {
598 diagnosis: ClientDeliveryDiagnosis,
599 error: RouterError,
600}
601
602#[derive(Debug)]
604pub(crate) struct ClientDeliveryDiagnosis {
605 pub frame_type: FrameType,
606 pub flags: Flags,
607 pub cause: &'static str,
608}
609
610impl fmt::Display for ClientDeliveryFailure {
611 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
612 self.error.fmt(f)
613 }
614}
615
616const ORPHAN_ROUTE_GOODBYE_INTERVAL: Duration = Duration::from_secs(5);
628
629const ORPHAN_ROUTE_GOODBYE_PRUNE_AT: usize = 256;
636
637#[derive(Debug, Default)]
641struct OrphanGoodbyeLimiter {
642 last_sent: Mutex<HashMap<ConnectionId, HashMap<u16, tokio::time::Instant>>>,
643}
644
645impl OrphanGoodbyeLimiter {
646 fn claim(&self, connection_id: ConnectionId, channel: u16) -> bool {
649 let now = tokio::time::Instant::now();
650 let mut last_sent = self
651 .last_sent
652 .lock()
653 .unwrap_or_else(std::sync::PoisonError::into_inner);
654 let channels = last_sent.entry(connection_id).or_default();
655 if channels.get(&channel).is_some_and(|sent| {
656 now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
657 }) {
658 return false;
659 }
660 if channels.len() >= ORPHAN_ROUTE_GOODBYE_PRUNE_AT {
661 channels.retain(|_, sent| {
662 now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
663 });
664 }
665 channels.insert(channel, now);
666 true
667 }
668
669 fn forget_connection(&self, connection_id: ConnectionId) {
670 self.last_sent
671 .lock()
672 .unwrap_or_else(std::sync::PoisonError::into_inner)
673 .remove(&connection_id);
674 }
675}
676
677#[derive(Debug, Clone)]
679pub struct RouteCtx {
680 pub connection_id: ConnectionId,
681 pub egress: FrameSink,
682}
683
684#[derive(Debug, Clone)]
690pub enum Backend {
691 Echo(EchoBackend),
692 Forward(ForwardBackend),
693}
694
695impl From<EchoBackend> for Backend {
696 fn from(backend: EchoBackend) -> Self {
697 Self::Echo(backend)
698 }
699}
700
701impl From<ForwardBackend> for Backend {
702 fn from(backend: ForwardBackend) -> Self {
703 Self::Forward(backend)
704 }
705}
706
707impl Backend {
708 pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
709 match self {
710 Self::Echo(backend) => backend.handle(ctx, frame).await,
711 Self::Forward(backend) => backend.handle(ctx, frame).await,
712 }
713 }
714}
715
716pub struct Router {
737 backends: HashMap<u16, Backend>,
738 control: Arc<ControlHandler>,
739 forwarding: Arc<ForwardingTable>,
740 forward_backend: ForwardBackend,
741 counters: DaemonCounters,
742 next_connection_id: AtomicU64,
743 orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
744}
745
746impl Router {
747 pub fn with_control_handler(control: Arc<ControlHandler>) -> Self {
748 control.install_swap_promotion_observer();
751 let forwarding = control.forwarding();
752 let counters = control.counters();
753 Self {
754 backends: HashMap::new(),
755 control,
756 forwarding: Arc::clone(&forwarding),
757 forward_backend: ForwardBackend::new(forwarding),
758 counters,
759 next_connection_id: AtomicU64::new(1),
761 orphan_goodbyes: Arc::default(),
762 }
763 }
764
765 pub fn with_default_self_handler() -> Self {
766 Self::with_control_handler(Arc::new(ControlHandler::default()))
767 }
768
769 pub fn forwarding(&self) -> Arc<ForwardingTable> {
770 Arc::clone(&self.forwarding)
771 }
772
773 pub fn register_backend(
774 &mut self,
775 channel: u16,
776 backend: impl Into<Backend>,
777 ) -> Result<(), RouterError> {
778 self.register_backend_arc(channel, Arc::new(backend.into()))
779 }
780
781 pub(crate) fn register_backend_arc(
782 &mut self,
783 channel: u16,
784 backend: Arc<Backend>,
785 ) -> Result<(), RouterError> {
786 if channel == 0 {
787 return Err(RouterError::ReservedChannelZero);
788 }
789 if self.backends.contains_key(&channel) {
790 return Err(RouterError::DuplicateChannel { channel });
791 }
792 self.backends.insert(channel, backend.as_ref().clone());
793 Ok(())
794 }
795
796 fn record_module_frame_drop(&self, connection_id: ConnectionId) -> Result<(), RouterError> {
799 let module_id = self
800 .forwarding
801 .module_id_for_connection(connection_id)
802 .map_err(RouterError::Forwarding)?;
803 self.counters
804 .increment_module_frames_dropped_no_route(module_id.as_deref());
805 Ok(())
806 }
807
808 fn handle_orphan_module_frame(&self, ctx: &RouteCtx, frame: &Frame) -> Result<(), RouterError> {
820 let channel = frame.header.channel;
821 let epoch = frame.header.epoch;
822 let module_id = self
823 .forwarding
824 .module_id_for_connection(ctx.connection_id)
825 .map_err(RouterError::Forwarding)?;
826 self.counters
827 .increment_module_frames_dropped_no_route(module_id.as_deref());
828 if self
829 .forwarding
830 .module_route_epoch_was_allocated(ctx.connection_id, channel, epoch)
831 .map_err(RouterError::Forwarding)?
832 {
833 self.counters
834 .increment_module_frames_dropped_released_route(module_id.as_deref());
835 }
836 if frame.header.ty == FrameType::Goodbye
837 || !self.orphan_goodbyes.claim(ctx.connection_id, channel)
838 {
839 return Ok(());
840 }
841 let goodbye = Frame::build_with_version(
842 frame.header.ver,
843 FrameType::Goodbye,
844 Flags::new(false, Priority::Passive, false),
845 channel,
846 epoch,
847 0,
848 Vec::new(),
849 )
850 .map_err(RouterError::FrameBuild)?;
851 match ctx.egress.try_send(goodbye) {
852 Ok(()) => {
853 self.counters.increment_module_orphan_route_goodbyes_sent();
854 debug!(
855 connection_id = ctx.connection_id.get(),
856 module_id = module_id.as_deref().unwrap_or("unknown"),
857 channel,
858 epoch,
859 "answered module frame on a route the daemon does not hold with a route GOODBYE"
860 );
861 }
862 Err(err) => {
863 self.forwarding.warn_frame_drop(
864 ctx.connection_id,
865 channel,
866 epoch,
867 "daemon_to_module",
868 if ctx.egress.is_closed() {
869 "endpoint_gone"
870 } else {
871 "egress_budget"
872 },
873 );
874 debug!(
875 connection_id = ctx.connection_id.get(),
876 module_id = module_id.as_deref().unwrap_or("unknown"),
877 channel,
878 epoch,
879 error = %err,
880 "could not enqueue route GOODBYE for module frame on a route the daemon does not hold; the next such frame retries"
881 );
882 }
883 }
884 Ok(())
885 }
886
887 pub fn begin_connection(&self) -> RouterConnection {
888 let raw = self.next_connection_id.fetch_add(1, Ordering::Relaxed);
889 let id = ConnectionId::new(raw);
890 let close_receiver = self.forwarding.register_connection_close(id);
891 RouterConnection {
892 id,
893 control_handler: Arc::clone(&self.control),
894 forwarding: Arc::clone(&self.forwarding),
895 close_receiver: Some(close_receiver),
896 orphan_goodbyes: Arc::clone(&self.orphan_goodbyes),
897 }
898 }
899
900 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
901 self.control.route_open_target(frame)
902 }
903
904 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
905 self.control.record_local_connection(connection);
906 }
907
908 pub(crate) fn record_captured_peer(
909 &self,
910 connection: ConnectionId,
911 peer: subc_os::local_peer::CapturedPeer,
912 ) {
913 self.control.record_captured_peer(connection, peer);
914 }
915
916 pub(crate) fn record_caller_endpoints(
917 &self,
918 connection: ConnectionId,
919 endpoints: Option<(std::net::SocketAddr, std::net::SocketAddr)>,
920 ) {
921 self.control.record_caller_endpoints(connection, endpoints);
922 }
923
924 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
925 self.control.defers_privileged_control(ctx, frame)
926 }
927
928 pub(crate) fn route_open_capacity_refusal(
929 &self,
930 ctx: &RouteCtx,
931 frame: &Frame,
932 target_module_id: &str,
933 in_flight: usize,
934 limit: usize,
935 ) -> Result<Frame, RouterError> {
936 self.control
937 .route_open_capacity_refusal(ctx, frame, target_module_id, in_flight, limit)
938 }
939
940 pub async fn route_for_connection(
941 &self,
942 ctx: &RouteCtx,
943 frame: Frame,
944 ) -> Result<(), RouterError> {
945 self.route_for_connection_started(ctx, frame, None).await
946 }
947
948 pub(crate) async fn route_for_connection_started(
949 &self,
950 ctx: &RouteCtx,
951 frame: Frame,
952 dispatch_started_at: Option<Instant>,
953 ) -> Result<(), RouterError> {
954 let channel = frame.header.channel;
955 let epoch = frame.header.epoch;
956 let corr = frame.header.corr;
957 if channel == 0 {
958 debug!(
959 connection_id = ctx.connection_id.get(),
960 corr,
961 frame_type = ?frame.header.ty,
962 "routing control frame"
963 );
964 let dispatch_started_at = (frame.header.ty == FrameType::Request)
970 .then(|| dispatch_started_at.unwrap_or_else(Instant::now));
971 let responses = self
972 .control
973 .handle_control_frame_timed(ctx, frame, dispatch_started_at)
974 .await?;
975 for response in responses {
976 ctx.egress.send(response).await?;
977 }
978 return Ok(());
979 }
980
981 let data_route = self
982 .forwarding
983 .lookup_data_route(ctx.connection_id, channel, epoch)
984 .map_err(RouterError::Forwarding)?;
985
986 match data_route {
987 DataRoute::Module(DataRouteState::EpochMismatch) => {
988 self.forwarding.warn_frame_drop(
989 ctx.connection_id,
990 channel,
991 epoch,
992 "module_to_client",
993 "stale_epoch",
994 );
995 if frame.header.ty == FrameType::Request {
996 self.counters
997 .increment_module_requests_dropped_stale_route();
998 let err = RouterError::StaleRouteEpoch {
999 channel,
1000 epoch,
1001 corr,
1002 };
1003 if let Some(error_frame) = err.to_error_frame() {
1004 ctx.egress.send(error_frame).await?;
1005 }
1006 } else {
1007 self.handle_orphan_module_frame(ctx, &frame)?;
1008 }
1009 debug!(
1010 connection_id = ctx.connection_id.get(),
1011 channel, epoch, corr, "dropping module frame for stale route epoch"
1012 );
1013 return Ok(());
1014 }
1015 DataRoute::Module(DataRouteState::Reserved) => {
1016 self.forwarding.warn_frame_drop(
1017 ctx.connection_id,
1018 channel,
1019 epoch,
1020 "module_to_client",
1021 "missing_binding",
1022 );
1023 if frame.header.ty == FrameType::Request {
1024 self.counters
1025 .increment_module_requests_dropped_stale_route();
1026 let err = RouterError::UnknownChannel {
1027 channel,
1028 epoch,
1029 corr,
1030 };
1031 if let Some(error_frame) = err.to_error_frame() {
1032 ctx.egress.send(error_frame).await?;
1033 }
1034 } else {
1035 self.record_module_frame_drop(ctx.connection_id)?;
1036 }
1037 debug!(
1038 connection_id = ctx.connection_id.get(),
1039 channel, epoch, corr, "dropping module frame for reserved route handle"
1040 );
1041 return Ok(());
1042 }
1043 DataRoute::Module(DataRouteState::Absent) => {
1044 self.forwarding.warn_frame_drop(
1045 ctx.connection_id,
1046 channel,
1047 epoch,
1048 "module_to_client",
1049 "unknown_channel",
1050 );
1051 if frame.header.ty == FrameType::Request {
1052 self.counters
1053 .increment_module_requests_dropped_stale_route();
1054 let err = RouterError::UnknownChannel {
1055 channel,
1056 epoch,
1057 corr,
1058 };
1059 if let Some(error_frame) = err.to_error_frame() {
1060 ctx.egress.send(error_frame).await?;
1061 }
1062 } else {
1063 self.handle_orphan_module_frame(ctx, &frame)?;
1064 }
1065 debug!(
1066 connection_id = ctx.connection_id.get(),
1067 channel, epoch, corr, "dropping module frame for absent route handle"
1068 );
1069 return Ok(());
1070 }
1071 DataRoute::Module(DataRouteState::Bound(route)) => {
1072 if frame.header.ty == FrameType::Goodbye {
1073 if let RouteRelease::Removed(target) = self
1074 .forwarding
1075 .release_module_route(ctx.connection_id, channel, epoch)
1076 .map_err(RouterError::Forwarding)?
1077 {
1078 let mut goodbye = frame;
1079 goodbye.header.channel = target.channel;
1080 goodbye.header.epoch = target.epoch;
1081 goodbye.header.flags = goodbye.header.flags.without_daemon_origin();
1085 if let Err(err) = target.sink.send_to_client(goodbye).await {
1086 if !target.close_on_delivery_failure() {
1087 self.forwarding.warn_frame_drop(
1088 target.connection_id,
1089 target.channel,
1090 target.epoch,
1091 "module_to_client",
1092 err.diagnosis.cause,
1093 );
1094 }
1095 if target.close_on_delivery_failure()
1096 && self
1097 .forwarding
1098 .escalate_client_delivery_failure_diagnosed(
1099 target.connection_id,
1100 target.channel,
1101 target.epoch,
1102 CloseReason::new(
1103 "route_goodbye_delivery_failed",
1104 format!(
1105 "failed to enqueue route GOODBYE for client channel {}: {err}",
1106 target.channel
1107 ),
1108 ),
1109 UndeliveredFrame {
1110 module_id: Some(&route.module_id),
1111 sink: &target.sink,
1112 },
1113 Some(&err.diagnosis),
1114 )
1115 .map_err(RouterError::Forwarding)?
1116 {
1117 self.counters.increment_goodbye_relay_client_failed();
1118 }
1119 }
1120 } else {
1121 self.forwarding.warn_frame_drop(
1122 ctx.connection_id,
1123 channel,
1124 epoch,
1125 "module_to_client",
1126 "missing_binding",
1127 );
1128 }
1129 return Ok(());
1130 }
1131
1132 let bytes = queued_frame_bytes(&frame);
1136 let limit = route.client_sink.accounting.byte_budget;
1137 if bytes > limit {
1138 let ty = frame.header.ty;
1139 let flags = frame.header.flags;
1140 drop(frame);
1141 return self
1142 .refuse_oversized_module_frame(ctx, &route, (ty, flags), corr, bytes)
1143 .await;
1144 }
1145
1146 let releases_credit = is_terminal_frame(frame.header.ty);
1152 if releases_credit {
1153 route.flow.release_corr(corr);
1154 }
1155 let mut frame = frame;
1156 frame.header.channel = route.client_channel;
1157 frame.header.epoch = route.client_epoch;
1158 frame.header.flags = frame.header.flags.without_daemon_origin();
1164 if let Err(err) = route.client_sink.send_to_client(frame).await {
1165 if self
1166 .forwarding
1167 .escalate_client_delivery_failure_diagnosed(
1168 route.client_connection_id,
1169 route.client_channel,
1170 route.client_epoch,
1171 CloseReason::new(
1172 "module_to_client_delivery_failed",
1173 format!(
1174 "failed to enqueue module frame for client channel {} corr {corr}: {err}",
1175 route.client_channel
1176 ),
1177 ),
1178 UndeliveredFrame {
1179 module_id: Some(&route.module_id),
1180 sink: &route.client_sink,
1181 },
1182 Some(&err.diagnosis),
1183 )
1184 .map_err(RouterError::Forwarding)?
1185 {
1186 self.counters
1187 .increment_client_egress_close_delivery_failed();
1188 }
1189 return Ok(());
1190 }
1191 return Ok(());
1192 }
1193 DataRoute::Client(DataRouteState::EpochMismatch) => {
1194 self.forwarding.warn_frame_drop(
1195 ctx.connection_id,
1196 channel,
1197 epoch,
1198 "client_to_module",
1199 "stale_epoch",
1200 );
1201 if frame.header.ty == FrameType::Request {
1202 self.counters.increment_client_frames_dropped_stale_route();
1203 let err = RouterError::StaleRouteEpoch {
1205 channel,
1206 epoch,
1207 corr,
1208 };
1209 if let Some(error_frame) = err.to_error_frame() {
1210 ctx.egress.send(error_frame).await?;
1211 }
1212 }
1213 debug!(
1214 connection_id = ctx.connection_id.get(),
1215 channel, epoch, corr, "dropping client frame for stale route epoch"
1216 );
1217 return Ok(());
1218 }
1219 DataRoute::Client(DataRouteState::Reserved) => {
1220 self.forwarding.warn_frame_drop(
1221 ctx.connection_id,
1222 channel,
1223 epoch,
1224 "client_to_module",
1225 "missing_binding",
1226 );
1227 if frame.header.ty == FrameType::Request {
1228 let err = RouterError::UnknownChannel {
1229 channel,
1230 epoch,
1231 corr,
1232 };
1233 if let Some(error_frame) = err.to_error_frame() {
1234 ctx.egress.send(error_frame).await?;
1235 }
1236 }
1237 return Ok(());
1238 }
1239 DataRoute::Client(DataRouteState::Bound(route)) => {
1240 if frame.header.ty == FrameType::Goodbye {
1241 let _ = self
1242 .control
1243 .handle_route_goodbye(ctx.connection_id, channel, epoch)?;
1244 return Ok(());
1245 }
1246 return self.forward_backend.handle_bound(frame, route).await;
1247 }
1248 DataRoute::Client(DataRouteState::Absent) => {}
1249 }
1250
1251 if let Some(backend) = self.backends.get(&channel) {
1252 return backend.handle(ctx.clone(), frame).await.inspect_err(|_| {
1253 self.forwarding.warn_frame_drop(
1254 ctx.connection_id,
1255 channel,
1256 epoch,
1257 "client_to_backend",
1258 "endpoint_gone",
1259 );
1260 });
1261 }
1262 self.forwarding.warn_frame_drop(
1263 ctx.connection_id,
1264 channel,
1265 epoch,
1266 "client_to_module",
1267 "unknown_channel",
1268 );
1269 if frame.header.ty == FrameType::Request {
1270 let err = RouterError::UnknownChannel {
1271 channel,
1272 epoch,
1273 corr,
1274 };
1275 if let Some(error_frame) = err.to_error_frame() {
1276 ctx.egress.send(error_frame).await?;
1277 }
1278 }
1279 Ok(())
1280 }
1281
1282 async fn refuse_oversized_module_frame(
1283 &self,
1284 ctx: &RouteCtx,
1285 route: &RouteBinding,
1286 header: (FrameType, Flags),
1287 corr: u64,
1288 bytes: usize,
1289 ) -> Result<(), RouterError> {
1290 let (ty, flags) = header;
1291 let limit = route.client_sink.accounting.byte_budget;
1292 if ty == FrameType::Push {
1293 let diagnosis = ClientDeliveryDiagnosis {
1297 frame_type: ty,
1298 flags,
1299 cause: "frame_too_large",
1300 };
1301 if self
1302 .forwarding
1303 .escalate_client_delivery_failure_diagnosed(
1304 route.client_connection_id,
1305 route.client_channel,
1306 route.client_epoch,
1307 CloseReason::new(
1308 error_codes::FRAME_TOO_LARGE,
1309 format!(
1310 "oversized push: size={bytes} limit={limit} (header plus body bytes)"
1311 ),
1312 ),
1313 UndeliveredFrame {
1314 module_id: Some(&route.module_id),
1315 sink: &route.client_sink,
1316 },
1317 Some(&diagnosis),
1318 )
1319 .map_err(RouterError::Forwarding)?
1320 {
1321 self.counters
1322 .increment_client_egress_close_delivery_failed();
1323 }
1324 return Ok(());
1325 }
1326
1327 warn!(
1328 connection_id = route.client_connection_id.get(),
1329 module_id = %route.module_id,
1330 client_principal = %principal_label(&route.principal),
1331 client_channel = route.client_channel,
1332 frame_type = ?ty,
1333 frame_bytes = bytes,
1334 byte_limit = limit,
1335 "refusing oversized module-to-client frame with ERROR frame_too_large"
1336 );
1337 if ty == FrameType::Request {
1338 ctx.egress
1341 .send(frame_too_large_error(
1342 route.module_channel,
1343 route.module_epoch,
1344 corr,
1345 bytes,
1346 limit,
1347 )?)
1348 .await?;
1349 } else {
1350 route.flow.release_corr(corr);
1353 let error = frame_too_large_error(
1354 route.client_channel,
1355 route.client_epoch,
1356 corr,
1357 bytes,
1358 limit,
1359 )?;
1360 if let Err(err) = route.client_sink.send_to_client(error).await {
1363 if self
1364 .forwarding
1365 .escalate_client_delivery_failure_diagnosed(
1366 route.client_connection_id,
1367 route.client_channel,
1368 route.client_epoch,
1369 CloseReason::new(
1370 "module_to_client_delivery_failed",
1371 format!(
1372 "failed to enqueue frame_too_large for client channel {} corr {corr}: {err}",
1373 route.client_channel
1374 ),
1375 ),
1376 UndeliveredFrame {
1377 module_id: Some(&route.module_id),
1378 sink: &route.client_sink,
1379 },
1380 Some(&err.diagnosis),
1381 )
1382 .map_err(RouterError::Forwarding)?
1383 {
1384 self.counters
1385 .increment_client_egress_close_delivery_failed();
1386 }
1387 }
1388 }
1389 Ok(())
1390 }
1391}
1392
1393impl Default for Router {
1394 fn default() -> Self {
1395 Self::with_default_self_handler()
1396 }
1397}
1398
1399#[must_use]
1401pub struct RouterConnection {
1402 id: ConnectionId,
1403 control_handler: Arc<ControlHandler>,
1404 forwarding: Arc<ForwardingTable>,
1405 close_receiver: Option<ConnectionCloseReceiver>,
1406 orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
1407}
1408
1409impl RouterConnection {
1410 pub fn id(&self) -> ConnectionId {
1411 self.id
1412 }
1413
1414 pub(crate) fn take_close_receiver(&mut self) -> ConnectionCloseReceiver {
1415 self.close_receiver
1416 .take()
1417 .expect("connection close receiver can only be taken once")
1418 }
1419}
1420
1421impl Drop for RouterConnection {
1422 fn drop(&mut self) {
1423 self.forwarding.unregister_connection_close(self.id);
1424 self.orphan_goodbyes.forget_connection(self.id);
1425 let _ = self.control_handler.cleanup_connection(self.id);
1428 }
1429}
1430
1431#[derive(Debug, Default, Clone, Copy)]
1434pub struct EchoBackend;
1435
1436impl EchoBackend {
1437 pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1438 let response = Frame::build_with_version(
1439 frame.header.ver,
1440 FrameType::Response,
1441 frame.header.flags,
1442 frame.header.channel,
1443 frame.header.epoch,
1444 frame.header.corr,
1445 frame.body,
1446 )
1447 .map_err(RouterError::FrameBuild)?;
1448 ctx.egress.send(response).await
1449 }
1450}
1451
1452#[derive(Debug, Clone)]
1454pub struct ForwardBackend {
1455 forwarding: Arc<ForwardingTable>,
1456}
1457
1458impl ForwardBackend {
1459 pub fn new(forwarding: Arc<ForwardingTable>) -> Self {
1460 Self { forwarding }
1461 }
1462
1463 pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1464 let channel = frame.header.channel;
1465 let corr = frame.header.corr;
1466 let route = match self
1467 .forwarding
1468 .lookup_data_route(ctx.connection_id, channel, frame.header.epoch)
1469 .map_err(RouterError::Forwarding)?
1470 {
1471 DataRoute::Client(DataRouteState::Bound(route)) => route,
1472 DataRoute::Client(_) | DataRoute::Module(_) => {
1473 self.forwarding.warn_frame_drop(
1474 ctx.connection_id,
1475 channel,
1476 frame.header.epoch,
1477 "client_to_module",
1478 "missing_binding",
1479 );
1480 return Err(RouterError::UnknownChannel {
1481 channel,
1482 epoch: frame.header.epoch,
1483 corr,
1484 });
1485 }
1486 };
1487 self.handle_bound(frame, route).await
1488 }
1489
1490 pub(crate) async fn handle_bound(
1491 &self,
1492 frame: Frame,
1493 route: Arc<RouteBinding>,
1494 ) -> Result<(), RouterError> {
1495 let channel = frame.header.channel;
1496 let corr = frame.header.corr;
1497 let frame_type = frame.header.ty;
1498
1499 let acquired_credit = frame_type == FrameType::Request;
1502 if acquired_credit {
1503 if let Err(err) = route
1504 .flow
1505 .acquire_tagged(corr, frame.header.flags.is_subscription())
1506 .await
1507 {
1508 self.forwarding.warn_frame_drop(
1509 route.client_connection_id,
1510 channel,
1511 frame.header.epoch,
1512 "client_to_module",
1513 "endpoint_gone",
1514 );
1515 if self
1522 .forwarding
1523 .endpoint_is_draining(route.module_endpoint)
1524 .map_err(RouterError::Forwarding)?
1525 {
1526 return Err(RouterError::route_error_with_epoch(
1527 channel,
1528 frame.header.epoch,
1529 corr,
1530 "module_reloading",
1531 format!("module endpoint for route channel {channel} is reloading"),
1532 ));
1533 }
1534 return Err(RouterError::backend_with_epoch(
1535 channel,
1536 frame.header.epoch,
1537 corr,
1538 format!("{err} for route channel {channel}"),
1539 ));
1540 }
1541 }
1542
1543 let mut frame = frame;
1544 frame.header.channel = route.module_channel;
1545 frame.header.epoch = route.module_epoch;
1546 frame.header.flags = frame.header.flags.without_daemon_origin();
1550 let result = route.module_sink.send(frame).await.map_err(|err| {
1551 self.forwarding.warn_frame_drop(
1552 route.client_connection_id,
1553 channel,
1554 route.client_epoch,
1555 "client_to_module",
1556 "endpoint_gone",
1557 );
1558 RouterError::backend_with_epoch(channel, route.client_epoch, corr, err.to_string())
1559 });
1560 if acquired_credit && result.is_err() {
1561 route.flow.release_corr(corr);
1562 }
1563 result
1564 }
1565}
1566
1567fn is_terminal_frame(frame_type: FrameType) -> bool {
1568 matches!(
1569 frame_type,
1570 FrameType::Response | FrameType::Error | FrameType::StreamEnd
1571 )
1572}
1573
1574#[derive(Debug, Clone, PartialEq, Eq)]
1577pub enum RouterError {
1578 ReservedChannelZero,
1579 DuplicateChannel {
1580 channel: u16,
1581 },
1582 UnknownChannel {
1583 channel: u16,
1584 epoch: u32,
1585 corr: u64,
1586 },
1587 StaleRouteEpoch {
1588 channel: u16,
1589 epoch: u32,
1590 corr: u64,
1591 },
1592 Backend {
1593 channel: u16,
1594 epoch: u32,
1595 corr: u64,
1596 message: String,
1597 },
1598 RouteError {
1599 channel: u16,
1600 epoch: u32,
1601 corr: u64,
1602 code: String,
1603 message: String,
1604 },
1605 FrameBuild(FrameBuildError),
1606 Forwarding(ForwardingError),
1607}
1608
1609impl RouterError {
1610 pub fn backend(channel: u16, corr: u64, message: impl Into<String>) -> Self {
1611 Self::backend_with_epoch(channel, 0, corr, message)
1612 }
1613
1614 pub fn backend_with_epoch(
1615 channel: u16,
1616 epoch: u32,
1617 corr: u64,
1618 message: impl Into<String>,
1619 ) -> Self {
1620 Self::Backend {
1621 channel,
1622 epoch,
1623 corr,
1624 message: message.into(),
1625 }
1626 }
1627
1628 pub fn route_error(
1629 channel: u16,
1630 corr: u64,
1631 code: impl Into<String>,
1632 message: impl Into<String>,
1633 ) -> Self {
1634 Self::route_error_with_epoch(channel, 0, corr, code, message)
1635 }
1636
1637 pub fn route_error_with_epoch(
1638 channel: u16,
1639 epoch: u32,
1640 corr: u64,
1641 code: impl Into<String>,
1642 message: impl Into<String>,
1643 ) -> Self {
1644 Self::RouteError {
1645 channel,
1646 epoch,
1647 corr,
1648 code: code.into(),
1649 message: message.into(),
1650 }
1651 }
1652
1653 pub fn to_error_frame(&self) -> Option<Frame> {
1655 match self {
1656 Self::UnknownChannel {
1657 channel,
1658 epoch,
1659 corr,
1660 } => error_frame(
1661 *channel,
1662 *epoch,
1663 *corr,
1664 error_codes::UNKNOWN_CHANNEL,
1665 format!("unknown channel {channel}"),
1666 ),
1667 Self::StaleRouteEpoch {
1668 channel,
1669 epoch,
1670 corr,
1671 } => error_frame(
1672 *channel,
1673 *epoch,
1674 *corr,
1675 error_codes::STALE_ROUTE_EPOCH,
1676 format!("stale route epoch for channel {channel}"),
1677 ),
1678 Self::Backend {
1679 channel,
1680 epoch,
1681 corr,
1682 message,
1683 } => error_frame(*channel, *epoch, *corr, "backend_error", message.clone()),
1684 Self::RouteError {
1685 channel,
1686 epoch,
1687 corr,
1688 code,
1689 message,
1690 } => error_frame(*channel, *epoch, *corr, code, message.clone()),
1691 Self::ReservedChannelZero
1692 | Self::DuplicateChannel { .. }
1693 | Self::FrameBuild(_)
1694 | Self::Forwarding(_) => None,
1695 }
1696 }
1697}
1698
1699pub(crate) fn daemon_error_flags() -> Flags {
1711 Flags::new(false, Priority::Passive, false).with_daemon_origin()
1712}
1713
1714fn error_frame(channel: u16, epoch: u32, corr: u64, code: &str, message: String) -> Option<Frame> {
1715 let body = serde_json::to_vec(&ErrorBody {
1716 code: code.to_string(),
1717 message,
1718 detail: None,
1719 })
1720 .ok()?;
1721
1722 Frame::build(
1723 FrameType::Error,
1724 daemon_error_flags(),
1725 channel,
1726 epoch,
1727 corr,
1728 body,
1729 )
1730 .ok()
1731}
1732
1733fn frame_too_large_error(
1734 channel: u16,
1735 epoch: u32,
1736 corr: u64,
1737 bytes: usize,
1738 limit: usize,
1739) -> Result<Frame, RouterError> {
1740 let body = serde_json::to_vec(&ErrorBody {
1741 code: error_codes::FRAME_TOO_LARGE.to_string(),
1742 message: format!(
1743 "module frame of {bytes} bytes exceeds client egress limit of {limit} bytes"
1744 ),
1745 detail: Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit })),
1746 })
1747 .expect("serializing a frame size error");
1748 Frame::build(
1749 FrameType::Error,
1750 daemon_error_flags(),
1751 channel,
1752 epoch,
1753 corr,
1754 body,
1755 )
1756 .map_err(RouterError::FrameBuild)
1757}
1758
1759impl fmt::Display for RouterError {
1760 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1761 match self {
1762 Self::ReservedChannelZero => write!(f, "channel 0 is reserved for subc"),
1763 Self::DuplicateChannel { channel } => {
1764 write!(f, "backend already registered for channel {channel}")
1765 }
1766 Self::UnknownChannel { channel, corr, .. } => {
1767 write!(f, "unknown channel {channel} for corr {corr}")
1768 }
1769 Self::StaleRouteEpoch { channel, corr, .. } => {
1770 write!(f, "stale route epoch for channel {channel} corr {corr}")
1771 }
1772 Self::Backend {
1773 channel,
1774 corr,
1775 message,
1776 ..
1777 } => write!(
1778 f,
1779 "backend error on channel {channel} corr {corr}: {message}"
1780 ),
1781 Self::RouteError {
1782 channel,
1783 corr,
1784 code,
1785 message,
1786 ..
1787 } => write!(
1788 f,
1789 "route error {code} on channel {channel} corr {corr}: {message}"
1790 ),
1791 Self::FrameBuild(err) => write!(f, "failed to build routed frame: {err}"),
1792 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
1793 }
1794 }
1795}
1796
1797impl Error for RouterError {
1798 fn source(&self) -> Option<&(dyn Error + 'static)> {
1799 match self {
1800 Self::FrameBuild(err) => Some(err),
1801 Self::Forwarding(err) => Some(err),
1802 Self::ReservedChannelZero
1803 | Self::DuplicateChannel { .. }
1804 | Self::UnknownChannel { .. }
1805 | Self::StaleRouteEpoch { .. }
1806 | Self::Backend { .. }
1807 | Self::RouteError { .. } => None,
1808 }
1809 }
1810}
1811
1812#[cfg(test)]
1813mod tests {
1814 use super::*;
1815 use crate::{
1816 forwarding::RouteBindRelayOutcome,
1817 supervise::{ModuleSpec, RestartPolicy, Supervisor, SupervisorHandle},
1818 ControlHandler, Registry,
1819 };
1820 use std::{
1821 sync::{mpsc as std_mpsc, Arc},
1822 time::Duration,
1823 };
1824 use subc_control::ModuleProtocol;
1825 use subc_protocol::{manifest::Concurrency, ErrorBody, Flags, FrameType, Priority};
1826 use tokio::sync::mpsc;
1827
1828 pub(crate) use crate::router::test_log::{captured_logs, log_capture};
1829
1830 fn logged_millis(logs: &str, field: &str) -> u64 {
1831 logs.split_whitespace()
1832 .find_map(|part| part.strip_prefix(field))
1833 .and_then(|value| value.parse().ok())
1834 .unwrap_or_else(|| panic!("missing numeric {field} in logs: {logs}"))
1835 }
1836
1837 fn request(channel: u16, corr: u64, body: &[u8]) -> Frame {
1838 Frame::build(
1839 FrameType::Request,
1840 Flags::new(true, Priority::Interactive, false),
1841 channel,
1842 0,
1843 corr,
1844 body.to_vec(),
1845 )
1846 .unwrap()
1847 }
1848
1849 fn ping(corr: u64) -> Frame {
1850 Frame::build(
1851 FrameType::Ping,
1852 Flags::new(false, Priority::Passive, false),
1853 0,
1854 0,
1855 corr,
1856 Vec::new(),
1857 )
1858 .unwrap()
1859 }
1860
1861 fn route_ctx() -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
1862 let (tx, rx) = mpsc::channel(8);
1863 (
1864 RouteCtx {
1865 connection_id: ConnectionId::LOCAL,
1866 egress: FrameSink::new(tx),
1867 },
1868 rx,
1869 )
1870 }
1871
1872 #[tokio::test]
1873 async fn echo_backend_returns_response_with_byte_identical_body() {
1874 let mut router = Router::with_default_self_handler();
1875 router.register_backend(7, EchoBackend).unwrap();
1876 let (ctx, mut rx) = route_ctx();
1877 let body = b"{not parsed}\0\xff";
1878
1879 router
1880 .route_for_connection(&ctx, request(7, 123, body))
1881 .await
1882 .unwrap();
1883 let response = rx.recv().await.unwrap();
1884
1885 assert_eq!(response.header.ty, FrameType::Response);
1886 assert_eq!(response.header.channel, 7);
1887 assert_eq!(response.header.corr, 123);
1888 assert_eq!(response.body, body);
1889 assert!(rx.try_recv().is_err());
1890 }
1891
1892 #[tokio::test]
1893 async fn unknown_channel_emits_canonical_error_frame() {
1894 let router = Router::with_default_self_handler();
1895 let (ctx, mut rx) = route_ctx();
1896
1897 router
1898 .route_for_connection(&ctx, request(99, 5, b"payload"))
1899 .await
1900 .unwrap();
1901 let error_frame = rx.recv().await.unwrap();
1902
1903 assert_eq!(error_frame.header.ty, FrameType::Error);
1904 assert_eq!(error_frame.header.channel, 99);
1905 assert_eq!(error_frame.header.corr, 5);
1906 let body: ErrorBody = serde_json::from_slice(&error_frame.body).unwrap();
1907 assert_eq!(body.code, "unknown_channel");
1908 assert_eq!(body.message, "unknown channel 99");
1909 }
1910
1911 #[tokio::test]
1916 async fn daemon_authored_data_plane_errors_carry_daemon_origin() {
1917 let router = Router::with_default_self_handler();
1918 let (ctx, mut rx) = route_ctx();
1919 router
1920 .route_for_connection(&ctx, request(99, 5, b"payload"))
1921 .await
1922 .unwrap();
1923 let unknown = rx.recv().await.unwrap();
1924 assert_eq!(unknown.header.ty, FrameType::Error);
1925 let body: ErrorBody = serde_json::from_slice(&unknown.body).unwrap();
1926 assert_eq!(body.code, "unknown_channel");
1927 assert!(
1928 unknown.header.flags.is_daemon_origin(),
1929 "unknown_channel must carry daemon origin: {:?}",
1930 unknown.header.flags
1931 );
1932
1933 let (
1934 router,
1935 _forwarding,
1936 client_ctx,
1937 mut client_rx,
1938 _module_ctx,
1939 _module_egress_rx,
1940 _module_rx,
1941 pending,
1942 ) = dynamic_route_fixture(true);
1943 let _ = client_rx.recv().await;
1944 router
1945 .route_for_connection(
1946 &client_ctx,
1947 route_frame(
1948 FrameType::Request,
1949 pending.client_channel,
1950 pending.client_epoch + 1,
1951 702,
1952 ),
1953 )
1954 .await
1955 .unwrap();
1956 let stale = client_rx.recv().await.unwrap();
1957 assert_eq!(stale.header.ty, FrameType::Error);
1958 let body: ErrorBody = serde_json::from_slice(&stale.body).unwrap();
1959 assert_eq!(body.code, "stale_route_epoch");
1960 assert!(
1961 stale.header.flags.is_daemon_origin(),
1962 "stale_route_epoch must carry daemon origin: {:?}",
1963 stale.header.flags
1964 );
1965 }
1966
1967 #[tokio::test]
1972 async fn module_error_with_daemon_bit_is_stripped_before_client_delivery() {
1973 let (
1974 router,
1975 _forwarding,
1976 client_ctx,
1977 mut client_rx,
1978 module_ctx,
1979 _module_egress_rx,
1980 mut module_rx,
1981 pending,
1982 ) = dynamic_route_fixture(true);
1983 let _ = client_rx.recv().await;
1984 router
1987 .route_for_connection(
1988 &client_ctx,
1989 route_frame(
1990 FrameType::Request,
1991 pending.client_channel,
1992 pending.client_epoch,
1993 701,
1994 ),
1995 )
1996 .await
1997 .unwrap();
1998 let _ = module_rx.recv().await;
1999
2000 let body = serde_json::to_vec(&ErrorBody::new("module_reloading", "try again")).unwrap();
2001 let spoof = Frame::build(
2002 FrameType::Error,
2003 Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2004 pending.module_channel,
2005 pending.module_epoch,
2006 701,
2007 body.clone(),
2008 )
2009 .unwrap();
2010 router
2011 .route_for_connection(&module_ctx, spoof)
2012 .await
2013 .unwrap();
2014 let delivered = client_rx.recv().await.unwrap();
2015 assert_eq!(delivered.header.ty, FrameType::Error);
2016 assert_eq!(delivered.header.channel, pending.client_channel);
2017 assert_eq!(delivered.header.epoch, pending.client_epoch);
2018 assert_eq!(delivered.header.corr, 701);
2019 assert!(
2020 !delivered.header.flags.is_daemon_origin(),
2021 "module-set daemon origin must be cleared: {:?}",
2022 delivered.header.flags
2023 );
2024 assert_eq!(delivered.body, body, "the body travels byte-identical");
2025 }
2026
2027 #[tokio::test]
2030 async fn module_response_with_daemon_bit_is_stripped_preserving_other_flags() {
2031 use subc_protocol::FLAG_SUBSCRIPTION;
2032 let (
2033 router,
2034 _forwarding,
2035 client_ctx,
2036 mut client_rx,
2037 module_ctx,
2038 _module_egress_rx,
2039 mut module_rx,
2040 pending,
2041 ) = dynamic_route_fixture(true);
2042 let _ = client_rx.recv().await;
2043 router
2044 .route_for_connection(
2045 &client_ctx,
2046 route_frame(
2047 FrameType::Request,
2048 pending.client_channel,
2049 pending.client_epoch,
2050 701,
2051 ),
2052 )
2053 .await
2054 .unwrap();
2055 let _ = module_rx.recv().await;
2056
2057 let sent_flags = Flags(
2058 Flags::new(true, Priority::Background, true).0
2059 | FLAG_SUBSCRIPTION
2060 | subc_protocol::FLAG_DAEMON_ORIGIN,
2061 );
2062 assert!(sent_flags.is_daemon_origin());
2063 let response = Frame::build(
2064 FrameType::Response,
2065 sent_flags,
2066 pending.module_channel,
2067 pending.module_epoch,
2068 701,
2069 b"route-body".to_vec(),
2070 )
2071 .unwrap();
2072 router
2073 .route_for_connection(&module_ctx, response)
2074 .await
2075 .unwrap();
2076 let delivered = client_rx.recv().await.unwrap();
2077 assert_eq!(delivered.header.ty, FrameType::Response);
2078 assert!(
2079 !delivered.header.flags.is_daemon_origin(),
2080 "module-set daemon origin must be cleared: {:?}",
2081 delivered.header.flags
2082 );
2083 assert!(
2084 delivered.header.flags.is_binary(),
2085 "binary flag must survive: {:?}",
2086 delivered.header.flags
2087 );
2088 assert!(
2089 delivered.header.flags.is_last(),
2090 "LAST flag must survive: {:?}",
2091 delivered.header.flags
2092 );
2093 assert!(
2094 delivered.header.flags.is_subscription(),
2095 "subscription flag must survive: {:?}",
2096 delivered.header.flags
2097 );
2098 assert_eq!(
2099 delivered.header.flags.priority(),
2100 Some(Priority::Background),
2101 "priority must survive"
2102 );
2103 }
2104
2105 #[tokio::test]
2108 async fn client_frame_with_daemon_bit_is_stripped_before_module_delivery() {
2109 let (
2110 router,
2111 _forwarding,
2112 client_ctx,
2113 mut client_rx,
2114 _module_ctx,
2115 _module_egress_rx,
2116 mut module_rx,
2117 pending,
2118 ) = dynamic_route_fixture(true);
2119 let _ = client_rx.recv().await;
2120 let spoof = Frame::build(
2121 FrameType::Request,
2122 Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2123 pending.client_channel,
2124 pending.client_epoch,
2125 701,
2126 b"route-body".to_vec(),
2127 )
2128 .unwrap();
2129 router
2130 .route_for_connection(&client_ctx, spoof)
2131 .await
2132 .unwrap();
2133 let delivered = module_rx.recv().await.unwrap();
2134 assert_eq!(delivered.header.ty, FrameType::Request);
2135 assert!(
2136 !delivered.header.flags.is_daemon_origin(),
2137 "client-set daemon origin must be cleared: {:?}",
2138 delivered.header.flags
2139 );
2140 }
2141
2142 #[tokio::test]
2143 async fn channel_zero_uses_control_handler_not_backend_registry() {
2144 let mut router = Router::with_default_self_handler();
2145 router.register_backend(1, EchoBackend).unwrap();
2146 let (ctx, mut rx) = route_ctx();
2147
2148 router.route_for_connection(&ctx, ping(77)).await.unwrap();
2149 let response = rx.recv().await.unwrap();
2150
2151 assert_eq!(response.header.ty, FrameType::Pong);
2152 assert_eq!(response.header.channel, 0);
2153 assert_eq!(response.header.corr, 77);
2154 assert!(response.body.is_empty());
2155 }
2156
2157 #[tokio::test]
2158 async fn slow_control_dispatch_logs_decoded_op_and_elapsed_time() {
2159 let control = Arc::new(
2160 ControlHandler::new(Arc::new(Registry::default()))
2161 .with_control_dispatch_delay(Duration::from_millis(1050)),
2162 );
2163 let router = Router::with_control_handler(control);
2164 let (ctx, mut rx) = route_ctx();
2165 let (output, guard) = log_capture(tracing::Level::WARN);
2166
2167 router
2168 .route_for_connection(&ctx, request(0, 41, br#"{"op":"server.describe"}"#))
2169 .await
2170 .expect("slow request routes");
2171 assert!(rx.recv().await.is_some(), "request receives a response");
2172 drop(guard);
2173
2174 let logs = captured_logs(&output);
2175 assert!(logs.contains("slow control dispatch"));
2176 assert!(logs.contains("op=server.describe"));
2177 assert!(logs.contains("connection_id=0"));
2178 assert!(logs.contains("corr=41"));
2179 assert!(
2180 logged_millis(&logs, "elapsed_ms=") >= 1050,
2181 "elapsed must include the injected handler delay: {logs}"
2182 );
2183 }
2184
2185 #[tokio::test]
2186 async fn fast_control_dispatch_emits_arrival_without_slow_warning() {
2187 let router = Router::with_default_self_handler();
2188 let (ctx, mut rx) = route_ctx();
2189 let (output, guard) = log_capture(tracing::Level::DEBUG);
2190
2191 router
2192 .route_for_connection(&ctx, request(0, 42, br#"{"op":"server.describe"}"#))
2193 .await
2194 .expect("fast request routes");
2195 assert!(rx.recv().await.is_some(), "request receives a response");
2196 drop(guard);
2197
2198 let logs = captured_logs(&output);
2199 assert!(logs.contains("control dispatch op=server.describe connection_id=0 corr=42"));
2200 assert!(!logs.contains("slow control dispatch"));
2201 }
2202
2203 #[tokio::test]
2204 async fn control_dispatch_arrival_is_hidden_at_info() {
2205 let router = Router::with_default_self_handler();
2206 let (ctx, mut rx) = route_ctx();
2207 let (output, guard) = log_capture(tracing::Level::INFO);
2208
2209 router
2210 .route_for_connection(&ctx, request(0, 43, br#"{"op":"server.describe"}"#))
2211 .await
2212 .expect("fast request routes");
2213 assert!(rx.recv().await.is_some(), "request receives a response");
2214 drop(guard);
2215
2216 assert!(
2217 !captured_logs(&output).contains("control dispatch"),
2218 "arrival logging must stay hidden at INFO"
2219 );
2220 }
2221
2222 #[tokio::test]
2223 async fn supervisor_list_logs_contended_snapshot_lock_only() {
2224 let registry = Arc::new(Registry::default());
2225 let handle = SupervisorHandle::new();
2226 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
2227 .with_handle(handle.clone());
2228 let module = supervisor
2229 .supervise_configured(
2230 ModuleSpec {
2231 module_id: "held-module".to_string(),
2232 program: "test-module".into(),
2233 args: Vec::new(),
2234 env: Vec::new(),
2235 reserved: false,
2236 reserved_prefixes: Vec::new(),
2237 protocol: ModuleProtocol::Subc,
2238 overlap: Default::default(),
2239 },
2240 false,
2241 )
2242 .expect("disabled test module is supervised");
2243 let router = Router::with_control_handler(Arc::new(
2244 ControlHandler::new(Arc::clone(®istry)).with_supervisor(handle),
2245 ));
2246 let (ctx, mut rx) = route_ctx();
2247 let (acquired, ready) = std_mpsc::channel();
2248 let holder = module.hold_snapshot_for_test(acquired, Duration::from_millis(400));
2249 ready.recv().expect("holder acquired snapshot lock");
2250 let (output, guard) = log_capture(tracing::Level::WARN);
2251
2252 router
2253 .route_for_connection(&ctx, request(0, 44, br#"{"op":"supervisor.list"}"#))
2254 .await
2255 .expect("list request routes after the lock releases");
2256 assert!(
2257 rx.recv().await.is_some(),
2258 "list request receives a response"
2259 );
2260 holder.join().expect("snapshot holder exits cleanly");
2261 drop(guard);
2262
2263 let logs = captured_logs(&output);
2264 assert!(logs.contains("slow snapshot lock"));
2265 assert!(logs.contains("module_id=held-module"));
2266 assert!(logs.contains("caller=list"));
2267 assert!(
2268 logged_millis(&logs, "waited_ms=") >= 250,
2269 "wait must exceed the slow-lock threshold: {logs}"
2270 );
2271
2272 let (output, guard) = log_capture(tracing::Level::WARN);
2273 router
2274 .route_for_connection(&ctx, request(0, 45, br#"{"op":"supervisor.list"}"#))
2275 .await
2276 .expect("uncontended list request routes");
2277 assert!(
2278 rx.recv().await.is_some(),
2279 "uncontended list receives a response"
2280 );
2281 drop(guard);
2282 assert!(
2283 !captured_logs(&output).contains("slow snapshot lock"),
2284 "uncontended list acquisition must not warn"
2285 );
2286 }
2287
2288 #[tokio::test]
2289 async fn full_module_to_client_sink_requests_client_close_without_erroring_module() {
2290 let forwarding = Arc::new(ForwardingTable::default());
2291 let control = Arc::new(ControlHandler::with_forwarding(
2292 Arc::new(crate::Registry::default()),
2293 Arc::clone(&forwarding),
2294 ));
2295 let router = Router::with_control_handler(control);
2296 let module_connection = ConnectionId::new(10);
2297 let client_connection = ConnectionId::new(20);
2298 let mut close_receiver = forwarding.register_connection_close(client_connection);
2299 let (module_tx, _module_rx) = mpsc::channel(1);
2300 forwarding
2301 .register_module_connection(
2302 module_connection,
2303 "full-sink-provider".to_string(),
2304 1,
2305 Concurrency::ModuleManaged,
2306 FrameSink::new(module_tx),
2307 )
2308 .unwrap();
2309 let (client_tx, mut client_rx) = mpsc::channel(1);
2310 let pending = forwarding
2311 .begin_route_bind_relay_for_test(
2312 client_connection,
2313 FrameSink::new(client_tx),
2314 700,
2315 "full-sink-provider",
2316 )
2317 .unwrap();
2318 forwarding
2319 .complete_pending_relay(
2320 module_connection,
2321 pending.corr,
2322 RouteBindRelayOutcome::Accepted,
2323 )
2324 .unwrap();
2325
2326 let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2327 let module_ctx = RouteCtx {
2328 connection_id: module_connection,
2329 egress: FrameSink::new(module_egress_tx),
2330 };
2331 let terminal = Frame::build(
2332 FrameType::Response,
2333 Flags::new(false, Priority::Interactive, true),
2334 pending.module_channel,
2335 pending.module_epoch,
2336 701,
2337 b"terminal".to_vec(),
2338 )
2339 .unwrap();
2340
2341 router
2342 .route_for_connection(&module_ctx, terminal)
2343 .await
2344 .unwrap();
2345 let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2346 .await
2347 .expect("close request should be sent for the full client sink")
2348 .expect("close sender should include a reason");
2349 assert!(
2350 reason
2351 .to_string()
2352 .contains("module_to_client_delivery_failed"),
2353 "unexpected close reason: {reason}"
2354 );
2355 assert_eq!(client_rx.try_recv().unwrap().header.corr, 700);
2356 assert!(client_rx.try_recv().is_err());
2357 assert_eq!(
2358 router.counters.snapshot()["client_egress_close_delivery_failed"],
2359 1
2360 );
2361 }
2362
2363 #[tokio::test]
2367 async fn terminal_frame_releases_its_credit_even_when_client_delivery_fails() {
2368 let forwarding = Arc::new(ForwardingTable::default());
2369 let control = Arc::new(ControlHandler::with_forwarding(
2370 Arc::new(crate::Registry::default()),
2371 Arc::clone(&forwarding),
2372 ));
2373 let router = Router::with_control_handler(control);
2374 let module_connection = ConnectionId::new(11);
2375 let client_connection = ConnectionId::new(21);
2376 let _close_receiver = forwarding.register_connection_close(client_connection);
2377 let (module_tx, _module_rx) = mpsc::channel(1);
2378 forwarding
2379 .register_module_connection(
2380 module_connection,
2381 "credit-provider".to_string(),
2382 1,
2383 Concurrency::ModuleManaged,
2384 FrameSink::new(module_tx),
2385 )
2386 .unwrap();
2387 let (client_tx, _client_rx) = mpsc::channel(1);
2390 let pending = forwarding
2391 .begin_route_bind_relay_for_test(
2392 client_connection,
2393 FrameSink::new(client_tx),
2394 800,
2395 "credit-provider",
2396 )
2397 .unwrap();
2398 forwarding
2399 .complete_pending_relay(
2400 module_connection,
2401 pending.corr,
2402 RouteBindRelayOutcome::Accepted,
2403 )
2404 .unwrap();
2405 let DataRoute::Client(DataRouteState::Bound(route)) = forwarding
2406 .lookup_data_route(
2407 client_connection,
2408 pending.client_channel,
2409 pending.client_epoch,
2410 )
2411 .unwrap()
2412 else {
2413 panic!("expected a bound client route");
2414 };
2415 route.flow.acquire_tagged(801, false).await.unwrap();
2416 assert_eq!(route.flow.drain_in_flight(), 1);
2417
2418 let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2419 let module_ctx = RouteCtx {
2420 connection_id: module_connection,
2421 egress: FrameSink::new(module_egress_tx),
2422 };
2423 let terminal = Frame::build(
2424 FrameType::Response,
2425 Flags::new(false, Priority::Interactive, true),
2426 pending.module_channel,
2427 pending.module_epoch,
2428 801,
2429 b"terminal".to_vec(),
2430 )
2431 .unwrap();
2432 router
2433 .route_for_connection(&module_ctx, terminal)
2434 .await
2435 .unwrap();
2436
2437 assert_eq!(
2438 router.counters.snapshot()["client_egress_close_delivery_failed"],
2439 1,
2440 "the client delivery must have failed for this test to mean anything"
2441 );
2442 assert_eq!(
2443 route.flow.drain_in_flight(),
2444 0,
2445 "the module's terminal frame must release its credit even though the client could not take it"
2446 );
2447 }
2448
2449 #[tokio::test]
2450 async fn full_route_goodbye_sink_requests_target_close_without_erroring_module() {
2451 let forwarding = Arc::new(ForwardingTable::default());
2452 let control = Arc::new(ControlHandler::with_forwarding(
2453 Arc::new(crate::Registry::default()),
2454 Arc::clone(&forwarding),
2455 ));
2456 let router = Router::with_control_handler(control);
2457 let module_connection = ConnectionId::new(30);
2458 let client_connection = ConnectionId::new(40);
2459 let mut close_receiver = forwarding.register_connection_close(client_connection);
2460 let (module_tx, _module_rx) = mpsc::channel(1);
2461 forwarding
2462 .register_module_connection(
2463 module_connection,
2464 "goodbye-full-provider".to_string(),
2465 1,
2466 Concurrency::ModuleManaged,
2467 FrameSink::new(module_tx),
2468 )
2469 .unwrap();
2470 let (client_tx, mut client_rx) = mpsc::channel(1);
2471 let pending = forwarding
2472 .begin_route_bind_relay_for_test(
2473 client_connection,
2474 FrameSink::new(client_tx),
2475 800,
2476 "goodbye-full-provider",
2477 )
2478 .unwrap();
2479 forwarding
2480 .complete_pending_relay(
2481 module_connection,
2482 pending.corr,
2483 RouteBindRelayOutcome::Accepted,
2484 )
2485 .unwrap();
2486
2487 let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2488 let module_ctx = RouteCtx {
2489 connection_id: module_connection,
2490 egress: FrameSink::new(module_egress_tx),
2491 };
2492 let goodbye = Frame::build(
2493 FrameType::Goodbye,
2494 Flags::new(false, Priority::Passive, true),
2495 pending.module_channel,
2496 pending.module_epoch,
2497 801,
2498 Vec::new(),
2499 )
2500 .unwrap();
2501
2502 router
2503 .route_for_connection(&module_ctx, goodbye)
2504 .await
2505 .unwrap();
2506 let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2507 .await
2508 .expect("close request should be sent for the full GOODBYE sink")
2509 .expect("close sender should include a reason");
2510 assert!(
2511 reason.to_string().contains("route_goodbye_delivery_failed"),
2512 "unexpected close reason: {reason}"
2513 );
2514 assert_eq!(client_rx.try_recv().unwrap().header.corr, 800);
2515 assert!(client_rx.try_recv().is_err());
2516 assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 1);
2517 assert_eq!(router.counters.snapshot()["route_released_epoch_fenced"], 1);
2518 }
2519
2520 async fn multi_route_client(
2525 module_id: &str,
2526 module_connection: ConnectionId,
2527 client_connection: ConnectionId,
2528 routes: usize,
2529 ) -> (
2530 Router,
2531 FrameSink,
2532 mpsc::Receiver<OutboundFrame>,
2533 RouteCtx,
2534 Vec<crate::forwarding::PendingRouteBindRelay>,
2535 ConnectionCloseReceiver,
2536 ) {
2537 let forwarding = Arc::new(ForwardingTable::default());
2538 let control = Arc::new(ControlHandler::with_forwarding(
2539 Arc::new(crate::Registry::default()),
2540 Arc::clone(&forwarding),
2541 ));
2542 let router = Router::with_control_handler(control);
2543 let close_receiver = forwarding.register_connection_close(client_connection);
2544 let (module_tx, _module_rx) = mpsc::channel(8);
2545 forwarding
2546 .register_module_connection(
2547 module_connection,
2548 module_id.to_string(),
2549 1,
2550 Concurrency::ModuleManaged,
2551 FrameSink::new(module_tx),
2552 )
2553 .unwrap();
2554 let (client_sink, mut client_rx) = crate::server::connection_egress();
2555 let mut bound = Vec::with_capacity(routes);
2556 for index in 0..routes {
2557 let pending = forwarding
2558 .begin_route_bind_relay_for_test(
2559 client_connection,
2560 client_sink.clone(),
2561 900 + index as u64,
2562 module_id,
2563 )
2564 .unwrap();
2565 forwarding
2566 .complete_pending_relay(
2567 module_connection,
2568 pending.corr,
2569 RouteBindRelayOutcome::Accepted,
2570 )
2571 .unwrap();
2572 assert_eq!(
2573 client_rx.recv().await.unwrap().header.corr,
2574 900 + index as u64
2575 );
2576 bound.push(pending);
2577 }
2578 let (module_egress_tx, _module_egress_rx) = mpsc::channel(8);
2579 let module_ctx = RouteCtx {
2580 connection_id: module_connection,
2581 egress: FrameSink::new(module_egress_tx),
2582 };
2583 (
2584 router,
2585 client_sink,
2586 client_rx,
2587 module_ctx,
2588 bound,
2589 close_receiver,
2590 )
2591 }
2592
2593 #[test]
2600 #[ignore = "timing measurement, run on demand"]
2601 fn egress_sink_per_frame_cost() {
2602 const FRAMES: usize = 1_000_000;
2603 const BATCH: usize = 1_000;
2604 let (sink, mut rx) = crate::server::connection_egress();
2605 let template = stream_frame(9, 1, 0, vec![b't'; 200]);
2606 let started = Instant::now();
2607 for _ in 0..FRAMES / BATCH {
2608 for _ in 0..BATCH {
2609 sink.try_send(template.clone()).unwrap();
2610 }
2611 for _ in 0..BATCH {
2612 let outbound = rx.try_recv().unwrap();
2613 if let Some(charge) = &outbound.charge {
2614 charge.taken_by_writer();
2615 }
2616 drop(std::hint::black_box(outbound));
2617 }
2618 }
2619 let elapsed = started.elapsed();
2620 assert_eq!(sink.backlog().queued_bytes, 0);
2621 println!(
2622 "egress sink: {FRAMES} frames in {elapsed:?}, {:.1} ns/frame",
2623 elapsed.as_nanos() as f64 / FRAMES as f64
2624 );
2625 }
2626
2627 fn stream_frame(channel: u16, epoch: u32, corr: u64, body: Vec<u8>) -> Frame {
2628 Frame::build(
2629 FrameType::StreamData,
2630 Flags::new(false, Priority::Interactive, false),
2631 channel,
2632 epoch,
2633 corr,
2634 body,
2635 )
2636 .unwrap()
2637 }
2638
2639 fn module_route_for_test(
2640 router: &Router,
2641 ctx: &RouteCtx,
2642 route: &crate::forwarding::PendingRouteBindRelay,
2643 ) -> Arc<RouteBinding> {
2644 match router
2645 .forwarding
2646 .lookup_data_route(ctx.connection_id, route.module_channel, route.module_epoch)
2647 .unwrap()
2648 {
2649 DataRoute::Module(DataRouteState::Bound(binding)) => binding,
2650 other => panic!("expected a bound module route, got {other:?}"),
2651 }
2652 }
2653
2654 fn assert_frame_size_error(frame: &Frame, channel: u16, epoch: u32, corr: u64, bytes: usize) {
2655 assert_eq!(frame.header.ty, FrameType::Error);
2656 assert_eq!(frame.header.channel, channel);
2657 assert_eq!(frame.header.epoch, epoch);
2658 assert_eq!(frame.header.corr, corr);
2659 assert!(frame.header.flags.is_daemon_origin());
2660 let body: ErrorBody = serde_json::from_slice(&frame.body).unwrap();
2661 assert_eq!(body.code, error_codes::FRAME_TOO_LARGE);
2662 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2663 assert!(body.message.contains(&bytes.to_string()));
2664 assert!(body.message.contains(&limit.to_string()));
2665 assert_eq!(
2666 body.detail,
2667 Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit }))
2668 );
2669 }
2670
2671 #[tokio::test]
2672 async fn module_egress_response_oversized_reply_keeps_connection_and_other_route() {
2673 let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
2674 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2675 "oversize-provider",
2676 ConnectionId::new(80),
2677 ConnectionId::new(81),
2678 2,
2679 )
2680 .await;
2681 let binding = module_route_for_test(&router, &ctx, &routes[0]);
2682 binding.flow.acquire_tagged(101, false).await.unwrap();
2683 module_route_for_test(&router, &ctx, &routes[1])
2684 .flow
2685 .acquire_tagged(102, false)
2686 .await
2687 .unwrap();
2688 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2689 let oversized = Frame::build(
2690 FrameType::Response,
2691 Flags::new(false, Priority::Interactive, true),
2692 routes[0].module_channel,
2693 routes[0].module_epoch,
2694 101,
2695 vec![b'x'; limit - subc_protocol::HEADER_LEN + 1],
2696 )
2697 .unwrap();
2698 assert_eq!(sink.backlog().queued_bytes, 0);
2699 router.route_for_connection(&ctx, oversized).await.unwrap();
2700
2701 router
2705 .route_for_connection(
2706 &ctx,
2707 route_frame(
2708 FrameType::Response,
2709 routes[1].module_channel,
2710 routes[1].module_epoch,
2711 102,
2712 ),
2713 )
2714 .await
2715 .unwrap();
2716 assert!(
2717 close.try_recv().is_err(),
2718 "an oversized reply must not close the multiplexed client"
2719 );
2720 let error = rx.try_recv().expect("the waiting client gets an ERROR");
2721 assert_frame_size_error(
2722 &error,
2723 routes[0].client_channel,
2724 routes[0].client_epoch,
2725 101,
2726 limit + 1,
2727 );
2728 let second = rx.try_recv().expect("the other route's reply arrives");
2729 assert_eq!(second.header.ty, FrameType::Response);
2730 assert_eq!(second.header.channel, routes[1].client_channel);
2731 assert_eq!(second.header.epoch, routes[1].client_epoch);
2732 assert_eq!(second.header.corr, 102);
2733 assert_eq!(second.body, b"route-body");
2734 assert_eq!(binding.flow.drain_in_flight(), 0);
2735 for route in &routes {
2736 module_route_for_test(&router, &ctx, route);
2737 }
2738 assert!(rx.try_recv().is_err());
2739 drop((error, second));
2740 assert_eq!(sink.backlog().queued_bytes, 0);
2741 assert_eq!(
2742 router.counters.snapshot()["client_egress_close_delivery_failed"],
2743 0
2744 );
2745
2746 let captured = test_log::captured_logs(&logs);
2747 let warnings: Vec<_> = captured
2748 .lines()
2749 .filter(|line| line.contains("WARN"))
2750 .collect();
2751 assert_eq!(warnings.len(), 1, "{captured}");
2752 let warning = warnings[0];
2753 for expected in [
2754 "module_id=oversize-provider".to_string(),
2755 "client_principal=direct".to_string(),
2756 format!("client_channel={}", routes[0].client_channel),
2757 format!("frame_bytes={}", limit + 1),
2758 format!("byte_limit={limit}"),
2759 ] {
2760 assert!(warning.contains(&expected), "missing {expected}: {warning}");
2761 }
2762 }
2763
2764 async fn poll_once<F: std::future::Future>(
2765 mut future: std::pin::Pin<&mut F>,
2766 ) -> std::task::Poll<F::Output> {
2767 std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
2768 }
2769
2770 fn fill_client_budget(sink: &FrameSink, route: &crate::forwarding::PendingRouteBindRelay) {
2771 sink.try_send(
2772 Frame::build(
2773 FrameType::Response,
2774 Flags::new(false, Priority::Interactive, true),
2775 route.client_channel,
2776 route.client_epoch,
2777 100,
2778 vec![
2779 b'x';
2780 crate::server::CONNECTION_EGRESS_BYTE_BUDGET - subc_protocol::HEADER_LEN
2781 ],
2782 )
2783 .unwrap(),
2784 )
2785 .unwrap();
2786 assert_eq!(
2787 sink.backlog().queued_bytes,
2788 crate::server::CONNECTION_EGRESS_BYTE_BUDGET
2789 );
2790 }
2791
2792 async fn assert_full_client_burst_survives(ty: FrameType, oversized: bool) {
2793 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2794 "draining-burst",
2795 ConnectionId::new(92),
2796 ConnectionId::new(93),
2797 2,
2798 )
2799 .await;
2800 fill_client_budget(&sink, &routes[0]);
2801 let frame = if oversized {
2802 Frame::build(
2803 ty,
2804 Flags::new(false, Priority::Interactive, false),
2805 routes[1].module_channel,
2806 routes[1].module_epoch,
2807 101,
2808 vec![b's'; crate::server::CONNECTION_EGRESS_BYTE_BUDGET],
2809 )
2810 .unwrap()
2811 } else {
2812 route_frame(ty, routes[1].module_channel, routes[1].module_epoch, 101)
2813 };
2814 let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2815 assert!(poll_once(delivery.as_mut()).await.is_pending());
2816 assert!(
2817 close.try_recv().is_err(),
2818 "fresh bursts must not close: {ty:?}"
2819 );
2820 tokio::time::advance(Duration::from_millis(50)).await;
2821 let queued = rx.try_recv().unwrap();
2822 queued.charge.as_ref().unwrap().taken_by_writer();
2823 drop(queued);
2824 assert!(matches!(
2825 poll_once(delivery.as_mut()).await,
2826 std::task::Poll::Ready(Ok(()))
2827 ));
2828 let delivered = rx
2829 .try_recv()
2830 .expect("the burst frame must be delivered after draining");
2831 assert_eq!(delivered.header.channel, routes[1].client_channel);
2832 assert_eq!(delivered.header.epoch, routes[1].client_epoch);
2833 assert_eq!(delivered.header.corr, 101);
2834 if oversized {
2835 assert_frame_size_error(
2836 &delivered,
2837 routes[1].client_channel,
2838 routes[1].client_epoch,
2839 101,
2840 crate::server::CONNECTION_EGRESS_BYTE_BUDGET + subc_protocol::HEADER_LEN,
2841 );
2842 } else {
2843 assert_eq!(delivered.header.ty, ty);
2844 }
2845 assert!(rx.try_recv().is_err());
2846 assert!(close.try_recv().is_err());
2847 assert_eq!(
2848 router.counters.snapshot()["client_egress_close_delivery_failed"],
2849 0
2850 );
2851 assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 0);
2852 }
2853
2854 #[tokio::test(start_paused = true)]
2855 async fn module_egress_fresh_full_queue_burst_survives() {
2856 for ty in [
2857 FrameType::Response,
2858 FrameType::StreamData,
2859 FrameType::Request,
2860 FrameType::Push,
2861 ] {
2862 assert_full_client_burst_survives(ty, false).await;
2863 }
2864 }
2865
2866 #[tokio::test(start_paused = true)]
2867 async fn module_goodbye_fresh_full_queue_burst_survives() {
2868 assert_full_client_burst_survives(FrameType::Goodbye, false).await;
2869 }
2870
2871 #[tokio::test(start_paused = true)]
2872 async fn frame_count_backstop_also_allows_a_draining_burst() {
2873 let (tx, mut rx) = mpsc::channel(1);
2874 let sink = FrameSink::with_byte_budget(tx, 2_000);
2875 sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
2876 .unwrap();
2877 let mut delivery = Box::pin(sink.send_to_client(stream_frame(1, 1, 2, vec![b'y'; 100])));
2878 assert!(poll_once(delivery.as_mut()).await.is_pending());
2879 assert!(
2880 sink.backlog().queued_bytes < 2_000,
2881 "only frame capacity is exhausted"
2882 );
2883 tokio::time::advance(Duration::from_millis(50)).await;
2884 drop(rx.try_recv().unwrap());
2885 assert!(matches!(
2886 poll_once(delivery.as_mut()).await,
2887 std::task::Poll::Ready(Ok(()))
2888 ));
2889 assert_eq!(rx.try_recv().unwrap().header.corr, 2);
2890 }
2891
2892 #[tokio::test(start_paused = true)]
2893 async fn oversized_reply_replacement_fresh_full_queue_burst_survives() {
2894 for ty in [FrameType::Response, FrameType::StreamData] {
2895 assert_full_client_burst_survives(ty, true).await;
2896 }
2897 }
2898
2899 #[tokio::test(start_paused = true)]
2900 async fn module_egress_stopped_reader_closes_without_waiting() {
2901 let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
2902 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2903 "oldest-stopped-reader",
2904 ConnectionId::new(94),
2905 ConnectionId::new(95),
2906 2,
2907 )
2908 .await;
2909 fill_client_budget(&sink, &routes[0]);
2910 let writing = rx.try_recv().unwrap();
2913 writing.charge.as_ref().unwrap().taken_by_writer();
2914 tokio::time::advance(Duration::from_millis(251)).await;
2915 assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(251)));
2916 assert_eq!(sink.backlog().queued_bytes, queued_frame_bytes(&writing));
2917 let started = tokio::time::Instant::now();
2918 let frame = stream_frame(
2919 routes[0].module_channel,
2920 routes[0].module_epoch,
2921 101,
2922 b"secret-body-never-log".to_vec(),
2923 );
2924 let flags = frame.header.flags.0;
2925 let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2926 assert!(
2927 matches!(
2928 poll_once(delivery.as_mut()).await,
2929 std::task::Poll::Ready(Ok(()))
2930 ),
2931 "an oldest frame older than 250 ms must close immediately, without a new wait"
2932 );
2933 assert_eq!(started.elapsed(), Duration::ZERO);
2934 assert!(close
2935 .try_recv()
2936 .unwrap()
2937 .to_string()
2938 .contains("module_to_client_delivery_failed"));
2939 let captured = test_log::captured_logs(&logs);
2940 for expected in [
2941 "frame_type=Some(StreamData)".to_string(),
2942 format!("frame_flags=Some({flags})"),
2943 "oldest_queued_ms=251".to_string(),
2944 "close_cause=\"oldest_frame_older_than_250_ms\"".to_string(),
2945 ] {
2946 assert!(
2947 captured.contains(&expected),
2948 "missing {expected}: {captured}"
2949 );
2950 }
2951 assert!(!captured.contains("secret-body-never-log"));
2952 }
2953
2954 #[tokio::test(start_paused = true)]
2955 async fn client_gone_mid_wait_resumes_module_reader_immediately() {
2956 for writer_gone in [false, true] {
2957 let (router, sink, mut rx, ctx, routes, _close) = multi_route_client(
2958 "gone-during-burst",
2959 ConnectionId::new(96),
2960 ConnectionId::new(97),
2961 1,
2962 )
2963 .await;
2964 fill_client_budget(&sink, &routes[0]);
2965 let frame = route_frame(
2966 FrameType::Response,
2967 routes[0].module_channel,
2968 routes[0].module_epoch,
2969 101,
2970 );
2971 let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2972 assert!(poll_once(delivery.as_mut()).await.is_pending());
2973 tokio::time::advance(Duration::from_millis(50)).await;
2974 let started = tokio::time::Instant::now();
2975 if writer_gone {
2976 rx.close();
2977 } else {
2978 sink.close_client_delivery();
2979 }
2980 assert!(
2981 matches!(
2982 poll_once(delivery.as_mut()).await,
2983 std::task::Poll::Ready(Ok(()))
2984 ),
2985 "client teardown must wake the module reader before the writer drains"
2986 );
2987 assert_eq!(started.elapsed(), Duration::ZERO);
2988 assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
2989 assert_eq!(rx.try_recv().unwrap().header.corr, 100);
2990 assert!(rx.try_recv().is_err());
2991 }
2992 }
2993
2994 #[tokio::test(start_paused = true)]
2995 async fn cancelled_module_delivery_releases_waiter_without_closing_client() {
2996 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2997 "cancelled-burst",
2998 ConnectionId::new(98),
2999 ConnectionId::new(99),
3000 1,
3001 )
3002 .await;
3003 fill_client_budget(&sink, &routes[0]);
3004 let frame = route_frame(
3005 FrameType::Response,
3006 routes[0].module_channel,
3007 routes[0].module_epoch,
3008 101,
3009 );
3010 let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3011 assert!(poll_once(delivery.as_mut()).await.is_pending());
3012 assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 1);
3013 drop(delivery);
3014 assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
3015 tokio::time::advance(Duration::from_secs(1)).await;
3016 assert!(close.try_recv().is_err());
3017 drop(rx.try_recv().unwrap());
3018 assert_eq!(sink.backlog().queued_bytes, 0);
3019 assert!(sink.backlog().oldest_age.is_none());
3020 }
3021
3022 #[tokio::test(start_paused = true)]
3023 async fn egress_atomic_age_follows_writer_and_clears_when_drained() {
3024 let (sink, mut rx) = crate::server::connection_egress();
3025 sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
3026 .unwrap();
3027 tokio::time::advance(Duration::from_millis(200)).await;
3028 sink.try_send(stream_frame(2, 1, 2, vec![b'y'; 100]))
3029 .unwrap();
3030 let writing = rx.try_recv().unwrap();
3031 writing.charge.as_ref().unwrap().taken_by_writer();
3032 assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3033 drop(writing);
3034 assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3037 tokio::time::advance(Duration::from_millis(50)).await;
3038 let next = rx.try_recv().unwrap();
3039 next.charge.as_ref().unwrap().taken_by_writer();
3040 assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(50)));
3041 assert!(
3042 sink.backlog().queued_bytes > 0,
3043 "in-flight bytes remain charged until written"
3044 );
3045 drop(next);
3046 assert_eq!(sink.backlog().queued_bytes, 0);
3047 assert_eq!(sink.backlog().oldest_age, None);
3048 }
3049
3050 #[tokio::test(start_paused = true)]
3051 async fn module_egress_response_full_client_waits_only_remaining_grace() {
3052 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3053 "backpressure-provider",
3054 ConnectionId::new(82),
3055 ConnectionId::new(83),
3056 2,
3057 )
3058 .await;
3059 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3060 router
3063 .route_for_connection(
3064 &ctx,
3065 Frame::build(
3066 FrameType::Response,
3067 Flags::new(false, Priority::Interactive, true),
3068 routes[0].module_channel,
3069 routes[0].module_epoch,
3070 201,
3071 vec![b'x'; limit - subc_protocol::HEADER_LEN],
3072 )
3073 .unwrap(),
3074 )
3075 .await
3076 .unwrap();
3077 assert_eq!(sink.backlog().queued_bytes, limit);
3078 assert_eq!(sink.backlog().queued_frames, 1);
3079 let ordinary = route_frame(
3080 FrameType::Response,
3081 routes[1].module_channel,
3082 routes[1].module_epoch,
3083 202,
3084 );
3085 assert!(queued_frame_bytes(&ordinary) < limit);
3086 tokio::time::advance(Duration::from_millis(100)).await;
3089 let mut delivery = Box::pin(router.route_for_connection(&ctx, ordinary));
3090 assert!(poll_once(delivery.as_mut()).await.is_pending());
3091 assert!(close.try_recv().is_err());
3092 tokio::time::advance(Duration::from_millis(149)).await;
3093 assert!(poll_once(delivery.as_mut()).await.is_pending());
3094 assert!(close.try_recv().is_err());
3095 tokio::time::advance(Duration::from_millis(1)).await;
3096 assert!(matches!(
3097 poll_once(delivery.as_mut()).await,
3098 std::task::Poll::Ready(Ok(()))
3099 ));
3100 drop(delivery);
3101 let reason = close.try_recv().expect("the full client must close");
3102 assert!(reason
3103 .to_string()
3104 .contains("module_to_client_delivery_failed"));
3105 assert!(!reason.to_string().contains("frame_too_large"));
3106 assert_eq!(
3107 router.counters.snapshot()["client_egress_close_delivery_failed"],
3108 1
3109 );
3110 let admitted = rx.try_recv().expect("the budget-sized frame was admitted");
3111 assert_eq!(admitted.header.ty, FrameType::Response);
3112 assert_eq!(admitted.header.corr, 201);
3113 assert!(
3114 rx.try_recv().is_err(),
3115 "no oversized-frame ERROR is substituted for backpressure"
3116 );
3117 }
3118
3119 #[tokio::test(start_paused = true)]
3120 async fn oversized_reply_to_a_full_client_waits_only_remaining_grace() {
3121 let (router, sink, _rx, ctx, routes, mut close) = multi_route_client(
3122 "full-oversize-provider",
3123 ConnectionId::new(90),
3124 ConnectionId::new(91),
3125 2,
3126 )
3127 .await;
3128 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3129 router
3131 .route_for_connection(
3132 &ctx,
3133 Frame::build(
3134 FrameType::Response,
3135 Flags::new(false, Priority::Interactive, true),
3136 routes[0].module_channel,
3137 routes[0].module_epoch,
3138 501,
3139 vec![b'x'; limit - subc_protocol::HEADER_LEN],
3140 )
3141 .unwrap(),
3142 )
3143 .await
3144 .unwrap();
3145 assert_eq!(sink.backlog().queued_bytes, limit);
3146 let oversized = Frame::build(
3147 FrameType::Response,
3148 Flags::new(false, Priority::Interactive, true),
3149 routes[1].module_channel,
3150 routes[1].module_epoch,
3151 502,
3152 vec![b'y'; limit - subc_protocol::HEADER_LEN + 1],
3153 )
3154 .unwrap();
3155 tokio::time::advance(Duration::from_millis(80)).await;
3158 let mut delivery = Box::pin(router.route_for_connection(&ctx, oversized));
3159 assert!(poll_once(delivery.as_mut()).await.is_pending());
3160 assert!(close.try_recv().is_err());
3161 tokio::time::advance(Duration::from_millis(169)).await;
3162 assert!(poll_once(delivery.as_mut()).await.is_pending());
3163 assert!(close.try_recv().is_err());
3164 tokio::time::advance(Duration::from_millis(1)).await;
3165 assert!(matches!(
3166 poll_once(delivery.as_mut()).await,
3167 std::task::Poll::Ready(Ok(()))
3168 ));
3169 drop(delivery);
3170 let reason = close.try_recv().expect("the full client must close");
3171 assert!(reason
3172 .to_string()
3173 .contains("module_to_client_delivery_failed"));
3174 }
3175
3176 #[tokio::test]
3177 async fn oversized_reverse_request_errors_module_without_reaching_client() {
3178 let (router, _sink, mut rx, mut ctx, routes, mut close) = multi_route_client(
3179 "reverse-oversize-provider",
3180 ConnectionId::new(84),
3181 ConnectionId::new(85),
3182 2,
3183 )
3184 .await;
3185 let (module_tx, mut module_rx) = mpsc::channel(8);
3186 ctx.egress = FrameSink::new(module_tx);
3187 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3188 router
3189 .route_for_connection(
3190 &ctx,
3191 Frame::build(
3192 FrameType::Request,
3193 Flags::new(false, Priority::Interactive, false),
3194 routes[0].module_channel,
3195 routes[0].module_epoch,
3196 301,
3197 vec![b'r'; limit - subc_protocol::HEADER_LEN + 1],
3198 )
3199 .unwrap(),
3200 )
3201 .await
3202 .unwrap();
3203 assert!(
3204 rx.try_recv().is_err(),
3205 "the reverse request never reaches the client"
3206 );
3207 let error = module_rx.try_recv().expect("the module gets the refusal");
3208 assert_frame_size_error(
3209 &error,
3210 routes[0].module_channel,
3211 routes[0].module_epoch,
3212 301,
3213 limit + 1,
3214 );
3215 assert!(module_rx.try_recv().is_err());
3216 for (index, route) in routes.iter().enumerate() {
3217 module_route_for_test(&router, &ctx, route);
3218 router
3219 .route_for_connection(
3220 &ctx,
3221 route_frame(
3222 FrameType::Response,
3223 route.module_channel,
3224 route.module_epoch,
3225 302 + index as u64,
3226 ),
3227 )
3228 .await
3229 .unwrap();
3230 let reply = rx.try_recv().expect("both routes still deliver");
3231 assert_eq!(reply.header.ty, FrameType::Response);
3232 assert_eq!(reply.header.channel, route.client_channel);
3233 }
3234 assert!(close.try_recv().is_err());
3235 assert_eq!(
3236 router.counters.snapshot()["client_egress_close_delivery_failed"],
3237 0
3238 );
3239 }
3240
3241 #[tokio::test]
3242 async fn oversized_push_closes_client_with_precise_warn() {
3243 let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3244 let (router, _sink, mut rx, ctx, routes, mut close) = multi_route_client(
3245 "push-oversize-provider",
3246 ConnectionId::new(86),
3247 ConnectionId::new(87),
3248 2,
3249 )
3250 .await;
3251 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3252 router
3253 .route_for_connection(
3254 &ctx,
3255 Frame::build(
3256 FrameType::Push,
3257 Flags::new(false, Priority::Interactive, false),
3258 routes[0].module_channel,
3259 routes[0].module_epoch,
3260 0,
3261 vec![b'p'; limit - subc_protocol::HEADER_LEN + 1],
3262 )
3263 .unwrap(),
3264 )
3265 .await
3266 .unwrap();
3267 let reason = close
3268 .try_recv()
3269 .expect("unsolicited oversize still closes the client");
3270 assert!(reason
3271 .to_string()
3272 .contains("frame_too_large: oversized push"));
3273 assert!(
3274 rx.try_recv().is_err(),
3275 "no part of the oversized push is forwarded"
3276 );
3277 let captured = test_log::captured_logs(&logs);
3278 let warnings: Vec<_> = captured
3279 .lines()
3280 .filter(|line| line.contains("WARN"))
3281 .collect();
3282 assert_eq!(warnings.len(), 1, "{captured}");
3283 let warning = warnings[0];
3284 for expected in [
3285 "oversized push".to_string(),
3286 "module_id=\"push-oversize-provider\"".to_string(),
3287 "principals=direct".to_string(),
3288 format!("client_channel={}", routes[0].client_channel),
3289 format!("size={}", limit + 1),
3290 format!("limit={limit}"),
3291 ] {
3292 assert!(warning.contains(&expected), "missing {expected}: {warning}");
3293 }
3294 assert_eq!(
3295 router.counters.snapshot()["client_egress_close_delivery_failed"],
3296 1
3297 );
3298 }
3299
3300 #[tokio::test]
3301 async fn oversized_stream_and_error_frames_end_request_even_with_queued_traffic() {
3302 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3303 "stream-oversize-provider",
3304 ConnectionId::new(88),
3305 ConnectionId::new(89),
3306 2,
3307 )
3308 .await;
3309 let binding = module_route_for_test(&router, &ctx, &routes[0]);
3310 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3311 for ty in [
3312 FrameType::StreamData,
3313 FrameType::StreamEnd,
3314 FrameType::Error,
3315 ] {
3316 binding.flow.acquire_tagged(401, false).await.unwrap();
3317 sink.try_send(route_frame(
3318 FrameType::Response,
3319 routes[1].client_channel,
3320 routes[1].client_epoch,
3321 402,
3322 ))
3323 .unwrap();
3324 router
3325 .route_for_connection(
3326 &ctx,
3327 Frame::build(
3328 ty,
3329 Flags::new(false, Priority::Interactive, false),
3330 routes[0].module_channel,
3331 routes[0].module_epoch,
3332 401,
3333 vec![b's'; limit - subc_protocol::HEADER_LEN + 1],
3334 )
3335 .unwrap(),
3336 )
3337 .await
3338 .unwrap();
3339 assert!(close.try_recv().is_err());
3340 assert_eq!(rx.try_recv().unwrap().header.corr, 402);
3341 let error = rx
3342 .try_recv()
3343 .expect("every correlated oversize frame becomes ERROR");
3344 assert_frame_size_error(
3345 &error,
3346 routes[0].client_channel,
3347 routes[0].client_epoch,
3348 401,
3349 limit + 1,
3350 );
3351 assert_eq!(binding.flow.drain_in_flight(), 0, "{ty:?} ends the request");
3352 assert!(rx.try_recv().is_err());
3353 }
3354 }
3355
3356 #[tokio::test]
3362 async fn awaited_send_parked_behind_a_full_byte_budget_wakes_when_bytes_free() {
3363 let (tx, mut rx) = mpsc::channel(1024);
3364 let sink = FrameSink::with_byte_budget(tx, 2_000);
3365 let mut queued = 0u64;
3366 while sink
3367 .try_send(stream_frame(7, 1, queued, vec![b'x'; 200]))
3368 .is_ok()
3369 {
3370 queued += 1;
3371 }
3372 assert!(queued > 0, "the budget admitted nothing");
3373
3374 let parked = tokio::spawn({
3375 let sink = sink.clone();
3376 async move { sink.send(stream_frame(0, 0, 999, vec![b'r'; 200])).await }
3377 });
3378 tokio::time::sleep(Duration::from_millis(100)).await;
3379 assert!(
3380 !parked.is_finished(),
3381 "the awaited send must wait while the byte budget is full"
3382 );
3383
3384 drop(rx.recv().await.expect("a queued frame"));
3386 tokio::time::timeout(Duration::from_secs(2), parked)
3387 .await
3388 .expect("the parked send was never woken after bytes were freed")
3389 .unwrap()
3390 .unwrap();
3391 }
3392
3393 #[tokio::test]
3398 async fn paused_client_reader_keeps_connection_through_small_frame_burst() {
3399 const ROUTES: usize = 4;
3400 const FRAMES_PER_ROUTE: usize = 250;
3401 let (router, client_sink, mut client_rx, module_ctx, routes, mut close_receiver) =
3402 multi_route_client(
3403 "burst-provider",
3404 ConnectionId::new(60),
3405 ConnectionId::new(61),
3406 ROUTES,
3407 )
3408 .await;
3409
3410 for seq in 0..FRAMES_PER_ROUTE as u64 {
3412 for (index, route) in routes.iter().enumerate() {
3413 let body = format!("route-{index}-token-{seq:05}-{}", "t".repeat(170));
3414 router
3415 .route_for_connection(
3416 &module_ctx,
3417 stream_frame(
3418 route.module_channel,
3419 route.module_epoch,
3420 seq,
3421 body.into_bytes(),
3422 ),
3423 )
3424 .await
3425 .unwrap();
3426 }
3427 }
3428
3429 let backlog = client_sink.backlog();
3430 assert_eq!(backlog.queued_frames, ROUTES * FRAMES_PER_ROUTE);
3431 assert!(backlog.queued_bytes < crate::server::CONNECTION_EGRESS_BYTE_BUDGET);
3432 assert!(
3433 close_receiver.try_recv().is_err(),
3434 "a paused reader under the byte budget must not be closed"
3435 );
3436 assert_eq!(
3437 router.counters.snapshot()["client_egress_close_delivery_failed"],
3438 0
3439 );
3440
3441 let mut next_seq = vec![0u64; ROUTES];
3443 for _ in 0..ROUTES * FRAMES_PER_ROUTE {
3444 let frame = client_rx.try_recv().expect("every queued frame arrives");
3445 let index = routes
3446 .iter()
3447 .position(|route| route.client_channel == frame.header.channel)
3448 .expect("frame arrives on one of the bound client channels");
3449 assert_eq!(frame.header.corr, next_seq[index], "route {index} order");
3450 let expected_prefix = format!("route-{index}-token-{:05}-", next_seq[index]);
3451 assert!(frame.body.starts_with(expected_prefix.as_bytes()));
3452 next_seq[index] += 1;
3453 }
3454 assert!(client_rx.try_recv().is_err());
3455 assert_eq!(next_seq, vec![FRAMES_PER_ROUTE as u64; ROUTES]);
3456 assert_eq!(client_sink.backlog().queued_bytes, 0);
3457 }
3458
3459 #[tokio::test(start_paused = true)]
3462 async fn never_reading_client_is_closed_at_byte_budget_with_warn_diagnosis() {
3463 let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3464 const BODY: usize = 16 * 1024;
3465 let (router, client_sink, _client_rx, module_ctx, routes, mut close_receiver) =
3466 multi_route_client(
3467 "stuck-reader-provider",
3468 ConnectionId::new(70),
3469 ConnectionId::new(71),
3470 2,
3471 )
3472 .await;
3473
3474 let mut admitted = 0usize;
3475 let mut sent = 0u64;
3476 let started = tokio::time::Instant::now();
3477 while router.counters.snapshot()["client_egress_close_delivery_failed"] == 0 {
3478 assert!(sent < 1_000, "the byte budget never refused a frame");
3479 let route = &routes[(sent % 2) as usize];
3480 router
3481 .route_for_connection(
3482 &module_ctx,
3483 stream_frame(
3484 route.module_channel,
3485 route.module_epoch,
3486 sent,
3487 vec![b'z'; BODY],
3488 ),
3489 )
3490 .await
3491 .unwrap();
3492 sent += 1;
3493 admitted = client_sink.backlog().queued_frames;
3494 }
3495 assert_eq!(
3496 started.elapsed(),
3497 crate::server::CONNECTION_EGRESS_STALL_GRACE
3498 );
3499 let frame_bytes = subc_protocol::HEADER_LEN + BODY;
3501 assert_eq!(
3502 admitted,
3503 crate::server::CONNECTION_EGRESS_BYTE_BUDGET / frame_bytes
3504 );
3505 let reason = close_receiver
3506 .try_recv()
3507 .expect("the client connection must be asked to close");
3508 assert!(reason
3509 .to_string()
3510 .contains("module_to_client_delivery_failed"));
3511
3512 router
3514 .route_for_connection(
3515 &module_ctx,
3516 stream_frame(
3517 routes[0].module_channel,
3518 routes[0].module_epoch,
3519 sent,
3520 vec![b'z'; BODY],
3521 ),
3522 )
3523 .await
3524 .unwrap();
3525
3526 let captured = test_log::captured_logs(&logs);
3527 let warn_lines = captured
3528 .lines()
3529 .filter(|line| {
3530 line.contains("closing client connection: its egress queue could not take a frame")
3531 })
3532 .collect::<Vec<_>>();
3533 assert_eq!(warn_lines.len(), 1, "exactly one WARN, got: {captured}");
3534 let line = warn_lines[0];
3535 assert!(line.contains("WARN"), "{line}");
3536 assert!(line.contains("connection_id=71"), "{line}");
3537 assert!(
3538 line.contains("module_id=\"stuck-reader-provider\""),
3539 "{line}"
3540 );
3541 assert!(line.contains("client_channel="), "{line}");
3542 assert!(line.contains("principals=direct"), "{line}");
3543 let queued_bytes: usize = line
3544 .split("queued_bytes=")
3545 .nth(1)
3546 .and_then(|rest| rest.split_whitespace().next())
3547 .and_then(|value| value.parse().ok())
3548 .expect("queued_bytes is logged");
3549 assert_eq!(queued_bytes, admitted * frame_bytes);
3550 assert!(
3551 line.contains(&format!("queued_frames={admitted}")),
3552 "{line}"
3553 );
3554 assert!(line.contains("oldest_queued_ms="), "{line}");
3555 assert!(line.contains("oldest_queued_ms=250"), "{line}");
3556 assert!(line.contains("frame_type=Some(StreamData)"), "{line}");
3557 assert!(
3558 line.contains(&format!(
3559 "frame_flags=Some({})",
3560 Flags::new(false, Priority::Interactive, false).0
3561 )),
3562 "{line}"
3563 );
3564 assert!(
3565 line.contains("close_cause=\"waited_250_ms_without_room\""),
3566 "{line}"
3567 );
3568 assert!(
3569 !line.contains(&"z".repeat(20)),
3570 "body bytes must never be logged: {line}"
3571 );
3572 }
3573
3574 fn route_frame(ty: FrameType, channel: u16, epoch: u32, corr: u64) -> Frame {
3575 Frame::build(
3576 ty,
3577 Flags::new(false, Priority::Interactive, false),
3578 channel,
3579 epoch,
3580 corr,
3581 if ty == FrameType::Request || ty == FrameType::Response {
3582 b"route-body".to_vec()
3583 } else {
3584 Vec::new()
3585 },
3586 )
3587 .unwrap()
3588 }
3589
3590 type DynamicRouteFixture = (
3591 Router,
3592 Arc<ForwardingTable>,
3593 RouteCtx,
3594 mpsc::Receiver<crate::router::OutboundFrame>,
3595 RouteCtx,
3596 mpsc::Receiver<crate::router::OutboundFrame>,
3597 mpsc::Receiver<crate::router::OutboundFrame>,
3598 crate::forwarding::PendingRouteBindRelay,
3599 );
3600
3601 fn dynamic_route_fixture(commit: bool) -> DynamicRouteFixture {
3602 let forwarding = Arc::new(ForwardingTable::default());
3603 let control = Arc::new(crate::ControlHandler::with_forwarding(
3604 Arc::new(crate::Registry::default()),
3605 Arc::clone(&forwarding),
3606 ));
3607 let router = Router::with_control_handler(control);
3608 let module_connection = ConnectionId::new(500);
3609 let client_connection = ConnectionId::new(501);
3610 let (module_tx, module_rx) = mpsc::channel(8);
3611 forwarding
3612 .register_module_connection(
3613 module_connection,
3614 "epoch-router".into(),
3615 2,
3616 Concurrency::ModuleManaged,
3617 FrameSink::new(module_tx),
3618 )
3619 .unwrap();
3620 let (client_tx, client_rx) = mpsc::channel(8);
3621 let client_sink = FrameSink::new(client_tx);
3622 let pending = forwarding
3623 .begin_route_bind_relay_for_test(
3624 client_connection,
3625 client_sink.clone(),
3626 700,
3627 "epoch-router",
3628 )
3629 .unwrap();
3630 if commit {
3631 forwarding
3632 .complete_pending_relay(
3633 module_connection,
3634 pending.corr,
3635 RouteBindRelayOutcome::Accepted,
3636 )
3637 .unwrap();
3638 }
3639 let (module_egress_tx, module_egress_rx) = mpsc::channel(8);
3640 (
3641 router,
3642 forwarding,
3643 RouteCtx {
3644 connection_id: client_connection,
3645 egress: client_sink,
3646 },
3647 client_rx,
3648 RouteCtx {
3649 connection_id: module_connection,
3650 egress: FrameSink::new(module_egress_tx),
3651 },
3652 module_egress_rx,
3653 module_rx,
3654 pending,
3655 )
3656 }
3657
3658 #[tokio::test]
3659 async fn route_epochs_validate_both_directions_and_rewrite_to_peer_handle() {
3660 let (
3661 router,
3662 _forwarding,
3663 client_ctx,
3664 mut client_rx,
3665 module_ctx,
3666 _module_egress_rx,
3667 mut module_rx,
3668 pending,
3669 ) = dynamic_route_fixture(true);
3670 let route_open = client_rx.recv().await.unwrap();
3671 assert_eq!(route_open.header.corr, 700);
3672
3673 router
3674 .route_for_connection(
3675 &client_ctx,
3676 route_frame(
3677 FrameType::Request,
3678 pending.client_channel,
3679 pending.client_epoch,
3680 701,
3681 ),
3682 )
3683 .await
3684 .unwrap();
3685 let forwarded = module_rx.recv().await.unwrap();
3686 assert_eq!(forwarded.header.channel, pending.module_channel);
3687 assert_eq!(forwarded.header.epoch, pending.module_epoch);
3688
3689 router
3690 .route_for_connection(
3691 &module_ctx,
3692 route_frame(
3693 FrameType::Response,
3694 pending.module_channel,
3695 pending.module_epoch,
3696 701,
3697 ),
3698 )
3699 .await
3700 .unwrap();
3701 let delivered = client_rx.recv().await.unwrap();
3702 assert_eq!(delivered.header.channel, pending.client_channel);
3703 assert_eq!(delivered.header.epoch, pending.client_epoch);
3704
3705 router
3706 .route_for_connection(
3707 &client_ctx,
3708 route_frame(
3709 FrameType::Request,
3710 pending.client_channel,
3711 pending.client_epoch + 1,
3712 702,
3713 ),
3714 )
3715 .await
3716 .unwrap();
3717 router
3718 .route_for_connection(
3719 &module_ctx,
3720 route_frame(
3721 FrameType::Response,
3722 pending.module_channel,
3723 pending.module_epoch + 1,
3724 703,
3725 ),
3726 )
3727 .await
3728 .unwrap();
3729 let stale_error = client_rx.recv().await.unwrap();
3730 assert_eq!(stale_error.header.ty, FrameType::Error);
3731 assert_eq!(stale_error.header.channel, pending.client_channel);
3732 assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
3733 assert_eq!(stale_error.header.corr, 702);
3734 let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
3735 assert_eq!(body.code, "stale_route_epoch");
3736 assert!(module_rx.try_recv().is_err());
3737 assert!(client_rx.try_recv().is_err());
3738 let counters = router.counters.snapshot();
3739 assert_eq!(counters["client_frames_dropped_stale_route"], 1);
3740 assert_eq!(counters["module_frames_dropped_no_route"], 1);
3741 }
3742
3743 #[tokio::test]
3744 async fn accepted_route_publishes_route_open_before_immediate_reverse_request() {
3745 let (
3746 router,
3747 _,
3748 _client_ctx,
3749 mut client_rx,
3750 module_ctx,
3751 _module_egress_rx,
3752 _module_rx,
3753 pending,
3754 ) = dynamic_route_fixture(true);
3755 router
3756 .route_for_connection(
3757 &module_ctx,
3758 route_frame(
3759 FrameType::Request,
3760 pending.module_channel,
3761 pending.module_epoch,
3762 800,
3763 ),
3764 )
3765 .await
3766 .unwrap();
3767
3768 let first = client_rx.recv().await.unwrap();
3769 let second = client_rx.recv().await.unwrap();
3770 assert_eq!(first.header.channel, 0);
3771 assert_eq!(first.header.corr, 700);
3772 assert_eq!(second.header.channel, pending.client_channel);
3773 assert_eq!(second.header.epoch, pending.client_epoch);
3774 assert_eq!(second.header.corr, 800);
3775 }
3776
3777 #[tokio::test]
3778 async fn reserved_slot_ingress_errors_only_matching_client_requests() {
3779 let (
3780 router,
3781 _forwarding,
3782 client_ctx,
3783 mut client_rx,
3784 _module_ctx,
3785 _module_egress_rx,
3786 mut module_rx,
3787 pending,
3788 ) = dynamic_route_fixture(false);
3789 router
3790 .route_for_connection(
3791 &client_ctx,
3792 route_frame(
3793 FrameType::Request,
3794 pending.client_channel,
3795 pending.client_epoch,
3796 900,
3797 ),
3798 )
3799 .await
3800 .unwrap();
3801 let error = client_rx.recv().await.unwrap();
3802 assert_eq!(error.header.ty, FrameType::Error);
3803 assert_eq!(error.header.channel, pending.client_channel);
3804 assert_eq!(error.header.epoch, pending.client_epoch);
3805 assert_eq!(error.header.corr, 900);
3806
3807 router
3808 .route_for_connection(
3809 &client_ctx,
3810 route_frame(
3811 FrameType::Response,
3812 pending.client_channel,
3813 pending.client_epoch,
3814 901,
3815 ),
3816 )
3817 .await
3818 .unwrap();
3819 router
3820 .route_for_connection(
3821 &client_ctx,
3822 route_frame(
3823 FrameType::Request,
3824 pending.client_channel,
3825 pending.client_epoch + 1,
3826 902,
3827 ),
3828 )
3829 .await
3830 .unwrap();
3831 let stale_error = client_rx.recv().await.unwrap();
3832 assert_eq!(stale_error.header.ty, FrameType::Error);
3833 assert_eq!(stale_error.header.channel, pending.client_channel);
3834 assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
3835 assert_eq!(stale_error.header.corr, 902);
3836 let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
3837 assert_eq!(body.code, "stale_route_epoch");
3838 assert!(module_rx.try_recv().is_err());
3839 let counters = router.counters.snapshot();
3840 assert_eq!(counters["client_frames_dropped_stale_route"], 1);
3841 assert_eq!(counters["module_frames_dropped_no_route"], 0);
3842 }
3843
3844 #[tokio::test]
3845 async fn dropped_module_route_goodbye_increments_counter() {
3846 let (
3847 router,
3848 _forwarding,
3849 client_ctx,
3850 mut client_rx,
3851 _module_ctx,
3852 _module_egress_rx,
3853 mut module_rx,
3854 pending,
3855 ) = dynamic_route_fixture(true);
3856 let _ = client_rx.recv().await;
3857 module_rx.close();
3858
3859 router
3860 .route_for_connection(
3861 &client_ctx,
3862 route_frame(
3863 FrameType::Goodbye,
3864 pending.client_channel,
3865 pending.client_epoch,
3866 999,
3867 ),
3868 )
3869 .await
3870 .unwrap();
3871
3872 let counters = router.counters.snapshot();
3873 assert_eq!(counters["goodbye_relay_module_dropped"], 1);
3874 assert_eq!(
3875 counters["goodbye_relay_module_dropped_by_module"],
3876 serde_json::json!({ "epoch-router": 1 })
3877 );
3878 assert_eq!(counters["route_released_epoch_fenced"], 1);
3879 }
3880
3881 #[tokio::test]
3882 async fn module_request_on_stale_epoch_receives_stale_route_epoch() {
3883 let (
3884 router,
3885 _forwarding,
3886 _client_ctx,
3887 _client_rx,
3888 module_ctx,
3889 mut module_egress_rx,
3890 mut module_rx,
3891 pending,
3892 ) = dynamic_route_fixture(true);
3893
3894 router
3895 .route_for_connection(
3896 &module_ctx,
3897 route_frame(
3898 FrameType::Request,
3899 pending.module_channel,
3900 pending.module_epoch + 1,
3901 1_000,
3902 ),
3903 )
3904 .await
3905 .unwrap();
3906
3907 let error = module_egress_rx.try_recv().unwrap();
3908 assert_eq!(error.header.ty, FrameType::Error);
3909 assert_eq!(error.header.channel, pending.module_channel);
3910 assert_eq!(error.header.epoch, pending.module_epoch + 1);
3911 assert_eq!(error.header.corr, 1_000);
3912 let body: ErrorBody = serde_json::from_slice(&error.body).unwrap();
3913 assert_eq!(body.code, "stale_route_epoch");
3914 assert!(module_rx.try_recv().is_err());
3915 let counters = router.counters.snapshot();
3916 assert_eq!(counters["module_requests_dropped_stale_route"], 1);
3917 assert_eq!(counters["module_frames_dropped_no_route"], 0);
3918 }
3919
3920 #[tokio::test(start_paused = true)]
3921 async fn stale_epoch_drop_warns_once_per_minute_with_suppressed_count() {
3922 let (output, _guard) = test_log::log_capture(tracing::Level::WARN);
3923 let (
3924 router,
3925 _forwarding,
3926 client_ctx,
3927 mut client_rx,
3928 _module_ctx,
3929 _module_egress_rx,
3930 mut module_rx,
3931 pending,
3932 ) = dynamic_route_fixture(true);
3933 let _ = client_rx.recv().await;
3934 for corr in 1..=3 {
3935 router
3936 .route_for_connection(
3937 &client_ctx,
3938 route_frame(
3939 FrameType::Request,
3940 pending.client_channel,
3941 pending.client_epoch + 1,
3942 corr,
3943 ),
3944 )
3945 .await
3946 .unwrap();
3947 assert_eq!(client_rx.recv().await.unwrap().header.ty, FrameType::Error);
3948 }
3949 assert!(module_rx.try_recv().is_err());
3950 let logs = test_log::captured_logs(&output);
3951 assert_eq!(
3952 logs.matches("dropping data-plane frame").count(),
3953 1,
3954 "{logs}"
3955 );
3956 assert!(logs.contains("WARN"), "{logs}");
3957 assert!(
3958 logs.contains(&format!("connection_id={}", client_ctx.connection_id.get())),
3959 "{logs}"
3960 );
3961 assert!(
3962 logs.contains(&format!("channel={}", pending.client_channel)),
3963 "{logs}"
3964 );
3965 assert!(
3966 logs.contains(&format!("epoch={}", pending.client_epoch + 1)),
3967 "{logs}"
3968 );
3969 assert!(
3970 logs.contains("client_to_module") && logs.contains("stale_epoch"),
3971 "{logs}"
3972 );
3973 assert!(logs.contains("suppressed=0"), "{logs}");
3974
3975 tokio::time::advance(Duration::from_secs(60)).await;
3976 router
3977 .route_for_connection(
3978 &client_ctx,
3979 route_frame(
3980 FrameType::Request,
3981 pending.client_channel,
3982 pending.client_epoch + 2,
3983 4,
3984 ),
3985 )
3986 .await
3987 .unwrap();
3988 let logs = test_log::captured_logs(&output);
3989 assert_eq!(
3990 logs.matches("dropping data-plane frame").count(),
3991 2,
3992 "{logs}"
3993 );
3994 assert!(logs.contains("suppressed=2"), "{logs}");
3995 }
3996
3997 #[tokio::test]
3998 async fn module_request_on_reserved_or_absent_route_receives_unknown_channel() {
3999 let (
4000 reserved_router,
4001 _forwarding,
4002 _client_ctx,
4003 _client_rx,
4004 reserved_module_ctx,
4005 mut reserved_module_egress_rx,
4006 _module_rx,
4007 reserved,
4008 ) = dynamic_route_fixture(false);
4009 reserved_router
4010 .route_for_connection(
4011 &reserved_module_ctx,
4012 route_frame(
4013 FrameType::Request,
4014 reserved.module_channel,
4015 reserved.module_epoch,
4016 1_001,
4017 ),
4018 )
4019 .await
4020 .unwrap();
4021 let reserved_error = reserved_module_egress_rx.try_recv().unwrap();
4022 let reserved_body: ErrorBody = serde_json::from_slice(&reserved_error.body).unwrap();
4023 assert_eq!(reserved_error.header.ty, FrameType::Error);
4024 assert_eq!(reserved_error.header.channel, reserved.module_channel);
4025 assert_eq!(reserved_error.header.epoch, reserved.module_epoch);
4026 assert_eq!(reserved_error.header.corr, 1_001);
4027 assert_eq!(reserved_body.code, "unknown_channel");
4028 assert_eq!(
4029 reserved_router.counters.snapshot()["module_requests_dropped_stale_route"],
4030 1
4031 );
4032
4033 let (
4034 absent_router,
4035 _forwarding,
4036 _client_ctx,
4037 _client_rx,
4038 absent_module_ctx,
4039 mut absent_module_egress_rx,
4040 _module_rx,
4041 absent,
4042 ) = dynamic_route_fixture(false);
4043 absent_router
4044 .route_for_connection(
4045 &absent_module_ctx,
4046 route_frame(
4047 FrameType::Request,
4048 absent.module_channel + 1,
4049 absent.module_epoch,
4050 1_002,
4051 ),
4052 )
4053 .await
4054 .unwrap();
4055 let absent_error = absent_module_egress_rx.try_recv().unwrap();
4056 let absent_body: ErrorBody = serde_json::from_slice(&absent_error.body).unwrap();
4057 assert_eq!(absent_error.header.ty, FrameType::Error);
4058 assert_eq!(absent_error.header.channel, absent.module_channel + 1);
4059 assert_eq!(absent_error.header.epoch, absent.module_epoch);
4060 assert_eq!(absent_error.header.corr, 1_002);
4061 assert_eq!(absent_body.code, "unknown_channel");
4062 assert_eq!(
4063 absent_router.counters.snapshot()["module_requests_dropped_stale_route"],
4064 1
4065 );
4066 }
4067
4068 #[tokio::test]
4069 async fn non_request_module_frame_on_dead_route_is_counted_without_error() {
4070 let (
4071 router,
4072 forwarding,
4073 client_ctx,
4074 mut client_rx,
4075 module_ctx,
4076 mut module_egress_rx,
4077 mut module_rx,
4078 pending,
4079 ) = dynamic_route_fixture(true);
4080 let (other_module_tx, _other_module_rx) = mpsc::channel(8);
4081 forwarding
4082 .register_module_connection(
4083 ConnectionId::new(502),
4084 "other-module".into(),
4085 2,
4086 Concurrency::ModuleManaged,
4087 FrameSink::new(other_module_tx),
4088 )
4089 .unwrap();
4090 let _ = client_rx.recv().await.unwrap();
4091
4092 router
4093 .route_for_connection(
4094 &client_ctx,
4095 route_frame(
4096 FrameType::Goodbye,
4097 pending.client_channel,
4098 pending.client_epoch,
4099 1_003,
4100 ),
4101 )
4102 .await
4103 .unwrap();
4104 let _ = module_rx.recv().await.unwrap();
4105
4106 router
4107 .route_for_connection(
4108 &module_ctx,
4109 route_frame(
4110 FrameType::StreamData,
4111 pending.module_channel,
4112 pending.module_epoch,
4113 1_004,
4114 ),
4115 )
4116 .await
4117 .unwrap();
4118
4119 let reply = module_egress_rx.try_recv().unwrap();
4122 assert_eq!(reply.header.ty, FrameType::Goodbye);
4123 assert!(module_egress_rx.try_recv().is_err());
4124 let counters = router.counters.snapshot();
4125 assert_eq!(counters["module_frames_dropped_no_route"], 1);
4126 assert_eq!(
4127 counters["module_frames_dropped_no_route_by_module"],
4128 serde_json::json!({ "epoch-router": 1 })
4129 );
4130 assert_eq!(counters["module_requests_dropped_stale_route"], 0);
4131 }
4132
4133 fn drain_now(rx: &mut mpsc::Receiver<OutboundFrame>) {
4136 while rx.try_recv().is_ok() {}
4137 }
4138
4139 #[tokio::test]
4145 async fn route_goodbye_refused_by_stalled_module_is_delivered_when_it_resumes_reading() {
4146 const BUDGET: usize = 4_096;
4147 let forwarding = Arc::new(ForwardingTable::default());
4148 let control = Arc::new(ControlHandler::with_forwarding(
4149 Arc::new(Registry::default()),
4150 Arc::clone(&forwarding),
4151 ));
4152 let router = Router::with_control_handler(control);
4153 let module_connection = ConnectionId::new(80);
4154 let client_connection = ConnectionId::new(81);
4155 let (module_tx, mut module_rx) = mpsc::channel(64);
4156 let module_sink = FrameSink::with_byte_budget(module_tx, BUDGET);
4157 forwarding
4158 .register_module_connection(
4159 module_connection,
4160 "stalled-provider".into(),
4161 2,
4162 Concurrency::ModuleManaged,
4163 module_sink.clone(),
4164 )
4165 .unwrap();
4166 let (client_tx, mut client_rx) = mpsc::channel(8);
4167 let client_sink = FrameSink::new(client_tx);
4168 let pending = forwarding
4169 .begin_route_bind_relay_for_test(
4170 client_connection,
4171 client_sink.clone(),
4172 1_100,
4173 "stalled-provider",
4174 )
4175 .unwrap();
4176 forwarding
4177 .complete_pending_relay(
4178 module_connection,
4179 pending.corr,
4180 RouteBindRelayOutcome::Accepted,
4181 )
4182 .unwrap();
4183 let _ = client_rx.recv().await.unwrap();
4184 drain_now(&mut module_rx);
4185
4186 module_sink
4189 .try_send(stream_frame(9, 1, 0, vec![b'f'; BUDGET]))
4190 .unwrap();
4191 assert!(module_sink
4192 .try_send(stream_frame(9, 1, 1, Vec::new()))
4193 .is_err());
4194
4195 let client_ctx = RouteCtx {
4196 connection_id: client_connection,
4197 egress: client_sink,
4198 };
4199 router
4200 .route_for_connection(
4201 &client_ctx,
4202 route_frame(
4203 FrameType::Goodbye,
4204 pending.client_channel,
4205 pending.client_epoch,
4206 1_101,
4207 ),
4208 )
4209 .await
4210 .unwrap();
4211 tokio::task::yield_now().await;
4212 assert_eq!(
4213 router.counters.snapshot()["goodbye_relay_module_dropped"],
4214 0,
4215 "a GOODBYE refused by a momentarily full module queue must not be dropped"
4216 );
4217
4218 let filler = module_rx.recv().await.unwrap();
4220 assert_eq!(filler.header.ty, FrameType::StreamData);
4221 drop(filler);
4222 let goodbye = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
4223 .await
4224 .expect("the refused GOODBYE must be delivered once the module frees room")
4225 .unwrap();
4226 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
4227 assert_eq!(goodbye.header.channel, pending.module_channel);
4228 assert_eq!(goodbye.header.epoch, pending.module_epoch);
4229 assert_eq!(
4230 router.counters.snapshot()["goodbye_relay_module_dropped"],
4231 0
4232 );
4233 }
4234
4235 #[tokio::test]
4241 async fn module_frame_on_route_the_daemon_does_not_hold_is_answered_with_goodbye() {
4242 let (
4243 router,
4244 _forwarding,
4245 client_ctx,
4246 mut client_rx,
4247 module_ctx,
4248 mut module_egress_rx,
4249 mut module_rx,
4250 pending,
4251 ) = dynamic_route_fixture(true);
4252 let _ = client_rx.recv().await.unwrap();
4253 router
4254 .route_for_connection(
4255 &client_ctx,
4256 route_frame(
4257 FrameType::Goodbye,
4258 pending.client_channel,
4259 pending.client_epoch,
4260 1_200,
4261 ),
4262 )
4263 .await
4264 .unwrap();
4265 drain_now(&mut module_rx);
4266
4267 let cases = [
4268 (pending.module_channel, pending.module_epoch),
4270 (pending.module_channel + 1, 1),
4272 ];
4273 for (channel, epoch) in cases {
4274 router
4275 .route_for_connection(
4276 &module_ctx,
4277 route_frame(FrameType::StreamData, channel, epoch, 1_201),
4278 )
4279 .await
4280 .unwrap();
4281 let reply = module_egress_rx
4282 .try_recv()
4283 .expect("a frame on a route the daemon does not hold is answered");
4284 assert_eq!(reply.header.ty, FrameType::Goodbye);
4285 assert_eq!(reply.header.channel, channel);
4286 assert_eq!(reply.header.epoch, epoch);
4287 assert_eq!(reply.header.corr, 0);
4288 assert!(module_egress_rx.try_recv().is_err());
4289 }
4290
4291 router
4294 .route_for_connection(
4295 &module_ctx,
4296 route_frame(FrameType::Goodbye, pending.module_channel + 2, 1, 0),
4297 )
4298 .await
4299 .unwrap();
4300 assert!(module_egress_rx.try_recv().is_err());
4301
4302 let counters = router.counters.snapshot();
4303 assert_eq!(counters["module_frames_dropped_no_route"], 3);
4304 assert_eq!(counters["module_frames_dropped_released_route"], 1);
4305 assert_eq!(
4306 counters["module_frames_dropped_released_route_by_module"],
4307 serde_json::json!({ "epoch-router": 1 })
4308 );
4309 assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4310 }
4311
4312 #[tokio::test(start_paused = true)]
4316 async fn burst_of_orphan_module_frames_is_answered_once_per_interval() {
4317 let (
4318 router,
4319 _forwarding,
4320 client_ctx,
4321 mut client_rx,
4322 module_ctx,
4323 mut module_egress_rx,
4324 mut module_rx,
4325 pending,
4326 ) = dynamic_route_fixture(true);
4327 let _ = client_rx.recv().await.unwrap();
4328 router
4329 .route_for_connection(
4330 &client_ctx,
4331 route_frame(
4332 FrameType::Goodbye,
4333 pending.client_channel,
4334 pending.client_epoch,
4335 1_300,
4336 ),
4337 )
4338 .await
4339 .unwrap();
4340 drain_now(&mut module_rx);
4341
4342 let send_burst = |corr: u64| {
4343 route_frame(
4344 FrameType::StreamData,
4345 pending.module_channel,
4346 pending.module_epoch,
4347 corr,
4348 )
4349 };
4350 for corr in 0..20 {
4351 router
4352 .route_for_connection(&module_ctx, send_burst(corr))
4353 .await
4354 .unwrap();
4355 }
4356 let mut replies = 0;
4357 while let Ok(reply) = module_egress_rx.try_recv() {
4358 assert_eq!(reply.header.ty, FrameType::Goodbye);
4359 replies += 1;
4360 }
4361 assert_eq!(replies, 1, "a burst within the interval gets one GOODBYE");
4362
4363 tokio::time::advance(ORPHAN_ROUTE_GOODBYE_INTERVAL).await;
4364 router
4365 .route_for_connection(&module_ctx, send_burst(20))
4366 .await
4367 .unwrap();
4368 let retry = module_egress_rx
4369 .try_recv()
4370 .expect("the first orphan frame after the interval is answered again");
4371 assert_eq!(retry.header.channel, pending.module_channel);
4372 assert_eq!(retry.header.epoch, pending.module_epoch);
4373
4374 let counters = router.counters.snapshot();
4375 assert_eq!(counters["module_frames_dropped_no_route"], 21);
4376 assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4377 }
4378
4379 #[test]
4381 fn orphan_goodbye_rate_limit_state_is_released_with_the_connection() {
4382 let router = Router::with_default_self_handler();
4383 let connection = router.begin_connection();
4384 let id = connection.id();
4385 assert!(router.orphan_goodbyes.claim(id, 7));
4386 assert!(!router.orphan_goodbyes.claim(id, 7));
4387 assert!(router.orphan_goodbyes.claim(id, 8));
4388 drop(connection);
4389 assert!(router
4390 .orphan_goodbyes
4391 .last_sent
4392 .lock()
4393 .unwrap()
4394 .get(&id)
4395 .is_none());
4396 }
4397
4398 #[test]
4399 fn channel_zero_cannot_be_registered_as_backend() {
4400 let mut router = Router::with_default_self_handler();
4401
4402 let err = router.register_backend(0, EchoBackend).unwrap_err();
4403
4404 assert_eq!(err, RouterError::ReservedChannelZero);
4405 }
4406}