1use std::{
2 collections::HashMap,
3 error::Error,
4 fmt,
5 sync::{
6 atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
7 Arc, Mutex,
8 },
9 time::{Duration, Instant},
10};
11
12use subc_protocol::{error_codes, ErrorBody, Flags, FrameType, Priority};
13use tokio::sync::{mpsc, Notify};
14use tracing::{debug, warn};
15
16use crate::{
17 control::ControlHandler,
18 forwarding::{
19 principal_label, CloseReason, ConnectionCloseReceiver, DataRoute, DataRouteState,
20 ForwardingError, ForwardingTable, RouteBinding, RouteRelease, UndeliveredFrame,
21 },
22 registry::ConnectionId,
23 DaemonCounters, Frame, FrameBuildError,
24};
25
26#[derive(Debug)]
35pub struct OutboundFrame {
36 pub frame: Frame,
37 pub enqueued_at: std::time::Instant,
38 pub(crate) flushed: Option<tokio::sync::oneshot::Sender<()>>,
39 pub(crate) charge: Option<EgressCharge>,
44}
45
46impl OutboundFrame {
47 fn charged(frame: Frame, charge: EgressCharge) -> Self {
48 Self {
49 frame,
50 enqueued_at: charge.enqueued_at,
51 flushed: None,
52 charge: Some(charge),
53 }
54 }
55}
56
57fn queued_frame_bytes(frame: &Frame) -> usize {
60 subc_protocol::HEADER_LEN + frame.body.len()
61}
62
63#[derive(Debug, Clone, Copy, PartialEq, Eq)]
66pub(crate) struct EgressBacklog {
67 pub queued_bytes: usize,
68 pub queued_frames: usize,
69 pub oldest_age: Option<Duration>,
76}
77
78#[derive(Debug)]
83struct EgressAccounting {
84 byte_budget: usize,
85 queued_bytes: AtomicUsize,
86 queued_frames: AtomicUsize,
87 time_base: Instant,
89 oldest_enqueued_nanos: AtomicU64,
92 waiters: AtomicUsize,
95 freed: Notify,
98 client_delivery_closed: AtomicBool,
99 client_delivery_stopped: Notify,
100}
101
102impl EgressAccounting {
103 fn new(byte_budget: usize) -> Self {
104 Self {
105 byte_budget,
106 queued_bytes: AtomicUsize::new(0),
107 queued_frames: AtomicUsize::new(0),
108 time_base: tokio::time::Instant::now().into_std(),
109 oldest_enqueued_nanos: AtomicU64::new(0),
110 waiters: AtomicUsize::new(0),
111 freed: Notify::new(),
112 client_delivery_closed: AtomicBool::new(false),
113 client_delivery_stopped: Notify::new(),
114 }
115 }
116
117 fn stamp(&self, at: Instant) -> u64 {
118 (at.saturating_duration_since(self.time_base).as_nanos() as u64).saturating_add(1)
119 }
120
121 fn try_charge(self: &Arc<Self>, bytes: usize) -> Option<EgressCharge> {
126 let mut current = self.queued_bytes.load(Ordering::SeqCst);
129 loop {
130 if current != 0 && current.saturating_add(bytes) > self.byte_budget {
131 return None;
132 }
133 match self.queued_bytes.compare_exchange_weak(
134 current,
135 current + bytes,
136 Ordering::SeqCst,
137 Ordering::SeqCst,
138 ) {
139 Ok(_) => return Some(self.record(bytes)),
140 Err(actual) => current = actual,
141 }
142 }
143 }
144
145 fn charge_unconditionally(self: &Arc<Self>, bytes: usize) -> EgressCharge {
148 self.queued_bytes.fetch_add(bytes, Ordering::SeqCst);
149 self.record(bytes)
150 }
151
152 fn record(self: &Arc<Self>, bytes: usize) -> EgressCharge {
153 let enqueued_at = tokio::time::Instant::now().into_std();
154 let stamp = self.stamp(enqueued_at);
155 if self.queued_frames.fetch_add(1, Ordering::AcqRel) == 0 {
156 self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
159 }
160 EgressCharge {
161 accounting: Arc::clone(self),
162 bytes,
163 stamp,
164 enqueued_at,
165 }
166 }
167
168 fn taken_by_writer(&self, stamp: u64) {
170 self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
171 }
172
173 fn release(&self, bytes: usize, stamp: u64) {
174 self.queued_bytes.fetch_sub(bytes, Ordering::SeqCst);
175 if self.queued_frames.fetch_sub(1, Ordering::AcqRel) == 1 {
176 let _ = self.oldest_enqueued_nanos.compare_exchange(
179 stamp,
180 0,
181 Ordering::AcqRel,
182 Ordering::Acquire,
183 );
184 }
185 if self.waiters.load(Ordering::SeqCst) != 0 {
186 self.freed.notify_waiters();
187 }
188 }
189
190 fn backlog(&self) -> EgressBacklog {
191 let queued_frames = self.queued_frames.load(Ordering::Acquire);
192 let stamp = self.oldest_enqueued_nanos.load(Ordering::Acquire);
193 let oldest_age = (queued_frames != 0 && stamp != 0).then(|| {
194 let enqueued = self.time_base + Duration::from_nanos(stamp - 1);
195 tokio::time::Instant::from_std(enqueued).elapsed()
196 });
197 EgressBacklog {
198 queued_bytes: self.queued_bytes.load(Ordering::Acquire),
199 queued_frames,
200 oldest_age,
201 }
202 }
203}
204
205#[derive(Debug)]
208pub(crate) struct EgressCharge {
209 accounting: Arc<EgressAccounting>,
210 bytes: usize,
211 stamp: u64,
212 enqueued_at: Instant,
213}
214
215impl EgressCharge {
216 pub(crate) fn taken_by_writer(&self) {
219 self.accounting.taken_by_writer(self.stamp);
220 }
221}
222
223impl Drop for EgressCharge {
224 fn drop(&mut self) {
225 self.accounting.release(self.bytes, self.stamp);
226 }
227}
228
229#[derive(Debug)]
235pub(crate) struct EgressPermit {
236 permit: mpsc::OwnedPermit<OutboundFrame>,
237 accounting: Arc<EgressAccounting>,
238}
239
240impl EgressPermit {
241 pub(crate) fn send(self, frame: Frame) -> bool {
244 let charge = self
245 .accounting
246 .charge_unconditionally(queued_frame_bytes(&frame));
247 let sender = self.permit.send(OutboundFrame::charged(frame, charge));
248 sender.is_closed()
249 }
250}
251
252impl std::ops::Deref for OutboundFrame {
255 type Target = Frame;
256
257 fn deref(&self) -> &Frame {
258 &self.frame
259 }
260}
261
262#[cfg(test)]
265pub(crate) mod test_log {
266 use std::{
267 io::Write,
268 sync::{Arc, Mutex, OnceLock},
269 };
270
271 #[derive(Clone)]
272 struct TestLogWriter(Arc<Mutex<Vec<u8>>>);
273
274 impl Write for TestLogWriter {
275 fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
276 self.0
277 .lock()
278 .expect("test log capture is not poisoned")
279 .extend(buffer);
280 Ok(buffer.len())
281 }
282
283 fn flush(&mut self) -> std::io::Result<()> {
284 Ok(())
285 }
286 }
287
288 pub(crate) fn log_capture(
289 level: tracing::Level,
290 ) -> (Arc<Mutex<Vec<u8>>>, tracing::dispatcher::DefaultGuard) {
291 static INTEREST_ANCHOR: OnceLock<tracing::Dispatch> = OnceLock::new();
298 INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
299 let output = Arc::new(Mutex::new(Vec::new()));
300 let writer = Arc::clone(&output);
301 let subscriber = tracing_subscriber::fmt()
302 .with_max_level(level)
303 .with_ansi(false)
304 .without_time()
305 .with_target(false)
306 .with_writer(move || TestLogWriter(Arc::clone(&writer)))
307 .finish();
308 let guard = tracing::subscriber::set_default(subscriber);
309 (output, guard)
310 }
311
312 pub(crate) fn captured_logs(output: &Arc<Mutex<Vec<u8>>>) -> String {
313 String::from_utf8(
314 output
315 .lock()
316 .expect("test log capture is not poisoned")
317 .clone(),
318 )
319 .expect("tracing output is UTF-8")
320 }
321
322 #[cfg(test)]
323 mod tests {
324 use super::*;
325
326 fn emit_capture_probe(source: &str) {
327 tracing::error!(source, "parallel capture probe");
328 }
329
330 #[test]
331 fn parallel_callsite_registration_keeps_the_test_capture_enabled() {
332 let (output, _guard) = log_capture(tracing::Level::ERROR);
333 std::thread::spawn(|| emit_capture_probe("other-thread"))
336 .join()
337 .expect("unsubscribed thread exits");
338 emit_capture_probe("captured-thread");
339
340 let logs = captured_logs(&output);
341 assert!(
342 logs.contains("captured-thread"),
343 "capture was disabled: {logs}"
344 );
345 assert!(!logs.contains("other-thread"), "capture leaked: {logs}");
346 }
347 }
348}
349
350#[derive(Debug, Clone)]
358pub struct FrameSink {
359 tx: mpsc::Sender<OutboundFrame>,
360 accounting: Arc<EgressAccounting>,
361}
362
363impl FrameSink {
364 pub fn new(tx: mpsc::Sender<OutboundFrame>) -> Self {
368 Self::with_byte_budget(tx, crate::server::CONNECTION_EGRESS_BYTE_BUDGET)
369 }
370
371 pub(crate) fn with_byte_budget(tx: mpsc::Sender<OutboundFrame>, byte_budget: usize) -> Self {
372 Self {
373 tx,
374 accounting: Arc::new(EgressAccounting::new(byte_budget)),
375 }
376 }
377
378 async fn charge_waiting(&self, bytes: usize) -> Option<EgressCharge> {
385 if let Some(charge) = self.accounting.try_charge(bytes) {
386 return Some(charge);
387 }
388 struct Waiting<'a>(&'a AtomicUsize);
391 impl Drop for Waiting<'_> {
392 fn drop(&mut self) {
393 self.0.fetch_sub(1, Ordering::SeqCst);
394 }
395 }
396 self.accounting.waiters.fetch_add(1, Ordering::SeqCst);
397 let _waiting = Waiting(&self.accounting.waiters);
398 loop {
399 let freed = self.accounting.freed.notified();
400 tokio::pin!(freed);
401 freed.as_mut().enable();
404 if let Some(charge) = self.accounting.try_charge(bytes) {
405 return Some(charge);
406 }
407 tokio::select! {
408 _ = &mut freed => {}
409 _ = self.tx.closed() => return None,
410 }
411 }
412 }
413
414 pub async fn send(&self, frame: Frame) -> Result<(), RouterError> {
415 let channel = frame.header.channel;
416 let epoch = frame.header.epoch;
417 let corr = frame.header.corr;
418 let closed =
419 || RouterError::backend_with_epoch(channel, epoch, corr, "connection writer closed");
420 let charge = self
421 .charge_waiting(queued_frame_bytes(&frame))
422 .await
423 .ok_or_else(closed)?;
424 self.tx
425 .send(OutboundFrame::charged(frame, charge))
426 .await
427 .map_err(|_| closed())
428 }
429
430 #[cfg(unix)]
437 pub async fn send_flushed(&self, frame: Frame) -> Result<(), RouterError> {
438 let (tx, rx) = tokio::sync::oneshot::channel();
439 let charge = self
440 .charge_waiting(queued_frame_bytes(&frame))
441 .await
442 .ok_or_else(|| RouterError::backend(0, 0, "connection writer closed"))?;
443 let mut outbound = OutboundFrame::charged(frame, charge);
444 outbound.flushed = Some(tx);
445 self.tx
446 .send(outbound)
447 .await
448 .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
449 rx.await
450 .map_err(|_| RouterError::backend(0, 0, "connection flush failed"))
451 }
452
453 pub(crate) async fn reserve_owned(&self) -> Result<EgressPermit, RouterError> {
454 let permit = self
455 .tx
456 .clone()
457 .reserve_owned()
458 .await
459 .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
460 Ok(EgressPermit {
461 permit,
462 accounting: Arc::clone(&self.accounting),
463 })
464 }
465
466 #[cfg(test)]
467 pub(crate) fn try_reserve_owned(&self) -> Result<EgressPermit, RouterError> {
468 let permit = self
469 .tx
470 .clone()
471 .try_reserve_owned()
472 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
473 Ok(EgressPermit {
474 permit,
475 accounting: Arc::clone(&self.accounting),
476 })
477 }
478
479 pub(crate) fn is_closed(&self) -> bool {
480 self.tx.is_closed()
481 }
482
483 pub(crate) fn backlog(&self) -> EgressBacklog {
485 self.accounting.backlog()
486 }
487
488 pub(crate) fn close_client_delivery(&self) {
491 self.accounting
492 .client_delivery_closed
493 .store(true, Ordering::Release);
494 self.accounting.client_delivery_stopped.notify_waiters();
495 }
496
497 async fn send_to_client(&self, frame: Frame) -> Result<(), ClientDeliveryFailure> {
500 let frame_type = frame.header.ty;
501 let flags = frame.header.flags;
502 let stopped = self.accounting.client_delivery_stopped.notified();
503 tokio::pin!(stopped);
504 stopped.as_mut().enable();
505 let failure = |cause, error| ClientDeliveryFailure {
506 diagnosis: ClientDeliveryDiagnosis {
507 frame_type,
508 flags,
509 cause,
510 },
511 error,
512 };
513 let closed = || {
514 RouterError::backend_with_epoch(
515 frame.header.channel,
516 frame.header.epoch,
517 frame.header.corr,
518 "client connection closed",
519 )
520 };
521 if self
522 .accounting
523 .client_delivery_closed
524 .load(Ordering::Acquire)
525 {
526 return Err(failure("client_closed", closed()));
527 }
528 let (frame, error) = match self.try_send_owned(frame) {
529 Ok(()) => return Ok(()),
530 Err(refused) => refused,
531 };
532 if self.is_closed() {
533 return Err(failure("client_closed", error));
534 }
535 let grace = crate::server::CONNECTION_EGRESS_STALL_GRACE;
536 let age = self.backlog().oldest_age.unwrap_or_default();
537 if age > grace {
538 return Err(failure("oldest_frame_older_than_250_ms", error));
539 }
540 let result = tokio::select! {
541 biased;
542 _ = &mut stopped => return Err(failure("client_closed", error)),
543 result = tokio::time::timeout(grace.saturating_sub(age), self.send(frame)) => result,
544 };
545 match result {
546 Ok(Ok(())) => Ok(()),
547 Ok(Err(error)) => Err(failure("client_closed", error)),
548 Err(_) => Err(failure("waited_250_ms_without_room", error)),
549 }
550 }
551
552 pub(crate) fn try_send(&self, frame: Frame) -> Result<(), RouterError> {
556 self.try_send_owned(frame).map_err(|(_, error)| error)
557 }
558
559 fn try_send_owned(&self, frame: Frame) -> Result<(), (Frame, RouterError)> {
560 let channel = frame.header.channel;
561 let epoch = frame.header.epoch;
562 let corr = frame.header.corr;
563 let unavailable = |why: String| {
564 RouterError::backend_with_epoch(
565 channel,
566 epoch,
567 corr,
568 format!("connection writer unavailable: {why}"),
569 )
570 };
571 let bytes = queued_frame_bytes(&frame);
572 let Some(charge) = self.accounting.try_charge(bytes) else {
573 if self.tx.is_closed() {
574 return Err((frame, unavailable("channel closed".to_string())));
575 }
576 return Err((
577 frame,
578 unavailable(format!(
579 "egress byte budget exhausted ({} queued bytes, frame of {bytes} bytes, budget {})",
580 self.accounting.queued_bytes.load(Ordering::Acquire),
581 self.accounting.byte_budget
582 )),
583 ));
584 };
585 self.tx
588 .try_send(OutboundFrame::charged(frame, charge))
589 .map_err(|err| {
590 let error = unavailable(err.to_string());
591 (err.into_inner().frame, error)
592 })
593 }
594}
595
596#[derive(Debug)]
597struct ClientDeliveryFailure {
598 diagnosis: ClientDeliveryDiagnosis,
599 error: RouterError,
600}
601
602#[derive(Debug)]
604pub(crate) struct ClientDeliveryDiagnosis {
605 pub frame_type: FrameType,
606 pub flags: Flags,
607 pub cause: &'static str,
608}
609
610impl fmt::Display for ClientDeliveryFailure {
611 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
612 self.error.fmt(f)
613 }
614}
615
616const ORPHAN_ROUTE_GOODBYE_INTERVAL: Duration = Duration::from_secs(5);
628
629const ORPHAN_ROUTE_GOODBYE_PRUNE_AT: usize = 256;
636
637#[derive(Debug, Default)]
641struct OrphanGoodbyeLimiter {
642 last_sent: Mutex<HashMap<ConnectionId, HashMap<u16, tokio::time::Instant>>>,
643}
644
645impl OrphanGoodbyeLimiter {
646 fn claim(&self, connection_id: ConnectionId, channel: u16) -> bool {
649 let now = tokio::time::Instant::now();
650 let mut last_sent = self
651 .last_sent
652 .lock()
653 .unwrap_or_else(std::sync::PoisonError::into_inner);
654 let channels = last_sent.entry(connection_id).or_default();
655 if channels.get(&channel).is_some_and(|sent| {
656 now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
657 }) {
658 return false;
659 }
660 if channels.len() >= ORPHAN_ROUTE_GOODBYE_PRUNE_AT {
661 channels.retain(|_, sent| {
662 now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
663 });
664 }
665 channels.insert(channel, now);
666 true
667 }
668
669 fn forget_connection(&self, connection_id: ConnectionId) {
670 self.last_sent
671 .lock()
672 .unwrap_or_else(std::sync::PoisonError::into_inner)
673 .remove(&connection_id);
674 }
675}
676
677#[derive(Debug, Clone)]
679pub struct RouteCtx {
680 pub connection_id: ConnectionId,
681 pub egress: FrameSink,
682}
683
684#[derive(Debug, Clone)]
690pub enum Backend {
691 Echo(EchoBackend),
692 Forward(ForwardBackend),
693}
694
695impl From<EchoBackend> for Backend {
696 fn from(backend: EchoBackend) -> Self {
697 Self::Echo(backend)
698 }
699}
700
701impl From<ForwardBackend> for Backend {
702 fn from(backend: ForwardBackend) -> Self {
703 Self::Forward(backend)
704 }
705}
706
707impl Backend {
708 pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
709 match self {
710 Self::Echo(backend) => backend.handle(ctx, frame).await,
711 Self::Forward(backend) => backend.handle(ctx, frame).await,
712 }
713 }
714}
715
716pub struct Router {
737 backends: HashMap<u16, Backend>,
738 control: Arc<ControlHandler>,
739 forwarding: Arc<ForwardingTable>,
740 forward_backend: ForwardBackend,
741 counters: DaemonCounters,
742 next_connection_id: AtomicU64,
743 orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
744}
745
746impl Router {
747 pub fn with_control_handler(control: Arc<ControlHandler>) -> Self {
748 control.install_swap_promotion_observer();
751 let forwarding = control.forwarding();
752 let counters = control.counters();
753 Self {
754 backends: HashMap::new(),
755 control,
756 forwarding: Arc::clone(&forwarding),
757 forward_backend: ForwardBackend::new(forwarding),
758 counters,
759 next_connection_id: AtomicU64::new(1),
761 orphan_goodbyes: Arc::default(),
762 }
763 }
764
765 pub fn with_default_self_handler() -> Self {
766 Self::with_control_handler(Arc::new(ControlHandler::default()))
767 }
768
769 pub fn forwarding(&self) -> Arc<ForwardingTable> {
770 Arc::clone(&self.forwarding)
771 }
772
773 pub fn register_backend(
774 &mut self,
775 channel: u16,
776 backend: impl Into<Backend>,
777 ) -> Result<(), RouterError> {
778 self.register_backend_arc(channel, Arc::new(backend.into()))
779 }
780
781 pub(crate) fn register_backend_arc(
782 &mut self,
783 channel: u16,
784 backend: Arc<Backend>,
785 ) -> Result<(), RouterError> {
786 if channel == 0 {
787 return Err(RouterError::ReservedChannelZero);
788 }
789 if self.backends.contains_key(&channel) {
790 return Err(RouterError::DuplicateChannel { channel });
791 }
792 self.backends.insert(channel, backend.as_ref().clone());
793 Ok(())
794 }
795
796 fn record_module_frame_drop(&self, connection_id: ConnectionId) -> Result<(), RouterError> {
799 let module_id = self
800 .forwarding
801 .module_id_for_connection(connection_id)
802 .map_err(RouterError::Forwarding)?;
803 self.counters
804 .increment_module_frames_dropped_no_route(module_id.as_deref());
805 Ok(())
806 }
807
808 fn handle_orphan_module_frame(&self, ctx: &RouteCtx, frame: &Frame) -> Result<(), RouterError> {
820 let channel = frame.header.channel;
821 let epoch = frame.header.epoch;
822 let module_id = self
823 .forwarding
824 .module_id_for_connection(ctx.connection_id)
825 .map_err(RouterError::Forwarding)?;
826 self.counters
827 .increment_module_frames_dropped_no_route(module_id.as_deref());
828 if self
829 .forwarding
830 .module_route_epoch_was_allocated(ctx.connection_id, channel, epoch)
831 .map_err(RouterError::Forwarding)?
832 {
833 self.counters
834 .increment_module_frames_dropped_released_route(module_id.as_deref());
835 }
836 if frame.header.ty == FrameType::Goodbye
837 || !self.orphan_goodbyes.claim(ctx.connection_id, channel)
838 {
839 return Ok(());
840 }
841 let goodbye = Frame::build_with_version(
842 frame.header.ver,
843 FrameType::Goodbye,
844 Flags::new(false, Priority::Passive, false),
845 channel,
846 epoch,
847 0,
848 Vec::new(),
849 )
850 .map_err(RouterError::FrameBuild)?;
851 match ctx.egress.try_send(goodbye) {
852 Ok(()) => {
853 self.counters.increment_module_orphan_route_goodbyes_sent();
854 debug!(
855 connection_id = ctx.connection_id.get(),
856 module_id = module_id.as_deref().unwrap_or("unknown"),
857 channel,
858 epoch,
859 "answered module frame on a route the daemon does not hold with a route GOODBYE"
860 );
861 }
862 Err(err) => {
863 self.forwarding.warn_frame_drop(
864 ctx.connection_id,
865 channel,
866 epoch,
867 "daemon_to_module",
868 if ctx.egress.is_closed() {
869 "endpoint_gone"
870 } else {
871 "egress_budget"
872 },
873 );
874 debug!(
875 connection_id = ctx.connection_id.get(),
876 module_id = module_id.as_deref().unwrap_or("unknown"),
877 channel,
878 epoch,
879 error = %err,
880 "could not enqueue route GOODBYE for module frame on a route the daemon does not hold; the next such frame retries"
881 );
882 }
883 }
884 Ok(())
885 }
886
887 pub fn begin_connection(&self) -> RouterConnection {
888 let raw = self.next_connection_id.fetch_add(1, Ordering::Relaxed);
889 let id = ConnectionId::new(raw);
890 let close_receiver = self.forwarding.register_connection_close(id);
891 RouterConnection {
892 id,
893 control_handler: Arc::clone(&self.control),
894 forwarding: Arc::clone(&self.forwarding),
895 close_receiver: Some(close_receiver),
896 orphan_goodbyes: Arc::clone(&self.orphan_goodbyes),
897 }
898 }
899
900 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
901 self.control.route_open_target(frame)
902 }
903
904 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
905 self.control.record_local_connection(connection);
906 }
907
908 pub(crate) fn record_captured_peer(
909 &self,
910 connection: ConnectionId,
911 peer: subc_os::local_peer::CapturedPeer,
912 ) {
913 self.control.record_captured_peer(connection, peer);
914 }
915
916 pub(crate) fn record_caller_endpoints(
917 &self,
918 connection: ConnectionId,
919 endpoints: Option<(std::net::SocketAddr, std::net::SocketAddr)>,
920 ) {
921 self.control.record_caller_endpoints(connection, endpoints);
922 }
923
924 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
925 self.control.defers_privileged_control(ctx, frame)
926 }
927
928 pub(crate) fn route_open_capacity_refusal(
929 &self,
930 ctx: &RouteCtx,
931 frame: &Frame,
932 target_module_id: &str,
933 in_flight: usize,
934 limit: usize,
935 ) -> Result<Frame, RouterError> {
936 self.control
937 .route_open_capacity_refusal(ctx, frame, target_module_id, in_flight, limit)
938 }
939
940 pub async fn route_for_connection(
941 &self,
942 ctx: &RouteCtx,
943 frame: Frame,
944 ) -> Result<(), RouterError> {
945 self.route_for_connection_started(ctx, frame, None).await
946 }
947
948 pub(crate) async fn observe_connection_wait<F: std::future::Future>(
949 &self,
950 ctx: &RouteCtx,
951 kind: &'static str,
952 future: F,
953 ) -> F::Output {
954 self.counters
955 .observe_reader_wait(
956 || {
957 let module = self
958 .forwarding
959 .module_id_for_connection(ctx.connection_id)
960 .ok()
961 .flatten();
962 let principal = if module.is_some() { "module" } else { "direct" };
963 (
964 module.unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
965 principal.into(),
966 )
967 },
968 kind,
969 future,
970 )
971 .await
972 }
973
974 pub(crate) async fn route_for_connection_started(
975 &self,
976 ctx: &RouteCtx,
977 frame: Frame,
978 dispatch_started_at: Option<Instant>,
979 ) -> Result<(), RouterError> {
980 let channel = frame.header.channel;
981 let epoch = frame.header.epoch;
982 let corr = frame.header.corr;
983 if channel == 0 {
984 debug!(
985 connection_id = ctx.connection_id.get(),
986 corr,
987 frame_type = ?frame.header.ty,
988 "routing control frame"
989 );
990 let dispatch_started_at = (frame.header.ty == FrameType::Request)
996 .then(|| dispatch_started_at.unwrap_or_else(Instant::now));
997 let responses = self
998 .control
999 .handle_control_frame_timed(ctx, frame, dispatch_started_at)
1000 .await?;
1001 for response in responses {
1002 self.observe_connection_wait(ctx, "reply_egress", ctx.egress.send(response))
1003 .await?;
1004 }
1005 return Ok(());
1006 }
1007
1008 let data_route = self
1009 .forwarding
1010 .lookup_data_route(ctx.connection_id, channel, epoch)
1011 .map_err(RouterError::Forwarding)?;
1012
1013 match data_route {
1014 DataRoute::Module(DataRouteState::EpochMismatch) => {
1015 self.forwarding.warn_frame_drop(
1016 ctx.connection_id,
1017 channel,
1018 epoch,
1019 "module_to_client",
1020 "stale_epoch",
1021 );
1022 if frame.header.ty == FrameType::Request {
1023 self.counters
1024 .increment_module_requests_dropped_stale_route();
1025 let err = RouterError::StaleRouteEpoch {
1026 channel,
1027 epoch,
1028 corr,
1029 };
1030 if let Some(error_frame) = err.to_error_frame() {
1031 self.observe_connection_wait(
1032 ctx,
1033 "reply_egress",
1034 ctx.egress.send(error_frame),
1035 )
1036 .await?;
1037 }
1038 } else {
1039 self.handle_orphan_module_frame(ctx, &frame)?;
1040 }
1041 debug!(
1042 connection_id = ctx.connection_id.get(),
1043 channel, epoch, corr, "dropping module frame for stale route epoch"
1044 );
1045 return Ok(());
1046 }
1047 DataRoute::Module(DataRouteState::Reserved) => {
1048 self.forwarding.warn_frame_drop(
1049 ctx.connection_id,
1050 channel,
1051 epoch,
1052 "module_to_client",
1053 "missing_binding",
1054 );
1055 if frame.header.ty == FrameType::Request {
1056 self.counters
1057 .increment_module_requests_dropped_stale_route();
1058 let err = RouterError::UnknownChannel {
1059 channel,
1060 epoch,
1061 corr,
1062 };
1063 if let Some(error_frame) = err.to_error_frame() {
1064 self.observe_connection_wait(
1065 ctx,
1066 "reply_egress",
1067 ctx.egress.send(error_frame),
1068 )
1069 .await?;
1070 }
1071 } else {
1072 self.record_module_frame_drop(ctx.connection_id)?;
1073 }
1074 debug!(
1075 connection_id = ctx.connection_id.get(),
1076 channel, epoch, corr, "dropping module frame for reserved route handle"
1077 );
1078 return Ok(());
1079 }
1080 DataRoute::Module(DataRouteState::Absent) => {
1081 self.forwarding.warn_frame_drop(
1082 ctx.connection_id,
1083 channel,
1084 epoch,
1085 "module_to_client",
1086 "unknown_channel",
1087 );
1088 if frame.header.ty == FrameType::Request {
1089 self.counters
1090 .increment_module_requests_dropped_stale_route();
1091 let err = RouterError::UnknownChannel {
1092 channel,
1093 epoch,
1094 corr,
1095 };
1096 if let Some(error_frame) = err.to_error_frame() {
1097 self.observe_connection_wait(
1098 ctx,
1099 "reply_egress",
1100 ctx.egress.send(error_frame),
1101 )
1102 .await?;
1103 }
1104 } else {
1105 self.handle_orphan_module_frame(ctx, &frame)?;
1106 }
1107 debug!(
1108 connection_id = ctx.connection_id.get(),
1109 channel, epoch, corr, "dropping module frame for absent route handle"
1110 );
1111 return Ok(());
1112 }
1113 DataRoute::Module(DataRouteState::Bound(route)) => {
1114 if frame.header.ty == FrameType::Goodbye {
1115 if let RouteRelease::Removed(target) = self
1116 .forwarding
1117 .release_module_route(ctx.connection_id, channel, epoch)
1118 .map_err(RouterError::Forwarding)?
1119 {
1120 let mut goodbye = frame;
1121 goodbye.header.channel = target.channel;
1122 goodbye.header.epoch = target.epoch;
1123 goodbye.header.flags = goodbye.header.flags.without_daemon_origin();
1127 if let Err(err) = self
1128 .observe_connection_wait(
1129 ctx,
1130 "module_to_client",
1131 target.sink.send_to_client(goodbye),
1132 )
1133 .await
1134 {
1135 if !target.close_on_delivery_failure() {
1136 self.forwarding.warn_frame_drop(
1137 target.connection_id,
1138 target.channel,
1139 target.epoch,
1140 "module_to_client",
1141 err.diagnosis.cause,
1142 );
1143 }
1144 if target.close_on_delivery_failure()
1145 && self
1146 .forwarding
1147 .escalate_client_delivery_failure_diagnosed(
1148 target.connection_id,
1149 target.channel,
1150 target.epoch,
1151 CloseReason::new(
1152 "route_goodbye_delivery_failed",
1153 format!(
1154 "failed to enqueue route GOODBYE for client channel {}: {err}",
1155 target.channel
1156 ),
1157 ),
1158 UndeliveredFrame {
1159 module_id: Some(&route.module_id),
1160 sink: &target.sink,
1161 },
1162 Some(&err.diagnosis),
1163 )
1164 .map_err(RouterError::Forwarding)?
1165 {
1166 self.counters.increment_goodbye_relay_client_failed();
1167 }
1168 }
1169 } else {
1170 self.forwarding.warn_frame_drop(
1171 ctx.connection_id,
1172 channel,
1173 epoch,
1174 "module_to_client",
1175 "missing_binding",
1176 );
1177 }
1178 return Ok(());
1179 }
1180
1181 let bytes = queued_frame_bytes(&frame);
1185 let limit = route.client_sink.accounting.byte_budget;
1186 if bytes > limit {
1187 let ty = frame.header.ty;
1188 let flags = frame.header.flags;
1189 drop(frame);
1190 return self
1191 .refuse_oversized_module_frame(ctx, &route, (ty, flags), corr, bytes)
1192 .await;
1193 }
1194
1195 let releases_credit = is_terminal_frame(frame.header.ty);
1201 if releases_credit {
1202 route.flow.release_corr(corr);
1203 }
1204 let mut frame = frame;
1205 frame.header.channel = route.client_channel;
1206 frame.header.epoch = route.client_epoch;
1207 frame.header.flags = frame.header.flags.without_daemon_origin();
1213 if let Err(err) = self
1214 .counters
1215 .observe_reader_wait(
1216 || (route.module_id.clone(), principal_label(&route.principal)),
1217 "module_to_client",
1218 route.client_sink.send_to_client(frame),
1219 )
1220 .await
1221 {
1222 if self
1223 .forwarding
1224 .escalate_client_delivery_failure_diagnosed(
1225 route.client_connection_id,
1226 route.client_channel,
1227 route.client_epoch,
1228 CloseReason::new(
1229 "module_to_client_delivery_failed",
1230 format!(
1231 "failed to enqueue module frame for client channel {} corr {corr}: {err}",
1232 route.client_channel
1233 ),
1234 ),
1235 UndeliveredFrame {
1236 module_id: Some(&route.module_id),
1237 sink: &route.client_sink,
1238 },
1239 Some(&err.diagnosis),
1240 )
1241 .map_err(RouterError::Forwarding)?
1242 {
1243 self.counters
1244 .increment_client_egress_close_delivery_failed();
1245 }
1246 return Ok(());
1247 }
1248 return Ok(());
1249 }
1250 DataRoute::Client(DataRouteState::EpochMismatch) => {
1251 self.forwarding.warn_frame_drop(
1252 ctx.connection_id,
1253 channel,
1254 epoch,
1255 "client_to_module",
1256 "stale_epoch",
1257 );
1258 if frame.header.ty == FrameType::Request {
1259 self.counters.increment_client_frames_dropped_stale_route();
1260 let err = RouterError::StaleRouteEpoch {
1262 channel,
1263 epoch,
1264 corr,
1265 };
1266 if let Some(error_frame) = err.to_error_frame() {
1267 self.observe_connection_wait(
1268 ctx,
1269 "reply_egress",
1270 ctx.egress.send(error_frame),
1271 )
1272 .await?;
1273 }
1274 }
1275 debug!(
1276 connection_id = ctx.connection_id.get(),
1277 channel, epoch, corr, "dropping client frame for stale route epoch"
1278 );
1279 return Ok(());
1280 }
1281 DataRoute::Client(DataRouteState::Reserved) => {
1282 self.forwarding.warn_frame_drop(
1283 ctx.connection_id,
1284 channel,
1285 epoch,
1286 "client_to_module",
1287 "missing_binding",
1288 );
1289 if frame.header.ty == FrameType::Request {
1290 let err = RouterError::UnknownChannel {
1291 channel,
1292 epoch,
1293 corr,
1294 };
1295 if let Some(error_frame) = err.to_error_frame() {
1296 self.observe_connection_wait(
1297 ctx,
1298 "reply_egress",
1299 ctx.egress.send(error_frame),
1300 )
1301 .await?;
1302 }
1303 }
1304 return Ok(());
1305 }
1306 DataRoute::Client(DataRouteState::Bound(route)) => {
1307 if frame.header.ty == FrameType::Goodbye {
1308 let _ = self
1309 .control
1310 .handle_route_goodbye(ctx.connection_id, channel, epoch)?;
1311 return Ok(());
1312 }
1313 return self.forward_backend.handle_bound(frame, route).await;
1314 }
1315 DataRoute::Client(DataRouteState::Absent) => {}
1316 }
1317
1318 if let Some(backend) = self.backends.get(&channel) {
1319 return backend.handle(ctx.clone(), frame).await.inspect_err(|_| {
1320 self.forwarding.warn_frame_drop(
1321 ctx.connection_id,
1322 channel,
1323 epoch,
1324 "client_to_backend",
1325 "endpoint_gone",
1326 );
1327 });
1328 }
1329 self.forwarding.warn_frame_drop(
1330 ctx.connection_id,
1331 channel,
1332 epoch,
1333 "client_to_module",
1334 "unknown_channel",
1335 );
1336 if frame.header.ty == FrameType::Request {
1337 let err = RouterError::UnknownChannel {
1338 channel,
1339 epoch,
1340 corr,
1341 };
1342 if let Some(error_frame) = err.to_error_frame() {
1343 self.observe_connection_wait(ctx, "reply_egress", ctx.egress.send(error_frame))
1344 .await?;
1345 }
1346 }
1347 Ok(())
1348 }
1349
1350 async fn refuse_oversized_module_frame(
1351 &self,
1352 ctx: &RouteCtx,
1353 route: &RouteBinding,
1354 header: (FrameType, Flags),
1355 corr: u64,
1356 bytes: usize,
1357 ) -> Result<(), RouterError> {
1358 let (ty, flags) = header;
1359 let limit = route.client_sink.accounting.byte_budget;
1360 if ty == FrameType::Push {
1361 let diagnosis = ClientDeliveryDiagnosis {
1365 frame_type: ty,
1366 flags,
1367 cause: "frame_too_large",
1368 };
1369 if self
1370 .forwarding
1371 .escalate_client_delivery_failure_diagnosed(
1372 route.client_connection_id,
1373 route.client_channel,
1374 route.client_epoch,
1375 CloseReason::new(
1376 error_codes::FRAME_TOO_LARGE,
1377 format!(
1378 "oversized push: size={bytes} limit={limit} (header plus body bytes)"
1379 ),
1380 ),
1381 UndeliveredFrame {
1382 module_id: Some(&route.module_id),
1383 sink: &route.client_sink,
1384 },
1385 Some(&diagnosis),
1386 )
1387 .map_err(RouterError::Forwarding)?
1388 {
1389 self.counters
1390 .increment_client_egress_close_delivery_failed();
1391 }
1392 return Ok(());
1393 }
1394
1395 warn!(
1396 connection_id = route.client_connection_id.get(),
1397 module_id = %route.module_id,
1398 client_principal = %principal_label(&route.principal),
1399 client_channel = route.client_channel,
1400 frame_type = ?ty,
1401 frame_bytes = bytes,
1402 byte_limit = limit,
1403 "refusing oversized module-to-client frame with ERROR frame_too_large"
1404 );
1405 if ty == FrameType::Request {
1406 let error = frame_too_large_error(
1409 route.module_channel,
1410 route.module_epoch,
1411 corr,
1412 bytes,
1413 limit,
1414 )?;
1415 self.observe_connection_wait(ctx, "reply_egress", ctx.egress.send(error))
1416 .await?;
1417 } else {
1418 route.flow.release_corr(corr);
1421 let error = frame_too_large_error(
1422 route.client_channel,
1423 route.client_epoch,
1424 corr,
1425 bytes,
1426 limit,
1427 )?;
1428 if let Err(err) = self
1431 .counters
1432 .observe_reader_wait(
1433 || (route.module_id.clone(), principal_label(&route.principal)),
1434 "module_to_client",
1435 route.client_sink.send_to_client(error),
1436 )
1437 .await
1438 {
1439 if self
1440 .forwarding
1441 .escalate_client_delivery_failure_diagnosed(
1442 route.client_connection_id,
1443 route.client_channel,
1444 route.client_epoch,
1445 CloseReason::new(
1446 "module_to_client_delivery_failed",
1447 format!(
1448 "failed to enqueue frame_too_large for client channel {} corr {corr}: {err}",
1449 route.client_channel
1450 ),
1451 ),
1452 UndeliveredFrame {
1453 module_id: Some(&route.module_id),
1454 sink: &route.client_sink,
1455 },
1456 Some(&err.diagnosis),
1457 )
1458 .map_err(RouterError::Forwarding)?
1459 {
1460 self.counters
1461 .increment_client_egress_close_delivery_failed();
1462 }
1463 }
1464 }
1465 Ok(())
1466 }
1467}
1468
1469impl Default for Router {
1470 fn default() -> Self {
1471 Self::with_default_self_handler()
1472 }
1473}
1474
1475#[must_use]
1477pub struct RouterConnection {
1478 id: ConnectionId,
1479 control_handler: Arc<ControlHandler>,
1480 forwarding: Arc<ForwardingTable>,
1481 close_receiver: Option<ConnectionCloseReceiver>,
1482 orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
1483}
1484
1485impl RouterConnection {
1486 pub fn id(&self) -> ConnectionId {
1487 self.id
1488 }
1489
1490 pub(crate) fn take_close_receiver(&mut self) -> ConnectionCloseReceiver {
1491 self.close_receiver
1492 .take()
1493 .expect("connection close receiver can only be taken once")
1494 }
1495}
1496
1497impl Drop for RouterConnection {
1498 fn drop(&mut self) {
1499 self.forwarding.unregister_connection_close(self.id);
1500 self.orphan_goodbyes.forget_connection(self.id);
1501 let _ = self.control_handler.cleanup_connection(self.id);
1504 }
1505}
1506
1507#[derive(Debug, Default, Clone, Copy)]
1510pub struct EchoBackend;
1511
1512impl EchoBackend {
1513 pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1514 let response = Frame::build_with_version(
1515 frame.header.ver,
1516 FrameType::Response,
1517 frame.header.flags,
1518 frame.header.channel,
1519 frame.header.epoch,
1520 frame.header.corr,
1521 frame.body,
1522 )
1523 .map_err(RouterError::FrameBuild)?;
1524 ctx.egress.send(response).await
1525 }
1526}
1527
1528#[derive(Debug, Clone)]
1530pub struct ForwardBackend {
1531 forwarding: Arc<ForwardingTable>,
1532}
1533
1534impl ForwardBackend {
1535 pub fn new(forwarding: Arc<ForwardingTable>) -> Self {
1536 Self { forwarding }
1537 }
1538
1539 pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1540 let channel = frame.header.channel;
1541 let corr = frame.header.corr;
1542 let route = match self
1543 .forwarding
1544 .lookup_data_route(ctx.connection_id, channel, frame.header.epoch)
1545 .map_err(RouterError::Forwarding)?
1546 {
1547 DataRoute::Client(DataRouteState::Bound(route)) => route,
1548 DataRoute::Client(_) | DataRoute::Module(_) => {
1549 self.forwarding.warn_frame_drop(
1550 ctx.connection_id,
1551 channel,
1552 frame.header.epoch,
1553 "client_to_module",
1554 "missing_binding",
1555 );
1556 return Err(RouterError::UnknownChannel {
1557 channel,
1558 epoch: frame.header.epoch,
1559 corr,
1560 });
1561 }
1562 };
1563 self.handle_bound(frame, route).await
1564 }
1565
1566 pub(crate) async fn handle_bound(
1567 &self,
1568 frame: Frame,
1569 route: Arc<RouteBinding>,
1570 ) -> Result<(), RouterError> {
1571 let channel = frame.header.channel;
1572 let corr = frame.header.corr;
1573 let frame_type = frame.header.ty;
1574
1575 let acquired_credit = frame_type == FrameType::Request;
1578 if acquired_credit && route.flow.busy_refusals() {
1579 use crate::forwarding::RequestAdmission;
1580 match self
1581 .forwarding
1582 .try_admit_request(
1583 &route,
1584 frame.header.epoch,
1585 corr,
1586 frame.header.flags.is_subscription(),
1587 )
1588 .map_err(RouterError::Forwarding)?
1589 {
1590 RequestAdmission::Admitted => {}
1591 RequestAdmission::Busy(detail) => {
1592 let principal = principal_label(&route.principal);
1593 let counters = self.forwarding.counters_ref();
1594 counters.increment_route_busy(&route.module_id, &principal);
1595 if route.flow.should_warn_busy() {
1596 warn!(module_id = %route.module_id, client_principal = %principal, client_channel = channel,
1597 route_epoch = route.client_epoch, ?detail, "request refused: route_busy");
1598 }
1599 let error = RouterError::RouteBusy {
1600 channel,
1601 epoch: frame.header.epoch,
1602 corr,
1603 detail: Box::new(detail),
1604 };
1605 let refusal = error
1606 .to_error_frame()
1607 .expect("route_busy is a serializable wire error");
1608 counters
1609 .observe_reader_wait(
1610 || (route.module_id.clone(), principal),
1611 "route_busy_egress",
1612 route.client_sink.send(refusal),
1613 )
1614 .await?;
1615 return Ok(());
1616 }
1617 RequestAdmission::ModuleReloading => {
1618 return Err(RouterError::route_error_with_epoch(
1619 channel,
1620 frame.header.epoch,
1621 corr,
1622 error_codes::MODULE_RELOADING,
1623 format!("module endpoint for route channel {channel} is reloading"),
1624 ));
1625 }
1626 RequestAdmission::Stale => {
1627 return Err(RouterError::StaleRouteEpoch {
1628 channel,
1629 epoch: frame.header.epoch,
1630 corr,
1631 })
1632 }
1633 }
1634 } else if acquired_credit {
1635 let counters = self.forwarding.counters_ref();
1636 if let Err(err) = counters
1637 .observe_reader_wait(
1638 || (route.module_id.clone(), principal_label(&route.principal)),
1639 "legacy_credit",
1640 route
1641 .flow
1642 .acquire_tagged(corr, frame.header.flags.is_subscription()),
1643 )
1644 .await
1645 {
1646 self.forwarding.warn_frame_drop(
1647 route.client_connection_id,
1648 channel,
1649 frame.header.epoch,
1650 "client_to_module",
1651 "endpoint_gone",
1652 );
1653 if self
1660 .forwarding
1661 .endpoint_is_draining(route.module_endpoint)
1662 .map_err(RouterError::Forwarding)?
1663 {
1664 return Err(RouterError::route_error_with_epoch(
1665 channel,
1666 frame.header.epoch,
1667 corr,
1668 "module_reloading",
1669 format!("module endpoint for route channel {channel} is reloading"),
1670 ));
1671 }
1672 return Err(RouterError::backend_with_epoch(
1673 channel,
1674 frame.header.epoch,
1675 corr,
1676 format!("{err} for route channel {channel}"),
1677 ));
1678 }
1679 }
1680
1681 let mut frame = frame;
1682 frame.header.channel = route.module_channel;
1683 frame.header.epoch = route.module_epoch;
1684 frame.header.flags = frame.header.flags.without_daemon_origin();
1688 let counters = self.forwarding.counters_ref();
1689 let result = counters
1690 .observe_reader_wait(
1691 || (route.module_id.clone(), principal_label(&route.principal)),
1692 "module_queue",
1693 route.module_sink.send(frame),
1694 )
1695 .await
1696 .map_err(|err| {
1697 self.forwarding.warn_frame_drop(
1698 route.client_connection_id,
1699 channel,
1700 route.client_epoch,
1701 "client_to_module",
1702 "endpoint_gone",
1703 );
1704 RouterError::backend_with_epoch(channel, route.client_epoch, corr, err.to_string())
1705 });
1706 if acquired_credit && result.is_err() {
1707 route.flow.release_corr(corr);
1708 }
1709 result
1710 }
1711}
1712
1713fn is_terminal_frame(frame_type: FrameType) -> bool {
1714 matches!(
1715 frame_type,
1716 FrameType::Response | FrameType::Error | FrameType::StreamEnd
1717 )
1718}
1719
1720#[derive(Debug, Clone, PartialEq, Eq)]
1723pub enum RouterError {
1724 ReservedChannelZero,
1725 DuplicateChannel {
1726 channel: u16,
1727 },
1728 UnknownChannel {
1729 channel: u16,
1730 epoch: u32,
1731 corr: u64,
1732 },
1733 StaleRouteEpoch {
1734 channel: u16,
1735 epoch: u32,
1736 corr: u64,
1737 },
1738 RouteBusy {
1739 channel: u16,
1740 epoch: u32,
1741 corr: u64,
1742 detail: Box<subc_protocol::RouteBusyDetail>,
1743 },
1744 Backend {
1745 channel: u16,
1746 epoch: u32,
1747 corr: u64,
1748 message: String,
1749 },
1750 RouteError {
1751 channel: u16,
1752 epoch: u32,
1753 corr: u64,
1754 code: String,
1755 message: String,
1756 },
1757 FrameBuild(FrameBuildError),
1758 Forwarding(ForwardingError),
1759}
1760
1761impl RouterError {
1762 pub fn backend(channel: u16, corr: u64, message: impl Into<String>) -> Self {
1763 Self::backend_with_epoch(channel, 0, corr, message)
1764 }
1765
1766 pub fn backend_with_epoch(
1767 channel: u16,
1768 epoch: u32,
1769 corr: u64,
1770 message: impl Into<String>,
1771 ) -> Self {
1772 Self::Backend {
1773 channel,
1774 epoch,
1775 corr,
1776 message: message.into(),
1777 }
1778 }
1779
1780 pub fn route_error(
1781 channel: u16,
1782 corr: u64,
1783 code: impl Into<String>,
1784 message: impl Into<String>,
1785 ) -> Self {
1786 Self::route_error_with_epoch(channel, 0, corr, code, message)
1787 }
1788
1789 pub fn route_error_with_epoch(
1790 channel: u16,
1791 epoch: u32,
1792 corr: u64,
1793 code: impl Into<String>,
1794 message: impl Into<String>,
1795 ) -> Self {
1796 Self::RouteError {
1797 channel,
1798 epoch,
1799 corr,
1800 code: code.into(),
1801 message: message.into(),
1802 }
1803 }
1804
1805 pub fn to_error_frame(&self) -> Option<Frame> {
1807 match self {
1808 Self::RouteBusy {
1809 channel,
1810 epoch,
1811 corr,
1812 detail,
1813 } => {
1814 let body = serde_json::to_vec(&ErrorBody {
1815 code: error_codes::ROUTE_BUSY.into(),
1816 message: "route has no free request or watch credit; retry on the same route"
1817 .into(),
1818 detail: Some(serde_json::to_value(detail).ok()?),
1819 })
1820 .ok()?;
1821 Frame::build(
1822 FrameType::Error,
1823 daemon_error_flags(),
1824 *channel,
1825 *epoch,
1826 *corr,
1827 body,
1828 )
1829 .ok()
1830 }
1831 Self::UnknownChannel {
1832 channel,
1833 epoch,
1834 corr,
1835 } => error_frame(
1836 *channel,
1837 *epoch,
1838 *corr,
1839 error_codes::UNKNOWN_CHANNEL,
1840 format!("unknown channel {channel}"),
1841 ),
1842 Self::StaleRouteEpoch {
1843 channel,
1844 epoch,
1845 corr,
1846 } => error_frame(
1847 *channel,
1848 *epoch,
1849 *corr,
1850 error_codes::STALE_ROUTE_EPOCH,
1851 format!("stale route epoch for channel {channel}"),
1852 ),
1853 Self::Backend {
1854 channel,
1855 epoch,
1856 corr,
1857 message,
1858 } => error_frame(*channel, *epoch, *corr, "backend_error", message.clone()),
1859 Self::RouteError {
1860 channel,
1861 epoch,
1862 corr,
1863 code,
1864 message,
1865 } => error_frame(*channel, *epoch, *corr, code, message.clone()),
1866 Self::ReservedChannelZero
1867 | Self::DuplicateChannel { .. }
1868 | Self::FrameBuild(_)
1869 | Self::Forwarding(_) => None,
1870 }
1871 }
1872}
1873
1874pub(crate) fn daemon_error_flags() -> Flags {
1886 Flags::new(false, Priority::Passive, false).with_daemon_origin()
1887}
1888
1889fn error_frame(channel: u16, epoch: u32, corr: u64, code: &str, message: String) -> Option<Frame> {
1890 let body = serde_json::to_vec(&ErrorBody {
1891 code: code.to_string(),
1892 message,
1893 detail: None,
1894 })
1895 .ok()?;
1896
1897 Frame::build(
1898 FrameType::Error,
1899 daemon_error_flags(),
1900 channel,
1901 epoch,
1902 corr,
1903 body,
1904 )
1905 .ok()
1906}
1907
1908fn frame_too_large_error(
1909 channel: u16,
1910 epoch: u32,
1911 corr: u64,
1912 bytes: usize,
1913 limit: usize,
1914) -> Result<Frame, RouterError> {
1915 let body = serde_json::to_vec(&ErrorBody {
1916 code: error_codes::FRAME_TOO_LARGE.to_string(),
1917 message: format!(
1918 "module frame of {bytes} bytes exceeds client egress limit of {limit} bytes"
1919 ),
1920 detail: Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit })),
1921 })
1922 .expect("serializing a frame size error");
1923 Frame::build(
1924 FrameType::Error,
1925 daemon_error_flags(),
1926 channel,
1927 epoch,
1928 corr,
1929 body,
1930 )
1931 .map_err(RouterError::FrameBuild)
1932}
1933
1934impl fmt::Display for RouterError {
1935 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1936 match self {
1937 Self::ReservedChannelZero => write!(f, "channel 0 is reserved for subc"),
1938 Self::DuplicateChannel { channel } => {
1939 write!(f, "backend already registered for channel {channel}")
1940 }
1941 Self::UnknownChannel { channel, corr, .. } => {
1942 write!(f, "unknown channel {channel} for corr {corr}")
1943 }
1944 Self::StaleRouteEpoch { channel, corr, .. } => {
1945 write!(f, "stale route epoch for channel {channel} corr {corr}")
1946 }
1947 Self::RouteBusy { channel, corr, .. } => {
1948 write!(f, "route busy on channel {channel} corr {corr}")
1949 }
1950 Self::Backend {
1951 channel,
1952 corr,
1953 message,
1954 ..
1955 } => write!(
1956 f,
1957 "backend error on channel {channel} corr {corr}: {message}"
1958 ),
1959 Self::RouteError {
1960 channel,
1961 corr,
1962 code,
1963 message,
1964 ..
1965 } => write!(
1966 f,
1967 "route error {code} on channel {channel} corr {corr}: {message}"
1968 ),
1969 Self::FrameBuild(err) => write!(f, "failed to build routed frame: {err}"),
1970 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
1971 }
1972 }
1973}
1974
1975impl Error for RouterError {
1976 fn source(&self) -> Option<&(dyn Error + 'static)> {
1977 match self {
1978 Self::FrameBuild(err) => Some(err),
1979 Self::Forwarding(err) => Some(err),
1980 Self::ReservedChannelZero
1981 | Self::DuplicateChannel { .. }
1982 | Self::UnknownChannel { .. }
1983 | Self::StaleRouteEpoch { .. }
1984 | Self::RouteBusy { .. }
1985 | Self::Backend { .. }
1986 | Self::RouteError { .. } => None,
1987 }
1988 }
1989}
1990
1991#[cfg(test)]
1992mod tests {
1993 include!("route_busy_router_tests.rs");
1994 use super::*;
1995 use crate::{
1996 forwarding::RouteBindRelayOutcome,
1997 supervise::{ModuleSpec, RestartPolicy, Supervisor, SupervisorHandle},
1998 ControlHandler, Registry,
1999 };
2000 use std::{
2001 sync::{mpsc as std_mpsc, Arc},
2002 time::Duration,
2003 };
2004 use subc_control::ModuleProtocol;
2005 use subc_protocol::{manifest::Concurrency, ErrorBody, Flags, FrameType, Priority};
2006 use tokio::sync::mpsc;
2007
2008 pub(crate) use crate::router::test_log::{captured_logs, log_capture};
2009
2010 fn logged_millis(logs: &str, field: &str) -> u64 {
2011 logs.split_whitespace()
2012 .find_map(|part| part.strip_prefix(field))
2013 .and_then(|value| value.parse().ok())
2014 .unwrap_or_else(|| panic!("missing numeric {field} in logs: {logs}"))
2015 }
2016
2017 fn request(channel: u16, corr: u64, body: &[u8]) -> Frame {
2018 Frame::build(
2019 FrameType::Request,
2020 Flags::new(true, Priority::Interactive, false),
2021 channel,
2022 0,
2023 corr,
2024 body.to_vec(),
2025 )
2026 .unwrap()
2027 }
2028
2029 fn ping(corr: u64) -> Frame {
2030 Frame::build(
2031 FrameType::Ping,
2032 Flags::new(false, Priority::Passive, false),
2033 0,
2034 0,
2035 corr,
2036 Vec::new(),
2037 )
2038 .unwrap()
2039 }
2040
2041 fn route_ctx() -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
2042 let (tx, rx) = mpsc::channel(8);
2043 (
2044 RouteCtx {
2045 connection_id: ConnectionId::LOCAL,
2046 egress: FrameSink::new(tx),
2047 },
2048 rx,
2049 )
2050 }
2051
2052 #[tokio::test]
2053 async fn echo_backend_returns_response_with_byte_identical_body() {
2054 let mut router = Router::with_default_self_handler();
2055 router.register_backend(7, EchoBackend).unwrap();
2056 let (ctx, mut rx) = route_ctx();
2057 let body = b"{not parsed}\0\xff";
2058
2059 router
2060 .route_for_connection(&ctx, request(7, 123, body))
2061 .await
2062 .unwrap();
2063 let response = rx.recv().await.unwrap();
2064
2065 assert_eq!(response.header.ty, FrameType::Response);
2066 assert_eq!(response.header.channel, 7);
2067 assert_eq!(response.header.corr, 123);
2068 assert_eq!(response.body, body);
2069 assert!(rx.try_recv().is_err());
2070 }
2071
2072 #[tokio::test]
2073 async fn unknown_channel_emits_canonical_error_frame() {
2074 let router = Router::with_default_self_handler();
2075 let (ctx, mut rx) = route_ctx();
2076
2077 router
2078 .route_for_connection(&ctx, request(99, 5, b"payload"))
2079 .await
2080 .unwrap();
2081 let error_frame = rx.recv().await.unwrap();
2082
2083 assert_eq!(error_frame.header.ty, FrameType::Error);
2084 assert_eq!(error_frame.header.channel, 99);
2085 assert_eq!(error_frame.header.corr, 5);
2086 let body: ErrorBody = serde_json::from_slice(&error_frame.body).unwrap();
2087 assert_eq!(body.code, "unknown_channel");
2088 assert_eq!(body.message, "unknown channel 99");
2089 }
2090
2091 #[tokio::test]
2096 async fn daemon_authored_data_plane_errors_carry_daemon_origin() {
2097 let router = Router::with_default_self_handler();
2098 let (ctx, mut rx) = route_ctx();
2099 router
2100 .route_for_connection(&ctx, request(99, 5, b"payload"))
2101 .await
2102 .unwrap();
2103 let unknown = rx.recv().await.unwrap();
2104 assert_eq!(unknown.header.ty, FrameType::Error);
2105 let body: ErrorBody = serde_json::from_slice(&unknown.body).unwrap();
2106 assert_eq!(body.code, "unknown_channel");
2107 assert!(
2108 unknown.header.flags.is_daemon_origin(),
2109 "unknown_channel must carry daemon origin: {:?}",
2110 unknown.header.flags
2111 );
2112
2113 let (
2114 router,
2115 _forwarding,
2116 client_ctx,
2117 mut client_rx,
2118 _module_ctx,
2119 _module_egress_rx,
2120 _module_rx,
2121 pending,
2122 ) = dynamic_route_fixture(true);
2123 let _ = client_rx.recv().await;
2124 router
2125 .route_for_connection(
2126 &client_ctx,
2127 route_frame(
2128 FrameType::Request,
2129 pending.client_channel,
2130 pending.client_epoch + 1,
2131 702,
2132 ),
2133 )
2134 .await
2135 .unwrap();
2136 let stale = client_rx.recv().await.unwrap();
2137 assert_eq!(stale.header.ty, FrameType::Error);
2138 let body: ErrorBody = serde_json::from_slice(&stale.body).unwrap();
2139 assert_eq!(body.code, "stale_route_epoch");
2140 assert!(
2141 stale.header.flags.is_daemon_origin(),
2142 "stale_route_epoch must carry daemon origin: {:?}",
2143 stale.header.flags
2144 );
2145 }
2146
2147 #[tokio::test]
2152 async fn module_error_with_daemon_bit_is_stripped_before_client_delivery() {
2153 let (
2154 router,
2155 _forwarding,
2156 client_ctx,
2157 mut client_rx,
2158 module_ctx,
2159 _module_egress_rx,
2160 mut module_rx,
2161 pending,
2162 ) = dynamic_route_fixture(true);
2163 let _ = client_rx.recv().await;
2164 router
2167 .route_for_connection(
2168 &client_ctx,
2169 route_frame(
2170 FrameType::Request,
2171 pending.client_channel,
2172 pending.client_epoch,
2173 701,
2174 ),
2175 )
2176 .await
2177 .unwrap();
2178 let _ = module_rx.recv().await;
2179
2180 let body = serde_json::to_vec(&ErrorBody::new("module_reloading", "try again")).unwrap();
2181 let spoof = Frame::build(
2182 FrameType::Error,
2183 Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2184 pending.module_channel,
2185 pending.module_epoch,
2186 701,
2187 body.clone(),
2188 )
2189 .unwrap();
2190 router
2191 .route_for_connection(&module_ctx, spoof)
2192 .await
2193 .unwrap();
2194 let delivered = client_rx.recv().await.unwrap();
2195 assert_eq!(delivered.header.ty, FrameType::Error);
2196 assert_eq!(delivered.header.channel, pending.client_channel);
2197 assert_eq!(delivered.header.epoch, pending.client_epoch);
2198 assert_eq!(delivered.header.corr, 701);
2199 assert!(
2200 !delivered.header.flags.is_daemon_origin(),
2201 "module-set daemon origin must be cleared: {:?}",
2202 delivered.header.flags
2203 );
2204 assert_eq!(delivered.body, body, "the body travels byte-identical");
2205 }
2206
2207 #[tokio::test]
2210 async fn module_response_with_daemon_bit_is_stripped_preserving_other_flags() {
2211 use subc_protocol::FLAG_SUBSCRIPTION;
2212 let (
2213 router,
2214 _forwarding,
2215 client_ctx,
2216 mut client_rx,
2217 module_ctx,
2218 _module_egress_rx,
2219 mut module_rx,
2220 pending,
2221 ) = dynamic_route_fixture(true);
2222 let _ = client_rx.recv().await;
2223 router
2224 .route_for_connection(
2225 &client_ctx,
2226 route_frame(
2227 FrameType::Request,
2228 pending.client_channel,
2229 pending.client_epoch,
2230 701,
2231 ),
2232 )
2233 .await
2234 .unwrap();
2235 let _ = module_rx.recv().await;
2236
2237 let sent_flags = Flags(
2238 Flags::new(true, Priority::Background, true).0
2239 | FLAG_SUBSCRIPTION
2240 | subc_protocol::FLAG_DAEMON_ORIGIN,
2241 );
2242 assert!(sent_flags.is_daemon_origin());
2243 let response = Frame::build(
2244 FrameType::Response,
2245 sent_flags,
2246 pending.module_channel,
2247 pending.module_epoch,
2248 701,
2249 b"route-body".to_vec(),
2250 )
2251 .unwrap();
2252 router
2253 .route_for_connection(&module_ctx, response)
2254 .await
2255 .unwrap();
2256 let delivered = client_rx.recv().await.unwrap();
2257 assert_eq!(delivered.header.ty, FrameType::Response);
2258 assert!(
2259 !delivered.header.flags.is_daemon_origin(),
2260 "module-set daemon origin must be cleared: {:?}",
2261 delivered.header.flags
2262 );
2263 assert!(
2264 delivered.header.flags.is_binary(),
2265 "binary flag must survive: {:?}",
2266 delivered.header.flags
2267 );
2268 assert!(
2269 delivered.header.flags.is_last(),
2270 "LAST flag must survive: {:?}",
2271 delivered.header.flags
2272 );
2273 assert!(
2274 delivered.header.flags.is_subscription(),
2275 "subscription flag must survive: {:?}",
2276 delivered.header.flags
2277 );
2278 assert_eq!(
2279 delivered.header.flags.priority(),
2280 Some(Priority::Background),
2281 "priority must survive"
2282 );
2283 }
2284
2285 #[tokio::test]
2288 async fn client_frame_with_daemon_bit_is_stripped_before_module_delivery() {
2289 let (
2290 router,
2291 _forwarding,
2292 client_ctx,
2293 mut client_rx,
2294 _module_ctx,
2295 _module_egress_rx,
2296 mut module_rx,
2297 pending,
2298 ) = dynamic_route_fixture(true);
2299 let _ = client_rx.recv().await;
2300 let spoof = Frame::build(
2301 FrameType::Request,
2302 Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2303 pending.client_channel,
2304 pending.client_epoch,
2305 701,
2306 b"route-body".to_vec(),
2307 )
2308 .unwrap();
2309 router
2310 .route_for_connection(&client_ctx, spoof)
2311 .await
2312 .unwrap();
2313 let delivered = module_rx.recv().await.unwrap();
2314 assert_eq!(delivered.header.ty, FrameType::Request);
2315 assert!(
2316 !delivered.header.flags.is_daemon_origin(),
2317 "client-set daemon origin must be cleared: {:?}",
2318 delivered.header.flags
2319 );
2320 }
2321
2322 #[tokio::test]
2323 async fn channel_zero_uses_control_handler_not_backend_registry() {
2324 let mut router = Router::with_default_self_handler();
2325 router.register_backend(1, EchoBackend).unwrap();
2326 let (ctx, mut rx) = route_ctx();
2327
2328 router.route_for_connection(&ctx, ping(77)).await.unwrap();
2329 let response = rx.recv().await.unwrap();
2330
2331 assert_eq!(response.header.ty, FrameType::Pong);
2332 assert_eq!(response.header.channel, 0);
2333 assert_eq!(response.header.corr, 77);
2334 assert!(response.body.is_empty());
2335 }
2336
2337 #[tokio::test]
2338 async fn slow_control_dispatch_logs_decoded_op_and_elapsed_time() {
2339 let control = Arc::new(
2340 ControlHandler::new(Arc::new(Registry::default()))
2341 .with_control_dispatch_delay(Duration::from_millis(1050)),
2342 );
2343 let router = Router::with_control_handler(control);
2344 let (ctx, mut rx) = route_ctx();
2345 let (output, guard) = log_capture(tracing::Level::WARN);
2346
2347 router
2348 .route_for_connection(&ctx, request(0, 41, br#"{"op":"server.describe"}"#))
2349 .await
2350 .expect("slow request routes");
2351 assert!(rx.recv().await.is_some(), "request receives a response");
2352 drop(guard);
2353
2354 let logs = captured_logs(&output);
2355 assert!(logs.contains("slow control dispatch"));
2356 assert!(logs.contains("op=server.describe"));
2357 assert!(logs.contains("connection_id=0"));
2358 assert!(logs.contains("corr=41"));
2359 assert!(
2360 logged_millis(&logs, "elapsed_ms=") >= 1050,
2361 "elapsed must include the injected handler delay: {logs}"
2362 );
2363 }
2364
2365 #[tokio::test]
2366 async fn fast_control_dispatch_emits_arrival_without_slow_warning() {
2367 let router = Router::with_default_self_handler();
2368 let (ctx, mut rx) = route_ctx();
2369 let (output, guard) = log_capture(tracing::Level::DEBUG);
2370
2371 router
2372 .route_for_connection(&ctx, request(0, 42, br#"{"op":"server.describe"}"#))
2373 .await
2374 .expect("fast request routes");
2375 assert!(rx.recv().await.is_some(), "request receives a response");
2376 drop(guard);
2377
2378 let logs = captured_logs(&output);
2379 assert!(logs.contains("control dispatch op=server.describe connection_id=0 corr=42"));
2380 assert!(!logs.contains("slow control dispatch"));
2381 }
2382
2383 #[tokio::test]
2384 async fn control_dispatch_arrival_is_hidden_at_info() {
2385 let router = Router::with_default_self_handler();
2386 let (ctx, mut rx) = route_ctx();
2387 let (output, guard) = log_capture(tracing::Level::INFO);
2388
2389 router
2390 .route_for_connection(&ctx, request(0, 43, br#"{"op":"server.describe"}"#))
2391 .await
2392 .expect("fast request routes");
2393 assert!(rx.recv().await.is_some(), "request receives a response");
2394 drop(guard);
2395
2396 assert!(
2397 !captured_logs(&output).contains("control dispatch"),
2398 "arrival logging must stay hidden at INFO"
2399 );
2400 }
2401
2402 #[tokio::test]
2403 async fn supervisor_list_logs_contended_snapshot_lock_only() {
2404 let registry = Arc::new(Registry::default());
2405 let handle = SupervisorHandle::new();
2406 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
2407 .with_handle(handle.clone());
2408 let module = supervisor
2409 .supervise_configured(
2410 ModuleSpec {
2411 carry_containment: false,
2412 module_id: "held-module".to_string(),
2413 program: "test-module".into(),
2414 args: Vec::new(),
2415 env: Vec::new(),
2416 reserved: false,
2417 reserved_prefixes: Vec::new(),
2418 protocol: ModuleProtocol::Subc,
2419 overlap: Default::default(),
2420 },
2421 false,
2422 )
2423 .expect("disabled test module is supervised");
2424 let router = Router::with_control_handler(Arc::new(
2425 ControlHandler::new(Arc::clone(®istry)).with_supervisor(handle),
2426 ));
2427 let (ctx, mut rx) = route_ctx();
2428 let (acquired, ready) = std_mpsc::channel();
2429 tokio::task::yield_now().await;
2432 let holder = module.hold_snapshot_for_test(acquired, Duration::from_millis(400));
2433 ready.recv().expect("holder acquired snapshot lock");
2434 let (output, guard) = log_capture(tracing::Level::WARN);
2435
2436 router
2437 .route_for_connection(&ctx, request(0, 44, br#"{"op":"supervisor.list"}"#))
2438 .await
2439 .expect("list request routes after the lock releases");
2440 assert!(
2441 rx.recv().await.is_some(),
2442 "list request receives a response"
2443 );
2444 holder.join().expect("snapshot holder exits cleanly");
2445 drop(guard);
2446
2447 let logs = captured_logs(&output);
2448 assert!(logs.contains("slow snapshot lock"));
2449 assert!(logs.contains("module_id=held-module"));
2450 assert!(logs.contains("caller=list"));
2451 assert!(
2452 logged_millis(&logs, "waited_ms=") >= 250,
2453 "wait must exceed the slow-lock threshold: {logs}"
2454 );
2455
2456 let (output, guard) = log_capture(tracing::Level::WARN);
2457 router
2458 .route_for_connection(&ctx, request(0, 45, br#"{"op":"supervisor.list"}"#))
2459 .await
2460 .expect("uncontended list request routes");
2461 assert!(
2462 rx.recv().await.is_some(),
2463 "uncontended list receives a response"
2464 );
2465 drop(guard);
2466 assert!(
2467 !captured_logs(&output).contains("slow snapshot lock"),
2468 "uncontended list acquisition must not warn"
2469 );
2470 }
2471
2472 #[tokio::test]
2473 async fn full_module_to_client_sink_requests_client_close_without_erroring_module() {
2474 let forwarding = Arc::new(ForwardingTable::default());
2475 let control = Arc::new(ControlHandler::with_forwarding(
2476 Arc::new(crate::Registry::default()),
2477 Arc::clone(&forwarding),
2478 ));
2479 let router = Router::with_control_handler(control);
2480 let module_connection = ConnectionId::new(10);
2481 let client_connection = ConnectionId::new(20);
2482 let mut close_receiver = forwarding.register_connection_close(client_connection);
2483 let (module_tx, _module_rx) = mpsc::channel(1);
2484 forwarding
2485 .register_module_connection(
2486 module_connection,
2487 "full-sink-provider".to_string(),
2488 1,
2489 Concurrency::ModuleManaged,
2490 FrameSink::new(module_tx),
2491 )
2492 .unwrap();
2493 let (client_tx, mut client_rx) = mpsc::channel(1);
2494 let pending = forwarding
2495 .begin_route_bind_relay_for_test(
2496 client_connection,
2497 FrameSink::new(client_tx),
2498 700,
2499 "full-sink-provider",
2500 )
2501 .unwrap();
2502 forwarding
2503 .complete_pending_relay(
2504 module_connection,
2505 pending.corr,
2506 RouteBindRelayOutcome::Accepted,
2507 )
2508 .unwrap();
2509
2510 let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2511 let module_ctx = RouteCtx {
2512 connection_id: module_connection,
2513 egress: FrameSink::new(module_egress_tx),
2514 };
2515 let terminal = Frame::build(
2516 FrameType::Response,
2517 Flags::new(false, Priority::Interactive, true),
2518 pending.module_channel,
2519 pending.module_epoch,
2520 701,
2521 b"terminal".to_vec(),
2522 )
2523 .unwrap();
2524
2525 router
2526 .route_for_connection(&module_ctx, terminal)
2527 .await
2528 .unwrap();
2529 let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2530 .await
2531 .expect("close request should be sent for the full client sink")
2532 .expect("close sender should include a reason");
2533 assert!(
2534 reason
2535 .to_string()
2536 .contains("module_to_client_delivery_failed"),
2537 "unexpected close reason: {reason}"
2538 );
2539 assert_eq!(client_rx.try_recv().unwrap().header.corr, 700);
2540 assert!(client_rx.try_recv().is_err());
2541 assert_eq!(
2542 router.counters.snapshot()["client_egress_close_delivery_failed"],
2543 1
2544 );
2545 }
2546
2547 #[tokio::test]
2551 async fn terminal_frame_releases_its_credit_even_when_client_delivery_fails() {
2552 let forwarding = Arc::new(ForwardingTable::default());
2553 let control = Arc::new(ControlHandler::with_forwarding(
2554 Arc::new(crate::Registry::default()),
2555 Arc::clone(&forwarding),
2556 ));
2557 let router = Router::with_control_handler(control);
2558 let module_connection = ConnectionId::new(11);
2559 let client_connection = ConnectionId::new(21);
2560 let _close_receiver = forwarding.register_connection_close(client_connection);
2561 let (module_tx, _module_rx) = mpsc::channel(1);
2562 forwarding
2563 .register_module_connection(
2564 module_connection,
2565 "credit-provider".to_string(),
2566 1,
2567 Concurrency::ModuleManaged,
2568 FrameSink::new(module_tx),
2569 )
2570 .unwrap();
2571 let (client_tx, _client_rx) = mpsc::channel(1);
2574 let pending = forwarding
2575 .begin_route_bind_relay_for_test(
2576 client_connection,
2577 FrameSink::new(client_tx),
2578 800,
2579 "credit-provider",
2580 )
2581 .unwrap();
2582 forwarding
2583 .complete_pending_relay(
2584 module_connection,
2585 pending.corr,
2586 RouteBindRelayOutcome::Accepted,
2587 )
2588 .unwrap();
2589 let DataRoute::Client(DataRouteState::Bound(route)) = forwarding
2590 .lookup_data_route(
2591 client_connection,
2592 pending.client_channel,
2593 pending.client_epoch,
2594 )
2595 .unwrap()
2596 else {
2597 panic!("expected a bound client route");
2598 };
2599 route.flow.acquire_tagged(801, false).await.unwrap();
2600 assert_eq!(route.flow.drain_in_flight(), 1);
2601
2602 let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2603 let module_ctx = RouteCtx {
2604 connection_id: module_connection,
2605 egress: FrameSink::new(module_egress_tx),
2606 };
2607 let terminal = Frame::build(
2608 FrameType::Response,
2609 Flags::new(false, Priority::Interactive, true),
2610 pending.module_channel,
2611 pending.module_epoch,
2612 801,
2613 b"terminal".to_vec(),
2614 )
2615 .unwrap();
2616 router
2617 .route_for_connection(&module_ctx, terminal)
2618 .await
2619 .unwrap();
2620
2621 assert_eq!(
2622 router.counters.snapshot()["client_egress_close_delivery_failed"],
2623 1,
2624 "the client delivery must have failed for this test to mean anything"
2625 );
2626 assert_eq!(
2627 route.flow.drain_in_flight(),
2628 0,
2629 "the module's terminal frame must release its credit even though the client could not take it"
2630 );
2631 }
2632
2633 #[tokio::test]
2634 async fn full_route_goodbye_sink_requests_target_close_without_erroring_module() {
2635 let forwarding = Arc::new(ForwardingTable::default());
2636 let control = Arc::new(ControlHandler::with_forwarding(
2637 Arc::new(crate::Registry::default()),
2638 Arc::clone(&forwarding),
2639 ));
2640 let router = Router::with_control_handler(control);
2641 let module_connection = ConnectionId::new(30);
2642 let client_connection = ConnectionId::new(40);
2643 let mut close_receiver = forwarding.register_connection_close(client_connection);
2644 let (module_tx, _module_rx) = mpsc::channel(1);
2645 forwarding
2646 .register_module_connection(
2647 module_connection,
2648 "goodbye-full-provider".to_string(),
2649 1,
2650 Concurrency::ModuleManaged,
2651 FrameSink::new(module_tx),
2652 )
2653 .unwrap();
2654 let (client_tx, mut client_rx) = mpsc::channel(1);
2655 let pending = forwarding
2656 .begin_route_bind_relay_for_test(
2657 client_connection,
2658 FrameSink::new(client_tx),
2659 800,
2660 "goodbye-full-provider",
2661 )
2662 .unwrap();
2663 forwarding
2664 .complete_pending_relay(
2665 module_connection,
2666 pending.corr,
2667 RouteBindRelayOutcome::Accepted,
2668 )
2669 .unwrap();
2670
2671 let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2672 let module_ctx = RouteCtx {
2673 connection_id: module_connection,
2674 egress: FrameSink::new(module_egress_tx),
2675 };
2676 let goodbye = Frame::build(
2677 FrameType::Goodbye,
2678 Flags::new(false, Priority::Passive, true),
2679 pending.module_channel,
2680 pending.module_epoch,
2681 801,
2682 Vec::new(),
2683 )
2684 .unwrap();
2685
2686 router
2687 .route_for_connection(&module_ctx, goodbye)
2688 .await
2689 .unwrap();
2690 let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2691 .await
2692 .expect("close request should be sent for the full GOODBYE sink")
2693 .expect("close sender should include a reason");
2694 assert!(
2695 reason.to_string().contains("route_goodbye_delivery_failed"),
2696 "unexpected close reason: {reason}"
2697 );
2698 assert_eq!(client_rx.try_recv().unwrap().header.corr, 800);
2699 assert!(client_rx.try_recv().is_err());
2700 assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 1);
2701 assert_eq!(router.counters.snapshot()["route_released_epoch_fenced"], 1);
2702 }
2703
2704 async fn multi_route_client(
2709 module_id: &str,
2710 module_connection: ConnectionId,
2711 client_connection: ConnectionId,
2712 routes: usize,
2713 ) -> (
2714 Router,
2715 FrameSink,
2716 mpsc::Receiver<OutboundFrame>,
2717 RouteCtx,
2718 Vec<crate::forwarding::PendingRouteBindRelay>,
2719 ConnectionCloseReceiver,
2720 ) {
2721 let forwarding = Arc::new(ForwardingTable::default());
2722 let control = Arc::new(ControlHandler::with_forwarding(
2723 Arc::new(crate::Registry::default()),
2724 Arc::clone(&forwarding),
2725 ));
2726 let router = Router::with_control_handler(control);
2727 let close_receiver = forwarding.register_connection_close(client_connection);
2728 let (module_tx, _module_rx) = mpsc::channel(8);
2729 forwarding
2730 .register_module_connection(
2731 module_connection,
2732 module_id.to_string(),
2733 1,
2734 Concurrency::ModuleManaged,
2735 FrameSink::new(module_tx),
2736 )
2737 .unwrap();
2738 let (client_sink, mut client_rx) = crate::server::connection_egress();
2739 let mut bound = Vec::with_capacity(routes);
2740 for index in 0..routes {
2741 let pending = forwarding
2742 .begin_route_bind_relay_for_test(
2743 client_connection,
2744 client_sink.clone(),
2745 900 + index as u64,
2746 module_id,
2747 )
2748 .unwrap();
2749 forwarding
2750 .complete_pending_relay(
2751 module_connection,
2752 pending.corr,
2753 RouteBindRelayOutcome::Accepted,
2754 )
2755 .unwrap();
2756 assert_eq!(
2757 client_rx.recv().await.unwrap().header.corr,
2758 900 + index as u64
2759 );
2760 bound.push(pending);
2761 }
2762 let (module_egress_tx, _module_egress_rx) = mpsc::channel(8);
2763 let module_ctx = RouteCtx {
2764 connection_id: module_connection,
2765 egress: FrameSink::new(module_egress_tx),
2766 };
2767 (
2768 router,
2769 client_sink,
2770 client_rx,
2771 module_ctx,
2772 bound,
2773 close_receiver,
2774 )
2775 }
2776
2777 #[test]
2784 #[ignore = "timing measurement, run on demand"]
2785 fn egress_sink_per_frame_cost() {
2786 const FRAMES: usize = 1_000_000;
2787 const BATCH: usize = 1_000;
2788 let (sink, mut rx) = crate::server::connection_egress();
2789 let template = stream_frame(9, 1, 0, vec![b't'; 200]);
2790 let started = Instant::now();
2791 for _ in 0..FRAMES / BATCH {
2792 for _ in 0..BATCH {
2793 sink.try_send(template.clone()).unwrap();
2794 }
2795 for _ in 0..BATCH {
2796 let outbound = rx.try_recv().unwrap();
2797 if let Some(charge) = &outbound.charge {
2798 charge.taken_by_writer();
2799 }
2800 drop(std::hint::black_box(outbound));
2801 }
2802 }
2803 let elapsed = started.elapsed();
2804 assert_eq!(sink.backlog().queued_bytes, 0);
2805 println!(
2806 "egress sink: {FRAMES} frames in {elapsed:?}, {:.1} ns/frame",
2807 elapsed.as_nanos() as f64 / FRAMES as f64
2808 );
2809 }
2810
2811 fn stream_frame(channel: u16, epoch: u32, corr: u64, body: Vec<u8>) -> Frame {
2812 Frame::build(
2813 FrameType::StreamData,
2814 Flags::new(false, Priority::Interactive, false),
2815 channel,
2816 epoch,
2817 corr,
2818 body,
2819 )
2820 .unwrap()
2821 }
2822
2823 fn module_route_for_test(
2824 router: &Router,
2825 ctx: &RouteCtx,
2826 route: &crate::forwarding::PendingRouteBindRelay,
2827 ) -> Arc<RouteBinding> {
2828 match router
2829 .forwarding
2830 .lookup_data_route(ctx.connection_id, route.module_channel, route.module_epoch)
2831 .unwrap()
2832 {
2833 DataRoute::Module(DataRouteState::Bound(binding)) => binding,
2834 other => panic!("expected a bound module route, got {other:?}"),
2835 }
2836 }
2837
2838 fn assert_frame_size_error(frame: &Frame, channel: u16, epoch: u32, corr: u64, bytes: usize) {
2839 assert_eq!(frame.header.ty, FrameType::Error);
2840 assert_eq!(frame.header.channel, channel);
2841 assert_eq!(frame.header.epoch, epoch);
2842 assert_eq!(frame.header.corr, corr);
2843 assert!(frame.header.flags.is_daemon_origin());
2844 let body: ErrorBody = serde_json::from_slice(&frame.body).unwrap();
2845 assert_eq!(body.code, error_codes::FRAME_TOO_LARGE);
2846 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2847 assert!(body.message.contains(&bytes.to_string()));
2848 assert!(body.message.contains(&limit.to_string()));
2849 assert_eq!(
2850 body.detail,
2851 Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit }))
2852 );
2853 }
2854
2855 #[tokio::test]
2856 async fn module_egress_response_oversized_reply_keeps_connection_and_other_route() {
2857 let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
2858 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2859 "oversize-provider",
2860 ConnectionId::new(80),
2861 ConnectionId::new(81),
2862 2,
2863 )
2864 .await;
2865 let binding = module_route_for_test(&router, &ctx, &routes[0]);
2866 binding.flow.acquire_tagged(101, false).await.unwrap();
2867 module_route_for_test(&router, &ctx, &routes[1])
2868 .flow
2869 .acquire_tagged(102, false)
2870 .await
2871 .unwrap();
2872 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2873 let oversized = Frame::build(
2874 FrameType::Response,
2875 Flags::new(false, Priority::Interactive, true),
2876 routes[0].module_channel,
2877 routes[0].module_epoch,
2878 101,
2879 vec![b'x'; limit - subc_protocol::HEADER_LEN + 1],
2880 )
2881 .unwrap();
2882 assert_eq!(sink.backlog().queued_bytes, 0);
2883 router.route_for_connection(&ctx, oversized).await.unwrap();
2884
2885 router
2889 .route_for_connection(
2890 &ctx,
2891 route_frame(
2892 FrameType::Response,
2893 routes[1].module_channel,
2894 routes[1].module_epoch,
2895 102,
2896 ),
2897 )
2898 .await
2899 .unwrap();
2900 assert!(
2901 close.try_recv().is_err(),
2902 "an oversized reply must not close the multiplexed client"
2903 );
2904 let error = rx.try_recv().expect("the waiting client gets an ERROR");
2905 assert_frame_size_error(
2906 &error,
2907 routes[0].client_channel,
2908 routes[0].client_epoch,
2909 101,
2910 limit + 1,
2911 );
2912 let second = rx.try_recv().expect("the other route's reply arrives");
2913 assert_eq!(second.header.ty, FrameType::Response);
2914 assert_eq!(second.header.channel, routes[1].client_channel);
2915 assert_eq!(second.header.epoch, routes[1].client_epoch);
2916 assert_eq!(second.header.corr, 102);
2917 assert_eq!(second.body, b"route-body");
2918 assert_eq!(binding.flow.drain_in_flight(), 0);
2919 for route in &routes {
2920 module_route_for_test(&router, &ctx, route);
2921 }
2922 assert!(rx.try_recv().is_err());
2923 drop((error, second));
2924 assert_eq!(sink.backlog().queued_bytes, 0);
2925 assert_eq!(
2926 router.counters.snapshot()["client_egress_close_delivery_failed"],
2927 0
2928 );
2929
2930 let captured = test_log::captured_logs(&logs);
2931 let warnings: Vec<_> = captured
2932 .lines()
2933 .filter(|line| line.contains("WARN"))
2934 .collect();
2935 assert_eq!(warnings.len(), 1, "{captured}");
2936 let warning = warnings[0];
2937 for expected in [
2938 "module_id=oversize-provider".to_string(),
2939 "client_principal=direct".to_string(),
2940 format!("client_channel={}", routes[0].client_channel),
2941 format!("frame_bytes={}", limit + 1),
2942 format!("byte_limit={limit}"),
2943 ] {
2944 assert!(warning.contains(&expected), "missing {expected}: {warning}");
2945 }
2946 }
2947
2948 async fn poll_once<F: std::future::Future>(
2949 mut future: std::pin::Pin<&mut F>,
2950 ) -> std::task::Poll<F::Output> {
2951 std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
2952 }
2953
2954 fn fill_client_budget(sink: &FrameSink, route: &crate::forwarding::PendingRouteBindRelay) {
2955 sink.try_send(
2956 Frame::build(
2957 FrameType::Response,
2958 Flags::new(false, Priority::Interactive, true),
2959 route.client_channel,
2960 route.client_epoch,
2961 100,
2962 vec![
2963 b'x';
2964 crate::server::CONNECTION_EGRESS_BYTE_BUDGET - subc_protocol::HEADER_LEN
2965 ],
2966 )
2967 .unwrap(),
2968 )
2969 .unwrap();
2970 assert_eq!(
2971 sink.backlog().queued_bytes,
2972 crate::server::CONNECTION_EGRESS_BYTE_BUDGET
2973 );
2974 }
2975
2976 async fn assert_full_client_burst_survives(ty: FrameType, oversized: bool) {
2977 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2978 "draining-burst",
2979 ConnectionId::new(92),
2980 ConnectionId::new(93),
2981 2,
2982 )
2983 .await;
2984 fill_client_budget(&sink, &routes[0]);
2985 let frame = if oversized {
2986 Frame::build(
2987 ty,
2988 Flags::new(false, Priority::Interactive, false),
2989 routes[1].module_channel,
2990 routes[1].module_epoch,
2991 101,
2992 vec![b's'; crate::server::CONNECTION_EGRESS_BYTE_BUDGET],
2993 )
2994 .unwrap()
2995 } else {
2996 route_frame(ty, routes[1].module_channel, routes[1].module_epoch, 101)
2997 };
2998 let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2999 assert!(poll_once(delivery.as_mut()).await.is_pending());
3000 assert!(
3001 close.try_recv().is_err(),
3002 "fresh bursts must not close: {ty:?}"
3003 );
3004 tokio::time::advance(Duration::from_millis(50)).await;
3005 let queued = rx.try_recv().unwrap();
3006 queued.charge.as_ref().unwrap().taken_by_writer();
3007 drop(queued);
3008 assert!(matches!(
3009 poll_once(delivery.as_mut()).await,
3010 std::task::Poll::Ready(Ok(()))
3011 ));
3012 let delivered = rx
3013 .try_recv()
3014 .expect("the burst frame must be delivered after draining");
3015 assert_eq!(delivered.header.channel, routes[1].client_channel);
3016 assert_eq!(delivered.header.epoch, routes[1].client_epoch);
3017 assert_eq!(delivered.header.corr, 101);
3018 if oversized {
3019 assert_frame_size_error(
3020 &delivered,
3021 routes[1].client_channel,
3022 routes[1].client_epoch,
3023 101,
3024 crate::server::CONNECTION_EGRESS_BYTE_BUDGET + subc_protocol::HEADER_LEN,
3025 );
3026 } else {
3027 assert_eq!(delivered.header.ty, ty);
3028 }
3029 assert!(rx.try_recv().is_err());
3030 assert!(close.try_recv().is_err());
3031 assert_eq!(
3032 router.counters.snapshot()["client_egress_close_delivery_failed"],
3033 0
3034 );
3035 assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 0);
3036 }
3037
3038 #[tokio::test(start_paused = true)]
3039 async fn module_egress_fresh_full_queue_burst_survives() {
3040 for ty in [
3041 FrameType::Response,
3042 FrameType::StreamData,
3043 FrameType::Request,
3044 FrameType::Push,
3045 ] {
3046 assert_full_client_burst_survives(ty, false).await;
3047 }
3048 }
3049
3050 #[tokio::test(start_paused = true)]
3051 async fn module_goodbye_fresh_full_queue_burst_survives() {
3052 assert_full_client_burst_survives(FrameType::Goodbye, false).await;
3053 }
3054
3055 #[tokio::test(start_paused = true)]
3056 async fn frame_count_backstop_also_allows_a_draining_burst() {
3057 let (tx, mut rx) = mpsc::channel(1);
3058 let sink = FrameSink::with_byte_budget(tx, 2_000);
3059 sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
3060 .unwrap();
3061 let mut delivery = Box::pin(sink.send_to_client(stream_frame(1, 1, 2, vec![b'y'; 100])));
3062 assert!(poll_once(delivery.as_mut()).await.is_pending());
3063 assert!(
3064 sink.backlog().queued_bytes < 2_000,
3065 "only frame capacity is exhausted"
3066 );
3067 tokio::time::advance(Duration::from_millis(50)).await;
3068 drop(rx.try_recv().unwrap());
3069 assert!(matches!(
3070 poll_once(delivery.as_mut()).await,
3071 std::task::Poll::Ready(Ok(()))
3072 ));
3073 assert_eq!(rx.try_recv().unwrap().header.corr, 2);
3074 }
3075
3076 #[tokio::test(start_paused = true)]
3077 async fn oversized_reply_replacement_fresh_full_queue_burst_survives() {
3078 for ty in [FrameType::Response, FrameType::StreamData] {
3079 assert_full_client_burst_survives(ty, true).await;
3080 }
3081 }
3082
3083 #[tokio::test(start_paused = true)]
3084 async fn module_egress_stopped_reader_closes_without_waiting() {
3085 let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3086 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3087 "oldest-stopped-reader",
3088 ConnectionId::new(94),
3089 ConnectionId::new(95),
3090 2,
3091 )
3092 .await;
3093 fill_client_budget(&sink, &routes[0]);
3094 let writing = rx.try_recv().unwrap();
3097 writing.charge.as_ref().unwrap().taken_by_writer();
3098 tokio::time::advance(Duration::from_millis(251)).await;
3099 assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(251)));
3100 assert_eq!(sink.backlog().queued_bytes, queued_frame_bytes(&writing));
3101 let started = tokio::time::Instant::now();
3102 let frame = stream_frame(
3103 routes[0].module_channel,
3104 routes[0].module_epoch,
3105 101,
3106 b"secret-body-never-log".to_vec(),
3107 );
3108 let flags = frame.header.flags.0;
3109 let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3110 assert!(
3111 matches!(
3112 poll_once(delivery.as_mut()).await,
3113 std::task::Poll::Ready(Ok(()))
3114 ),
3115 "an oldest frame older than 250 ms must close immediately, without a new wait"
3116 );
3117 assert_eq!(started.elapsed(), Duration::ZERO);
3118 assert!(close
3119 .try_recv()
3120 .unwrap()
3121 .to_string()
3122 .contains("module_to_client_delivery_failed"));
3123 let captured = test_log::captured_logs(&logs);
3124 for expected in [
3125 "frame_type=Some(StreamData)".to_string(),
3126 format!("frame_flags=Some({flags})"),
3127 "oldest_queued_ms=251".to_string(),
3128 "close_cause=\"oldest_frame_older_than_250_ms\"".to_string(),
3129 ] {
3130 assert!(
3131 captured.contains(&expected),
3132 "missing {expected}: {captured}"
3133 );
3134 }
3135 assert!(!captured.contains("secret-body-never-log"));
3136 }
3137
3138 #[tokio::test(start_paused = true)]
3139 async fn client_gone_mid_wait_resumes_module_reader_immediately() {
3140 for writer_gone in [false, true] {
3141 let (router, sink, mut rx, ctx, routes, _close) = multi_route_client(
3142 "gone-during-burst",
3143 ConnectionId::new(96),
3144 ConnectionId::new(97),
3145 1,
3146 )
3147 .await;
3148 fill_client_budget(&sink, &routes[0]);
3149 let frame = route_frame(
3150 FrameType::Response,
3151 routes[0].module_channel,
3152 routes[0].module_epoch,
3153 101,
3154 );
3155 let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3156 assert!(poll_once(delivery.as_mut()).await.is_pending());
3157 tokio::time::advance(Duration::from_millis(50)).await;
3158 let started = tokio::time::Instant::now();
3159 if writer_gone {
3160 rx.close();
3161 } else {
3162 sink.close_client_delivery();
3163 }
3164 assert!(
3165 matches!(
3166 poll_once(delivery.as_mut()).await,
3167 std::task::Poll::Ready(Ok(()))
3168 ),
3169 "client teardown must wake the module reader before the writer drains"
3170 );
3171 assert_eq!(started.elapsed(), Duration::ZERO);
3172 assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
3173 assert_eq!(rx.try_recv().unwrap().header.corr, 100);
3174 assert!(rx.try_recv().is_err());
3175 }
3176 }
3177
3178 #[tokio::test(start_paused = true)]
3179 async fn cancelled_module_delivery_releases_waiter_without_closing_client() {
3180 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3181 "cancelled-burst",
3182 ConnectionId::new(98),
3183 ConnectionId::new(99),
3184 1,
3185 )
3186 .await;
3187 fill_client_budget(&sink, &routes[0]);
3188 let frame = route_frame(
3189 FrameType::Response,
3190 routes[0].module_channel,
3191 routes[0].module_epoch,
3192 101,
3193 );
3194 let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3195 assert!(poll_once(delivery.as_mut()).await.is_pending());
3196 assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 1);
3197 drop(delivery);
3198 assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
3199 tokio::time::advance(Duration::from_secs(1)).await;
3200 assert!(close.try_recv().is_err());
3201 drop(rx.try_recv().unwrap());
3202 assert_eq!(sink.backlog().queued_bytes, 0);
3203 assert!(sink.backlog().oldest_age.is_none());
3204 }
3205
3206 #[tokio::test(start_paused = true)]
3207 async fn egress_atomic_age_follows_writer_and_clears_when_drained() {
3208 let (sink, mut rx) = crate::server::connection_egress();
3209 sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
3210 .unwrap();
3211 tokio::time::advance(Duration::from_millis(200)).await;
3212 sink.try_send(stream_frame(2, 1, 2, vec![b'y'; 100]))
3213 .unwrap();
3214 let writing = rx.try_recv().unwrap();
3215 writing.charge.as_ref().unwrap().taken_by_writer();
3216 assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3217 drop(writing);
3218 assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3221 tokio::time::advance(Duration::from_millis(50)).await;
3222 let next = rx.try_recv().unwrap();
3223 next.charge.as_ref().unwrap().taken_by_writer();
3224 assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(50)));
3225 assert!(
3226 sink.backlog().queued_bytes > 0,
3227 "in-flight bytes remain charged until written"
3228 );
3229 drop(next);
3230 assert_eq!(sink.backlog().queued_bytes, 0);
3231 assert_eq!(sink.backlog().oldest_age, None);
3232 }
3233
3234 #[tokio::test(start_paused = true)]
3235 async fn module_egress_response_full_client_waits_only_remaining_grace() {
3236 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3237 "backpressure-provider",
3238 ConnectionId::new(82),
3239 ConnectionId::new(83),
3240 2,
3241 )
3242 .await;
3243 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3244 router
3247 .route_for_connection(
3248 &ctx,
3249 Frame::build(
3250 FrameType::Response,
3251 Flags::new(false, Priority::Interactive, true),
3252 routes[0].module_channel,
3253 routes[0].module_epoch,
3254 201,
3255 vec![b'x'; limit - subc_protocol::HEADER_LEN],
3256 )
3257 .unwrap(),
3258 )
3259 .await
3260 .unwrap();
3261 assert_eq!(sink.backlog().queued_bytes, limit);
3262 assert_eq!(sink.backlog().queued_frames, 1);
3263 let ordinary = route_frame(
3264 FrameType::Response,
3265 routes[1].module_channel,
3266 routes[1].module_epoch,
3267 202,
3268 );
3269 assert!(queued_frame_bytes(&ordinary) < limit);
3270 tokio::time::advance(Duration::from_millis(100)).await;
3273 let mut delivery = Box::pin(router.route_for_connection(&ctx, ordinary));
3274 assert!(poll_once(delivery.as_mut()).await.is_pending());
3275 assert!(close.try_recv().is_err());
3276 tokio::time::advance(Duration::from_millis(149)).await;
3277 assert!(poll_once(delivery.as_mut()).await.is_pending());
3278 assert!(close.try_recv().is_err());
3279 tokio::time::advance(Duration::from_millis(1)).await;
3280 assert!(matches!(
3281 poll_once(delivery.as_mut()).await,
3282 std::task::Poll::Ready(Ok(()))
3283 ));
3284 drop(delivery);
3285 let reason = close.try_recv().expect("the full client must close");
3286 assert!(reason
3287 .to_string()
3288 .contains("module_to_client_delivery_failed"));
3289 assert!(!reason.to_string().contains("frame_too_large"));
3290 assert_eq!(
3291 router.counters.snapshot()["client_egress_close_delivery_failed"],
3292 1
3293 );
3294 let admitted = rx.try_recv().expect("the budget-sized frame was admitted");
3295 assert_eq!(admitted.header.ty, FrameType::Response);
3296 assert_eq!(admitted.header.corr, 201);
3297 assert!(
3298 rx.try_recv().is_err(),
3299 "no oversized-frame ERROR is substituted for backpressure"
3300 );
3301 }
3302
3303 #[tokio::test(start_paused = true)]
3304 async fn oversized_reply_to_a_full_client_waits_only_remaining_grace() {
3305 let (router, sink, _rx, ctx, routes, mut close) = multi_route_client(
3306 "full-oversize-provider",
3307 ConnectionId::new(90),
3308 ConnectionId::new(91),
3309 2,
3310 )
3311 .await;
3312 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3313 router
3315 .route_for_connection(
3316 &ctx,
3317 Frame::build(
3318 FrameType::Response,
3319 Flags::new(false, Priority::Interactive, true),
3320 routes[0].module_channel,
3321 routes[0].module_epoch,
3322 501,
3323 vec![b'x'; limit - subc_protocol::HEADER_LEN],
3324 )
3325 .unwrap(),
3326 )
3327 .await
3328 .unwrap();
3329 assert_eq!(sink.backlog().queued_bytes, limit);
3330 let oversized = Frame::build(
3331 FrameType::Response,
3332 Flags::new(false, Priority::Interactive, true),
3333 routes[1].module_channel,
3334 routes[1].module_epoch,
3335 502,
3336 vec![b'y'; limit - subc_protocol::HEADER_LEN + 1],
3337 )
3338 .unwrap();
3339 tokio::time::advance(Duration::from_millis(80)).await;
3342 let mut delivery = Box::pin(router.route_for_connection(&ctx, oversized));
3343 assert!(poll_once(delivery.as_mut()).await.is_pending());
3344 assert!(close.try_recv().is_err());
3345 tokio::time::advance(Duration::from_millis(169)).await;
3346 assert!(poll_once(delivery.as_mut()).await.is_pending());
3347 assert!(close.try_recv().is_err());
3348 tokio::time::advance(Duration::from_millis(1)).await;
3349 assert!(matches!(
3350 poll_once(delivery.as_mut()).await,
3351 std::task::Poll::Ready(Ok(()))
3352 ));
3353 drop(delivery);
3354 let reason = close.try_recv().expect("the full client must close");
3355 assert!(reason
3356 .to_string()
3357 .contains("module_to_client_delivery_failed"));
3358 }
3359
3360 #[tokio::test]
3361 async fn oversized_reverse_request_errors_module_without_reaching_client() {
3362 let (router, _sink, mut rx, mut ctx, routes, mut close) = multi_route_client(
3363 "reverse-oversize-provider",
3364 ConnectionId::new(84),
3365 ConnectionId::new(85),
3366 2,
3367 )
3368 .await;
3369 let (module_tx, mut module_rx) = mpsc::channel(8);
3370 ctx.egress = FrameSink::new(module_tx);
3371 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3372 router
3373 .route_for_connection(
3374 &ctx,
3375 Frame::build(
3376 FrameType::Request,
3377 Flags::new(false, Priority::Interactive, false),
3378 routes[0].module_channel,
3379 routes[0].module_epoch,
3380 301,
3381 vec![b'r'; limit - subc_protocol::HEADER_LEN + 1],
3382 )
3383 .unwrap(),
3384 )
3385 .await
3386 .unwrap();
3387 assert!(
3388 rx.try_recv().is_err(),
3389 "the reverse request never reaches the client"
3390 );
3391 let error = module_rx.try_recv().expect("the module gets the refusal");
3392 assert_frame_size_error(
3393 &error,
3394 routes[0].module_channel,
3395 routes[0].module_epoch,
3396 301,
3397 limit + 1,
3398 );
3399 assert!(module_rx.try_recv().is_err());
3400 for (index, route) in routes.iter().enumerate() {
3401 module_route_for_test(&router, &ctx, route);
3402 router
3403 .route_for_connection(
3404 &ctx,
3405 route_frame(
3406 FrameType::Response,
3407 route.module_channel,
3408 route.module_epoch,
3409 302 + index as u64,
3410 ),
3411 )
3412 .await
3413 .unwrap();
3414 let reply = rx.try_recv().expect("both routes still deliver");
3415 assert_eq!(reply.header.ty, FrameType::Response);
3416 assert_eq!(reply.header.channel, route.client_channel);
3417 }
3418 assert!(close.try_recv().is_err());
3419 assert_eq!(
3420 router.counters.snapshot()["client_egress_close_delivery_failed"],
3421 0
3422 );
3423 }
3424
3425 #[tokio::test]
3426 async fn oversized_push_closes_client_with_precise_warn() {
3427 let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3428 let (router, _sink, mut rx, ctx, routes, mut close) = multi_route_client(
3429 "push-oversize-provider",
3430 ConnectionId::new(86),
3431 ConnectionId::new(87),
3432 2,
3433 )
3434 .await;
3435 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3436 router
3437 .route_for_connection(
3438 &ctx,
3439 Frame::build(
3440 FrameType::Push,
3441 Flags::new(false, Priority::Interactive, false),
3442 routes[0].module_channel,
3443 routes[0].module_epoch,
3444 0,
3445 vec![b'p'; limit - subc_protocol::HEADER_LEN + 1],
3446 )
3447 .unwrap(),
3448 )
3449 .await
3450 .unwrap();
3451 let reason = close
3452 .try_recv()
3453 .expect("unsolicited oversize still closes the client");
3454 assert!(reason
3455 .to_string()
3456 .contains("frame_too_large: oversized push"));
3457 assert!(
3458 rx.try_recv().is_err(),
3459 "no part of the oversized push is forwarded"
3460 );
3461 let captured = test_log::captured_logs(&logs);
3462 let warnings: Vec<_> = captured
3463 .lines()
3464 .filter(|line| line.contains("WARN"))
3465 .collect();
3466 assert_eq!(warnings.len(), 1, "{captured}");
3467 let warning = warnings[0];
3468 for expected in [
3469 "oversized push".to_string(),
3470 "module_id=\"push-oversize-provider\"".to_string(),
3471 "principals=direct".to_string(),
3472 format!("client_channel={}", routes[0].client_channel),
3473 format!("size={}", limit + 1),
3474 format!("limit={limit}"),
3475 ] {
3476 assert!(warning.contains(&expected), "missing {expected}: {warning}");
3477 }
3478 assert_eq!(
3479 router.counters.snapshot()["client_egress_close_delivery_failed"],
3480 1
3481 );
3482 }
3483
3484 #[tokio::test]
3485 async fn oversized_stream_and_error_frames_end_request_even_with_queued_traffic() {
3486 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3487 "stream-oversize-provider",
3488 ConnectionId::new(88),
3489 ConnectionId::new(89),
3490 2,
3491 )
3492 .await;
3493 let binding = module_route_for_test(&router, &ctx, &routes[0]);
3494 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3495 for ty in [
3496 FrameType::StreamData,
3497 FrameType::StreamEnd,
3498 FrameType::Error,
3499 ] {
3500 binding.flow.acquire_tagged(401, false).await.unwrap();
3501 sink.try_send(route_frame(
3502 FrameType::Response,
3503 routes[1].client_channel,
3504 routes[1].client_epoch,
3505 402,
3506 ))
3507 .unwrap();
3508 router
3509 .route_for_connection(
3510 &ctx,
3511 Frame::build(
3512 ty,
3513 Flags::new(false, Priority::Interactive, false),
3514 routes[0].module_channel,
3515 routes[0].module_epoch,
3516 401,
3517 vec![b's'; limit - subc_protocol::HEADER_LEN + 1],
3518 )
3519 .unwrap(),
3520 )
3521 .await
3522 .unwrap();
3523 assert!(close.try_recv().is_err());
3524 assert_eq!(rx.try_recv().unwrap().header.corr, 402);
3525 let error = rx
3526 .try_recv()
3527 .expect("every correlated oversize frame becomes ERROR");
3528 assert_frame_size_error(
3529 &error,
3530 routes[0].client_channel,
3531 routes[0].client_epoch,
3532 401,
3533 limit + 1,
3534 );
3535 assert_eq!(binding.flow.drain_in_flight(), 0, "{ty:?} ends the request");
3536 assert!(rx.try_recv().is_err());
3537 }
3538 }
3539
3540 #[tokio::test]
3546 async fn awaited_send_parked_behind_a_full_byte_budget_wakes_when_bytes_free() {
3547 let (tx, mut rx) = mpsc::channel(1024);
3548 let sink = FrameSink::with_byte_budget(tx, 2_000);
3549 let mut queued = 0u64;
3550 while sink
3551 .try_send(stream_frame(7, 1, queued, vec![b'x'; 200]))
3552 .is_ok()
3553 {
3554 queued += 1;
3555 }
3556 assert!(queued > 0, "the budget admitted nothing");
3557
3558 let parked = tokio::spawn({
3559 let sink = sink.clone();
3560 async move { sink.send(stream_frame(0, 0, 999, vec![b'r'; 200])).await }
3561 });
3562 tokio::time::sleep(Duration::from_millis(100)).await;
3563 assert!(
3564 !parked.is_finished(),
3565 "the awaited send must wait while the byte budget is full"
3566 );
3567
3568 drop(rx.recv().await.expect("a queued frame"));
3570 tokio::time::timeout(Duration::from_secs(2), parked)
3571 .await
3572 .expect("the parked send was never woken after bytes were freed")
3573 .unwrap()
3574 .unwrap();
3575 }
3576
3577 #[tokio::test]
3582 async fn paused_client_reader_keeps_connection_through_small_frame_burst() {
3583 const ROUTES: usize = 4;
3584 const FRAMES_PER_ROUTE: usize = 250;
3585 let (router, client_sink, mut client_rx, module_ctx, routes, mut close_receiver) =
3586 multi_route_client(
3587 "burst-provider",
3588 ConnectionId::new(60),
3589 ConnectionId::new(61),
3590 ROUTES,
3591 )
3592 .await;
3593
3594 for seq in 0..FRAMES_PER_ROUTE as u64 {
3596 for (index, route) in routes.iter().enumerate() {
3597 let body = format!("route-{index}-token-{seq:05}-{}", "t".repeat(170));
3598 router
3599 .route_for_connection(
3600 &module_ctx,
3601 stream_frame(
3602 route.module_channel,
3603 route.module_epoch,
3604 seq,
3605 body.into_bytes(),
3606 ),
3607 )
3608 .await
3609 .unwrap();
3610 }
3611 }
3612
3613 let backlog = client_sink.backlog();
3614 assert_eq!(backlog.queued_frames, ROUTES * FRAMES_PER_ROUTE);
3615 assert!(backlog.queued_bytes < crate::server::CONNECTION_EGRESS_BYTE_BUDGET);
3616 assert!(
3617 close_receiver.try_recv().is_err(),
3618 "a paused reader under the byte budget must not be closed"
3619 );
3620 assert_eq!(
3621 router.counters.snapshot()["client_egress_close_delivery_failed"],
3622 0
3623 );
3624
3625 let mut next_seq = vec![0u64; ROUTES];
3627 for _ in 0..ROUTES * FRAMES_PER_ROUTE {
3628 let frame = client_rx.try_recv().expect("every queued frame arrives");
3629 let index = routes
3630 .iter()
3631 .position(|route| route.client_channel == frame.header.channel)
3632 .expect("frame arrives on one of the bound client channels");
3633 assert_eq!(frame.header.corr, next_seq[index], "route {index} order");
3634 let expected_prefix = format!("route-{index}-token-{:05}-", next_seq[index]);
3635 assert!(frame.body.starts_with(expected_prefix.as_bytes()));
3636 next_seq[index] += 1;
3637 }
3638 assert!(client_rx.try_recv().is_err());
3639 assert_eq!(next_seq, vec![FRAMES_PER_ROUTE as u64; ROUTES]);
3640 assert_eq!(client_sink.backlog().queued_bytes, 0);
3641 }
3642
3643 #[tokio::test(start_paused = true)]
3646 async fn never_reading_client_is_closed_at_byte_budget_with_warn_diagnosis() {
3647 let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3648 const BODY: usize = 16 * 1024;
3649 let (router, client_sink, _client_rx, module_ctx, routes, mut close_receiver) =
3650 multi_route_client(
3651 "stuck-reader-provider",
3652 ConnectionId::new(70),
3653 ConnectionId::new(71),
3654 2,
3655 )
3656 .await;
3657
3658 let mut admitted = 0usize;
3659 let mut sent = 0u64;
3660 let started = tokio::time::Instant::now();
3661 while router.counters.snapshot()["client_egress_close_delivery_failed"] == 0 {
3662 assert!(sent < 1_000, "the byte budget never refused a frame");
3663 let route = &routes[(sent % 2) as usize];
3664 router
3665 .route_for_connection(
3666 &module_ctx,
3667 stream_frame(
3668 route.module_channel,
3669 route.module_epoch,
3670 sent,
3671 vec![b'z'; BODY],
3672 ),
3673 )
3674 .await
3675 .unwrap();
3676 sent += 1;
3677 admitted = client_sink.backlog().queued_frames;
3678 }
3679 assert_eq!(
3680 started.elapsed(),
3681 crate::server::CONNECTION_EGRESS_STALL_GRACE
3682 );
3683 let frame_bytes = subc_protocol::HEADER_LEN + BODY;
3685 assert_eq!(
3686 admitted,
3687 crate::server::CONNECTION_EGRESS_BYTE_BUDGET / frame_bytes
3688 );
3689 let reason = close_receiver
3690 .try_recv()
3691 .expect("the client connection must be asked to close");
3692 assert!(reason
3693 .to_string()
3694 .contains("module_to_client_delivery_failed"));
3695
3696 router
3698 .route_for_connection(
3699 &module_ctx,
3700 stream_frame(
3701 routes[0].module_channel,
3702 routes[0].module_epoch,
3703 sent,
3704 vec![b'z'; BODY],
3705 ),
3706 )
3707 .await
3708 .unwrap();
3709
3710 let captured = test_log::captured_logs(&logs);
3711 let warn_lines = captured
3712 .lines()
3713 .filter(|line| {
3714 line.contains("closing client connection: its egress queue could not take a frame")
3715 })
3716 .collect::<Vec<_>>();
3717 assert_eq!(warn_lines.len(), 1, "exactly one WARN, got: {captured}");
3718 let line = warn_lines[0];
3719 assert!(line.contains("WARN"), "{line}");
3720 assert!(line.contains("connection_id=71"), "{line}");
3721 assert!(
3722 line.contains("module_id=\"stuck-reader-provider\""),
3723 "{line}"
3724 );
3725 assert!(line.contains("client_channel="), "{line}");
3726 assert!(line.contains("principals=direct"), "{line}");
3727 let queued_bytes: usize = line
3728 .split("queued_bytes=")
3729 .nth(1)
3730 .and_then(|rest| rest.split_whitespace().next())
3731 .and_then(|value| value.parse().ok())
3732 .expect("queued_bytes is logged");
3733 assert_eq!(queued_bytes, admitted * frame_bytes);
3734 assert!(
3735 line.contains(&format!("queued_frames={admitted}")),
3736 "{line}"
3737 );
3738 assert!(line.contains("oldest_queued_ms="), "{line}");
3739 assert!(line.contains("oldest_queued_ms=250"), "{line}");
3740 assert!(line.contains("frame_type=Some(StreamData)"), "{line}");
3741 assert!(
3742 line.contains(&format!(
3743 "frame_flags=Some({})",
3744 Flags::new(false, Priority::Interactive, false).0
3745 )),
3746 "{line}"
3747 );
3748 assert!(
3749 line.contains("close_cause=\"waited_250_ms_without_room\""),
3750 "{line}"
3751 );
3752 assert!(
3753 !line.contains(&"z".repeat(20)),
3754 "body bytes must never be logged: {line}"
3755 );
3756 }
3757
3758 fn route_frame(ty: FrameType, channel: u16, epoch: u32, corr: u64) -> Frame {
3759 Frame::build(
3760 ty,
3761 Flags::new(false, Priority::Interactive, false),
3762 channel,
3763 epoch,
3764 corr,
3765 if ty == FrameType::Request || ty == FrameType::Response {
3766 b"route-body".to_vec()
3767 } else {
3768 Vec::new()
3769 },
3770 )
3771 .unwrap()
3772 }
3773
3774 type DynamicRouteFixture = (
3775 Router,
3776 Arc<ForwardingTable>,
3777 RouteCtx,
3778 mpsc::Receiver<crate::router::OutboundFrame>,
3779 RouteCtx,
3780 mpsc::Receiver<crate::router::OutboundFrame>,
3781 mpsc::Receiver<crate::router::OutboundFrame>,
3782 crate::forwarding::PendingRouteBindRelay,
3783 );
3784
3785 fn dynamic_route_fixture(commit: bool) -> DynamicRouteFixture {
3786 dynamic_route_fixture_with_busy(commit, false)
3787 }
3788
3789 fn dynamic_route_fixture_with_busy(commit: bool, busy_refusals: bool) -> DynamicRouteFixture {
3790 let forwarding = Arc::new(ForwardingTable::default());
3791 let control = Arc::new(crate::ControlHandler::with_forwarding(
3792 Arc::new(crate::Registry::default()),
3793 Arc::clone(&forwarding),
3794 ));
3795 let router = Router::with_control_handler(control);
3796 let module_connection = ConnectionId::new(500);
3797 let client_connection = ConnectionId::new(501);
3798 let (module_tx, module_rx) = mpsc::channel(if busy_refusals { 64 } else { 8 });
3799 forwarding
3800 .register_module_connection(
3801 module_connection,
3802 "epoch-router".into(),
3803 2,
3804 Concurrency::ModuleManaged,
3805 FrameSink::new(module_tx),
3806 )
3807 .unwrap();
3808 let (client_tx, client_rx) = mpsc::channel(if busy_refusals { 64 } else { 8 });
3809 let client_sink = FrameSink::new(client_tx);
3810 let pending = forwarding
3811 .begin_route_bind_relay_with_busy_for_test(
3812 client_connection,
3813 client_sink.clone(),
3814 700,
3815 "epoch-router",
3816 busy_refusals,
3817 )
3818 .unwrap();
3819 if commit {
3820 forwarding
3821 .complete_pending_relay(
3822 module_connection,
3823 pending.corr,
3824 RouteBindRelayOutcome::Accepted,
3825 )
3826 .unwrap();
3827 }
3828 let (module_egress_tx, module_egress_rx) = mpsc::channel(8);
3829 (
3830 router,
3831 forwarding,
3832 RouteCtx {
3833 connection_id: client_connection,
3834 egress: client_sink,
3835 },
3836 client_rx,
3837 RouteCtx {
3838 connection_id: module_connection,
3839 egress: FrameSink::new(module_egress_tx),
3840 },
3841 module_egress_rx,
3842 module_rx,
3843 pending,
3844 )
3845 }
3846
3847 #[tokio::test]
3848 async fn route_epochs_validate_both_directions_and_rewrite_to_peer_handle() {
3849 let (
3850 router,
3851 _forwarding,
3852 client_ctx,
3853 mut client_rx,
3854 module_ctx,
3855 _module_egress_rx,
3856 mut module_rx,
3857 pending,
3858 ) = dynamic_route_fixture(true);
3859 let route_open = client_rx.recv().await.unwrap();
3860 assert_eq!(route_open.header.corr, 700);
3861
3862 router
3863 .route_for_connection(
3864 &client_ctx,
3865 route_frame(
3866 FrameType::Request,
3867 pending.client_channel,
3868 pending.client_epoch,
3869 701,
3870 ),
3871 )
3872 .await
3873 .unwrap();
3874 let forwarded = module_rx.recv().await.unwrap();
3875 assert_eq!(forwarded.header.channel, pending.module_channel);
3876 assert_eq!(forwarded.header.epoch, pending.module_epoch);
3877
3878 router
3879 .route_for_connection(
3880 &module_ctx,
3881 route_frame(
3882 FrameType::Response,
3883 pending.module_channel,
3884 pending.module_epoch,
3885 701,
3886 ),
3887 )
3888 .await
3889 .unwrap();
3890 let delivered = client_rx.recv().await.unwrap();
3891 assert_eq!(delivered.header.channel, pending.client_channel);
3892 assert_eq!(delivered.header.epoch, pending.client_epoch);
3893
3894 router
3895 .route_for_connection(
3896 &client_ctx,
3897 route_frame(
3898 FrameType::Request,
3899 pending.client_channel,
3900 pending.client_epoch + 1,
3901 702,
3902 ),
3903 )
3904 .await
3905 .unwrap();
3906 router
3907 .route_for_connection(
3908 &module_ctx,
3909 route_frame(
3910 FrameType::Response,
3911 pending.module_channel,
3912 pending.module_epoch + 1,
3913 703,
3914 ),
3915 )
3916 .await
3917 .unwrap();
3918 let stale_error = client_rx.recv().await.unwrap();
3919 assert_eq!(stale_error.header.ty, FrameType::Error);
3920 assert_eq!(stale_error.header.channel, pending.client_channel);
3921 assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
3922 assert_eq!(stale_error.header.corr, 702);
3923 let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
3924 assert_eq!(body.code, "stale_route_epoch");
3925 assert!(module_rx.try_recv().is_err());
3926 assert!(client_rx.try_recv().is_err());
3927 let counters = router.counters.snapshot();
3928 assert_eq!(counters["client_frames_dropped_stale_route"], 1);
3929 assert_eq!(counters["module_frames_dropped_no_route"], 1);
3930 }
3931
3932 #[tokio::test]
3933 async fn accepted_route_publishes_route_open_before_immediate_reverse_request() {
3934 let (
3935 router,
3936 _,
3937 _client_ctx,
3938 mut client_rx,
3939 module_ctx,
3940 _module_egress_rx,
3941 _module_rx,
3942 pending,
3943 ) = dynamic_route_fixture(true);
3944 router
3945 .route_for_connection(
3946 &module_ctx,
3947 route_frame(
3948 FrameType::Request,
3949 pending.module_channel,
3950 pending.module_epoch,
3951 800,
3952 ),
3953 )
3954 .await
3955 .unwrap();
3956
3957 let first = client_rx.recv().await.unwrap();
3958 let second = client_rx.recv().await.unwrap();
3959 assert_eq!(first.header.channel, 0);
3960 assert_eq!(first.header.corr, 700);
3961 assert_eq!(second.header.channel, pending.client_channel);
3962 assert_eq!(second.header.epoch, pending.client_epoch);
3963 assert_eq!(second.header.corr, 800);
3964 }
3965
3966 #[tokio::test]
3967 async fn reserved_slot_ingress_errors_only_matching_client_requests() {
3968 let (
3969 router,
3970 _forwarding,
3971 client_ctx,
3972 mut client_rx,
3973 _module_ctx,
3974 _module_egress_rx,
3975 mut module_rx,
3976 pending,
3977 ) = dynamic_route_fixture(false);
3978 router
3979 .route_for_connection(
3980 &client_ctx,
3981 route_frame(
3982 FrameType::Request,
3983 pending.client_channel,
3984 pending.client_epoch,
3985 900,
3986 ),
3987 )
3988 .await
3989 .unwrap();
3990 let error = client_rx.recv().await.unwrap();
3991 assert_eq!(error.header.ty, FrameType::Error);
3992 assert_eq!(error.header.channel, pending.client_channel);
3993 assert_eq!(error.header.epoch, pending.client_epoch);
3994 assert_eq!(error.header.corr, 900);
3995
3996 router
3997 .route_for_connection(
3998 &client_ctx,
3999 route_frame(
4000 FrameType::Response,
4001 pending.client_channel,
4002 pending.client_epoch,
4003 901,
4004 ),
4005 )
4006 .await
4007 .unwrap();
4008 router
4009 .route_for_connection(
4010 &client_ctx,
4011 route_frame(
4012 FrameType::Request,
4013 pending.client_channel,
4014 pending.client_epoch + 1,
4015 902,
4016 ),
4017 )
4018 .await
4019 .unwrap();
4020 let stale_error = client_rx.recv().await.unwrap();
4021 assert_eq!(stale_error.header.ty, FrameType::Error);
4022 assert_eq!(stale_error.header.channel, pending.client_channel);
4023 assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
4024 assert_eq!(stale_error.header.corr, 902);
4025 let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
4026 assert_eq!(body.code, "stale_route_epoch");
4027 assert!(module_rx.try_recv().is_err());
4028 let counters = router.counters.snapshot();
4029 assert_eq!(counters["client_frames_dropped_stale_route"], 1);
4030 assert_eq!(counters["module_frames_dropped_no_route"], 0);
4031 }
4032
4033 #[tokio::test]
4034 async fn dropped_module_route_goodbye_increments_counter() {
4035 let (
4036 router,
4037 _forwarding,
4038 client_ctx,
4039 mut client_rx,
4040 _module_ctx,
4041 _module_egress_rx,
4042 mut module_rx,
4043 pending,
4044 ) = dynamic_route_fixture(true);
4045 let _ = client_rx.recv().await;
4046 module_rx.close();
4047
4048 router
4049 .route_for_connection(
4050 &client_ctx,
4051 route_frame(
4052 FrameType::Goodbye,
4053 pending.client_channel,
4054 pending.client_epoch,
4055 999,
4056 ),
4057 )
4058 .await
4059 .unwrap();
4060
4061 let counters = router.counters.snapshot();
4062 assert_eq!(counters["goodbye_relay_module_dropped"], 1);
4063 assert_eq!(
4064 counters["goodbye_relay_module_dropped_by_module"],
4065 serde_json::json!({ "epoch-router": 1 })
4066 );
4067 assert_eq!(counters["route_released_epoch_fenced"], 1);
4068 }
4069
4070 #[tokio::test]
4071 async fn module_request_on_stale_epoch_receives_stale_route_epoch() {
4072 let (
4073 router,
4074 _forwarding,
4075 _client_ctx,
4076 _client_rx,
4077 module_ctx,
4078 mut module_egress_rx,
4079 mut module_rx,
4080 pending,
4081 ) = dynamic_route_fixture(true);
4082
4083 router
4084 .route_for_connection(
4085 &module_ctx,
4086 route_frame(
4087 FrameType::Request,
4088 pending.module_channel,
4089 pending.module_epoch + 1,
4090 1_000,
4091 ),
4092 )
4093 .await
4094 .unwrap();
4095
4096 let error = module_egress_rx.try_recv().unwrap();
4097 assert_eq!(error.header.ty, FrameType::Error);
4098 assert_eq!(error.header.channel, pending.module_channel);
4099 assert_eq!(error.header.epoch, pending.module_epoch + 1);
4100 assert_eq!(error.header.corr, 1_000);
4101 let body: ErrorBody = serde_json::from_slice(&error.body).unwrap();
4102 assert_eq!(body.code, "stale_route_epoch");
4103 assert!(module_rx.try_recv().is_err());
4104 let counters = router.counters.snapshot();
4105 assert_eq!(counters["module_requests_dropped_stale_route"], 1);
4106 assert_eq!(counters["module_frames_dropped_no_route"], 0);
4107 }
4108
4109 #[tokio::test(start_paused = true)]
4110 async fn stale_epoch_drop_warns_once_per_minute_with_suppressed_count() {
4111 let (output, _guard) = test_log::log_capture(tracing::Level::WARN);
4112 let (
4113 router,
4114 _forwarding,
4115 client_ctx,
4116 mut client_rx,
4117 _module_ctx,
4118 _module_egress_rx,
4119 mut module_rx,
4120 pending,
4121 ) = dynamic_route_fixture(true);
4122 let _ = client_rx.recv().await;
4123 for corr in 1..=3 {
4124 router
4125 .route_for_connection(
4126 &client_ctx,
4127 route_frame(
4128 FrameType::Request,
4129 pending.client_channel,
4130 pending.client_epoch + 1,
4131 corr,
4132 ),
4133 )
4134 .await
4135 .unwrap();
4136 assert_eq!(client_rx.recv().await.unwrap().header.ty, FrameType::Error);
4137 }
4138 assert!(module_rx.try_recv().is_err());
4139 let logs = test_log::captured_logs(&output);
4140 assert_eq!(
4141 logs.matches("dropping data-plane frame").count(),
4142 1,
4143 "{logs}"
4144 );
4145 assert!(logs.contains("WARN"), "{logs}");
4146 assert!(
4147 logs.contains(&format!("connection_id={}", client_ctx.connection_id.get())),
4148 "{logs}"
4149 );
4150 assert!(
4151 logs.contains(&format!("channel={}", pending.client_channel)),
4152 "{logs}"
4153 );
4154 assert!(
4155 logs.contains(&format!("epoch={}", pending.client_epoch + 1)),
4156 "{logs}"
4157 );
4158 assert!(
4159 logs.contains("client_to_module") && logs.contains("stale_epoch"),
4160 "{logs}"
4161 );
4162 assert!(logs.contains("suppressed=0"), "{logs}");
4163
4164 tokio::time::advance(Duration::from_secs(60)).await;
4165 router
4166 .route_for_connection(
4167 &client_ctx,
4168 route_frame(
4169 FrameType::Request,
4170 pending.client_channel,
4171 pending.client_epoch + 2,
4172 4,
4173 ),
4174 )
4175 .await
4176 .unwrap();
4177 let logs = test_log::captured_logs(&output);
4178 assert_eq!(
4179 logs.matches("dropping data-plane frame").count(),
4180 2,
4181 "{logs}"
4182 );
4183 assert!(logs.contains("suppressed=2"), "{logs}");
4184 }
4185
4186 #[tokio::test]
4187 async fn module_request_on_reserved_or_absent_route_receives_unknown_channel() {
4188 let (
4189 reserved_router,
4190 _forwarding,
4191 _client_ctx,
4192 _client_rx,
4193 reserved_module_ctx,
4194 mut reserved_module_egress_rx,
4195 _module_rx,
4196 reserved,
4197 ) = dynamic_route_fixture(false);
4198 reserved_router
4199 .route_for_connection(
4200 &reserved_module_ctx,
4201 route_frame(
4202 FrameType::Request,
4203 reserved.module_channel,
4204 reserved.module_epoch,
4205 1_001,
4206 ),
4207 )
4208 .await
4209 .unwrap();
4210 let reserved_error = reserved_module_egress_rx.try_recv().unwrap();
4211 let reserved_body: ErrorBody = serde_json::from_slice(&reserved_error.body).unwrap();
4212 assert_eq!(reserved_error.header.ty, FrameType::Error);
4213 assert_eq!(reserved_error.header.channel, reserved.module_channel);
4214 assert_eq!(reserved_error.header.epoch, reserved.module_epoch);
4215 assert_eq!(reserved_error.header.corr, 1_001);
4216 assert_eq!(reserved_body.code, "unknown_channel");
4217 assert_eq!(
4218 reserved_router.counters.snapshot()["module_requests_dropped_stale_route"],
4219 1
4220 );
4221
4222 let (
4223 absent_router,
4224 _forwarding,
4225 _client_ctx,
4226 _client_rx,
4227 absent_module_ctx,
4228 mut absent_module_egress_rx,
4229 _module_rx,
4230 absent,
4231 ) = dynamic_route_fixture(false);
4232 absent_router
4233 .route_for_connection(
4234 &absent_module_ctx,
4235 route_frame(
4236 FrameType::Request,
4237 absent.module_channel + 1,
4238 absent.module_epoch,
4239 1_002,
4240 ),
4241 )
4242 .await
4243 .unwrap();
4244 let absent_error = absent_module_egress_rx.try_recv().unwrap();
4245 let absent_body: ErrorBody = serde_json::from_slice(&absent_error.body).unwrap();
4246 assert_eq!(absent_error.header.ty, FrameType::Error);
4247 assert_eq!(absent_error.header.channel, absent.module_channel + 1);
4248 assert_eq!(absent_error.header.epoch, absent.module_epoch);
4249 assert_eq!(absent_error.header.corr, 1_002);
4250 assert_eq!(absent_body.code, "unknown_channel");
4251 assert_eq!(
4252 absent_router.counters.snapshot()["module_requests_dropped_stale_route"],
4253 1
4254 );
4255 }
4256
4257 #[tokio::test]
4258 async fn non_request_module_frame_on_dead_route_is_counted_without_error() {
4259 let (
4260 router,
4261 forwarding,
4262 client_ctx,
4263 mut client_rx,
4264 module_ctx,
4265 mut module_egress_rx,
4266 mut module_rx,
4267 pending,
4268 ) = dynamic_route_fixture(true);
4269 let (other_module_tx, _other_module_rx) = mpsc::channel(8);
4270 forwarding
4271 .register_module_connection(
4272 ConnectionId::new(502),
4273 "other-module".into(),
4274 2,
4275 Concurrency::ModuleManaged,
4276 FrameSink::new(other_module_tx),
4277 )
4278 .unwrap();
4279 let _ = client_rx.recv().await.unwrap();
4280
4281 router
4282 .route_for_connection(
4283 &client_ctx,
4284 route_frame(
4285 FrameType::Goodbye,
4286 pending.client_channel,
4287 pending.client_epoch,
4288 1_003,
4289 ),
4290 )
4291 .await
4292 .unwrap();
4293 let _ = module_rx.recv().await.unwrap();
4294
4295 router
4296 .route_for_connection(
4297 &module_ctx,
4298 route_frame(
4299 FrameType::StreamData,
4300 pending.module_channel,
4301 pending.module_epoch,
4302 1_004,
4303 ),
4304 )
4305 .await
4306 .unwrap();
4307
4308 let reply = module_egress_rx.try_recv().unwrap();
4311 assert_eq!(reply.header.ty, FrameType::Goodbye);
4312 assert!(module_egress_rx.try_recv().is_err());
4313 let counters = router.counters.snapshot();
4314 assert_eq!(counters["module_frames_dropped_no_route"], 1);
4315 assert_eq!(
4316 counters["module_frames_dropped_no_route_by_module"],
4317 serde_json::json!({ "epoch-router": 1 })
4318 );
4319 assert_eq!(counters["module_requests_dropped_stale_route"], 0);
4320 }
4321
4322 fn drain_now(rx: &mut mpsc::Receiver<OutboundFrame>) {
4325 while rx.try_recv().is_ok() {}
4326 }
4327
4328 #[tokio::test]
4334 async fn route_goodbye_refused_by_stalled_module_is_delivered_when_it_resumes_reading() {
4335 const BUDGET: usize = 4_096;
4336 let forwarding = Arc::new(ForwardingTable::default());
4337 let control = Arc::new(ControlHandler::with_forwarding(
4338 Arc::new(Registry::default()),
4339 Arc::clone(&forwarding),
4340 ));
4341 let router = Router::with_control_handler(control);
4342 let module_connection = ConnectionId::new(80);
4343 let client_connection = ConnectionId::new(81);
4344 let (module_tx, mut module_rx) = mpsc::channel(64);
4345 let module_sink = FrameSink::with_byte_budget(module_tx, BUDGET);
4346 forwarding
4347 .register_module_connection(
4348 module_connection,
4349 "stalled-provider".into(),
4350 2,
4351 Concurrency::ModuleManaged,
4352 module_sink.clone(),
4353 )
4354 .unwrap();
4355 let (client_tx, mut client_rx) = mpsc::channel(8);
4356 let client_sink = FrameSink::new(client_tx);
4357 let pending = forwarding
4358 .begin_route_bind_relay_for_test(
4359 client_connection,
4360 client_sink.clone(),
4361 1_100,
4362 "stalled-provider",
4363 )
4364 .unwrap();
4365 forwarding
4366 .complete_pending_relay(
4367 module_connection,
4368 pending.corr,
4369 RouteBindRelayOutcome::Accepted,
4370 )
4371 .unwrap();
4372 let _ = client_rx.recv().await.unwrap();
4373 drain_now(&mut module_rx);
4374
4375 module_sink
4378 .try_send(stream_frame(9, 1, 0, vec![b'f'; BUDGET]))
4379 .unwrap();
4380 assert!(module_sink
4381 .try_send(stream_frame(9, 1, 1, Vec::new()))
4382 .is_err());
4383
4384 let client_ctx = RouteCtx {
4385 connection_id: client_connection,
4386 egress: client_sink,
4387 };
4388 router
4389 .route_for_connection(
4390 &client_ctx,
4391 route_frame(
4392 FrameType::Goodbye,
4393 pending.client_channel,
4394 pending.client_epoch,
4395 1_101,
4396 ),
4397 )
4398 .await
4399 .unwrap();
4400 tokio::task::yield_now().await;
4401 assert_eq!(
4402 router.counters.snapshot()["goodbye_relay_module_dropped"],
4403 0,
4404 "a GOODBYE refused by a momentarily full module queue must not be dropped"
4405 );
4406
4407 let filler = module_rx.recv().await.unwrap();
4409 assert_eq!(filler.header.ty, FrameType::StreamData);
4410 drop(filler);
4411 let goodbye = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
4412 .await
4413 .expect("the refused GOODBYE must be delivered once the module frees room")
4414 .unwrap();
4415 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
4416 assert_eq!(goodbye.header.channel, pending.module_channel);
4417 assert_eq!(goodbye.header.epoch, pending.module_epoch);
4418 assert_eq!(
4419 router.counters.snapshot()["goodbye_relay_module_dropped"],
4420 0
4421 );
4422 }
4423
4424 #[tokio::test]
4430 async fn module_frame_on_route_the_daemon_does_not_hold_is_answered_with_goodbye() {
4431 let (
4432 router,
4433 _forwarding,
4434 client_ctx,
4435 mut client_rx,
4436 module_ctx,
4437 mut module_egress_rx,
4438 mut module_rx,
4439 pending,
4440 ) = dynamic_route_fixture(true);
4441 let _ = client_rx.recv().await.unwrap();
4442 router
4443 .route_for_connection(
4444 &client_ctx,
4445 route_frame(
4446 FrameType::Goodbye,
4447 pending.client_channel,
4448 pending.client_epoch,
4449 1_200,
4450 ),
4451 )
4452 .await
4453 .unwrap();
4454 drain_now(&mut module_rx);
4455
4456 let cases = [
4457 (pending.module_channel, pending.module_epoch),
4459 (pending.module_channel + 1, 1),
4461 ];
4462 for (channel, epoch) in cases {
4463 router
4464 .route_for_connection(
4465 &module_ctx,
4466 route_frame(FrameType::StreamData, channel, epoch, 1_201),
4467 )
4468 .await
4469 .unwrap();
4470 let reply = module_egress_rx
4471 .try_recv()
4472 .expect("a frame on a route the daemon does not hold is answered");
4473 assert_eq!(reply.header.ty, FrameType::Goodbye);
4474 assert_eq!(reply.header.channel, channel);
4475 assert_eq!(reply.header.epoch, epoch);
4476 assert_eq!(reply.header.corr, 0);
4477 assert!(module_egress_rx.try_recv().is_err());
4478 }
4479
4480 router
4483 .route_for_connection(
4484 &module_ctx,
4485 route_frame(FrameType::Goodbye, pending.module_channel + 2, 1, 0),
4486 )
4487 .await
4488 .unwrap();
4489 assert!(module_egress_rx.try_recv().is_err());
4490
4491 let counters = router.counters.snapshot();
4492 assert_eq!(counters["module_frames_dropped_no_route"], 3);
4493 assert_eq!(counters["module_frames_dropped_released_route"], 1);
4494 assert_eq!(
4495 counters["module_frames_dropped_released_route_by_module"],
4496 serde_json::json!({ "epoch-router": 1 })
4497 );
4498 assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4499 }
4500
4501 #[tokio::test(start_paused = true)]
4505 async fn burst_of_orphan_module_frames_is_answered_once_per_interval() {
4506 let (
4507 router,
4508 _forwarding,
4509 client_ctx,
4510 mut client_rx,
4511 module_ctx,
4512 mut module_egress_rx,
4513 mut module_rx,
4514 pending,
4515 ) = dynamic_route_fixture(true);
4516 let _ = client_rx.recv().await.unwrap();
4517 router
4518 .route_for_connection(
4519 &client_ctx,
4520 route_frame(
4521 FrameType::Goodbye,
4522 pending.client_channel,
4523 pending.client_epoch,
4524 1_300,
4525 ),
4526 )
4527 .await
4528 .unwrap();
4529 drain_now(&mut module_rx);
4530
4531 let send_burst = |corr: u64| {
4532 route_frame(
4533 FrameType::StreamData,
4534 pending.module_channel,
4535 pending.module_epoch,
4536 corr,
4537 )
4538 };
4539 for corr in 0..20 {
4540 router
4541 .route_for_connection(&module_ctx, send_burst(corr))
4542 .await
4543 .unwrap();
4544 }
4545 let mut replies = 0;
4546 while let Ok(reply) = module_egress_rx.try_recv() {
4547 assert_eq!(reply.header.ty, FrameType::Goodbye);
4548 replies += 1;
4549 }
4550 assert_eq!(replies, 1, "a burst within the interval gets one GOODBYE");
4551
4552 tokio::time::advance(ORPHAN_ROUTE_GOODBYE_INTERVAL).await;
4553 router
4554 .route_for_connection(&module_ctx, send_burst(20))
4555 .await
4556 .unwrap();
4557 let retry = module_egress_rx
4558 .try_recv()
4559 .expect("the first orphan frame after the interval is answered again");
4560 assert_eq!(retry.header.channel, pending.module_channel);
4561 assert_eq!(retry.header.epoch, pending.module_epoch);
4562
4563 let counters = router.counters.snapshot();
4564 assert_eq!(counters["module_frames_dropped_no_route"], 21);
4565 assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4566 }
4567
4568 #[test]
4570 fn orphan_goodbye_rate_limit_state_is_released_with_the_connection() {
4571 let router = Router::with_default_self_handler();
4572 let connection = router.begin_connection();
4573 let id = connection.id();
4574 assert!(router.orphan_goodbyes.claim(id, 7));
4575 assert!(!router.orphan_goodbyes.claim(id, 7));
4576 assert!(router.orphan_goodbyes.claim(id, 8));
4577 drop(connection);
4578 assert!(router
4579 .orphan_goodbyes
4580 .last_sent
4581 .lock()
4582 .unwrap()
4583 .get(&id)
4584 .is_none());
4585 }
4586
4587 #[test]
4588 fn channel_zero_cannot_be_registered_as_backend() {
4589 let mut router = Router::with_default_self_handler();
4590
4591 let err = router.register_backend(0, EchoBackend).unwrap_err();
4592
4593 assert_eq!(err, RouterError::ReservedChannelZero);
4594 }
4595}