1use std::{
2 collections::HashMap,
3 error::Error,
4 fmt,
5 sync::{
6 atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
7 Arc, Mutex,
8 },
9 time::{Duration, Instant},
10};
11
12use subc_protocol::{error_codes, ErrorBody, Flags, FrameType, Priority};
13use tokio::sync::{mpsc, Notify};
14use tracing::{debug, warn};
15
16use crate::{
17 control::ControlHandler,
18 forwarding::{
19 principal_label, CloseReason, ConnectionCloseReceiver, DataRoute, DataRouteState,
20 ForwardingError, ForwardingTable, RouteBinding, RouteRelease, UndeliveredFrame,
21 },
22 registry::ConnectionId,
23 DaemonCounters, Frame, FrameBuildError,
24};
25
26#[derive(Debug)]
35pub struct OutboundFrame {
36 pub frame: Frame,
37 pub enqueued_at: std::time::Instant,
38 pub(crate) flushed: Option<tokio::sync::oneshot::Sender<()>>,
39 pub(crate) charge: Option<EgressCharge>,
44}
45
46impl OutboundFrame {
47 fn charged(frame: Frame, charge: EgressCharge) -> Self {
48 Self {
49 frame,
50 enqueued_at: charge.enqueued_at,
51 flushed: None,
52 charge: Some(charge),
53 }
54 }
55}
56
57fn queued_frame_bytes(frame: &Frame) -> usize {
60 subc_protocol::HEADER_LEN + frame.body.len()
61}
62
63#[derive(Debug, Clone, Copy, PartialEq, Eq)]
66pub(crate) struct EgressBacklog {
67 pub queued_bytes: usize,
68 pub queued_frames: usize,
69 pub oldest_age: Option<Duration>,
76}
77
78#[derive(Debug)]
83struct EgressAccounting {
84 byte_budget: usize,
85 queued_bytes: AtomicUsize,
86 queued_frames: AtomicUsize,
87 time_base: Instant,
89 oldest_enqueued_nanos: AtomicU64,
92 waiters: AtomicUsize,
95 freed: Notify,
98 client_delivery_closed: AtomicBool,
99 client_delivery_stopped: Notify,
100}
101
102impl EgressAccounting {
103 fn new(byte_budget: usize) -> Self {
104 Self {
105 byte_budget,
106 queued_bytes: AtomicUsize::new(0),
107 queued_frames: AtomicUsize::new(0),
108 time_base: tokio::time::Instant::now().into_std(),
109 oldest_enqueued_nanos: AtomicU64::new(0),
110 waiters: AtomicUsize::new(0),
111 freed: Notify::new(),
112 client_delivery_closed: AtomicBool::new(false),
113 client_delivery_stopped: Notify::new(),
114 }
115 }
116
117 fn stamp(&self, at: Instant) -> u64 {
118 (at.saturating_duration_since(self.time_base).as_nanos() as u64).saturating_add(1)
119 }
120
121 fn try_charge(self: &Arc<Self>, bytes: usize) -> Option<EgressCharge> {
126 let mut current = self.queued_bytes.load(Ordering::SeqCst);
129 loop {
130 if current != 0 && current.saturating_add(bytes) > self.byte_budget {
131 return None;
132 }
133 match self.queued_bytes.compare_exchange_weak(
134 current,
135 current + bytes,
136 Ordering::SeqCst,
137 Ordering::SeqCst,
138 ) {
139 Ok(_) => return Some(self.record(bytes)),
140 Err(actual) => current = actual,
141 }
142 }
143 }
144
145 fn charge_unconditionally(self: &Arc<Self>, bytes: usize) -> EgressCharge {
148 self.queued_bytes.fetch_add(bytes, Ordering::SeqCst);
149 self.record(bytes)
150 }
151
152 fn record(self: &Arc<Self>, bytes: usize) -> EgressCharge {
153 let enqueued_at = tokio::time::Instant::now().into_std();
154 let stamp = self.stamp(enqueued_at);
155 if self.queued_frames.fetch_add(1, Ordering::AcqRel) == 0 {
156 self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
159 }
160 EgressCharge {
161 accounting: Arc::clone(self),
162 bytes,
163 stamp,
164 enqueued_at,
165 }
166 }
167
168 fn taken_by_writer(&self, stamp: u64) {
170 self.oldest_enqueued_nanos.store(stamp, Ordering::Release);
171 }
172
173 fn release(&self, bytes: usize, stamp: u64) {
174 self.queued_bytes.fetch_sub(bytes, Ordering::SeqCst);
175 if self.queued_frames.fetch_sub(1, Ordering::AcqRel) == 1 {
176 let _ = self.oldest_enqueued_nanos.compare_exchange(
179 stamp,
180 0,
181 Ordering::AcqRel,
182 Ordering::Acquire,
183 );
184 }
185 if self.waiters.load(Ordering::SeqCst) != 0 {
186 self.freed.notify_waiters();
187 }
188 }
189
190 fn backlog(&self) -> EgressBacklog {
191 let queued_frames = self.queued_frames.load(Ordering::Acquire);
192 let stamp = self.oldest_enqueued_nanos.load(Ordering::Acquire);
193 let oldest_age = (queued_frames != 0 && stamp != 0).then(|| {
194 let enqueued = self.time_base + Duration::from_nanos(stamp - 1);
195 tokio::time::Instant::from_std(enqueued).elapsed()
196 });
197 EgressBacklog {
198 queued_bytes: self.queued_bytes.load(Ordering::Acquire),
199 queued_frames,
200 oldest_age,
201 }
202 }
203}
204
205#[derive(Debug)]
208pub(crate) struct EgressCharge {
209 accounting: Arc<EgressAccounting>,
210 bytes: usize,
211 stamp: u64,
212 enqueued_at: Instant,
213}
214
215impl EgressCharge {
216 pub(crate) fn taken_by_writer(&self) {
219 self.accounting.taken_by_writer(self.stamp);
220 }
221}
222
223impl Drop for EgressCharge {
224 fn drop(&mut self) {
225 self.accounting.release(self.bytes, self.stamp);
226 }
227}
228
229#[derive(Debug)]
235pub(crate) struct EgressPermit {
236 permit: mpsc::OwnedPermit<OutboundFrame>,
237 accounting: Arc<EgressAccounting>,
238}
239
240impl EgressPermit {
241 pub(crate) fn send(self, frame: Frame) -> bool {
244 let charge = self
245 .accounting
246 .charge_unconditionally(queued_frame_bytes(&frame));
247 let sender = self.permit.send(OutboundFrame::charged(frame, charge));
248 sender.is_closed()
249 }
250}
251
252impl std::ops::Deref for OutboundFrame {
255 type Target = Frame;
256
257 fn deref(&self) -> &Frame {
258 &self.frame
259 }
260}
261
262#[cfg(test)]
265pub(crate) mod test_log {
266 use std::{
267 io::Write,
268 sync::{Arc, Mutex, OnceLock},
269 };
270
271 #[derive(Clone)]
272 struct TestLogWriter(Arc<Mutex<Vec<u8>>>);
273
274 impl Write for TestLogWriter {
275 fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
276 self.0
277 .lock()
278 .expect("test log capture is not poisoned")
279 .extend(buffer);
280 Ok(buffer.len())
281 }
282
283 fn flush(&mut self) -> std::io::Result<()> {
284 Ok(())
285 }
286 }
287
288 pub(crate) fn log_capture(
289 level: tracing::Level,
290 ) -> (Arc<Mutex<Vec<u8>>>, tracing::dispatcher::DefaultGuard) {
291 static INTEREST_ANCHOR: OnceLock<tracing::Dispatch> = OnceLock::new();
298 INTEREST_ANCHOR.get_or_init(|| tracing::Dispatch::new(tracing_subscriber::registry()));
299 let output = Arc::new(Mutex::new(Vec::new()));
300 let writer = Arc::clone(&output);
301 let subscriber = tracing_subscriber::fmt()
302 .with_max_level(level)
303 .with_ansi(false)
304 .without_time()
305 .with_target(false)
306 .with_writer(move || TestLogWriter(Arc::clone(&writer)))
307 .finish();
308 let guard = tracing::subscriber::set_default(subscriber);
309 (output, guard)
310 }
311
312 pub(crate) fn captured_logs(output: &Arc<Mutex<Vec<u8>>>) -> String {
313 String::from_utf8(
314 output
315 .lock()
316 .expect("test log capture is not poisoned")
317 .clone(),
318 )
319 .expect("tracing output is UTF-8")
320 }
321
322 #[cfg(test)]
323 mod tests {
324 use super::*;
325
326 fn emit_capture_probe(source: &str) {
327 tracing::error!(source, "parallel capture probe");
328 }
329
330 #[test]
331 fn parallel_callsite_registration_keeps_the_test_capture_enabled() {
332 let (output, _guard) = log_capture(tracing::Level::ERROR);
333 std::thread::spawn(|| emit_capture_probe("other-thread"))
336 .join()
337 .expect("unsubscribed thread exits");
338 emit_capture_probe("captured-thread");
339
340 let logs = captured_logs(&output);
341 assert!(
342 logs.contains("captured-thread"),
343 "capture was disabled: {logs}"
344 );
345 assert!(!logs.contains("other-thread"), "capture leaked: {logs}");
346 }
347 }
348}
349
350#[derive(Debug, Clone)]
358pub struct FrameSink {
359 tx: mpsc::Sender<OutboundFrame>,
360 accounting: Arc<EgressAccounting>,
361}
362
363impl FrameSink {
364 pub fn new(tx: mpsc::Sender<OutboundFrame>) -> Self {
368 Self::with_byte_budget(tx, crate::server::CONNECTION_EGRESS_BYTE_BUDGET)
369 }
370
371 pub(crate) fn with_byte_budget(tx: mpsc::Sender<OutboundFrame>, byte_budget: usize) -> Self {
372 Self {
373 tx,
374 accounting: Arc::new(EgressAccounting::new(byte_budget)),
375 }
376 }
377
378 async fn charge_waiting(&self, bytes: usize) -> Option<EgressCharge> {
385 if let Some(charge) = self.accounting.try_charge(bytes) {
386 return Some(charge);
387 }
388 struct Waiting<'a>(&'a AtomicUsize);
391 impl Drop for Waiting<'_> {
392 fn drop(&mut self) {
393 self.0.fetch_sub(1, Ordering::SeqCst);
394 }
395 }
396 self.accounting.waiters.fetch_add(1, Ordering::SeqCst);
397 let _waiting = Waiting(&self.accounting.waiters);
398 loop {
399 let freed = self.accounting.freed.notified();
400 tokio::pin!(freed);
401 freed.as_mut().enable();
404 if let Some(charge) = self.accounting.try_charge(bytes) {
405 return Some(charge);
406 }
407 tokio::select! {
408 _ = &mut freed => {}
409 _ = self.tx.closed() => return None,
410 }
411 }
412 }
413
414 pub async fn send(&self, frame: Frame) -> Result<(), RouterError> {
415 let channel = frame.header.channel;
416 let epoch = frame.header.epoch;
417 let corr = frame.header.corr;
418 let closed =
419 || RouterError::backend_with_epoch(channel, epoch, corr, "connection writer closed");
420 let charge = self
421 .charge_waiting(queued_frame_bytes(&frame))
422 .await
423 .ok_or_else(closed)?;
424 self.tx
425 .send(OutboundFrame::charged(frame, charge))
426 .await
427 .map_err(|_| closed())
428 }
429
430 #[cfg(unix)]
437 pub async fn send_flushed(&self, frame: Frame) -> Result<(), RouterError> {
438 let (tx, rx) = tokio::sync::oneshot::channel();
439 let charge = self
440 .charge_waiting(queued_frame_bytes(&frame))
441 .await
442 .ok_or_else(|| RouterError::backend(0, 0, "connection writer closed"))?;
443 let mut outbound = OutboundFrame::charged(frame, charge);
444 outbound.flushed = Some(tx);
445 self.tx
446 .send(outbound)
447 .await
448 .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
449 rx.await
450 .map_err(|_| RouterError::backend(0, 0, "connection flush failed"))
451 }
452
453 pub(crate) async fn reserve_owned(&self) -> Result<EgressPermit, RouterError> {
454 let permit = self
455 .tx
456 .clone()
457 .reserve_owned()
458 .await
459 .map_err(|_| RouterError::backend(0, 0, "connection writer closed"))?;
460 Ok(EgressPermit {
461 permit,
462 accounting: Arc::clone(&self.accounting),
463 })
464 }
465
466 #[cfg(test)]
467 pub(crate) fn try_reserve_owned(&self) -> Result<EgressPermit, RouterError> {
468 let permit = self
469 .tx
470 .clone()
471 .try_reserve_owned()
472 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
473 Ok(EgressPermit {
474 permit,
475 accounting: Arc::clone(&self.accounting),
476 })
477 }
478
479 pub(crate) fn is_closed(&self) -> bool {
480 self.tx.is_closed()
481 }
482
483 pub(crate) fn backlog(&self) -> EgressBacklog {
485 self.accounting.backlog()
486 }
487
488 pub(crate) fn close_client_delivery(&self) {
491 self.accounting
492 .client_delivery_closed
493 .store(true, Ordering::Release);
494 self.accounting.client_delivery_stopped.notify_waiters();
495 }
496
497 async fn send_to_client(&self, frame: Frame) -> Result<(), ClientDeliveryFailure> {
500 let frame_type = frame.header.ty;
501 let flags = frame.header.flags;
502 let stopped = self.accounting.client_delivery_stopped.notified();
503 tokio::pin!(stopped);
504 stopped.as_mut().enable();
505 let failure = |cause, error| ClientDeliveryFailure {
506 diagnosis: ClientDeliveryDiagnosis {
507 frame_type,
508 flags,
509 cause,
510 },
511 error,
512 };
513 let closed = || {
514 RouterError::backend_with_epoch(
515 frame.header.channel,
516 frame.header.epoch,
517 frame.header.corr,
518 "client connection closed",
519 )
520 };
521 if self
522 .accounting
523 .client_delivery_closed
524 .load(Ordering::Acquire)
525 {
526 return Err(failure("client_closed", closed()));
527 }
528 let (frame, error) = match self.try_send_owned(frame) {
529 Ok(()) => return Ok(()),
530 Err(refused) => refused,
531 };
532 if self.is_closed() {
533 return Err(failure("client_closed", error));
534 }
535 let grace = crate::server::CONNECTION_EGRESS_STALL_GRACE;
536 let age = self.backlog().oldest_age.unwrap_or_default();
537 if age > grace {
538 return Err(failure("oldest_frame_older_than_250_ms", error));
539 }
540 let result = tokio::select! {
541 biased;
542 _ = &mut stopped => return Err(failure("client_closed", error)),
543 result = tokio::time::timeout(grace.saturating_sub(age), self.send(frame)) => result,
544 };
545 match result {
546 Ok(Ok(())) => Ok(()),
547 Ok(Err(error)) => Err(failure("client_closed", error)),
548 Err(_) => Err(failure("waited_250_ms_without_room", error)),
549 }
550 }
551
552 pub(crate) fn try_send(&self, frame: Frame) -> Result<(), RouterError> {
556 self.try_send_owned(frame).map_err(|(_, error)| error)
557 }
558
559 fn try_send_owned(&self, frame: Frame) -> Result<(), (Frame, RouterError)> {
560 let channel = frame.header.channel;
561 let epoch = frame.header.epoch;
562 let corr = frame.header.corr;
563 let unavailable = |why: String| {
564 RouterError::backend_with_epoch(
565 channel,
566 epoch,
567 corr,
568 format!("connection writer unavailable: {why}"),
569 )
570 };
571 let bytes = queued_frame_bytes(&frame);
572 let Some(charge) = self.accounting.try_charge(bytes) else {
573 if self.tx.is_closed() {
574 return Err((frame, unavailable("channel closed".to_string())));
575 }
576 return Err((
577 frame,
578 unavailable(format!(
579 "egress byte budget exhausted ({} queued bytes, frame of {bytes} bytes, budget {})",
580 self.accounting.queued_bytes.load(Ordering::Acquire),
581 self.accounting.byte_budget
582 )),
583 ));
584 };
585 self.tx
588 .try_send(OutboundFrame::charged(frame, charge))
589 .map_err(|err| {
590 let error = unavailable(err.to_string());
591 (err.into_inner().frame, error)
592 })
593 }
594}
595
596#[derive(Debug)]
597struct ClientDeliveryFailure {
598 diagnosis: ClientDeliveryDiagnosis,
599 error: RouterError,
600}
601
602#[derive(Debug)]
604pub(crate) struct ClientDeliveryDiagnosis {
605 pub frame_type: FrameType,
606 pub flags: Flags,
607 pub cause: &'static str,
608}
609
610impl fmt::Display for ClientDeliveryFailure {
611 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
612 self.error.fmt(f)
613 }
614}
615
616const ORPHAN_ROUTE_GOODBYE_INTERVAL: Duration = Duration::from_secs(5);
628
629const ORPHAN_ROUTE_GOODBYE_PRUNE_AT: usize = 256;
636
637#[derive(Debug, Default)]
641struct OrphanGoodbyeLimiter {
642 last_sent: Mutex<HashMap<ConnectionId, HashMap<u16, tokio::time::Instant>>>,
643}
644
645impl OrphanGoodbyeLimiter {
646 fn claim(&self, connection_id: ConnectionId, channel: u16) -> bool {
649 let now = tokio::time::Instant::now();
650 let mut last_sent = self
651 .last_sent
652 .lock()
653 .unwrap_or_else(std::sync::PoisonError::into_inner);
654 let channels = last_sent.entry(connection_id).or_default();
655 if channels.get(&channel).is_some_and(|sent| {
656 now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
657 }) {
658 return false;
659 }
660 if channels.len() >= ORPHAN_ROUTE_GOODBYE_PRUNE_AT {
661 channels.retain(|_, sent| {
662 now.saturating_duration_since(*sent) < ORPHAN_ROUTE_GOODBYE_INTERVAL
663 });
664 }
665 channels.insert(channel, now);
666 true
667 }
668
669 fn forget_connection(&self, connection_id: ConnectionId) {
670 self.last_sent
671 .lock()
672 .unwrap_or_else(std::sync::PoisonError::into_inner)
673 .remove(&connection_id);
674 }
675}
676
677#[derive(Debug, Clone)]
679pub struct RouteCtx {
680 pub connection_id: ConnectionId,
681 pub egress: FrameSink,
682}
683
684#[derive(Debug, Clone)]
690pub enum Backend {
691 Echo(EchoBackend),
692 Forward(ForwardBackend),
693}
694
695impl From<EchoBackend> for Backend {
696 fn from(backend: EchoBackend) -> Self {
697 Self::Echo(backend)
698 }
699}
700
701impl From<ForwardBackend> for Backend {
702 fn from(backend: ForwardBackend) -> Self {
703 Self::Forward(backend)
704 }
705}
706
707impl Backend {
708 pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
709 match self {
710 Self::Echo(backend) => backend.handle(ctx, frame).await,
711 Self::Forward(backend) => backend.handle(ctx, frame).await,
712 }
713 }
714}
715
716pub struct Router {
737 backends: HashMap<u16, Backend>,
738 control: Arc<ControlHandler>,
739 forwarding: Arc<ForwardingTable>,
740 forward_backend: ForwardBackend,
741 counters: DaemonCounters,
742 next_connection_id: AtomicU64,
743 orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
744}
745
746impl Router {
747 pub fn with_control_handler(control: Arc<ControlHandler>) -> Self {
748 control.install_swap_promotion_observer();
751 let forwarding = control.forwarding();
752 let counters = control.counters();
753 Self {
754 backends: HashMap::new(),
755 control,
756 forwarding: Arc::clone(&forwarding),
757 forward_backend: ForwardBackend::new(forwarding),
758 counters,
759 next_connection_id: AtomicU64::new(1),
761 orphan_goodbyes: Arc::default(),
762 }
763 }
764
765 pub fn with_default_self_handler() -> Self {
766 Self::with_control_handler(Arc::new(ControlHandler::default()))
767 }
768
769 pub fn forwarding(&self) -> Arc<ForwardingTable> {
770 Arc::clone(&self.forwarding)
771 }
772
773 pub fn register_backend(
774 &mut self,
775 channel: u16,
776 backend: impl Into<Backend>,
777 ) -> Result<(), RouterError> {
778 self.register_backend_arc(channel, Arc::new(backend.into()))
779 }
780
781 pub(crate) fn register_backend_arc(
782 &mut self,
783 channel: u16,
784 backend: Arc<Backend>,
785 ) -> Result<(), RouterError> {
786 if channel == 0 {
787 return Err(RouterError::ReservedChannelZero);
788 }
789 if self.backends.contains_key(&channel) {
790 return Err(RouterError::DuplicateChannel { channel });
791 }
792 self.backends.insert(channel, backend.as_ref().clone());
793 Ok(())
794 }
795
796 fn record_module_frame_drop(&self, connection_id: ConnectionId) -> Result<(), RouterError> {
799 let module_id = self
800 .forwarding
801 .module_id_for_connection(connection_id)
802 .map_err(RouterError::Forwarding)?;
803 self.counters
804 .increment_module_frames_dropped_no_route(module_id.as_deref());
805 Ok(())
806 }
807
808 fn handle_orphan_module_frame(&self, ctx: &RouteCtx, frame: &Frame) -> Result<(), RouterError> {
820 let channel = frame.header.channel;
821 let epoch = frame.header.epoch;
822 let module_id = self
823 .forwarding
824 .module_id_for_connection(ctx.connection_id)
825 .map_err(RouterError::Forwarding)?;
826 self.counters
827 .increment_module_frames_dropped_no_route(module_id.as_deref());
828 if self
829 .forwarding
830 .module_route_epoch_was_allocated(ctx.connection_id, channel, epoch)
831 .map_err(RouterError::Forwarding)?
832 {
833 self.counters
834 .increment_module_frames_dropped_released_route(module_id.as_deref());
835 }
836 if frame.header.ty == FrameType::Goodbye
837 || !self.orphan_goodbyes.claim(ctx.connection_id, channel)
838 {
839 return Ok(());
840 }
841 let goodbye = Frame::build_with_version(
842 frame.header.ver,
843 FrameType::Goodbye,
844 Flags::new(false, Priority::Passive, false),
845 channel,
846 epoch,
847 0,
848 Vec::new(),
849 )
850 .map_err(RouterError::FrameBuild)?;
851 match ctx.egress.try_send(goodbye) {
852 Ok(()) => {
853 self.counters.increment_module_orphan_route_goodbyes_sent();
854 debug!(
855 connection_id = ctx.connection_id.get(),
856 module_id = module_id.as_deref().unwrap_or("unknown"),
857 channel,
858 epoch,
859 "answered module frame on a route the daemon does not hold with a route GOODBYE"
860 );
861 }
862 Err(err) => {
863 self.forwarding.warn_frame_drop(
864 ctx.connection_id,
865 channel,
866 epoch,
867 "daemon_to_module",
868 if ctx.egress.is_closed() {
869 "endpoint_gone"
870 } else {
871 "egress_budget"
872 },
873 );
874 debug!(
875 connection_id = ctx.connection_id.get(),
876 module_id = module_id.as_deref().unwrap_or("unknown"),
877 channel,
878 epoch,
879 error = %err,
880 "could not enqueue route GOODBYE for module frame on a route the daemon does not hold; the next such frame retries"
881 );
882 }
883 }
884 Ok(())
885 }
886
887 pub fn begin_connection(&self) -> RouterConnection {
888 let raw = self.next_connection_id.fetch_add(1, Ordering::Relaxed);
889 let id = ConnectionId::new(raw);
890 let close_receiver = self.forwarding.register_connection_close(id);
891 RouterConnection {
892 id,
893 control_handler: Arc::clone(&self.control),
894 forwarding: Arc::clone(&self.forwarding),
895 close_receiver: Some(close_receiver),
896 orphan_goodbyes: Arc::clone(&self.orphan_goodbyes),
897 }
898 }
899
900 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
901 self.control.route_open_target(frame)
902 }
903
904 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
905 self.control.record_local_connection(connection);
906 }
907
908 pub(crate) fn record_captured_peer(
909 &self,
910 connection: ConnectionId,
911 peer: subc_os::local_peer::CapturedPeer,
912 ) {
913 self.control.record_captured_peer(connection, peer);
914 }
915
916 pub(crate) fn record_caller_endpoints(
917 &self,
918 connection: ConnectionId,
919 endpoints: Option<(std::net::SocketAddr, std::net::SocketAddr)>,
920 ) {
921 self.control.record_caller_endpoints(connection, endpoints);
922 }
923
924 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
925 self.control.defers_privileged_control(ctx, frame)
926 }
927
928 pub(crate) fn route_open_capacity_refusal(
929 &self,
930 ctx: &RouteCtx,
931 frame: &Frame,
932 target_module_id: &str,
933 in_flight: usize,
934 limit: usize,
935 ) -> Result<Frame, RouterError> {
936 self.control
937 .route_open_capacity_refusal(ctx, frame, target_module_id, in_flight, limit)
938 }
939
940 pub async fn route_for_connection(
941 &self,
942 ctx: &RouteCtx,
943 frame: Frame,
944 ) -> Result<(), RouterError> {
945 self.route_for_connection_started(ctx, frame, None).await
946 }
947
948 pub(crate) async fn observe_connection_wait<F: std::future::Future>(
949 &self,
950 ctx: &RouteCtx,
951 kind: &'static str,
952 future: F,
953 ) -> F::Output {
954 self.counters
955 .observe_reader_wait(
956 || {
957 let module = self
958 .forwarding
959 .module_id_for_connection(ctx.connection_id)
960 .ok()
961 .flatten();
962 let principal = if module.is_some() { "module" } else { "direct" };
963 (
964 module.unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
965 principal.into(),
966 )
967 },
968 kind,
969 future,
970 )
971 .await
972 }
973
974 pub(crate) async fn route_for_connection_started(
975 &self,
976 ctx: &RouteCtx,
977 frame: Frame,
978 dispatch_started_at: Option<Instant>,
979 ) -> Result<(), RouterError> {
980 let channel = frame.header.channel;
981 let epoch = frame.header.epoch;
982 let corr = frame.header.corr;
983 if channel == 0 {
984 debug!(
985 connection_id = ctx.connection_id.get(),
986 corr,
987 frame_type = ?frame.header.ty,
988 "routing control frame"
989 );
990 let dispatch_started_at = (frame.header.ty == FrameType::Request)
996 .then(|| dispatch_started_at.unwrap_or_else(Instant::now));
997 let responses = self
998 .control
999 .handle_control_frame_timed(ctx, frame, dispatch_started_at)
1000 .await?;
1001 for response in responses {
1002 self.observe_connection_wait(ctx, "reply_egress", ctx.egress.send(response))
1003 .await?;
1004 }
1005 return Ok(());
1006 }
1007
1008 let data_route = self
1009 .forwarding
1010 .lookup_data_route(ctx.connection_id, channel, epoch)
1011 .map_err(RouterError::Forwarding)?;
1012
1013 match data_route {
1014 DataRoute::Module(DataRouteState::EpochMismatch) => {
1015 self.forwarding.warn_frame_drop(
1016 ctx.connection_id,
1017 channel,
1018 epoch,
1019 "module_to_client",
1020 "stale_epoch",
1021 );
1022 if frame.header.ty == FrameType::Request {
1023 self.counters
1024 .increment_module_requests_dropped_stale_route();
1025 let err = RouterError::StaleRouteEpoch {
1026 channel,
1027 epoch,
1028 corr,
1029 };
1030 if let Some(error_frame) = err.to_error_frame() {
1031 self.observe_connection_wait(
1032 ctx,
1033 "reply_egress",
1034 ctx.egress.send(error_frame),
1035 )
1036 .await?;
1037 }
1038 } else {
1039 self.handle_orphan_module_frame(ctx, &frame)?;
1040 }
1041 debug!(
1042 connection_id = ctx.connection_id.get(),
1043 channel, epoch, corr, "dropping module frame for stale route epoch"
1044 );
1045 return Ok(());
1046 }
1047 DataRoute::Module(DataRouteState::Reserved) => {
1048 self.forwarding.warn_frame_drop(
1049 ctx.connection_id,
1050 channel,
1051 epoch,
1052 "module_to_client",
1053 "missing_binding",
1054 );
1055 if frame.header.ty == FrameType::Request {
1056 self.counters
1057 .increment_module_requests_dropped_stale_route();
1058 let err = RouterError::UnknownChannel {
1059 channel,
1060 epoch,
1061 corr,
1062 };
1063 if let Some(error_frame) = err.to_error_frame() {
1064 self.observe_connection_wait(
1065 ctx,
1066 "reply_egress",
1067 ctx.egress.send(error_frame),
1068 )
1069 .await?;
1070 }
1071 } else {
1072 self.record_module_frame_drop(ctx.connection_id)?;
1073 }
1074 debug!(
1075 connection_id = ctx.connection_id.get(),
1076 channel, epoch, corr, "dropping module frame for reserved route handle"
1077 );
1078 return Ok(());
1079 }
1080 DataRoute::Module(DataRouteState::Absent) => {
1081 self.forwarding.warn_frame_drop(
1082 ctx.connection_id,
1083 channel,
1084 epoch,
1085 "module_to_client",
1086 "unknown_channel",
1087 );
1088 if frame.header.ty == FrameType::Request {
1089 self.counters
1090 .increment_module_requests_dropped_stale_route();
1091 let err = RouterError::UnknownChannel {
1092 channel,
1093 epoch,
1094 corr,
1095 };
1096 if let Some(error_frame) = err.to_error_frame() {
1097 self.observe_connection_wait(
1098 ctx,
1099 "reply_egress",
1100 ctx.egress.send(error_frame),
1101 )
1102 .await?;
1103 }
1104 } else {
1105 self.handle_orphan_module_frame(ctx, &frame)?;
1106 }
1107 debug!(
1108 connection_id = ctx.connection_id.get(),
1109 channel, epoch, corr, "dropping module frame for absent route handle"
1110 );
1111 return Ok(());
1112 }
1113 DataRoute::Module(DataRouteState::Bound(route)) => {
1114 if frame.header.ty == FrameType::Goodbye {
1115 if let RouteRelease::Removed(target) = self
1116 .forwarding
1117 .release_module_route(ctx.connection_id, channel, epoch)
1118 .map_err(RouterError::Forwarding)?
1119 {
1120 let mut goodbye = frame;
1121 goodbye.header.channel = target.channel;
1122 goodbye.header.epoch = target.epoch;
1123 goodbye.header.flags = goodbye.header.flags.without_daemon_origin();
1127 if let Err(err) = self
1128 .observe_connection_wait(
1129 ctx,
1130 "module_to_client",
1131 target.sink.send_to_client(goodbye),
1132 )
1133 .await
1134 {
1135 if !target.close_on_delivery_failure() {
1136 self.forwarding.warn_frame_drop(
1137 target.connection_id,
1138 target.channel,
1139 target.epoch,
1140 "module_to_client",
1141 err.diagnosis.cause,
1142 );
1143 }
1144 if target.close_on_delivery_failure()
1145 && self
1146 .forwarding
1147 .escalate_client_delivery_failure_diagnosed(
1148 target.connection_id,
1149 target.channel,
1150 target.epoch,
1151 CloseReason::new(
1152 "route_goodbye_delivery_failed",
1153 format!(
1154 "failed to enqueue route GOODBYE for client channel {}: {err}",
1155 target.channel
1156 ),
1157 ),
1158 UndeliveredFrame {
1159 module_id: Some(&route.module_id),
1160 sink: &target.sink,
1161 },
1162 Some(&err.diagnosis),
1163 )
1164 .map_err(RouterError::Forwarding)?
1165 {
1166 self.counters.increment_goodbye_relay_client_failed();
1167 }
1168 }
1169 } else {
1170 self.forwarding.warn_frame_drop(
1171 ctx.connection_id,
1172 channel,
1173 epoch,
1174 "module_to_client",
1175 "missing_binding",
1176 );
1177 }
1178 return Ok(());
1179 }
1180
1181 let bytes = queued_frame_bytes(&frame);
1185 let limit = route.client_sink.accounting.byte_budget;
1186 if bytes > limit {
1187 let ty = frame.header.ty;
1188 let flags = frame.header.flags;
1189 drop(frame);
1190 return self
1191 .refuse_oversized_module_frame(ctx, &route, (ty, flags), corr, bytes)
1192 .await;
1193 }
1194
1195 let releases_credit = is_terminal_frame(frame.header.ty);
1201 if releases_credit {
1202 route.flow.release_corr(corr);
1203 }
1204 let mut frame = frame;
1205 frame.header.channel = route.client_channel;
1206 frame.header.epoch = route.client_epoch;
1207 frame.header.flags = frame.header.flags.without_daemon_origin();
1213 if let Err(err) = self
1214 .counters
1215 .observe_reader_wait(
1216 || (route.module_id.clone(), principal_label(&route.principal)),
1217 "module_to_client",
1218 route.client_sink.send_to_client(frame),
1219 )
1220 .await
1221 {
1222 if self
1223 .forwarding
1224 .escalate_client_delivery_failure_diagnosed(
1225 route.client_connection_id,
1226 route.client_channel,
1227 route.client_epoch,
1228 CloseReason::new(
1229 "module_to_client_delivery_failed",
1230 format!(
1231 "failed to enqueue module frame for client channel {} corr {corr}: {err}",
1232 route.client_channel
1233 ),
1234 ),
1235 UndeliveredFrame {
1236 module_id: Some(&route.module_id),
1237 sink: &route.client_sink,
1238 },
1239 Some(&err.diagnosis),
1240 )
1241 .map_err(RouterError::Forwarding)?
1242 {
1243 self.counters
1244 .increment_client_egress_close_delivery_failed();
1245 }
1246 return Ok(());
1247 }
1248 return Ok(());
1249 }
1250 DataRoute::Client(DataRouteState::EpochMismatch) => {
1251 self.forwarding.warn_frame_drop(
1252 ctx.connection_id,
1253 channel,
1254 epoch,
1255 "client_to_module",
1256 "stale_epoch",
1257 );
1258 if frame.header.ty == FrameType::Request {
1259 self.counters.increment_client_frames_dropped_stale_route();
1260 let err = RouterError::StaleRouteEpoch {
1262 channel,
1263 epoch,
1264 corr,
1265 };
1266 if let Some(error_frame) = err.to_error_frame() {
1267 self.observe_connection_wait(
1268 ctx,
1269 "reply_egress",
1270 ctx.egress.send(error_frame),
1271 )
1272 .await?;
1273 }
1274 }
1275 debug!(
1276 connection_id = ctx.connection_id.get(),
1277 channel, epoch, corr, "dropping client frame for stale route epoch"
1278 );
1279 return Ok(());
1280 }
1281 DataRoute::Client(DataRouteState::Reserved) => {
1282 self.forwarding.warn_frame_drop(
1283 ctx.connection_id,
1284 channel,
1285 epoch,
1286 "client_to_module",
1287 "missing_binding",
1288 );
1289 if frame.header.ty == FrameType::Request {
1290 let err = RouterError::UnknownChannel {
1291 channel,
1292 epoch,
1293 corr,
1294 };
1295 if let Some(error_frame) = err.to_error_frame() {
1296 self.observe_connection_wait(
1297 ctx,
1298 "reply_egress",
1299 ctx.egress.send(error_frame),
1300 )
1301 .await?;
1302 }
1303 }
1304 return Ok(());
1305 }
1306 DataRoute::Client(DataRouteState::Bound(route)) => {
1307 if frame.header.ty == FrameType::Goodbye {
1308 let _ = self
1309 .control
1310 .handle_route_goodbye(ctx.connection_id, channel, epoch)?;
1311 return Ok(());
1312 }
1313 return self.forward_backend.handle_bound(frame, route).await;
1314 }
1315 DataRoute::Client(DataRouteState::Absent) => {}
1316 }
1317
1318 if let Some(backend) = self.backends.get(&channel) {
1319 return backend.handle(ctx.clone(), frame).await.inspect_err(|_| {
1320 self.forwarding.warn_frame_drop(
1321 ctx.connection_id,
1322 channel,
1323 epoch,
1324 "client_to_backend",
1325 "endpoint_gone",
1326 );
1327 });
1328 }
1329 self.forwarding.warn_frame_drop(
1330 ctx.connection_id,
1331 channel,
1332 epoch,
1333 "client_to_module",
1334 "unknown_channel",
1335 );
1336 if frame.header.ty == FrameType::Request {
1337 let err = RouterError::UnknownChannel {
1338 channel,
1339 epoch,
1340 corr,
1341 };
1342 if let Some(error_frame) = err.to_error_frame() {
1343 self.observe_connection_wait(ctx, "reply_egress", ctx.egress.send(error_frame))
1344 .await?;
1345 }
1346 }
1347 Ok(())
1348 }
1349
1350 async fn refuse_oversized_module_frame(
1351 &self,
1352 ctx: &RouteCtx,
1353 route: &RouteBinding,
1354 header: (FrameType, Flags),
1355 corr: u64,
1356 bytes: usize,
1357 ) -> Result<(), RouterError> {
1358 let (ty, flags) = header;
1359 let limit = route.client_sink.accounting.byte_budget;
1360 if ty == FrameType::Push {
1361 let diagnosis = ClientDeliveryDiagnosis {
1365 frame_type: ty,
1366 flags,
1367 cause: "frame_too_large",
1368 };
1369 if self
1370 .forwarding
1371 .escalate_client_delivery_failure_diagnosed(
1372 route.client_connection_id,
1373 route.client_channel,
1374 route.client_epoch,
1375 CloseReason::new(
1376 error_codes::FRAME_TOO_LARGE,
1377 format!(
1378 "oversized push: size={bytes} limit={limit} (header plus body bytes)"
1379 ),
1380 ),
1381 UndeliveredFrame {
1382 module_id: Some(&route.module_id),
1383 sink: &route.client_sink,
1384 },
1385 Some(&diagnosis),
1386 )
1387 .map_err(RouterError::Forwarding)?
1388 {
1389 self.counters
1390 .increment_client_egress_close_delivery_failed();
1391 }
1392 return Ok(());
1393 }
1394
1395 warn!(
1396 connection_id = route.client_connection_id.get(),
1397 module_id = %route.module_id,
1398 client_principal = %principal_label(&route.principal),
1399 client_channel = route.client_channel,
1400 frame_type = ?ty,
1401 frame_bytes = bytes,
1402 byte_limit = limit,
1403 "refusing oversized module-to-client frame with ERROR frame_too_large"
1404 );
1405 if ty == FrameType::Request {
1406 let error = frame_too_large_error(
1409 route.module_channel,
1410 route.module_epoch,
1411 corr,
1412 bytes,
1413 limit,
1414 )?;
1415 self.observe_connection_wait(ctx, "reply_egress", ctx.egress.send(error))
1416 .await?;
1417 } else {
1418 route.flow.release_corr(corr);
1421 let error = frame_too_large_error(
1422 route.client_channel,
1423 route.client_epoch,
1424 corr,
1425 bytes,
1426 limit,
1427 )?;
1428 if let Err(err) = self
1431 .counters
1432 .observe_reader_wait(
1433 || (route.module_id.clone(), principal_label(&route.principal)),
1434 "module_to_client",
1435 route.client_sink.send_to_client(error),
1436 )
1437 .await
1438 {
1439 if self
1440 .forwarding
1441 .escalate_client_delivery_failure_diagnosed(
1442 route.client_connection_id,
1443 route.client_channel,
1444 route.client_epoch,
1445 CloseReason::new(
1446 "module_to_client_delivery_failed",
1447 format!(
1448 "failed to enqueue frame_too_large for client channel {} corr {corr}: {err}",
1449 route.client_channel
1450 ),
1451 ),
1452 UndeliveredFrame {
1453 module_id: Some(&route.module_id),
1454 sink: &route.client_sink,
1455 },
1456 Some(&err.diagnosis),
1457 )
1458 .map_err(RouterError::Forwarding)?
1459 {
1460 self.counters
1461 .increment_client_egress_close_delivery_failed();
1462 }
1463 }
1464 }
1465 Ok(())
1466 }
1467}
1468
1469impl Default for Router {
1470 fn default() -> Self {
1471 Self::with_default_self_handler()
1472 }
1473}
1474
1475#[must_use]
1477pub struct RouterConnection {
1478 id: ConnectionId,
1479 control_handler: Arc<ControlHandler>,
1480 forwarding: Arc<ForwardingTable>,
1481 close_receiver: Option<ConnectionCloseReceiver>,
1482 orphan_goodbyes: Arc<OrphanGoodbyeLimiter>,
1483}
1484
1485impl RouterConnection {
1486 pub fn id(&self) -> ConnectionId {
1487 self.id
1488 }
1489
1490 pub(crate) fn take_close_receiver(&mut self) -> ConnectionCloseReceiver {
1491 self.close_receiver
1492 .take()
1493 .expect("connection close receiver can only be taken once")
1494 }
1495}
1496
1497impl Drop for RouterConnection {
1498 fn drop(&mut self) {
1499 self.forwarding.unregister_connection_close(self.id);
1500 self.orphan_goodbyes.forget_connection(self.id);
1501 let _ = self.control_handler.cleanup_connection(self.id);
1504 }
1505}
1506
1507#[derive(Debug, Default, Clone, Copy)]
1510pub struct EchoBackend;
1511
1512impl EchoBackend {
1513 pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1514 let response = Frame::build_with_version(
1515 frame.header.ver,
1516 FrameType::Response,
1517 frame.header.flags,
1518 frame.header.channel,
1519 frame.header.epoch,
1520 frame.header.corr,
1521 frame.body,
1522 )
1523 .map_err(RouterError::FrameBuild)?;
1524 ctx.egress.send(response).await
1525 }
1526}
1527
1528#[derive(Debug, Clone)]
1530pub struct ForwardBackend {
1531 forwarding: Arc<ForwardingTable>,
1532}
1533
1534impl ForwardBackend {
1535 pub fn new(forwarding: Arc<ForwardingTable>) -> Self {
1536 Self { forwarding }
1537 }
1538
1539 pub async fn handle(&self, ctx: RouteCtx, frame: Frame) -> Result<(), RouterError> {
1540 let channel = frame.header.channel;
1541 let corr = frame.header.corr;
1542 let route = match self
1543 .forwarding
1544 .lookup_data_route(ctx.connection_id, channel, frame.header.epoch)
1545 .map_err(RouterError::Forwarding)?
1546 {
1547 DataRoute::Client(DataRouteState::Bound(route)) => route,
1548 DataRoute::Client(_) | DataRoute::Module(_) => {
1549 self.forwarding.warn_frame_drop(
1550 ctx.connection_id,
1551 channel,
1552 frame.header.epoch,
1553 "client_to_module",
1554 "missing_binding",
1555 );
1556 return Err(RouterError::UnknownChannel {
1557 channel,
1558 epoch: frame.header.epoch,
1559 corr,
1560 });
1561 }
1562 };
1563 self.handle_bound(frame, route).await
1564 }
1565
1566 pub(crate) async fn handle_bound(
1567 &self,
1568 frame: Frame,
1569 route: Arc<RouteBinding>,
1570 ) -> Result<(), RouterError> {
1571 let channel = frame.header.channel;
1572 let corr = frame.header.corr;
1573 let frame_type = frame.header.ty;
1574
1575 let acquired_credit = frame_type == FrameType::Request;
1578 if acquired_credit && route.flow.busy_refusals() {
1579 use crate::forwarding::RequestAdmission;
1580 match self
1581 .forwarding
1582 .try_admit_request(
1583 &route,
1584 frame.header.epoch,
1585 corr,
1586 frame.header.flags.is_subscription(),
1587 )
1588 .map_err(RouterError::Forwarding)?
1589 {
1590 RequestAdmission::Admitted => {}
1591 RequestAdmission::Busy(detail) => {
1592 let principal = principal_label(&route.principal);
1593 let counters = self.forwarding.counters_ref();
1594 counters.increment_route_busy(&route.module_id, &principal);
1595 if route.flow.should_warn_busy() {
1596 warn!(module_id = %route.module_id, client_principal = %principal, client_channel = channel,
1597 route_epoch = route.client_epoch, ?detail, "request refused: route_busy");
1598 }
1599 let error = RouterError::RouteBusy {
1600 channel,
1601 epoch: frame.header.epoch,
1602 corr,
1603 detail: Box::new(detail),
1604 };
1605 let refusal = error
1606 .to_error_frame()
1607 .expect("route_busy is a serializable wire error");
1608 counters
1609 .observe_reader_wait(
1610 || (route.module_id.clone(), principal),
1611 "route_busy_egress",
1612 route.client_sink.send(refusal),
1613 )
1614 .await?;
1615 return Ok(());
1616 }
1617 RequestAdmission::ModuleReloading => {
1618 return Err(RouterError::route_error_with_epoch(
1619 channel,
1620 frame.header.epoch,
1621 corr,
1622 error_codes::MODULE_RELOADING,
1623 format!("module endpoint for route channel {channel} is reloading"),
1624 ));
1625 }
1626 RequestAdmission::Stale => {
1627 return Err(RouterError::StaleRouteEpoch {
1628 channel,
1629 epoch: frame.header.epoch,
1630 corr,
1631 })
1632 }
1633 }
1634 } else if acquired_credit {
1635 let counters = self.forwarding.counters_ref();
1636 if let Err(err) = counters
1637 .observe_reader_wait(
1638 || (route.module_id.clone(), principal_label(&route.principal)),
1639 "legacy_credit",
1640 route
1641 .flow
1642 .acquire_tagged(corr, frame.header.flags.is_subscription()),
1643 )
1644 .await
1645 {
1646 self.forwarding.warn_frame_drop(
1647 route.client_connection_id,
1648 channel,
1649 frame.header.epoch,
1650 "client_to_module",
1651 "endpoint_gone",
1652 );
1653 if self
1660 .forwarding
1661 .endpoint_is_draining(route.module_endpoint)
1662 .map_err(RouterError::Forwarding)?
1663 {
1664 return Err(RouterError::route_error_with_epoch(
1665 channel,
1666 frame.header.epoch,
1667 corr,
1668 "module_reloading",
1669 format!("module endpoint for route channel {channel} is reloading"),
1670 ));
1671 }
1672 return Err(RouterError::backend_with_epoch(
1673 channel,
1674 frame.header.epoch,
1675 corr,
1676 format!("{err} for route channel {channel}"),
1677 ));
1678 }
1679 }
1680
1681 let mut frame = frame;
1682 frame.header.channel = route.module_channel;
1683 frame.header.epoch = route.module_epoch;
1684 frame.header.flags = frame.header.flags.without_daemon_origin();
1688 let counters = self.forwarding.counters_ref();
1689 let result = counters
1690 .observe_reader_wait(
1691 || (route.module_id.clone(), principal_label(&route.principal)),
1692 "module_queue",
1693 route.module_sink.send(frame),
1694 )
1695 .await
1696 .map_err(|err| {
1697 self.forwarding.warn_frame_drop(
1698 route.client_connection_id,
1699 channel,
1700 route.client_epoch,
1701 "client_to_module",
1702 "endpoint_gone",
1703 );
1704 RouterError::backend_with_epoch(channel, route.client_epoch, corr, err.to_string())
1705 });
1706 if acquired_credit && result.is_err() {
1707 route.flow.release_corr(corr);
1708 }
1709 result
1710 }
1711}
1712
1713fn is_terminal_frame(frame_type: FrameType) -> bool {
1714 matches!(
1715 frame_type,
1716 FrameType::Response | FrameType::Error | FrameType::StreamEnd
1717 )
1718}
1719
1720#[derive(Debug, Clone, PartialEq, Eq)]
1723pub enum RouterError {
1724 ReservedChannelZero,
1725 DuplicateChannel {
1726 channel: u16,
1727 },
1728 UnknownChannel {
1729 channel: u16,
1730 epoch: u32,
1731 corr: u64,
1732 },
1733 StaleRouteEpoch {
1734 channel: u16,
1735 epoch: u32,
1736 corr: u64,
1737 },
1738 RouteBusy {
1739 channel: u16,
1740 epoch: u32,
1741 corr: u64,
1742 detail: Box<subc_protocol::RouteBusyDetail>,
1743 },
1744 Backend {
1745 channel: u16,
1746 epoch: u32,
1747 corr: u64,
1748 message: String,
1749 },
1750 RouteError {
1751 channel: u16,
1752 epoch: u32,
1753 corr: u64,
1754 code: String,
1755 message: String,
1756 },
1757 FrameBuild(FrameBuildError),
1758 Forwarding(ForwardingError),
1759}
1760
1761impl RouterError {
1762 pub fn backend(channel: u16, corr: u64, message: impl Into<String>) -> Self {
1763 Self::backend_with_epoch(channel, 0, corr, message)
1764 }
1765
1766 pub fn backend_with_epoch(
1767 channel: u16,
1768 epoch: u32,
1769 corr: u64,
1770 message: impl Into<String>,
1771 ) -> Self {
1772 Self::Backend {
1773 channel,
1774 epoch,
1775 corr,
1776 message: message.into(),
1777 }
1778 }
1779
1780 pub fn route_error(
1781 channel: u16,
1782 corr: u64,
1783 code: impl Into<String>,
1784 message: impl Into<String>,
1785 ) -> Self {
1786 Self::route_error_with_epoch(channel, 0, corr, code, message)
1787 }
1788
1789 pub fn route_error_with_epoch(
1790 channel: u16,
1791 epoch: u32,
1792 corr: u64,
1793 code: impl Into<String>,
1794 message: impl Into<String>,
1795 ) -> Self {
1796 Self::RouteError {
1797 channel,
1798 epoch,
1799 corr,
1800 code: code.into(),
1801 message: message.into(),
1802 }
1803 }
1804
1805 pub fn to_error_frame(&self) -> Option<Frame> {
1807 match self {
1808 Self::RouteBusy {
1809 channel,
1810 epoch,
1811 corr,
1812 detail,
1813 } => {
1814 let body = serde_json::to_vec(&ErrorBody {
1815 code: error_codes::ROUTE_BUSY.into(),
1816 message: "route has no free request or watch credit; retry on the same route"
1817 .into(),
1818 detail: Some(serde_json::to_value(detail).ok()?),
1819 })
1820 .ok()?;
1821 Frame::build(
1822 FrameType::Error,
1823 daemon_error_flags(),
1824 *channel,
1825 *epoch,
1826 *corr,
1827 body,
1828 )
1829 .ok()
1830 }
1831 Self::UnknownChannel {
1832 channel,
1833 epoch,
1834 corr,
1835 } => error_frame(
1836 *channel,
1837 *epoch,
1838 *corr,
1839 error_codes::UNKNOWN_CHANNEL,
1840 format!("unknown channel {channel}"),
1841 ),
1842 Self::StaleRouteEpoch {
1843 channel,
1844 epoch,
1845 corr,
1846 } => error_frame(
1847 *channel,
1848 *epoch,
1849 *corr,
1850 error_codes::STALE_ROUTE_EPOCH,
1851 format!("stale route epoch for channel {channel}"),
1852 ),
1853 Self::Backend {
1854 channel,
1855 epoch,
1856 corr,
1857 message,
1858 } => error_frame(*channel, *epoch, *corr, "backend_error", message.clone()),
1859 Self::RouteError {
1860 channel,
1861 epoch,
1862 corr,
1863 code,
1864 message,
1865 } => error_frame(*channel, *epoch, *corr, code, message.clone()),
1866 Self::ReservedChannelZero
1867 | Self::DuplicateChannel { .. }
1868 | Self::FrameBuild(_)
1869 | Self::Forwarding(_) => None,
1870 }
1871 }
1872}
1873
1874pub(crate) fn daemon_error_flags() -> Flags {
1886 Flags::new(false, Priority::Passive, false).with_daemon_origin()
1887}
1888
1889fn error_frame(channel: u16, epoch: u32, corr: u64, code: &str, message: String) -> Option<Frame> {
1890 let body = serde_json::to_vec(&ErrorBody {
1891 code: code.to_string(),
1892 message,
1893 detail: None,
1894 })
1895 .ok()?;
1896
1897 Frame::build(
1898 FrameType::Error,
1899 daemon_error_flags(),
1900 channel,
1901 epoch,
1902 corr,
1903 body,
1904 )
1905 .ok()
1906}
1907
1908fn frame_too_large_error(
1909 channel: u16,
1910 epoch: u32,
1911 corr: u64,
1912 bytes: usize,
1913 limit: usize,
1914) -> Result<Frame, RouterError> {
1915 let body = serde_json::to_vec(&ErrorBody {
1916 code: error_codes::FRAME_TOO_LARGE.to_string(),
1917 message: format!(
1918 "module frame of {bytes} bytes exceeds client egress limit of {limit} bytes"
1919 ),
1920 detail: Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit })),
1921 })
1922 .expect("serializing a frame size error");
1923 Frame::build(
1924 FrameType::Error,
1925 daemon_error_flags(),
1926 channel,
1927 epoch,
1928 corr,
1929 body,
1930 )
1931 .map_err(RouterError::FrameBuild)
1932}
1933
1934impl fmt::Display for RouterError {
1935 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1936 match self {
1937 Self::ReservedChannelZero => write!(f, "channel 0 is reserved for subc"),
1938 Self::DuplicateChannel { channel } => {
1939 write!(f, "backend already registered for channel {channel}")
1940 }
1941 Self::UnknownChannel { channel, corr, .. } => {
1942 write!(f, "unknown channel {channel} for corr {corr}")
1943 }
1944 Self::StaleRouteEpoch { channel, corr, .. } => {
1945 write!(f, "stale route epoch for channel {channel} corr {corr}")
1946 }
1947 Self::RouteBusy { channel, corr, .. } => {
1948 write!(f, "route busy on channel {channel} corr {corr}")
1949 }
1950 Self::Backend {
1951 channel,
1952 corr,
1953 message,
1954 ..
1955 } => write!(
1956 f,
1957 "backend error on channel {channel} corr {corr}: {message}"
1958 ),
1959 Self::RouteError {
1960 channel,
1961 corr,
1962 code,
1963 message,
1964 ..
1965 } => write!(
1966 f,
1967 "route error {code} on channel {channel} corr {corr}: {message}"
1968 ),
1969 Self::FrameBuild(err) => write!(f, "failed to build routed frame: {err}"),
1970 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
1971 }
1972 }
1973}
1974
1975impl Error for RouterError {
1976 fn source(&self) -> Option<&(dyn Error + 'static)> {
1977 match self {
1978 Self::FrameBuild(err) => Some(err),
1979 Self::Forwarding(err) => Some(err),
1980 Self::ReservedChannelZero
1981 | Self::DuplicateChannel { .. }
1982 | Self::UnknownChannel { .. }
1983 | Self::StaleRouteEpoch { .. }
1984 | Self::RouteBusy { .. }
1985 | Self::Backend { .. }
1986 | Self::RouteError { .. } => None,
1987 }
1988 }
1989}
1990
1991#[cfg(test)]
1992mod tests {
1993 include!("route_busy_router_tests.rs");
1994 use super::*;
1995 use crate::{
1996 forwarding::RouteBindRelayOutcome,
1997 supervise::{ModuleSpec, RestartPolicy, Supervisor, SupervisorHandle},
1998 ControlHandler, Registry,
1999 };
2000 use std::{
2001 sync::{mpsc as std_mpsc, Arc},
2002 time::Duration,
2003 };
2004 use subc_control::ModuleProtocol;
2005 use subc_protocol::{manifest::Concurrency, ErrorBody, Flags, FrameType, Priority};
2006 use tokio::sync::mpsc;
2007
2008 pub(crate) use crate::router::test_log::{captured_logs, log_capture};
2009
2010 fn logged_millis(logs: &str, field: &str) -> u64 {
2011 logs.split_whitespace()
2012 .find_map(|part| part.strip_prefix(field))
2013 .and_then(|value| value.parse().ok())
2014 .unwrap_or_else(|| panic!("missing numeric {field} in logs: {logs}"))
2015 }
2016
2017 fn request(channel: u16, corr: u64, body: &[u8]) -> Frame {
2018 Frame::build(
2019 FrameType::Request,
2020 Flags::new(true, Priority::Interactive, false),
2021 channel,
2022 0,
2023 corr,
2024 body.to_vec(),
2025 )
2026 .unwrap()
2027 }
2028
2029 fn ping(corr: u64) -> Frame {
2030 Frame::build(
2031 FrameType::Ping,
2032 Flags::new(false, Priority::Passive, false),
2033 0,
2034 0,
2035 corr,
2036 Vec::new(),
2037 )
2038 .unwrap()
2039 }
2040
2041 fn route_ctx() -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
2042 let (tx, rx) = mpsc::channel(8);
2043 (
2044 RouteCtx {
2045 connection_id: ConnectionId::LOCAL,
2046 egress: FrameSink::new(tx),
2047 },
2048 rx,
2049 )
2050 }
2051
2052 #[tokio::test]
2053 async fn echo_backend_returns_response_with_byte_identical_body() {
2054 let mut router = Router::with_default_self_handler();
2055 router.register_backend(7, EchoBackend).unwrap();
2056 let (ctx, mut rx) = route_ctx();
2057 let body = b"{not parsed}\0\xff";
2058
2059 router
2060 .route_for_connection(&ctx, request(7, 123, body))
2061 .await
2062 .unwrap();
2063 let response = rx.recv().await.unwrap();
2064
2065 assert_eq!(response.header.ty, FrameType::Response);
2066 assert_eq!(response.header.channel, 7);
2067 assert_eq!(response.header.corr, 123);
2068 assert_eq!(response.body, body);
2069 assert!(rx.try_recv().is_err());
2070 }
2071
2072 #[tokio::test]
2073 async fn unknown_channel_emits_canonical_error_frame() {
2074 let router = Router::with_default_self_handler();
2075 let (ctx, mut rx) = route_ctx();
2076
2077 router
2078 .route_for_connection(&ctx, request(99, 5, b"payload"))
2079 .await
2080 .unwrap();
2081 let error_frame = rx.recv().await.unwrap();
2082
2083 assert_eq!(error_frame.header.ty, FrameType::Error);
2084 assert_eq!(error_frame.header.channel, 99);
2085 assert_eq!(error_frame.header.corr, 5);
2086 let body: ErrorBody = serde_json::from_slice(&error_frame.body).unwrap();
2087 assert_eq!(body.code, "unknown_channel");
2088 assert_eq!(body.message, "unknown channel 99");
2089 }
2090
2091 #[tokio::test]
2096 async fn daemon_authored_data_plane_errors_carry_daemon_origin() {
2097 let router = Router::with_default_self_handler();
2098 let (ctx, mut rx) = route_ctx();
2099 router
2100 .route_for_connection(&ctx, request(99, 5, b"payload"))
2101 .await
2102 .unwrap();
2103 let unknown = rx.recv().await.unwrap();
2104 assert_eq!(unknown.header.ty, FrameType::Error);
2105 let body: ErrorBody = serde_json::from_slice(&unknown.body).unwrap();
2106 assert_eq!(body.code, "unknown_channel");
2107 assert!(
2108 unknown.header.flags.is_daemon_origin(),
2109 "unknown_channel must carry daemon origin: {:?}",
2110 unknown.header.flags
2111 );
2112
2113 let (
2114 router,
2115 _forwarding,
2116 client_ctx,
2117 mut client_rx,
2118 _module_ctx,
2119 _module_egress_rx,
2120 _module_rx,
2121 pending,
2122 ) = dynamic_route_fixture(true);
2123 let _ = client_rx.recv().await;
2124 router
2125 .route_for_connection(
2126 &client_ctx,
2127 route_frame(
2128 FrameType::Request,
2129 pending.client_channel,
2130 pending.client_epoch + 1,
2131 702,
2132 ),
2133 )
2134 .await
2135 .unwrap();
2136 let stale = client_rx.recv().await.unwrap();
2137 assert_eq!(stale.header.ty, FrameType::Error);
2138 let body: ErrorBody = serde_json::from_slice(&stale.body).unwrap();
2139 assert_eq!(body.code, "stale_route_epoch");
2140 assert!(
2141 stale.header.flags.is_daemon_origin(),
2142 "stale_route_epoch must carry daemon origin: {:?}",
2143 stale.header.flags
2144 );
2145 }
2146
2147 #[tokio::test]
2152 async fn module_error_with_daemon_bit_is_stripped_before_client_delivery() {
2153 let (
2154 router,
2155 _forwarding,
2156 client_ctx,
2157 mut client_rx,
2158 module_ctx,
2159 _module_egress_rx,
2160 mut module_rx,
2161 pending,
2162 ) = dynamic_route_fixture(true);
2163 let _ = client_rx.recv().await;
2164 router
2167 .route_for_connection(
2168 &client_ctx,
2169 route_frame(
2170 FrameType::Request,
2171 pending.client_channel,
2172 pending.client_epoch,
2173 701,
2174 ),
2175 )
2176 .await
2177 .unwrap();
2178 let _ = module_rx.recv().await;
2179
2180 let body = serde_json::to_vec(&ErrorBody::new("module_reloading", "try again")).unwrap();
2181 let spoof = Frame::build(
2182 FrameType::Error,
2183 Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2184 pending.module_channel,
2185 pending.module_epoch,
2186 701,
2187 body.clone(),
2188 )
2189 .unwrap();
2190 router
2191 .route_for_connection(&module_ctx, spoof)
2192 .await
2193 .unwrap();
2194 let delivered = client_rx.recv().await.unwrap();
2195 assert_eq!(delivered.header.ty, FrameType::Error);
2196 assert_eq!(delivered.header.channel, pending.client_channel);
2197 assert_eq!(delivered.header.epoch, pending.client_epoch);
2198 assert_eq!(delivered.header.corr, 701);
2199 assert!(
2200 !delivered.header.flags.is_daemon_origin(),
2201 "module-set daemon origin must be cleared: {:?}",
2202 delivered.header.flags
2203 );
2204 assert_eq!(delivered.body, body, "the body travels byte-identical");
2205 }
2206
2207 #[tokio::test]
2210 async fn module_response_with_daemon_bit_is_stripped_preserving_other_flags() {
2211 use subc_protocol::FLAG_SUBSCRIPTION;
2212 let (
2213 router,
2214 _forwarding,
2215 client_ctx,
2216 mut client_rx,
2217 module_ctx,
2218 _module_egress_rx,
2219 mut module_rx,
2220 pending,
2221 ) = dynamic_route_fixture(true);
2222 let _ = client_rx.recv().await;
2223 router
2224 .route_for_connection(
2225 &client_ctx,
2226 route_frame(
2227 FrameType::Request,
2228 pending.client_channel,
2229 pending.client_epoch,
2230 701,
2231 ),
2232 )
2233 .await
2234 .unwrap();
2235 let _ = module_rx.recv().await;
2236
2237 let sent_flags = Flags(
2238 Flags::new(true, Priority::Background, true).0
2239 | FLAG_SUBSCRIPTION
2240 | subc_protocol::FLAG_DAEMON_ORIGIN,
2241 );
2242 assert!(sent_flags.is_daemon_origin());
2243 let response = Frame::build(
2244 FrameType::Response,
2245 sent_flags,
2246 pending.module_channel,
2247 pending.module_epoch,
2248 701,
2249 b"route-body".to_vec(),
2250 )
2251 .unwrap();
2252 router
2253 .route_for_connection(&module_ctx, response)
2254 .await
2255 .unwrap();
2256 let delivered = client_rx.recv().await.unwrap();
2257 assert_eq!(delivered.header.ty, FrameType::Response);
2258 assert!(
2259 !delivered.header.flags.is_daemon_origin(),
2260 "module-set daemon origin must be cleared: {:?}",
2261 delivered.header.flags
2262 );
2263 assert!(
2264 delivered.header.flags.is_binary(),
2265 "binary flag must survive: {:?}",
2266 delivered.header.flags
2267 );
2268 assert!(
2269 delivered.header.flags.is_last(),
2270 "LAST flag must survive: {:?}",
2271 delivered.header.flags
2272 );
2273 assert!(
2274 delivered.header.flags.is_subscription(),
2275 "subscription flag must survive: {:?}",
2276 delivered.header.flags
2277 );
2278 assert_eq!(
2279 delivered.header.flags.priority(),
2280 Some(Priority::Background),
2281 "priority must survive"
2282 );
2283 }
2284
2285 #[tokio::test]
2288 async fn client_frame_with_daemon_bit_is_stripped_before_module_delivery() {
2289 let (
2290 router,
2291 _forwarding,
2292 client_ctx,
2293 mut client_rx,
2294 _module_ctx,
2295 _module_egress_rx,
2296 mut module_rx,
2297 pending,
2298 ) = dynamic_route_fixture(true);
2299 let _ = client_rx.recv().await;
2300 let spoof = Frame::build(
2301 FrameType::Request,
2302 Flags::new(false, Priority::Interactive, false).with_daemon_origin(),
2303 pending.client_channel,
2304 pending.client_epoch,
2305 701,
2306 b"route-body".to_vec(),
2307 )
2308 .unwrap();
2309 router
2310 .route_for_connection(&client_ctx, spoof)
2311 .await
2312 .unwrap();
2313 let delivered = module_rx.recv().await.unwrap();
2314 assert_eq!(delivered.header.ty, FrameType::Request);
2315 assert!(
2316 !delivered.header.flags.is_daemon_origin(),
2317 "client-set daemon origin must be cleared: {:?}",
2318 delivered.header.flags
2319 );
2320 }
2321
2322 #[tokio::test]
2323 async fn channel_zero_uses_control_handler_not_backend_registry() {
2324 let mut router = Router::with_default_self_handler();
2325 router.register_backend(1, EchoBackend).unwrap();
2326 let (ctx, mut rx) = route_ctx();
2327
2328 router.route_for_connection(&ctx, ping(77)).await.unwrap();
2329 let response = rx.recv().await.unwrap();
2330
2331 assert_eq!(response.header.ty, FrameType::Pong);
2332 assert_eq!(response.header.channel, 0);
2333 assert_eq!(response.header.corr, 77);
2334 assert!(response.body.is_empty());
2335 }
2336
2337 #[tokio::test]
2338 async fn slow_control_dispatch_logs_decoded_op_and_elapsed_time() {
2339 let control = Arc::new(
2340 ControlHandler::new(Arc::new(Registry::default()))
2341 .with_control_dispatch_delay(Duration::from_millis(1050)),
2342 );
2343 let router = Router::with_control_handler(control);
2344 let (ctx, mut rx) = route_ctx();
2345 let (output, guard) = log_capture(tracing::Level::WARN);
2346
2347 router
2348 .route_for_connection(&ctx, request(0, 41, br#"{"op":"server.describe"}"#))
2349 .await
2350 .expect("slow request routes");
2351 assert!(rx.recv().await.is_some(), "request receives a response");
2352 drop(guard);
2353
2354 let logs = captured_logs(&output);
2355 assert!(logs.contains("slow control dispatch"));
2356 assert!(logs.contains("op=server.describe"));
2357 assert!(logs.contains("connection_id=0"));
2358 assert!(logs.contains("corr=41"));
2359 assert!(
2360 logged_millis(&logs, "elapsed_ms=") >= 1050,
2361 "elapsed must include the injected handler delay: {logs}"
2362 );
2363 }
2364
2365 #[tokio::test]
2366 async fn fast_control_dispatch_emits_arrival_without_slow_warning() {
2367 let router = Router::with_default_self_handler();
2368 let (ctx, mut rx) = route_ctx();
2369 let (output, guard) = log_capture(tracing::Level::DEBUG);
2370
2371 router
2372 .route_for_connection(&ctx, request(0, 42, br#"{"op":"server.describe"}"#))
2373 .await
2374 .expect("fast request routes");
2375 assert!(rx.recv().await.is_some(), "request receives a response");
2376 drop(guard);
2377
2378 let logs = captured_logs(&output);
2379 assert!(logs.contains("control dispatch op=server.describe connection_id=0 corr=42"));
2380 assert!(!logs.contains("slow control dispatch"));
2381 }
2382
2383 #[tokio::test]
2384 async fn control_dispatch_arrival_is_hidden_at_info() {
2385 let router = Router::with_default_self_handler();
2386 let (ctx, mut rx) = route_ctx();
2387 let (output, guard) = log_capture(tracing::Level::INFO);
2388
2389 router
2390 .route_for_connection(&ctx, request(0, 43, br#"{"op":"server.describe"}"#))
2391 .await
2392 .expect("fast request routes");
2393 assert!(rx.recv().await.is_some(), "request receives a response");
2394 drop(guard);
2395
2396 assert!(
2397 !captured_logs(&output).contains("control dispatch"),
2398 "arrival logging must stay hidden at INFO"
2399 );
2400 }
2401
2402 #[tokio::test]
2403 async fn supervisor_list_logs_contended_snapshot_lock_only() {
2404 let registry = Arc::new(Registry::default());
2405 let handle = SupervisorHandle::new();
2406 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
2407 .with_handle(handle.clone());
2408 let module = supervisor
2409 .supervise_configured(
2410 ModuleSpec {
2411 module_id: "held-module".to_string(),
2412 program: "test-module".into(),
2413 args: Vec::new(),
2414 env: Vec::new(),
2415 reserved: false,
2416 reserved_prefixes: Vec::new(),
2417 protocol: ModuleProtocol::Subc,
2418 overlap: Default::default(),
2419 },
2420 false,
2421 )
2422 .expect("disabled test module is supervised");
2423 let router = Router::with_control_handler(Arc::new(
2424 ControlHandler::new(Arc::clone(®istry)).with_supervisor(handle),
2425 ));
2426 let (ctx, mut rx) = route_ctx();
2427 let (acquired, ready) = std_mpsc::channel();
2428 tokio::task::yield_now().await;
2431 let holder = module.hold_snapshot_for_test(acquired, Duration::from_millis(400));
2432 ready.recv().expect("holder acquired snapshot lock");
2433 let (output, guard) = log_capture(tracing::Level::WARN);
2434
2435 router
2436 .route_for_connection(&ctx, request(0, 44, br#"{"op":"supervisor.list"}"#))
2437 .await
2438 .expect("list request routes after the lock releases");
2439 assert!(
2440 rx.recv().await.is_some(),
2441 "list request receives a response"
2442 );
2443 holder.join().expect("snapshot holder exits cleanly");
2444 drop(guard);
2445
2446 let logs = captured_logs(&output);
2447 assert!(logs.contains("slow snapshot lock"));
2448 assert!(logs.contains("module_id=held-module"));
2449 assert!(logs.contains("caller=list"));
2450 assert!(
2451 logged_millis(&logs, "waited_ms=") >= 250,
2452 "wait must exceed the slow-lock threshold: {logs}"
2453 );
2454
2455 let (output, guard) = log_capture(tracing::Level::WARN);
2456 router
2457 .route_for_connection(&ctx, request(0, 45, br#"{"op":"supervisor.list"}"#))
2458 .await
2459 .expect("uncontended list request routes");
2460 assert!(
2461 rx.recv().await.is_some(),
2462 "uncontended list receives a response"
2463 );
2464 drop(guard);
2465 assert!(
2466 !captured_logs(&output).contains("slow snapshot lock"),
2467 "uncontended list acquisition must not warn"
2468 );
2469 }
2470
2471 #[tokio::test]
2472 async fn full_module_to_client_sink_requests_client_close_without_erroring_module() {
2473 let forwarding = Arc::new(ForwardingTable::default());
2474 let control = Arc::new(ControlHandler::with_forwarding(
2475 Arc::new(crate::Registry::default()),
2476 Arc::clone(&forwarding),
2477 ));
2478 let router = Router::with_control_handler(control);
2479 let module_connection = ConnectionId::new(10);
2480 let client_connection = ConnectionId::new(20);
2481 let mut close_receiver = forwarding.register_connection_close(client_connection);
2482 let (module_tx, _module_rx) = mpsc::channel(1);
2483 forwarding
2484 .register_module_connection(
2485 module_connection,
2486 "full-sink-provider".to_string(),
2487 1,
2488 Concurrency::ModuleManaged,
2489 FrameSink::new(module_tx),
2490 )
2491 .unwrap();
2492 let (client_tx, mut client_rx) = mpsc::channel(1);
2493 let pending = forwarding
2494 .begin_route_bind_relay_for_test(
2495 client_connection,
2496 FrameSink::new(client_tx),
2497 700,
2498 "full-sink-provider",
2499 )
2500 .unwrap();
2501 forwarding
2502 .complete_pending_relay(
2503 module_connection,
2504 pending.corr,
2505 RouteBindRelayOutcome::Accepted,
2506 )
2507 .unwrap();
2508
2509 let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2510 let module_ctx = RouteCtx {
2511 connection_id: module_connection,
2512 egress: FrameSink::new(module_egress_tx),
2513 };
2514 let terminal = Frame::build(
2515 FrameType::Response,
2516 Flags::new(false, Priority::Interactive, true),
2517 pending.module_channel,
2518 pending.module_epoch,
2519 701,
2520 b"terminal".to_vec(),
2521 )
2522 .unwrap();
2523
2524 router
2525 .route_for_connection(&module_ctx, terminal)
2526 .await
2527 .unwrap();
2528 let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2529 .await
2530 .expect("close request should be sent for the full client sink")
2531 .expect("close sender should include a reason");
2532 assert!(
2533 reason
2534 .to_string()
2535 .contains("module_to_client_delivery_failed"),
2536 "unexpected close reason: {reason}"
2537 );
2538 assert_eq!(client_rx.try_recv().unwrap().header.corr, 700);
2539 assert!(client_rx.try_recv().is_err());
2540 assert_eq!(
2541 router.counters.snapshot()["client_egress_close_delivery_failed"],
2542 1
2543 );
2544 }
2545
2546 #[tokio::test]
2550 async fn terminal_frame_releases_its_credit_even_when_client_delivery_fails() {
2551 let forwarding = Arc::new(ForwardingTable::default());
2552 let control = Arc::new(ControlHandler::with_forwarding(
2553 Arc::new(crate::Registry::default()),
2554 Arc::clone(&forwarding),
2555 ));
2556 let router = Router::with_control_handler(control);
2557 let module_connection = ConnectionId::new(11);
2558 let client_connection = ConnectionId::new(21);
2559 let _close_receiver = forwarding.register_connection_close(client_connection);
2560 let (module_tx, _module_rx) = mpsc::channel(1);
2561 forwarding
2562 .register_module_connection(
2563 module_connection,
2564 "credit-provider".to_string(),
2565 1,
2566 Concurrency::ModuleManaged,
2567 FrameSink::new(module_tx),
2568 )
2569 .unwrap();
2570 let (client_tx, _client_rx) = mpsc::channel(1);
2573 let pending = forwarding
2574 .begin_route_bind_relay_for_test(
2575 client_connection,
2576 FrameSink::new(client_tx),
2577 800,
2578 "credit-provider",
2579 )
2580 .unwrap();
2581 forwarding
2582 .complete_pending_relay(
2583 module_connection,
2584 pending.corr,
2585 RouteBindRelayOutcome::Accepted,
2586 )
2587 .unwrap();
2588 let DataRoute::Client(DataRouteState::Bound(route)) = forwarding
2589 .lookup_data_route(
2590 client_connection,
2591 pending.client_channel,
2592 pending.client_epoch,
2593 )
2594 .unwrap()
2595 else {
2596 panic!("expected a bound client route");
2597 };
2598 route.flow.acquire_tagged(801, false).await.unwrap();
2599 assert_eq!(route.flow.drain_in_flight(), 1);
2600
2601 let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2602 let module_ctx = RouteCtx {
2603 connection_id: module_connection,
2604 egress: FrameSink::new(module_egress_tx),
2605 };
2606 let terminal = Frame::build(
2607 FrameType::Response,
2608 Flags::new(false, Priority::Interactive, true),
2609 pending.module_channel,
2610 pending.module_epoch,
2611 801,
2612 b"terminal".to_vec(),
2613 )
2614 .unwrap();
2615 router
2616 .route_for_connection(&module_ctx, terminal)
2617 .await
2618 .unwrap();
2619
2620 assert_eq!(
2621 router.counters.snapshot()["client_egress_close_delivery_failed"],
2622 1,
2623 "the client delivery must have failed for this test to mean anything"
2624 );
2625 assert_eq!(
2626 route.flow.drain_in_flight(),
2627 0,
2628 "the module's terminal frame must release its credit even though the client could not take it"
2629 );
2630 }
2631
2632 #[tokio::test]
2633 async fn full_route_goodbye_sink_requests_target_close_without_erroring_module() {
2634 let forwarding = Arc::new(ForwardingTable::default());
2635 let control = Arc::new(ControlHandler::with_forwarding(
2636 Arc::new(crate::Registry::default()),
2637 Arc::clone(&forwarding),
2638 ));
2639 let router = Router::with_control_handler(control);
2640 let module_connection = ConnectionId::new(30);
2641 let client_connection = ConnectionId::new(40);
2642 let mut close_receiver = forwarding.register_connection_close(client_connection);
2643 let (module_tx, _module_rx) = mpsc::channel(1);
2644 forwarding
2645 .register_module_connection(
2646 module_connection,
2647 "goodbye-full-provider".to_string(),
2648 1,
2649 Concurrency::ModuleManaged,
2650 FrameSink::new(module_tx),
2651 )
2652 .unwrap();
2653 let (client_tx, mut client_rx) = mpsc::channel(1);
2654 let pending = forwarding
2655 .begin_route_bind_relay_for_test(
2656 client_connection,
2657 FrameSink::new(client_tx),
2658 800,
2659 "goodbye-full-provider",
2660 )
2661 .unwrap();
2662 forwarding
2663 .complete_pending_relay(
2664 module_connection,
2665 pending.corr,
2666 RouteBindRelayOutcome::Accepted,
2667 )
2668 .unwrap();
2669
2670 let (module_egress_tx, _module_egress_rx) = mpsc::channel(1);
2671 let module_ctx = RouteCtx {
2672 connection_id: module_connection,
2673 egress: FrameSink::new(module_egress_tx),
2674 };
2675 let goodbye = Frame::build(
2676 FrameType::Goodbye,
2677 Flags::new(false, Priority::Passive, true),
2678 pending.module_channel,
2679 pending.module_epoch,
2680 801,
2681 Vec::new(),
2682 )
2683 .unwrap();
2684
2685 router
2686 .route_for_connection(&module_ctx, goodbye)
2687 .await
2688 .unwrap();
2689 let reason = tokio::time::timeout(Duration::from_secs(1), &mut close_receiver)
2690 .await
2691 .expect("close request should be sent for the full GOODBYE sink")
2692 .expect("close sender should include a reason");
2693 assert!(
2694 reason.to_string().contains("route_goodbye_delivery_failed"),
2695 "unexpected close reason: {reason}"
2696 );
2697 assert_eq!(client_rx.try_recv().unwrap().header.corr, 800);
2698 assert!(client_rx.try_recv().is_err());
2699 assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 1);
2700 assert_eq!(router.counters.snapshot()["route_released_epoch_fenced"], 1);
2701 }
2702
2703 async fn multi_route_client(
2708 module_id: &str,
2709 module_connection: ConnectionId,
2710 client_connection: ConnectionId,
2711 routes: usize,
2712 ) -> (
2713 Router,
2714 FrameSink,
2715 mpsc::Receiver<OutboundFrame>,
2716 RouteCtx,
2717 Vec<crate::forwarding::PendingRouteBindRelay>,
2718 ConnectionCloseReceiver,
2719 ) {
2720 let forwarding = Arc::new(ForwardingTable::default());
2721 let control = Arc::new(ControlHandler::with_forwarding(
2722 Arc::new(crate::Registry::default()),
2723 Arc::clone(&forwarding),
2724 ));
2725 let router = Router::with_control_handler(control);
2726 let close_receiver = forwarding.register_connection_close(client_connection);
2727 let (module_tx, _module_rx) = mpsc::channel(8);
2728 forwarding
2729 .register_module_connection(
2730 module_connection,
2731 module_id.to_string(),
2732 1,
2733 Concurrency::ModuleManaged,
2734 FrameSink::new(module_tx),
2735 )
2736 .unwrap();
2737 let (client_sink, mut client_rx) = crate::server::connection_egress();
2738 let mut bound = Vec::with_capacity(routes);
2739 for index in 0..routes {
2740 let pending = forwarding
2741 .begin_route_bind_relay_for_test(
2742 client_connection,
2743 client_sink.clone(),
2744 900 + index as u64,
2745 module_id,
2746 )
2747 .unwrap();
2748 forwarding
2749 .complete_pending_relay(
2750 module_connection,
2751 pending.corr,
2752 RouteBindRelayOutcome::Accepted,
2753 )
2754 .unwrap();
2755 assert_eq!(
2756 client_rx.recv().await.unwrap().header.corr,
2757 900 + index as u64
2758 );
2759 bound.push(pending);
2760 }
2761 let (module_egress_tx, _module_egress_rx) = mpsc::channel(8);
2762 let module_ctx = RouteCtx {
2763 connection_id: module_connection,
2764 egress: FrameSink::new(module_egress_tx),
2765 };
2766 (
2767 router,
2768 client_sink,
2769 client_rx,
2770 module_ctx,
2771 bound,
2772 close_receiver,
2773 )
2774 }
2775
2776 #[test]
2783 #[ignore = "timing measurement, run on demand"]
2784 fn egress_sink_per_frame_cost() {
2785 const FRAMES: usize = 1_000_000;
2786 const BATCH: usize = 1_000;
2787 let (sink, mut rx) = crate::server::connection_egress();
2788 let template = stream_frame(9, 1, 0, vec![b't'; 200]);
2789 let started = Instant::now();
2790 for _ in 0..FRAMES / BATCH {
2791 for _ in 0..BATCH {
2792 sink.try_send(template.clone()).unwrap();
2793 }
2794 for _ in 0..BATCH {
2795 let outbound = rx.try_recv().unwrap();
2796 if let Some(charge) = &outbound.charge {
2797 charge.taken_by_writer();
2798 }
2799 drop(std::hint::black_box(outbound));
2800 }
2801 }
2802 let elapsed = started.elapsed();
2803 assert_eq!(sink.backlog().queued_bytes, 0);
2804 println!(
2805 "egress sink: {FRAMES} frames in {elapsed:?}, {:.1} ns/frame",
2806 elapsed.as_nanos() as f64 / FRAMES as f64
2807 );
2808 }
2809
2810 fn stream_frame(channel: u16, epoch: u32, corr: u64, body: Vec<u8>) -> Frame {
2811 Frame::build(
2812 FrameType::StreamData,
2813 Flags::new(false, Priority::Interactive, false),
2814 channel,
2815 epoch,
2816 corr,
2817 body,
2818 )
2819 .unwrap()
2820 }
2821
2822 fn module_route_for_test(
2823 router: &Router,
2824 ctx: &RouteCtx,
2825 route: &crate::forwarding::PendingRouteBindRelay,
2826 ) -> Arc<RouteBinding> {
2827 match router
2828 .forwarding
2829 .lookup_data_route(ctx.connection_id, route.module_channel, route.module_epoch)
2830 .unwrap()
2831 {
2832 DataRoute::Module(DataRouteState::Bound(binding)) => binding,
2833 other => panic!("expected a bound module route, got {other:?}"),
2834 }
2835 }
2836
2837 fn assert_frame_size_error(frame: &Frame, channel: u16, epoch: u32, corr: u64, bytes: usize) {
2838 assert_eq!(frame.header.ty, FrameType::Error);
2839 assert_eq!(frame.header.channel, channel);
2840 assert_eq!(frame.header.epoch, epoch);
2841 assert_eq!(frame.header.corr, corr);
2842 assert!(frame.header.flags.is_daemon_origin());
2843 let body: ErrorBody = serde_json::from_slice(&frame.body).unwrap();
2844 assert_eq!(body.code, error_codes::FRAME_TOO_LARGE);
2845 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2846 assert!(body.message.contains(&bytes.to_string()));
2847 assert!(body.message.contains(&limit.to_string()));
2848 assert_eq!(
2849 body.detail,
2850 Some(serde_json::json!({ "frame_bytes": bytes, "byte_limit": limit }))
2851 );
2852 }
2853
2854 #[tokio::test]
2855 async fn module_egress_response_oversized_reply_keeps_connection_and_other_route() {
2856 let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
2857 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2858 "oversize-provider",
2859 ConnectionId::new(80),
2860 ConnectionId::new(81),
2861 2,
2862 )
2863 .await;
2864 let binding = module_route_for_test(&router, &ctx, &routes[0]);
2865 binding.flow.acquire_tagged(101, false).await.unwrap();
2866 module_route_for_test(&router, &ctx, &routes[1])
2867 .flow
2868 .acquire_tagged(102, false)
2869 .await
2870 .unwrap();
2871 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
2872 let oversized = Frame::build(
2873 FrameType::Response,
2874 Flags::new(false, Priority::Interactive, true),
2875 routes[0].module_channel,
2876 routes[0].module_epoch,
2877 101,
2878 vec![b'x'; limit - subc_protocol::HEADER_LEN + 1],
2879 )
2880 .unwrap();
2881 assert_eq!(sink.backlog().queued_bytes, 0);
2882 router.route_for_connection(&ctx, oversized).await.unwrap();
2883
2884 router
2888 .route_for_connection(
2889 &ctx,
2890 route_frame(
2891 FrameType::Response,
2892 routes[1].module_channel,
2893 routes[1].module_epoch,
2894 102,
2895 ),
2896 )
2897 .await
2898 .unwrap();
2899 assert!(
2900 close.try_recv().is_err(),
2901 "an oversized reply must not close the multiplexed client"
2902 );
2903 let error = rx.try_recv().expect("the waiting client gets an ERROR");
2904 assert_frame_size_error(
2905 &error,
2906 routes[0].client_channel,
2907 routes[0].client_epoch,
2908 101,
2909 limit + 1,
2910 );
2911 let second = rx.try_recv().expect("the other route's reply arrives");
2912 assert_eq!(second.header.ty, FrameType::Response);
2913 assert_eq!(second.header.channel, routes[1].client_channel);
2914 assert_eq!(second.header.epoch, routes[1].client_epoch);
2915 assert_eq!(second.header.corr, 102);
2916 assert_eq!(second.body, b"route-body");
2917 assert_eq!(binding.flow.drain_in_flight(), 0);
2918 for route in &routes {
2919 module_route_for_test(&router, &ctx, route);
2920 }
2921 assert!(rx.try_recv().is_err());
2922 drop((error, second));
2923 assert_eq!(sink.backlog().queued_bytes, 0);
2924 assert_eq!(
2925 router.counters.snapshot()["client_egress_close_delivery_failed"],
2926 0
2927 );
2928
2929 let captured = test_log::captured_logs(&logs);
2930 let warnings: Vec<_> = captured
2931 .lines()
2932 .filter(|line| line.contains("WARN"))
2933 .collect();
2934 assert_eq!(warnings.len(), 1, "{captured}");
2935 let warning = warnings[0];
2936 for expected in [
2937 "module_id=oversize-provider".to_string(),
2938 "client_principal=direct".to_string(),
2939 format!("client_channel={}", routes[0].client_channel),
2940 format!("frame_bytes={}", limit + 1),
2941 format!("byte_limit={limit}"),
2942 ] {
2943 assert!(warning.contains(&expected), "missing {expected}: {warning}");
2944 }
2945 }
2946
2947 async fn poll_once<F: std::future::Future>(
2948 mut future: std::pin::Pin<&mut F>,
2949 ) -> std::task::Poll<F::Output> {
2950 std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
2951 }
2952
2953 fn fill_client_budget(sink: &FrameSink, route: &crate::forwarding::PendingRouteBindRelay) {
2954 sink.try_send(
2955 Frame::build(
2956 FrameType::Response,
2957 Flags::new(false, Priority::Interactive, true),
2958 route.client_channel,
2959 route.client_epoch,
2960 100,
2961 vec![
2962 b'x';
2963 crate::server::CONNECTION_EGRESS_BYTE_BUDGET - subc_protocol::HEADER_LEN
2964 ],
2965 )
2966 .unwrap(),
2967 )
2968 .unwrap();
2969 assert_eq!(
2970 sink.backlog().queued_bytes,
2971 crate::server::CONNECTION_EGRESS_BYTE_BUDGET
2972 );
2973 }
2974
2975 async fn assert_full_client_burst_survives(ty: FrameType, oversized: bool) {
2976 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
2977 "draining-burst",
2978 ConnectionId::new(92),
2979 ConnectionId::new(93),
2980 2,
2981 )
2982 .await;
2983 fill_client_budget(&sink, &routes[0]);
2984 let frame = if oversized {
2985 Frame::build(
2986 ty,
2987 Flags::new(false, Priority::Interactive, false),
2988 routes[1].module_channel,
2989 routes[1].module_epoch,
2990 101,
2991 vec![b's'; crate::server::CONNECTION_EGRESS_BYTE_BUDGET],
2992 )
2993 .unwrap()
2994 } else {
2995 route_frame(ty, routes[1].module_channel, routes[1].module_epoch, 101)
2996 };
2997 let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
2998 assert!(poll_once(delivery.as_mut()).await.is_pending());
2999 assert!(
3000 close.try_recv().is_err(),
3001 "fresh bursts must not close: {ty:?}"
3002 );
3003 tokio::time::advance(Duration::from_millis(50)).await;
3004 let queued = rx.try_recv().unwrap();
3005 queued.charge.as_ref().unwrap().taken_by_writer();
3006 drop(queued);
3007 assert!(matches!(
3008 poll_once(delivery.as_mut()).await,
3009 std::task::Poll::Ready(Ok(()))
3010 ));
3011 let delivered = rx
3012 .try_recv()
3013 .expect("the burst frame must be delivered after draining");
3014 assert_eq!(delivered.header.channel, routes[1].client_channel);
3015 assert_eq!(delivered.header.epoch, routes[1].client_epoch);
3016 assert_eq!(delivered.header.corr, 101);
3017 if oversized {
3018 assert_frame_size_error(
3019 &delivered,
3020 routes[1].client_channel,
3021 routes[1].client_epoch,
3022 101,
3023 crate::server::CONNECTION_EGRESS_BYTE_BUDGET + subc_protocol::HEADER_LEN,
3024 );
3025 } else {
3026 assert_eq!(delivered.header.ty, ty);
3027 }
3028 assert!(rx.try_recv().is_err());
3029 assert!(close.try_recv().is_err());
3030 assert_eq!(
3031 router.counters.snapshot()["client_egress_close_delivery_failed"],
3032 0
3033 );
3034 assert_eq!(router.counters.snapshot()["goodbye_relay_client_failed"], 0);
3035 }
3036
3037 #[tokio::test(start_paused = true)]
3038 async fn module_egress_fresh_full_queue_burst_survives() {
3039 for ty in [
3040 FrameType::Response,
3041 FrameType::StreamData,
3042 FrameType::Request,
3043 FrameType::Push,
3044 ] {
3045 assert_full_client_burst_survives(ty, false).await;
3046 }
3047 }
3048
3049 #[tokio::test(start_paused = true)]
3050 async fn module_goodbye_fresh_full_queue_burst_survives() {
3051 assert_full_client_burst_survives(FrameType::Goodbye, false).await;
3052 }
3053
3054 #[tokio::test(start_paused = true)]
3055 async fn frame_count_backstop_also_allows_a_draining_burst() {
3056 let (tx, mut rx) = mpsc::channel(1);
3057 let sink = FrameSink::with_byte_budget(tx, 2_000);
3058 sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
3059 .unwrap();
3060 let mut delivery = Box::pin(sink.send_to_client(stream_frame(1, 1, 2, vec![b'y'; 100])));
3061 assert!(poll_once(delivery.as_mut()).await.is_pending());
3062 assert!(
3063 sink.backlog().queued_bytes < 2_000,
3064 "only frame capacity is exhausted"
3065 );
3066 tokio::time::advance(Duration::from_millis(50)).await;
3067 drop(rx.try_recv().unwrap());
3068 assert!(matches!(
3069 poll_once(delivery.as_mut()).await,
3070 std::task::Poll::Ready(Ok(()))
3071 ));
3072 assert_eq!(rx.try_recv().unwrap().header.corr, 2);
3073 }
3074
3075 #[tokio::test(start_paused = true)]
3076 async fn oversized_reply_replacement_fresh_full_queue_burst_survives() {
3077 for ty in [FrameType::Response, FrameType::StreamData] {
3078 assert_full_client_burst_survives(ty, true).await;
3079 }
3080 }
3081
3082 #[tokio::test(start_paused = true)]
3083 async fn module_egress_stopped_reader_closes_without_waiting() {
3084 let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3085 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3086 "oldest-stopped-reader",
3087 ConnectionId::new(94),
3088 ConnectionId::new(95),
3089 2,
3090 )
3091 .await;
3092 fill_client_budget(&sink, &routes[0]);
3093 let writing = rx.try_recv().unwrap();
3096 writing.charge.as_ref().unwrap().taken_by_writer();
3097 tokio::time::advance(Duration::from_millis(251)).await;
3098 assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(251)));
3099 assert_eq!(sink.backlog().queued_bytes, queued_frame_bytes(&writing));
3100 let started = tokio::time::Instant::now();
3101 let frame = stream_frame(
3102 routes[0].module_channel,
3103 routes[0].module_epoch,
3104 101,
3105 b"secret-body-never-log".to_vec(),
3106 );
3107 let flags = frame.header.flags.0;
3108 let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3109 assert!(
3110 matches!(
3111 poll_once(delivery.as_mut()).await,
3112 std::task::Poll::Ready(Ok(()))
3113 ),
3114 "an oldest frame older than 250 ms must close immediately, without a new wait"
3115 );
3116 assert_eq!(started.elapsed(), Duration::ZERO);
3117 assert!(close
3118 .try_recv()
3119 .unwrap()
3120 .to_string()
3121 .contains("module_to_client_delivery_failed"));
3122 let captured = test_log::captured_logs(&logs);
3123 for expected in [
3124 "frame_type=Some(StreamData)".to_string(),
3125 format!("frame_flags=Some({flags})"),
3126 "oldest_queued_ms=251".to_string(),
3127 "close_cause=\"oldest_frame_older_than_250_ms\"".to_string(),
3128 ] {
3129 assert!(
3130 captured.contains(&expected),
3131 "missing {expected}: {captured}"
3132 );
3133 }
3134 assert!(!captured.contains("secret-body-never-log"));
3135 }
3136
3137 #[tokio::test(start_paused = true)]
3138 async fn client_gone_mid_wait_resumes_module_reader_immediately() {
3139 for writer_gone in [false, true] {
3140 let (router, sink, mut rx, ctx, routes, _close) = multi_route_client(
3141 "gone-during-burst",
3142 ConnectionId::new(96),
3143 ConnectionId::new(97),
3144 1,
3145 )
3146 .await;
3147 fill_client_budget(&sink, &routes[0]);
3148 let frame = route_frame(
3149 FrameType::Response,
3150 routes[0].module_channel,
3151 routes[0].module_epoch,
3152 101,
3153 );
3154 let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3155 assert!(poll_once(delivery.as_mut()).await.is_pending());
3156 tokio::time::advance(Duration::from_millis(50)).await;
3157 let started = tokio::time::Instant::now();
3158 if writer_gone {
3159 rx.close();
3160 } else {
3161 sink.close_client_delivery();
3162 }
3163 assert!(
3164 matches!(
3165 poll_once(delivery.as_mut()).await,
3166 std::task::Poll::Ready(Ok(()))
3167 ),
3168 "client teardown must wake the module reader before the writer drains"
3169 );
3170 assert_eq!(started.elapsed(), Duration::ZERO);
3171 assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
3172 assert_eq!(rx.try_recv().unwrap().header.corr, 100);
3173 assert!(rx.try_recv().is_err());
3174 }
3175 }
3176
3177 #[tokio::test(start_paused = true)]
3178 async fn cancelled_module_delivery_releases_waiter_without_closing_client() {
3179 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3180 "cancelled-burst",
3181 ConnectionId::new(98),
3182 ConnectionId::new(99),
3183 1,
3184 )
3185 .await;
3186 fill_client_budget(&sink, &routes[0]);
3187 let frame = route_frame(
3188 FrameType::Response,
3189 routes[0].module_channel,
3190 routes[0].module_epoch,
3191 101,
3192 );
3193 let mut delivery = Box::pin(router.route_for_connection(&ctx, frame));
3194 assert!(poll_once(delivery.as_mut()).await.is_pending());
3195 assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 1);
3196 drop(delivery);
3197 assert_eq!(sink.accounting.waiters.load(Ordering::SeqCst), 0);
3198 tokio::time::advance(Duration::from_secs(1)).await;
3199 assert!(close.try_recv().is_err());
3200 drop(rx.try_recv().unwrap());
3201 assert_eq!(sink.backlog().queued_bytes, 0);
3202 assert!(sink.backlog().oldest_age.is_none());
3203 }
3204
3205 #[tokio::test(start_paused = true)]
3206 async fn egress_atomic_age_follows_writer_and_clears_when_drained() {
3207 let (sink, mut rx) = crate::server::connection_egress();
3208 sink.try_send(stream_frame(1, 1, 1, vec![b'x'; 100]))
3209 .unwrap();
3210 tokio::time::advance(Duration::from_millis(200)).await;
3211 sink.try_send(stream_frame(2, 1, 2, vec![b'y'; 100]))
3212 .unwrap();
3213 let writing = rx.try_recv().unwrap();
3214 writing.charge.as_ref().unwrap().taken_by_writer();
3215 assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3216 drop(writing);
3217 assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(200)));
3220 tokio::time::advance(Duration::from_millis(50)).await;
3221 let next = rx.try_recv().unwrap();
3222 next.charge.as_ref().unwrap().taken_by_writer();
3223 assert_eq!(sink.backlog().oldest_age, Some(Duration::from_millis(50)));
3224 assert!(
3225 sink.backlog().queued_bytes > 0,
3226 "in-flight bytes remain charged until written"
3227 );
3228 drop(next);
3229 assert_eq!(sink.backlog().queued_bytes, 0);
3230 assert_eq!(sink.backlog().oldest_age, None);
3231 }
3232
3233 #[tokio::test(start_paused = true)]
3234 async fn module_egress_response_full_client_waits_only_remaining_grace() {
3235 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3236 "backpressure-provider",
3237 ConnectionId::new(82),
3238 ConnectionId::new(83),
3239 2,
3240 )
3241 .await;
3242 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3243 router
3246 .route_for_connection(
3247 &ctx,
3248 Frame::build(
3249 FrameType::Response,
3250 Flags::new(false, Priority::Interactive, true),
3251 routes[0].module_channel,
3252 routes[0].module_epoch,
3253 201,
3254 vec![b'x'; limit - subc_protocol::HEADER_LEN],
3255 )
3256 .unwrap(),
3257 )
3258 .await
3259 .unwrap();
3260 assert_eq!(sink.backlog().queued_bytes, limit);
3261 assert_eq!(sink.backlog().queued_frames, 1);
3262 let ordinary = route_frame(
3263 FrameType::Response,
3264 routes[1].module_channel,
3265 routes[1].module_epoch,
3266 202,
3267 );
3268 assert!(queued_frame_bytes(&ordinary) < limit);
3269 tokio::time::advance(Duration::from_millis(100)).await;
3272 let mut delivery = Box::pin(router.route_for_connection(&ctx, ordinary));
3273 assert!(poll_once(delivery.as_mut()).await.is_pending());
3274 assert!(close.try_recv().is_err());
3275 tokio::time::advance(Duration::from_millis(149)).await;
3276 assert!(poll_once(delivery.as_mut()).await.is_pending());
3277 assert!(close.try_recv().is_err());
3278 tokio::time::advance(Duration::from_millis(1)).await;
3279 assert!(matches!(
3280 poll_once(delivery.as_mut()).await,
3281 std::task::Poll::Ready(Ok(()))
3282 ));
3283 drop(delivery);
3284 let reason = close.try_recv().expect("the full client must close");
3285 assert!(reason
3286 .to_string()
3287 .contains("module_to_client_delivery_failed"));
3288 assert!(!reason.to_string().contains("frame_too_large"));
3289 assert_eq!(
3290 router.counters.snapshot()["client_egress_close_delivery_failed"],
3291 1
3292 );
3293 let admitted = rx.try_recv().expect("the budget-sized frame was admitted");
3294 assert_eq!(admitted.header.ty, FrameType::Response);
3295 assert_eq!(admitted.header.corr, 201);
3296 assert!(
3297 rx.try_recv().is_err(),
3298 "no oversized-frame ERROR is substituted for backpressure"
3299 );
3300 }
3301
3302 #[tokio::test(start_paused = true)]
3303 async fn oversized_reply_to_a_full_client_waits_only_remaining_grace() {
3304 let (router, sink, _rx, ctx, routes, mut close) = multi_route_client(
3305 "full-oversize-provider",
3306 ConnectionId::new(90),
3307 ConnectionId::new(91),
3308 2,
3309 )
3310 .await;
3311 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3312 router
3314 .route_for_connection(
3315 &ctx,
3316 Frame::build(
3317 FrameType::Response,
3318 Flags::new(false, Priority::Interactive, true),
3319 routes[0].module_channel,
3320 routes[0].module_epoch,
3321 501,
3322 vec![b'x'; limit - subc_protocol::HEADER_LEN],
3323 )
3324 .unwrap(),
3325 )
3326 .await
3327 .unwrap();
3328 assert_eq!(sink.backlog().queued_bytes, limit);
3329 let oversized = Frame::build(
3330 FrameType::Response,
3331 Flags::new(false, Priority::Interactive, true),
3332 routes[1].module_channel,
3333 routes[1].module_epoch,
3334 502,
3335 vec![b'y'; limit - subc_protocol::HEADER_LEN + 1],
3336 )
3337 .unwrap();
3338 tokio::time::advance(Duration::from_millis(80)).await;
3341 let mut delivery = Box::pin(router.route_for_connection(&ctx, oversized));
3342 assert!(poll_once(delivery.as_mut()).await.is_pending());
3343 assert!(close.try_recv().is_err());
3344 tokio::time::advance(Duration::from_millis(169)).await;
3345 assert!(poll_once(delivery.as_mut()).await.is_pending());
3346 assert!(close.try_recv().is_err());
3347 tokio::time::advance(Duration::from_millis(1)).await;
3348 assert!(matches!(
3349 poll_once(delivery.as_mut()).await,
3350 std::task::Poll::Ready(Ok(()))
3351 ));
3352 drop(delivery);
3353 let reason = close.try_recv().expect("the full client must close");
3354 assert!(reason
3355 .to_string()
3356 .contains("module_to_client_delivery_failed"));
3357 }
3358
3359 #[tokio::test]
3360 async fn oversized_reverse_request_errors_module_without_reaching_client() {
3361 let (router, _sink, mut rx, mut ctx, routes, mut close) = multi_route_client(
3362 "reverse-oversize-provider",
3363 ConnectionId::new(84),
3364 ConnectionId::new(85),
3365 2,
3366 )
3367 .await;
3368 let (module_tx, mut module_rx) = mpsc::channel(8);
3369 ctx.egress = FrameSink::new(module_tx);
3370 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3371 router
3372 .route_for_connection(
3373 &ctx,
3374 Frame::build(
3375 FrameType::Request,
3376 Flags::new(false, Priority::Interactive, false),
3377 routes[0].module_channel,
3378 routes[0].module_epoch,
3379 301,
3380 vec![b'r'; limit - subc_protocol::HEADER_LEN + 1],
3381 )
3382 .unwrap(),
3383 )
3384 .await
3385 .unwrap();
3386 assert!(
3387 rx.try_recv().is_err(),
3388 "the reverse request never reaches the client"
3389 );
3390 let error = module_rx.try_recv().expect("the module gets the refusal");
3391 assert_frame_size_error(
3392 &error,
3393 routes[0].module_channel,
3394 routes[0].module_epoch,
3395 301,
3396 limit + 1,
3397 );
3398 assert!(module_rx.try_recv().is_err());
3399 for (index, route) in routes.iter().enumerate() {
3400 module_route_for_test(&router, &ctx, route);
3401 router
3402 .route_for_connection(
3403 &ctx,
3404 route_frame(
3405 FrameType::Response,
3406 route.module_channel,
3407 route.module_epoch,
3408 302 + index as u64,
3409 ),
3410 )
3411 .await
3412 .unwrap();
3413 let reply = rx.try_recv().expect("both routes still deliver");
3414 assert_eq!(reply.header.ty, FrameType::Response);
3415 assert_eq!(reply.header.channel, route.client_channel);
3416 }
3417 assert!(close.try_recv().is_err());
3418 assert_eq!(
3419 router.counters.snapshot()["client_egress_close_delivery_failed"],
3420 0
3421 );
3422 }
3423
3424 #[tokio::test]
3425 async fn oversized_push_closes_client_with_precise_warn() {
3426 let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3427 let (router, _sink, mut rx, ctx, routes, mut close) = multi_route_client(
3428 "push-oversize-provider",
3429 ConnectionId::new(86),
3430 ConnectionId::new(87),
3431 2,
3432 )
3433 .await;
3434 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3435 router
3436 .route_for_connection(
3437 &ctx,
3438 Frame::build(
3439 FrameType::Push,
3440 Flags::new(false, Priority::Interactive, false),
3441 routes[0].module_channel,
3442 routes[0].module_epoch,
3443 0,
3444 vec![b'p'; limit - subc_protocol::HEADER_LEN + 1],
3445 )
3446 .unwrap(),
3447 )
3448 .await
3449 .unwrap();
3450 let reason = close
3451 .try_recv()
3452 .expect("unsolicited oversize still closes the client");
3453 assert!(reason
3454 .to_string()
3455 .contains("frame_too_large: oversized push"));
3456 assert!(
3457 rx.try_recv().is_err(),
3458 "no part of the oversized push is forwarded"
3459 );
3460 let captured = test_log::captured_logs(&logs);
3461 let warnings: Vec<_> = captured
3462 .lines()
3463 .filter(|line| line.contains("WARN"))
3464 .collect();
3465 assert_eq!(warnings.len(), 1, "{captured}");
3466 let warning = warnings[0];
3467 for expected in [
3468 "oversized push".to_string(),
3469 "module_id=\"push-oversize-provider\"".to_string(),
3470 "principals=direct".to_string(),
3471 format!("client_channel={}", routes[0].client_channel),
3472 format!("size={}", limit + 1),
3473 format!("limit={limit}"),
3474 ] {
3475 assert!(warning.contains(&expected), "missing {expected}: {warning}");
3476 }
3477 assert_eq!(
3478 router.counters.snapshot()["client_egress_close_delivery_failed"],
3479 1
3480 );
3481 }
3482
3483 #[tokio::test]
3484 async fn oversized_stream_and_error_frames_end_request_even_with_queued_traffic() {
3485 let (router, sink, mut rx, ctx, routes, mut close) = multi_route_client(
3486 "stream-oversize-provider",
3487 ConnectionId::new(88),
3488 ConnectionId::new(89),
3489 2,
3490 )
3491 .await;
3492 let binding = module_route_for_test(&router, &ctx, &routes[0]);
3493 let limit = crate::server::CONNECTION_EGRESS_BYTE_BUDGET;
3494 for ty in [
3495 FrameType::StreamData,
3496 FrameType::StreamEnd,
3497 FrameType::Error,
3498 ] {
3499 binding.flow.acquire_tagged(401, false).await.unwrap();
3500 sink.try_send(route_frame(
3501 FrameType::Response,
3502 routes[1].client_channel,
3503 routes[1].client_epoch,
3504 402,
3505 ))
3506 .unwrap();
3507 router
3508 .route_for_connection(
3509 &ctx,
3510 Frame::build(
3511 ty,
3512 Flags::new(false, Priority::Interactive, false),
3513 routes[0].module_channel,
3514 routes[0].module_epoch,
3515 401,
3516 vec![b's'; limit - subc_protocol::HEADER_LEN + 1],
3517 )
3518 .unwrap(),
3519 )
3520 .await
3521 .unwrap();
3522 assert!(close.try_recv().is_err());
3523 assert_eq!(rx.try_recv().unwrap().header.corr, 402);
3524 let error = rx
3525 .try_recv()
3526 .expect("every correlated oversize frame becomes ERROR");
3527 assert_frame_size_error(
3528 &error,
3529 routes[0].client_channel,
3530 routes[0].client_epoch,
3531 401,
3532 limit + 1,
3533 );
3534 assert_eq!(binding.flow.drain_in_flight(), 0, "{ty:?} ends the request");
3535 assert!(rx.try_recv().is_err());
3536 }
3537 }
3538
3539 #[tokio::test]
3545 async fn awaited_send_parked_behind_a_full_byte_budget_wakes_when_bytes_free() {
3546 let (tx, mut rx) = mpsc::channel(1024);
3547 let sink = FrameSink::with_byte_budget(tx, 2_000);
3548 let mut queued = 0u64;
3549 while sink
3550 .try_send(stream_frame(7, 1, queued, vec![b'x'; 200]))
3551 .is_ok()
3552 {
3553 queued += 1;
3554 }
3555 assert!(queued > 0, "the budget admitted nothing");
3556
3557 let parked = tokio::spawn({
3558 let sink = sink.clone();
3559 async move { sink.send(stream_frame(0, 0, 999, vec![b'r'; 200])).await }
3560 });
3561 tokio::time::sleep(Duration::from_millis(100)).await;
3562 assert!(
3563 !parked.is_finished(),
3564 "the awaited send must wait while the byte budget is full"
3565 );
3566
3567 drop(rx.recv().await.expect("a queued frame"));
3569 tokio::time::timeout(Duration::from_secs(2), parked)
3570 .await
3571 .expect("the parked send was never woken after bytes were freed")
3572 .unwrap()
3573 .unwrap();
3574 }
3575
3576 #[tokio::test]
3581 async fn paused_client_reader_keeps_connection_through_small_frame_burst() {
3582 const ROUTES: usize = 4;
3583 const FRAMES_PER_ROUTE: usize = 250;
3584 let (router, client_sink, mut client_rx, module_ctx, routes, mut close_receiver) =
3585 multi_route_client(
3586 "burst-provider",
3587 ConnectionId::new(60),
3588 ConnectionId::new(61),
3589 ROUTES,
3590 )
3591 .await;
3592
3593 for seq in 0..FRAMES_PER_ROUTE as u64 {
3595 for (index, route) in routes.iter().enumerate() {
3596 let body = format!("route-{index}-token-{seq:05}-{}", "t".repeat(170));
3597 router
3598 .route_for_connection(
3599 &module_ctx,
3600 stream_frame(
3601 route.module_channel,
3602 route.module_epoch,
3603 seq,
3604 body.into_bytes(),
3605 ),
3606 )
3607 .await
3608 .unwrap();
3609 }
3610 }
3611
3612 let backlog = client_sink.backlog();
3613 assert_eq!(backlog.queued_frames, ROUTES * FRAMES_PER_ROUTE);
3614 assert!(backlog.queued_bytes < crate::server::CONNECTION_EGRESS_BYTE_BUDGET);
3615 assert!(
3616 close_receiver.try_recv().is_err(),
3617 "a paused reader under the byte budget must not be closed"
3618 );
3619 assert_eq!(
3620 router.counters.snapshot()["client_egress_close_delivery_failed"],
3621 0
3622 );
3623
3624 let mut next_seq = vec![0u64; ROUTES];
3626 for _ in 0..ROUTES * FRAMES_PER_ROUTE {
3627 let frame = client_rx.try_recv().expect("every queued frame arrives");
3628 let index = routes
3629 .iter()
3630 .position(|route| route.client_channel == frame.header.channel)
3631 .expect("frame arrives on one of the bound client channels");
3632 assert_eq!(frame.header.corr, next_seq[index], "route {index} order");
3633 let expected_prefix = format!("route-{index}-token-{:05}-", next_seq[index]);
3634 assert!(frame.body.starts_with(expected_prefix.as_bytes()));
3635 next_seq[index] += 1;
3636 }
3637 assert!(client_rx.try_recv().is_err());
3638 assert_eq!(next_seq, vec![FRAMES_PER_ROUTE as u64; ROUTES]);
3639 assert_eq!(client_sink.backlog().queued_bytes, 0);
3640 }
3641
3642 #[tokio::test(start_paused = true)]
3645 async fn never_reading_client_is_closed_at_byte_budget_with_warn_diagnosis() {
3646 let (logs, _guard) = test_log::log_capture(tracing::Level::WARN);
3647 const BODY: usize = 16 * 1024;
3648 let (router, client_sink, _client_rx, module_ctx, routes, mut close_receiver) =
3649 multi_route_client(
3650 "stuck-reader-provider",
3651 ConnectionId::new(70),
3652 ConnectionId::new(71),
3653 2,
3654 )
3655 .await;
3656
3657 let mut admitted = 0usize;
3658 let mut sent = 0u64;
3659 let started = tokio::time::Instant::now();
3660 while router.counters.snapshot()["client_egress_close_delivery_failed"] == 0 {
3661 assert!(sent < 1_000, "the byte budget never refused a frame");
3662 let route = &routes[(sent % 2) as usize];
3663 router
3664 .route_for_connection(
3665 &module_ctx,
3666 stream_frame(
3667 route.module_channel,
3668 route.module_epoch,
3669 sent,
3670 vec![b'z'; BODY],
3671 ),
3672 )
3673 .await
3674 .unwrap();
3675 sent += 1;
3676 admitted = client_sink.backlog().queued_frames;
3677 }
3678 assert_eq!(
3679 started.elapsed(),
3680 crate::server::CONNECTION_EGRESS_STALL_GRACE
3681 );
3682 let frame_bytes = subc_protocol::HEADER_LEN + BODY;
3684 assert_eq!(
3685 admitted,
3686 crate::server::CONNECTION_EGRESS_BYTE_BUDGET / frame_bytes
3687 );
3688 let reason = close_receiver
3689 .try_recv()
3690 .expect("the client connection must be asked to close");
3691 assert!(reason
3692 .to_string()
3693 .contains("module_to_client_delivery_failed"));
3694
3695 router
3697 .route_for_connection(
3698 &module_ctx,
3699 stream_frame(
3700 routes[0].module_channel,
3701 routes[0].module_epoch,
3702 sent,
3703 vec![b'z'; BODY],
3704 ),
3705 )
3706 .await
3707 .unwrap();
3708
3709 let captured = test_log::captured_logs(&logs);
3710 let warn_lines = captured
3711 .lines()
3712 .filter(|line| {
3713 line.contains("closing client connection: its egress queue could not take a frame")
3714 })
3715 .collect::<Vec<_>>();
3716 assert_eq!(warn_lines.len(), 1, "exactly one WARN, got: {captured}");
3717 let line = warn_lines[0];
3718 assert!(line.contains("WARN"), "{line}");
3719 assert!(line.contains("connection_id=71"), "{line}");
3720 assert!(
3721 line.contains("module_id=\"stuck-reader-provider\""),
3722 "{line}"
3723 );
3724 assert!(line.contains("client_channel="), "{line}");
3725 assert!(line.contains("principals=direct"), "{line}");
3726 let queued_bytes: usize = line
3727 .split("queued_bytes=")
3728 .nth(1)
3729 .and_then(|rest| rest.split_whitespace().next())
3730 .and_then(|value| value.parse().ok())
3731 .expect("queued_bytes is logged");
3732 assert_eq!(queued_bytes, admitted * frame_bytes);
3733 assert!(
3734 line.contains(&format!("queued_frames={admitted}")),
3735 "{line}"
3736 );
3737 assert!(line.contains("oldest_queued_ms="), "{line}");
3738 assert!(line.contains("oldest_queued_ms=250"), "{line}");
3739 assert!(line.contains("frame_type=Some(StreamData)"), "{line}");
3740 assert!(
3741 line.contains(&format!(
3742 "frame_flags=Some({})",
3743 Flags::new(false, Priority::Interactive, false).0
3744 )),
3745 "{line}"
3746 );
3747 assert!(
3748 line.contains("close_cause=\"waited_250_ms_without_room\""),
3749 "{line}"
3750 );
3751 assert!(
3752 !line.contains(&"z".repeat(20)),
3753 "body bytes must never be logged: {line}"
3754 );
3755 }
3756
3757 fn route_frame(ty: FrameType, channel: u16, epoch: u32, corr: u64) -> Frame {
3758 Frame::build(
3759 ty,
3760 Flags::new(false, Priority::Interactive, false),
3761 channel,
3762 epoch,
3763 corr,
3764 if ty == FrameType::Request || ty == FrameType::Response {
3765 b"route-body".to_vec()
3766 } else {
3767 Vec::new()
3768 },
3769 )
3770 .unwrap()
3771 }
3772
3773 type DynamicRouteFixture = (
3774 Router,
3775 Arc<ForwardingTable>,
3776 RouteCtx,
3777 mpsc::Receiver<crate::router::OutboundFrame>,
3778 RouteCtx,
3779 mpsc::Receiver<crate::router::OutboundFrame>,
3780 mpsc::Receiver<crate::router::OutboundFrame>,
3781 crate::forwarding::PendingRouteBindRelay,
3782 );
3783
3784 fn dynamic_route_fixture(commit: bool) -> DynamicRouteFixture {
3785 dynamic_route_fixture_with_busy(commit, false)
3786 }
3787
3788 fn dynamic_route_fixture_with_busy(commit: bool, busy_refusals: bool) -> DynamicRouteFixture {
3789 let forwarding = Arc::new(ForwardingTable::default());
3790 let control = Arc::new(crate::ControlHandler::with_forwarding(
3791 Arc::new(crate::Registry::default()),
3792 Arc::clone(&forwarding),
3793 ));
3794 let router = Router::with_control_handler(control);
3795 let module_connection = ConnectionId::new(500);
3796 let client_connection = ConnectionId::new(501);
3797 let (module_tx, module_rx) = mpsc::channel(if busy_refusals { 64 } else { 8 });
3798 forwarding
3799 .register_module_connection(
3800 module_connection,
3801 "epoch-router".into(),
3802 2,
3803 Concurrency::ModuleManaged,
3804 FrameSink::new(module_tx),
3805 )
3806 .unwrap();
3807 let (client_tx, client_rx) = mpsc::channel(if busy_refusals { 64 } else { 8 });
3808 let client_sink = FrameSink::new(client_tx);
3809 let pending = forwarding
3810 .begin_route_bind_relay_with_busy_for_test(
3811 client_connection,
3812 client_sink.clone(),
3813 700,
3814 "epoch-router",
3815 busy_refusals,
3816 )
3817 .unwrap();
3818 if commit {
3819 forwarding
3820 .complete_pending_relay(
3821 module_connection,
3822 pending.corr,
3823 RouteBindRelayOutcome::Accepted,
3824 )
3825 .unwrap();
3826 }
3827 let (module_egress_tx, module_egress_rx) = mpsc::channel(8);
3828 (
3829 router,
3830 forwarding,
3831 RouteCtx {
3832 connection_id: client_connection,
3833 egress: client_sink,
3834 },
3835 client_rx,
3836 RouteCtx {
3837 connection_id: module_connection,
3838 egress: FrameSink::new(module_egress_tx),
3839 },
3840 module_egress_rx,
3841 module_rx,
3842 pending,
3843 )
3844 }
3845
3846 #[tokio::test]
3847 async fn route_epochs_validate_both_directions_and_rewrite_to_peer_handle() {
3848 let (
3849 router,
3850 _forwarding,
3851 client_ctx,
3852 mut client_rx,
3853 module_ctx,
3854 _module_egress_rx,
3855 mut module_rx,
3856 pending,
3857 ) = dynamic_route_fixture(true);
3858 let route_open = client_rx.recv().await.unwrap();
3859 assert_eq!(route_open.header.corr, 700);
3860
3861 router
3862 .route_for_connection(
3863 &client_ctx,
3864 route_frame(
3865 FrameType::Request,
3866 pending.client_channel,
3867 pending.client_epoch,
3868 701,
3869 ),
3870 )
3871 .await
3872 .unwrap();
3873 let forwarded = module_rx.recv().await.unwrap();
3874 assert_eq!(forwarded.header.channel, pending.module_channel);
3875 assert_eq!(forwarded.header.epoch, pending.module_epoch);
3876
3877 router
3878 .route_for_connection(
3879 &module_ctx,
3880 route_frame(
3881 FrameType::Response,
3882 pending.module_channel,
3883 pending.module_epoch,
3884 701,
3885 ),
3886 )
3887 .await
3888 .unwrap();
3889 let delivered = client_rx.recv().await.unwrap();
3890 assert_eq!(delivered.header.channel, pending.client_channel);
3891 assert_eq!(delivered.header.epoch, pending.client_epoch);
3892
3893 router
3894 .route_for_connection(
3895 &client_ctx,
3896 route_frame(
3897 FrameType::Request,
3898 pending.client_channel,
3899 pending.client_epoch + 1,
3900 702,
3901 ),
3902 )
3903 .await
3904 .unwrap();
3905 router
3906 .route_for_connection(
3907 &module_ctx,
3908 route_frame(
3909 FrameType::Response,
3910 pending.module_channel,
3911 pending.module_epoch + 1,
3912 703,
3913 ),
3914 )
3915 .await
3916 .unwrap();
3917 let stale_error = client_rx.recv().await.unwrap();
3918 assert_eq!(stale_error.header.ty, FrameType::Error);
3919 assert_eq!(stale_error.header.channel, pending.client_channel);
3920 assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
3921 assert_eq!(stale_error.header.corr, 702);
3922 let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
3923 assert_eq!(body.code, "stale_route_epoch");
3924 assert!(module_rx.try_recv().is_err());
3925 assert!(client_rx.try_recv().is_err());
3926 let counters = router.counters.snapshot();
3927 assert_eq!(counters["client_frames_dropped_stale_route"], 1);
3928 assert_eq!(counters["module_frames_dropped_no_route"], 1);
3929 }
3930
3931 #[tokio::test]
3932 async fn accepted_route_publishes_route_open_before_immediate_reverse_request() {
3933 let (
3934 router,
3935 _,
3936 _client_ctx,
3937 mut client_rx,
3938 module_ctx,
3939 _module_egress_rx,
3940 _module_rx,
3941 pending,
3942 ) = dynamic_route_fixture(true);
3943 router
3944 .route_for_connection(
3945 &module_ctx,
3946 route_frame(
3947 FrameType::Request,
3948 pending.module_channel,
3949 pending.module_epoch,
3950 800,
3951 ),
3952 )
3953 .await
3954 .unwrap();
3955
3956 let first = client_rx.recv().await.unwrap();
3957 let second = client_rx.recv().await.unwrap();
3958 assert_eq!(first.header.channel, 0);
3959 assert_eq!(first.header.corr, 700);
3960 assert_eq!(second.header.channel, pending.client_channel);
3961 assert_eq!(second.header.epoch, pending.client_epoch);
3962 assert_eq!(second.header.corr, 800);
3963 }
3964
3965 #[tokio::test]
3966 async fn reserved_slot_ingress_errors_only_matching_client_requests() {
3967 let (
3968 router,
3969 _forwarding,
3970 client_ctx,
3971 mut client_rx,
3972 _module_ctx,
3973 _module_egress_rx,
3974 mut module_rx,
3975 pending,
3976 ) = dynamic_route_fixture(false);
3977 router
3978 .route_for_connection(
3979 &client_ctx,
3980 route_frame(
3981 FrameType::Request,
3982 pending.client_channel,
3983 pending.client_epoch,
3984 900,
3985 ),
3986 )
3987 .await
3988 .unwrap();
3989 let error = client_rx.recv().await.unwrap();
3990 assert_eq!(error.header.ty, FrameType::Error);
3991 assert_eq!(error.header.channel, pending.client_channel);
3992 assert_eq!(error.header.epoch, pending.client_epoch);
3993 assert_eq!(error.header.corr, 900);
3994
3995 router
3996 .route_for_connection(
3997 &client_ctx,
3998 route_frame(
3999 FrameType::Response,
4000 pending.client_channel,
4001 pending.client_epoch,
4002 901,
4003 ),
4004 )
4005 .await
4006 .unwrap();
4007 router
4008 .route_for_connection(
4009 &client_ctx,
4010 route_frame(
4011 FrameType::Request,
4012 pending.client_channel,
4013 pending.client_epoch + 1,
4014 902,
4015 ),
4016 )
4017 .await
4018 .unwrap();
4019 let stale_error = client_rx.recv().await.unwrap();
4020 assert_eq!(stale_error.header.ty, FrameType::Error);
4021 assert_eq!(stale_error.header.channel, pending.client_channel);
4022 assert_eq!(stale_error.header.epoch, pending.client_epoch + 1);
4023 assert_eq!(stale_error.header.corr, 902);
4024 let body: ErrorBody = serde_json::from_slice(&stale_error.body).unwrap();
4025 assert_eq!(body.code, "stale_route_epoch");
4026 assert!(module_rx.try_recv().is_err());
4027 let counters = router.counters.snapshot();
4028 assert_eq!(counters["client_frames_dropped_stale_route"], 1);
4029 assert_eq!(counters["module_frames_dropped_no_route"], 0);
4030 }
4031
4032 #[tokio::test]
4033 async fn dropped_module_route_goodbye_increments_counter() {
4034 let (
4035 router,
4036 _forwarding,
4037 client_ctx,
4038 mut client_rx,
4039 _module_ctx,
4040 _module_egress_rx,
4041 mut module_rx,
4042 pending,
4043 ) = dynamic_route_fixture(true);
4044 let _ = client_rx.recv().await;
4045 module_rx.close();
4046
4047 router
4048 .route_for_connection(
4049 &client_ctx,
4050 route_frame(
4051 FrameType::Goodbye,
4052 pending.client_channel,
4053 pending.client_epoch,
4054 999,
4055 ),
4056 )
4057 .await
4058 .unwrap();
4059
4060 let counters = router.counters.snapshot();
4061 assert_eq!(counters["goodbye_relay_module_dropped"], 1);
4062 assert_eq!(
4063 counters["goodbye_relay_module_dropped_by_module"],
4064 serde_json::json!({ "epoch-router": 1 })
4065 );
4066 assert_eq!(counters["route_released_epoch_fenced"], 1);
4067 }
4068
4069 #[tokio::test]
4070 async fn module_request_on_stale_epoch_receives_stale_route_epoch() {
4071 let (
4072 router,
4073 _forwarding,
4074 _client_ctx,
4075 _client_rx,
4076 module_ctx,
4077 mut module_egress_rx,
4078 mut module_rx,
4079 pending,
4080 ) = dynamic_route_fixture(true);
4081
4082 router
4083 .route_for_connection(
4084 &module_ctx,
4085 route_frame(
4086 FrameType::Request,
4087 pending.module_channel,
4088 pending.module_epoch + 1,
4089 1_000,
4090 ),
4091 )
4092 .await
4093 .unwrap();
4094
4095 let error = module_egress_rx.try_recv().unwrap();
4096 assert_eq!(error.header.ty, FrameType::Error);
4097 assert_eq!(error.header.channel, pending.module_channel);
4098 assert_eq!(error.header.epoch, pending.module_epoch + 1);
4099 assert_eq!(error.header.corr, 1_000);
4100 let body: ErrorBody = serde_json::from_slice(&error.body).unwrap();
4101 assert_eq!(body.code, "stale_route_epoch");
4102 assert!(module_rx.try_recv().is_err());
4103 let counters = router.counters.snapshot();
4104 assert_eq!(counters["module_requests_dropped_stale_route"], 1);
4105 assert_eq!(counters["module_frames_dropped_no_route"], 0);
4106 }
4107
4108 #[tokio::test(start_paused = true)]
4109 async fn stale_epoch_drop_warns_once_per_minute_with_suppressed_count() {
4110 let (output, _guard) = test_log::log_capture(tracing::Level::WARN);
4111 let (
4112 router,
4113 _forwarding,
4114 client_ctx,
4115 mut client_rx,
4116 _module_ctx,
4117 _module_egress_rx,
4118 mut module_rx,
4119 pending,
4120 ) = dynamic_route_fixture(true);
4121 let _ = client_rx.recv().await;
4122 for corr in 1..=3 {
4123 router
4124 .route_for_connection(
4125 &client_ctx,
4126 route_frame(
4127 FrameType::Request,
4128 pending.client_channel,
4129 pending.client_epoch + 1,
4130 corr,
4131 ),
4132 )
4133 .await
4134 .unwrap();
4135 assert_eq!(client_rx.recv().await.unwrap().header.ty, FrameType::Error);
4136 }
4137 assert!(module_rx.try_recv().is_err());
4138 let logs = test_log::captured_logs(&output);
4139 assert_eq!(
4140 logs.matches("dropping data-plane frame").count(),
4141 1,
4142 "{logs}"
4143 );
4144 assert!(logs.contains("WARN"), "{logs}");
4145 assert!(
4146 logs.contains(&format!("connection_id={}", client_ctx.connection_id.get())),
4147 "{logs}"
4148 );
4149 assert!(
4150 logs.contains(&format!("channel={}", pending.client_channel)),
4151 "{logs}"
4152 );
4153 assert!(
4154 logs.contains(&format!("epoch={}", pending.client_epoch + 1)),
4155 "{logs}"
4156 );
4157 assert!(
4158 logs.contains("client_to_module") && logs.contains("stale_epoch"),
4159 "{logs}"
4160 );
4161 assert!(logs.contains("suppressed=0"), "{logs}");
4162
4163 tokio::time::advance(Duration::from_secs(60)).await;
4164 router
4165 .route_for_connection(
4166 &client_ctx,
4167 route_frame(
4168 FrameType::Request,
4169 pending.client_channel,
4170 pending.client_epoch + 2,
4171 4,
4172 ),
4173 )
4174 .await
4175 .unwrap();
4176 let logs = test_log::captured_logs(&output);
4177 assert_eq!(
4178 logs.matches("dropping data-plane frame").count(),
4179 2,
4180 "{logs}"
4181 );
4182 assert!(logs.contains("suppressed=2"), "{logs}");
4183 }
4184
4185 #[tokio::test]
4186 async fn module_request_on_reserved_or_absent_route_receives_unknown_channel() {
4187 let (
4188 reserved_router,
4189 _forwarding,
4190 _client_ctx,
4191 _client_rx,
4192 reserved_module_ctx,
4193 mut reserved_module_egress_rx,
4194 _module_rx,
4195 reserved,
4196 ) = dynamic_route_fixture(false);
4197 reserved_router
4198 .route_for_connection(
4199 &reserved_module_ctx,
4200 route_frame(
4201 FrameType::Request,
4202 reserved.module_channel,
4203 reserved.module_epoch,
4204 1_001,
4205 ),
4206 )
4207 .await
4208 .unwrap();
4209 let reserved_error = reserved_module_egress_rx.try_recv().unwrap();
4210 let reserved_body: ErrorBody = serde_json::from_slice(&reserved_error.body).unwrap();
4211 assert_eq!(reserved_error.header.ty, FrameType::Error);
4212 assert_eq!(reserved_error.header.channel, reserved.module_channel);
4213 assert_eq!(reserved_error.header.epoch, reserved.module_epoch);
4214 assert_eq!(reserved_error.header.corr, 1_001);
4215 assert_eq!(reserved_body.code, "unknown_channel");
4216 assert_eq!(
4217 reserved_router.counters.snapshot()["module_requests_dropped_stale_route"],
4218 1
4219 );
4220
4221 let (
4222 absent_router,
4223 _forwarding,
4224 _client_ctx,
4225 _client_rx,
4226 absent_module_ctx,
4227 mut absent_module_egress_rx,
4228 _module_rx,
4229 absent,
4230 ) = dynamic_route_fixture(false);
4231 absent_router
4232 .route_for_connection(
4233 &absent_module_ctx,
4234 route_frame(
4235 FrameType::Request,
4236 absent.module_channel + 1,
4237 absent.module_epoch,
4238 1_002,
4239 ),
4240 )
4241 .await
4242 .unwrap();
4243 let absent_error = absent_module_egress_rx.try_recv().unwrap();
4244 let absent_body: ErrorBody = serde_json::from_slice(&absent_error.body).unwrap();
4245 assert_eq!(absent_error.header.ty, FrameType::Error);
4246 assert_eq!(absent_error.header.channel, absent.module_channel + 1);
4247 assert_eq!(absent_error.header.epoch, absent.module_epoch);
4248 assert_eq!(absent_error.header.corr, 1_002);
4249 assert_eq!(absent_body.code, "unknown_channel");
4250 assert_eq!(
4251 absent_router.counters.snapshot()["module_requests_dropped_stale_route"],
4252 1
4253 );
4254 }
4255
4256 #[tokio::test]
4257 async fn non_request_module_frame_on_dead_route_is_counted_without_error() {
4258 let (
4259 router,
4260 forwarding,
4261 client_ctx,
4262 mut client_rx,
4263 module_ctx,
4264 mut module_egress_rx,
4265 mut module_rx,
4266 pending,
4267 ) = dynamic_route_fixture(true);
4268 let (other_module_tx, _other_module_rx) = mpsc::channel(8);
4269 forwarding
4270 .register_module_connection(
4271 ConnectionId::new(502),
4272 "other-module".into(),
4273 2,
4274 Concurrency::ModuleManaged,
4275 FrameSink::new(other_module_tx),
4276 )
4277 .unwrap();
4278 let _ = client_rx.recv().await.unwrap();
4279
4280 router
4281 .route_for_connection(
4282 &client_ctx,
4283 route_frame(
4284 FrameType::Goodbye,
4285 pending.client_channel,
4286 pending.client_epoch,
4287 1_003,
4288 ),
4289 )
4290 .await
4291 .unwrap();
4292 let _ = module_rx.recv().await.unwrap();
4293
4294 router
4295 .route_for_connection(
4296 &module_ctx,
4297 route_frame(
4298 FrameType::StreamData,
4299 pending.module_channel,
4300 pending.module_epoch,
4301 1_004,
4302 ),
4303 )
4304 .await
4305 .unwrap();
4306
4307 let reply = module_egress_rx.try_recv().unwrap();
4310 assert_eq!(reply.header.ty, FrameType::Goodbye);
4311 assert!(module_egress_rx.try_recv().is_err());
4312 let counters = router.counters.snapshot();
4313 assert_eq!(counters["module_frames_dropped_no_route"], 1);
4314 assert_eq!(
4315 counters["module_frames_dropped_no_route_by_module"],
4316 serde_json::json!({ "epoch-router": 1 })
4317 );
4318 assert_eq!(counters["module_requests_dropped_stale_route"], 0);
4319 }
4320
4321 fn drain_now(rx: &mut mpsc::Receiver<OutboundFrame>) {
4324 while rx.try_recv().is_ok() {}
4325 }
4326
4327 #[tokio::test]
4333 async fn route_goodbye_refused_by_stalled_module_is_delivered_when_it_resumes_reading() {
4334 const BUDGET: usize = 4_096;
4335 let forwarding = Arc::new(ForwardingTable::default());
4336 let control = Arc::new(ControlHandler::with_forwarding(
4337 Arc::new(Registry::default()),
4338 Arc::clone(&forwarding),
4339 ));
4340 let router = Router::with_control_handler(control);
4341 let module_connection = ConnectionId::new(80);
4342 let client_connection = ConnectionId::new(81);
4343 let (module_tx, mut module_rx) = mpsc::channel(64);
4344 let module_sink = FrameSink::with_byte_budget(module_tx, BUDGET);
4345 forwarding
4346 .register_module_connection(
4347 module_connection,
4348 "stalled-provider".into(),
4349 2,
4350 Concurrency::ModuleManaged,
4351 module_sink.clone(),
4352 )
4353 .unwrap();
4354 let (client_tx, mut client_rx) = mpsc::channel(8);
4355 let client_sink = FrameSink::new(client_tx);
4356 let pending = forwarding
4357 .begin_route_bind_relay_for_test(
4358 client_connection,
4359 client_sink.clone(),
4360 1_100,
4361 "stalled-provider",
4362 )
4363 .unwrap();
4364 forwarding
4365 .complete_pending_relay(
4366 module_connection,
4367 pending.corr,
4368 RouteBindRelayOutcome::Accepted,
4369 )
4370 .unwrap();
4371 let _ = client_rx.recv().await.unwrap();
4372 drain_now(&mut module_rx);
4373
4374 module_sink
4377 .try_send(stream_frame(9, 1, 0, vec![b'f'; BUDGET]))
4378 .unwrap();
4379 assert!(module_sink
4380 .try_send(stream_frame(9, 1, 1, Vec::new()))
4381 .is_err());
4382
4383 let client_ctx = RouteCtx {
4384 connection_id: client_connection,
4385 egress: client_sink,
4386 };
4387 router
4388 .route_for_connection(
4389 &client_ctx,
4390 route_frame(
4391 FrameType::Goodbye,
4392 pending.client_channel,
4393 pending.client_epoch,
4394 1_101,
4395 ),
4396 )
4397 .await
4398 .unwrap();
4399 tokio::task::yield_now().await;
4400 assert_eq!(
4401 router.counters.snapshot()["goodbye_relay_module_dropped"],
4402 0,
4403 "a GOODBYE refused by a momentarily full module queue must not be dropped"
4404 );
4405
4406 let filler = module_rx.recv().await.unwrap();
4408 assert_eq!(filler.header.ty, FrameType::StreamData);
4409 drop(filler);
4410 let goodbye = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
4411 .await
4412 .expect("the refused GOODBYE must be delivered once the module frees room")
4413 .unwrap();
4414 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
4415 assert_eq!(goodbye.header.channel, pending.module_channel);
4416 assert_eq!(goodbye.header.epoch, pending.module_epoch);
4417 assert_eq!(
4418 router.counters.snapshot()["goodbye_relay_module_dropped"],
4419 0
4420 );
4421 }
4422
4423 #[tokio::test]
4429 async fn module_frame_on_route_the_daemon_does_not_hold_is_answered_with_goodbye() {
4430 let (
4431 router,
4432 _forwarding,
4433 client_ctx,
4434 mut client_rx,
4435 module_ctx,
4436 mut module_egress_rx,
4437 mut module_rx,
4438 pending,
4439 ) = dynamic_route_fixture(true);
4440 let _ = client_rx.recv().await.unwrap();
4441 router
4442 .route_for_connection(
4443 &client_ctx,
4444 route_frame(
4445 FrameType::Goodbye,
4446 pending.client_channel,
4447 pending.client_epoch,
4448 1_200,
4449 ),
4450 )
4451 .await
4452 .unwrap();
4453 drain_now(&mut module_rx);
4454
4455 let cases = [
4456 (pending.module_channel, pending.module_epoch),
4458 (pending.module_channel + 1, 1),
4460 ];
4461 for (channel, epoch) in cases {
4462 router
4463 .route_for_connection(
4464 &module_ctx,
4465 route_frame(FrameType::StreamData, channel, epoch, 1_201),
4466 )
4467 .await
4468 .unwrap();
4469 let reply = module_egress_rx
4470 .try_recv()
4471 .expect("a frame on a route the daemon does not hold is answered");
4472 assert_eq!(reply.header.ty, FrameType::Goodbye);
4473 assert_eq!(reply.header.channel, channel);
4474 assert_eq!(reply.header.epoch, epoch);
4475 assert_eq!(reply.header.corr, 0);
4476 assert!(module_egress_rx.try_recv().is_err());
4477 }
4478
4479 router
4482 .route_for_connection(
4483 &module_ctx,
4484 route_frame(FrameType::Goodbye, pending.module_channel + 2, 1, 0),
4485 )
4486 .await
4487 .unwrap();
4488 assert!(module_egress_rx.try_recv().is_err());
4489
4490 let counters = router.counters.snapshot();
4491 assert_eq!(counters["module_frames_dropped_no_route"], 3);
4492 assert_eq!(counters["module_frames_dropped_released_route"], 1);
4493 assert_eq!(
4494 counters["module_frames_dropped_released_route_by_module"],
4495 serde_json::json!({ "epoch-router": 1 })
4496 );
4497 assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4498 }
4499
4500 #[tokio::test(start_paused = true)]
4504 async fn burst_of_orphan_module_frames_is_answered_once_per_interval() {
4505 let (
4506 router,
4507 _forwarding,
4508 client_ctx,
4509 mut client_rx,
4510 module_ctx,
4511 mut module_egress_rx,
4512 mut module_rx,
4513 pending,
4514 ) = dynamic_route_fixture(true);
4515 let _ = client_rx.recv().await.unwrap();
4516 router
4517 .route_for_connection(
4518 &client_ctx,
4519 route_frame(
4520 FrameType::Goodbye,
4521 pending.client_channel,
4522 pending.client_epoch,
4523 1_300,
4524 ),
4525 )
4526 .await
4527 .unwrap();
4528 drain_now(&mut module_rx);
4529
4530 let send_burst = |corr: u64| {
4531 route_frame(
4532 FrameType::StreamData,
4533 pending.module_channel,
4534 pending.module_epoch,
4535 corr,
4536 )
4537 };
4538 for corr in 0..20 {
4539 router
4540 .route_for_connection(&module_ctx, send_burst(corr))
4541 .await
4542 .unwrap();
4543 }
4544 let mut replies = 0;
4545 while let Ok(reply) = module_egress_rx.try_recv() {
4546 assert_eq!(reply.header.ty, FrameType::Goodbye);
4547 replies += 1;
4548 }
4549 assert_eq!(replies, 1, "a burst within the interval gets one GOODBYE");
4550
4551 tokio::time::advance(ORPHAN_ROUTE_GOODBYE_INTERVAL).await;
4552 router
4553 .route_for_connection(&module_ctx, send_burst(20))
4554 .await
4555 .unwrap();
4556 let retry = module_egress_rx
4557 .try_recv()
4558 .expect("the first orphan frame after the interval is answered again");
4559 assert_eq!(retry.header.channel, pending.module_channel);
4560 assert_eq!(retry.header.epoch, pending.module_epoch);
4561
4562 let counters = router.counters.snapshot();
4563 assert_eq!(counters["module_frames_dropped_no_route"], 21);
4564 assert_eq!(counters["module_orphan_route_goodbyes_sent"], 2);
4565 }
4566
4567 #[test]
4569 fn orphan_goodbye_rate_limit_state_is_released_with_the_connection() {
4570 let router = Router::with_default_self_handler();
4571 let connection = router.begin_connection();
4572 let id = connection.id();
4573 assert!(router.orphan_goodbyes.claim(id, 7));
4574 assert!(!router.orphan_goodbyes.claim(id, 7));
4575 assert!(router.orphan_goodbyes.claim(id, 8));
4576 drop(connection);
4577 assert!(router
4578 .orphan_goodbyes
4579 .last_sent
4580 .lock()
4581 .unwrap()
4582 .get(&id)
4583 .is_none());
4584 }
4585
4586 #[test]
4587 fn channel_zero_cannot_be_registered_as_backend() {
4588 let mut router = Router::with_default_self_handler();
4589
4590 let err = router.register_backend(0, EchoBackend).unwrap_err();
4591
4592 assert_eq!(err, RouterError::ReservedChannelZero);
4593 }
4594}