1use std::{
2 any::Any,
3 error::Error,
4 fmt, io,
5 net::SocketAddr,
6 sync::{
7 atomic::{AtomicU64, Ordering as AtomicOrdering},
8 Arc,
9 },
10 time::{Duration, Instant},
11};
12
13use subc_transport::{
14 authenticate_server, AuthError, AuthStage, DAEMON_ID_LEN, WATCHDOG_CLIENT_ROLE,
15};
16use tokio::{
17 io::{AsyncRead, AsyncWrite, AsyncWriteExt, BufReader, BufWriter},
18 net::TcpListener,
19 sync::{mpsc, Semaphore},
20 task::{JoinHandle, JoinSet},
21 time::timeout,
22};
23use tracing::{debug, info, warn};
24
25use crate::{
26 forwarding::{CloseReason, ConnectionCloseReceiver},
27 observability::ConnectedClients,
28 read_frame,
29 router::{FrameSink, RouteCtx, Router},
30 write_frame, FrameIoError, RouterError,
31};
32
33pub const CONNECTION_EGRESS_BYTE_BUDGET: usize = subc_protocol::MAX_DELIVERED_FRAME_LEN;
52pub const CONNECTION_EGRESS_STALL_GRACE: Duration = Duration::from_millis(250);
59pub const CONNECTION_EGRESS_FRAME_CAP: usize = 32 * 1024;
68pub const MAX_PENDING_ROUTE_OPENS_PER_CONNECTION: usize = 8;
74pub(crate) const MAX_PENDING_ROUTE_BINDS_PER_TARGET: usize =
78 MAX_PENDING_ROUTE_OPENS_PER_CONNECTION * 2;
79pub const DEFAULT_AUTH_DEADLINE: Duration = Duration::from_secs(2);
80pub const DEFAULT_MAX_UNAUTHENTICATED_CONNECTIONS: usize = 256;
85const CLOSE_DRAIN_GRACE: Duration = Duration::from_secs(2);
86const PRE_AUTH_REJECT_LOG_INTERVAL: Duration = Duration::from_secs(1);
87
88struct ClientDeliveryGuard(FrameSink);
91
92impl Drop for ClientDeliveryGuard {
93 fn drop(&mut self) {
94 self.0.close_client_delivery();
95 }
96}
97
98#[derive(Clone, Copy, Debug)]
99enum PreAuthRejectStage {
100 WaitingForHandshakeSlot,
101 HandshakeDeadline,
102 InvalidProof,
103 MalformedMessage,
104 PeerClosed,
105 OtherAuthenticationFailure,
106}
107
108impl PreAuthRejectStage {
109 const fn index(self) -> usize {
110 match self {
111 Self::WaitingForHandshakeSlot => 0,
112 Self::HandshakeDeadline => 1,
113 Self::InvalidProof => 2,
114 Self::MalformedMessage => 3,
115 Self::PeerClosed => 4,
116 Self::OtherAuthenticationFailure => 5,
117 }
118 }
119
120 const fn stage(self) -> &'static str {
121 match self {
122 Self::WaitingForHandshakeSlot => "waiting_for_handshake_slot",
123 Self::HandshakeDeadline => "handshake_deadline",
124 Self::InvalidProof => "client_authentication",
125 Self::MalformedMessage => "malformed_message",
126 Self::PeerClosed => "peer_closed",
127 Self::OtherAuthenticationFailure => "authentication",
128 }
129 }
130
131 const fn reason(self) -> &'static str {
132 match self {
133 Self::WaitingForHandshakeSlot => "capacity_wait_timed_out",
134 Self::HandshakeDeadline => "handshake_deadline_expired",
135 Self::InvalidProof => "client_proof_mismatch",
136 Self::MalformedMessage => "malformed_or_oversized_message",
137 Self::PeerClosed => "peer_closed_before_authentication_completed",
138 Self::OtherAuthenticationFailure => "authentication_failed_before_routing",
139 }
140 }
141}
142
143#[derive(Debug)]
144struct PreAuthRejectRateLimit {
145 next_allowed_ns: AtomicU64,
146 suppressed: AtomicU64,
147}
148
149impl PreAuthRejectRateLimit {
150 const fn new() -> Self {
151 Self {
152 next_allowed_ns: AtomicU64::new(0),
153 suppressed: AtomicU64::new(0),
154 }
155 }
156}
157
158#[derive(Debug)]
161struct PreAuthRejectLog {
162 epoch: Instant,
163 limiter: PreAuthRejectLimiter,
164}
165
166impl PreAuthRejectLog {
167 fn new() -> Self {
168 Self {
169 epoch: Instant::now(),
170 limiter: PreAuthRejectLimiter::new(),
171 }
172 }
173}
174
175#[derive(Debug)]
177struct PreAuthRejectLimiter {
178 stages: [PreAuthRejectRateLimit; 6],
179}
180
181impl PreAuthRejectLimiter {
182 fn new() -> Self {
183 Self {
184 stages: std::array::from_fn(|_| PreAuthRejectRateLimit::new()),
185 }
186 }
187
188 fn allow_at(&self, stage: PreAuthRejectStage, now: Duration) -> Option<u64> {
189 let slot = &self.stages[stage.index()];
190 let now_ns = now.as_nanos().min(u64::MAX as u128) as u64;
191 let interval_ns = PRE_AUTH_REJECT_LOG_INTERVAL.as_nanos() as u64;
192 let mut next_allowed_ns = slot.next_allowed_ns.load(AtomicOrdering::Relaxed);
193
194 loop {
195 if now_ns < next_allowed_ns {
196 slot.suppressed.fetch_add(1, AtomicOrdering::Relaxed);
197 return None;
198 }
199
200 match slot.next_allowed_ns.compare_exchange_weak(
201 next_allowed_ns,
202 now_ns.saturating_add(interval_ns),
203 AtomicOrdering::Relaxed,
204 AtomicOrdering::Relaxed,
205 ) {
206 Ok(_) => return Some(slot.suppressed.swap(0, AtomicOrdering::Relaxed)),
207 Err(actual) => next_allowed_ns = actual,
208 }
209 }
210 }
211}
212
213fn log_pre_auth_rejection(
214 error: &ConnectionError,
215 peer_addr: SocketAddr,
216 elapsed: Duration,
217 now: Duration,
218 limiter: &PreAuthRejectLimiter,
219) {
220 let Some(stage) = error.pre_auth_reject_stage() else {
221 return;
222 };
223 let Some(suppressed_since_last) = limiter.allow_at(stage, now) else {
224 return;
225 };
226 let elapsed_ms = elapsed.as_millis().min(u64::MAX as u128) as u64;
227
228 info!(
229 peer_addr = %peer_addr,
230 elapsed_ms,
231 stage = stage.stage(),
232 reason = stage.reason(),
233 auth_stage = ?error.auth_stage(),
234 suppressed_since_last,
235 error = %error,
236 "subc TCP connection rejected before routing"
237 );
238}
239
240async fn handle_accepted_connection<S>(
241 stream: S,
242 router: Arc<Router>,
243 auth: ServerAuth,
244 peer_addr: SocketAddr,
245 accepted_at: Instant,
246) -> Result<(), ConnectionError>
247where
248 S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
249{
250 let reject_log = Arc::clone(&auth.pre_auth_reject_log);
251 let result = handle_connection(stream, router, auth).await;
252 if let Err(error) = &result {
253 if error.is_quiet_reject() {
254 log_pre_auth_rejection(
255 error,
256 peer_addr,
257 accepted_at.elapsed(),
258 reject_log.epoch.elapsed(),
259 &reject_log.limiter,
260 );
261 } else {
262 warn!(?peer_addr, error = %error, "subc connection ended with error");
263 }
264 }
265 result
266}
267
268#[derive(Clone)]
271pub struct ServerAuth {
272 key: Arc<[u8]>,
273 daemon_id: [u8; DAEMON_ID_LEN],
274 daemon_ver: Arc<str>,
275 deadline: Duration,
276 unauthenticated: Arc<Semaphore>,
277 connected_clients: ConnectedClients,
278 pre_auth_reject_log: Arc<PreAuthRejectLog>,
279}
280
281impl ServerAuth {
282 pub fn new(
283 key: Vec<u8>,
284 daemon_id: [u8; DAEMON_ID_LEN],
285 daemon_ver: impl Into<String>,
286 ) -> Self {
287 Self::with_limits(
288 key,
289 daemon_id,
290 daemon_ver,
291 DEFAULT_AUTH_DEADLINE,
292 DEFAULT_MAX_UNAUTHENTICATED_CONNECTIONS,
293 )
294 }
295
296 pub fn with_limits(
298 key: Vec<u8>,
299 daemon_id: [u8; DAEMON_ID_LEN],
300 daemon_ver: impl Into<String>,
301 deadline: Duration,
302 max_unauthenticated: usize,
303 ) -> Self {
304 Self {
305 key: Arc::from(key),
306 daemon_id,
307 daemon_ver: Arc::from(daemon_ver.into()),
308 deadline,
309 unauthenticated: Arc::new(Semaphore::new(max_unauthenticated.max(1))),
310 connected_clients: ConnectedClients::new(),
311 pre_auth_reject_log: Arc::new(PreAuthRejectLog::new()),
312 }
313 }
314
315 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
316 self.connected_clients = connected_clients;
317 self
318 }
319}
320
321impl fmt::Debug for ServerAuth {
322 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
323 f.debug_struct("ServerAuth")
324 .field("key", &"<redacted>")
325 .field("daemon_id", &self.daemon_id)
326 .field("daemon_ver", &self.daemon_ver)
327 .field("deadline", &self.deadline)
328 .finish_non_exhaustive()
329 }
330}
331
332pub async fn serve_listener(
335 listener: TcpListener,
336 router: Arc<Router>,
337 auth: ServerAuth,
338) -> Result<(), ServerError> {
339 serve_listener_with_accept(listener.local_addr().ok(), router, auth, || {
340 listener.accept()
341 })
342 .await
343}
344
345async fn serve_listener_with_accept<A, F>(
346 local_addr: Option<SocketAddr>,
347 router: Arc<Router>,
348 auth: ServerAuth,
349 mut accept: A,
350) -> Result<(), ServerError>
351where
352 A: FnMut() -> F,
353 F: std::future::Future<Output = io::Result<(tokio::net::TcpStream, SocketAddr)>>,
354{
355 loop {
356 let (stream, peer_addr) = match accept().await {
357 Ok(accepted) => accepted,
358 Err(source) => {
359 let kind = source.kind();
360 let exhausted = accept_resource_exhausted(&source);
364 if matches!(
365 kind,
366 io::ErrorKind::ConnectionAborted
367 | io::ErrorKind::ConnectionReset
368 | io::ErrorKind::Interrupted
369 ) || exhausted
370 {
371 warn!(?local_addr, error = %source, "temporary TCP accept failure");
372 if exhausted {
373 tokio::time::sleep(Duration::from_millis(75)).await;
374 }
375 continue;
376 }
377 return Err(ServerError::Accept { local_addr, source });
378 }
379 };
380 let accepted_at = Instant::now();
381 if let Err(source) = stream.set_nodelay(true) {
397 warn!(?peer_addr, error = %source, "could not disable Nagle on accepted connection");
398 }
399 debug!(?peer_addr, ?local_addr, "accepted subc TCP connection");
400 let router = Arc::clone(&router);
401 let auth = auth.clone();
402 tokio::spawn(async move {
403 let _ = handle_accepted_connection(stream, router, auth, peer_addr, accepted_at).await;
404 });
405 }
406}
407
408#[cfg_attr(not(unix), allow(dead_code))]
409fn temporary_accept_failure(source: &io::Error) -> bool {
410 matches!(
411 source.kind(),
412 io::ErrorKind::ConnectionAborted
413 | io::ErrorKind::ConnectionReset
414 | io::ErrorKind::Interrupted
415 ) || accept_resource_exhausted(source)
416}
417
418fn accept_resource_exhausted(error: &io::Error) -> bool {
419 #[cfg(unix)]
424 {
425 use rustix::io::Errno;
426 let exhausted = [Errno::NFILE, Errno::MFILE, Errno::NOBUFS];
427 error
428 .raw_os_error()
429 .is_some_and(|code| exhausted.iter().any(|e| e.raw_os_error() == code))
430 }
431 #[cfg(windows)]
432 {
433 matches!(error.raw_os_error(), Some(10024 | 10055))
434 }
435 #[cfg(not(any(unix, windows)))]
436 {
437 let _ = error;
438 false
439 }
440}
441
442pub async fn serve_local_listener(
453 bound: crate::local_listener::BoundLocalListener,
454 router: Arc<Router>,
455 auth: ServerAuth,
456) -> Result<(), ServerError> {
457 match bound {
458 #[cfg(unix)]
459 crate::local_listener::BoundLocalListener::Unix { listener, path } => {
460 serve_unix_local(listener, path, router, auth).await
461 }
462 #[cfg(windows)]
463 crate::local_listener::BoundLocalListener::Windows {
464 pipe_name,
465 first_instance,
466 } => serve_pipe_local(pipe_name, first_instance, router, auth).await,
467 #[allow(unreachable_patterns)]
469 _ => Err(ServerError::NoListeners),
470 }
471}
472
473#[cfg(unix)]
474async fn serve_unix_local(
475 listener: std::os::unix::net::UnixListener,
476 path: std::path::PathBuf,
477 router: Arc<Router>,
478 auth: ServerAuth,
479) -> Result<(), ServerError> {
480 let _release = SocketFileRelease(path.clone());
483 listener
484 .set_nonblocking(true)
485 .map_err(|source| local_accept_error(&path, source))?;
486 let listener = tokio::net::UnixListener::from_std(listener)
487 .map_err(|source| local_accept_error(&path, source))?;
488 loop {
489 let (stream, _addr) = match listener.accept().await {
490 Ok(accepted) => accepted,
491 Err(source) if temporary_accept_failure(&source) => {
492 warn!(error = %source, "temporary local-socket accept failure");
493 if accept_resource_exhausted(&source) {
494 tokio::time::sleep(Duration::from_millis(75)).await;
495 }
496 continue;
497 }
498 Err(source) => return Err(local_accept_error(&path, source)),
499 };
500 debug!(?path, "accepted a local-socket connection");
501 let peer = subc_os::local_peer::capture_unix_peer(&stream);
502 let router = Arc::clone(&router);
503 let auth = auth.clone();
504 tokio::spawn(async move {
505 let _ = handle_accepted_local_connection(stream, router, auth, peer).await;
506 });
507 }
508}
509
510#[cfg(unix)]
511fn local_accept_error(endpoint: &std::path::Path, source: io::Error) -> ServerError {
512 ServerError::LocalAccept {
513 endpoint: endpoint.display().to_string(),
514 source,
515 }
516}
517
518#[cfg(unix)]
519struct SocketFileRelease(std::path::PathBuf);
520
521#[cfg(unix)]
522impl Drop for SocketFileRelease {
523 fn drop(&mut self) {
524 crate::local_listener::remove_unix_socket(&self.0);
525 }
526}
527
528#[cfg(windows)]
529async fn serve_pipe_local(
530 pipe_name: String,
531 first_instance: Option<tokio::net::windows::named_pipe::NamedPipeServer>,
532 router: Arc<Router>,
533 auth: ServerAuth,
534) -> Result<(), ServerError> {
535 let mut free = first_instance;
539 loop {
540 let server = match free.take() {
541 Some(server) => server,
542 None => create_pipe_instance(&pipe_name)?,
543 };
544 if let Err(source) = server.connect().await {
549 if temporary_accept_failure(&source) {
550 continue;
551 }
552 return Err(ServerError::LocalAccept {
553 endpoint: pipe_name.clone(),
554 source,
555 });
556 }
557 free = Some(create_pipe_instance(&pipe_name)?);
560 let router = Arc::clone(&router);
561 let auth = auth.clone();
562 if let Err(error) = subc_os::named_pipe::reject_remote_client(&server) {
565 warn!(%error, "refused a non-local named-pipe client on the local endpoint");
566 continue;
567 }
568 let peer = subc_os::local_peer::observe_pipe_client(&server).into();
569 tokio::spawn(async move {
570 let _ = handle_accepted_local_connection(server, router, auth, peer).await;
571 });
572 }
573}
574
575#[cfg(windows)]
576fn create_pipe_instance(
577 pipe_name: &str,
578) -> Result<tokio::net::windows::named_pipe::NamedPipeServer, ServerError> {
579 crate::local_listener::create_named_pipe_end(pipe_name).map_err(|source| {
580 ServerError::LocalAccept {
581 endpoint: pipe_name.to_owned(),
582 source,
583 }
584 })
585}
586
587async fn handle_accepted_local_connection<S>(
590 stream: S,
591 router: Arc<Router>,
592 auth: ServerAuth,
593 peer: subc_os::local_peer::CapturedPeer,
594) -> Result<(), ConnectionError>
595where
596 S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
597{
598 let result =
599 handle_connection_with_peer(stream, router, auth, LocalOrigin::Local, Some(peer)).await;
600 if let Err(error) = &result {
601 debug!(error = %error, "local-endpoint connection ended");
602 }
603 result
604}
605
606#[derive(Debug, Clone, Copy, PartialEq, Eq)]
610pub(crate) enum LocalOrigin {
611 Tcp,
612 Local,
613}
614
615pub async fn serve_listeners(
617 listeners: Vec<TcpListener>,
618 router: Arc<Router>,
619 auth: ServerAuth,
620) -> Result<(), ServerError> {
621 if listeners.is_empty() {
622 return Err(ServerError::NoListeners);
623 }
624
625 let (tx, mut rx) = mpsc::channel(listeners.len());
626 let mut accept_tasks = AbortTasksOnDrop::default();
627 for listener in listeners {
628 let router = Arc::clone(&router);
629 let auth = auth.clone();
630 let tx = tx.clone();
631 accept_tasks.push(tokio::spawn(async move {
632 let result = serve_listener(listener, router, auth).await;
633 let _ = tx.send(result).await;
634 }));
635 }
636 drop(tx);
637
638 rx.recv().await.unwrap_or(Ok(()))
639}
640
641#[derive(Default)]
642struct AbortTasksOnDrop {
643 handles: Vec<JoinHandle<()>>,
644}
645
646impl AbortTasksOnDrop {
647 fn push(&mut self, handle: JoinHandle<()>) {
648 self.handles.push(handle);
649 }
650}
651
652impl Drop for AbortTasksOnDrop {
653 fn drop(&mut self) {
654 for handle in &self.handles {
655 if !handle.is_finished() {
656 handle.abort();
657 }
658 }
659 }
660}
661
662#[derive(Debug)]
663enum ConnectionLoopExit {
664 PeerClosed,
665 CloseRequested(CloseReason),
666}
667
668pub async fn handle_connection<S>(
677 stream: S,
678 router: Arc<Router>,
679 auth: ServerAuth,
680) -> Result<(), ConnectionError>
681where
682 S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
683{
684 handle_connection_with_origin(stream, router, auth, LocalOrigin::Tcp).await
685}
686
687pub(crate) async fn handle_connection_with_origin<S>(
690 stream: S,
691 router: Arc<Router>,
692 auth: ServerAuth,
693 origin: LocalOrigin,
694) -> Result<(), ConnectionError>
695where
696 S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
697{
698 handle_connection_with_peer(stream, router, auth, origin, None).await
699}
700
701async fn handle_connection_with_peer<S>(
702 mut stream: S,
703 router: Arc<Router>,
704 auth: ServerAuth,
705 origin: LocalOrigin,
706 peer: Option<subc_os::local_peer::CapturedPeer>,
707) -> Result<(), ConnectionError>
708where
709 S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
710{
711 let endpoints = (&stream as &dyn Any)
714 .downcast_ref::<tokio::net::TcpStream>()
715 .and_then(|tcp| tcp.local_addr().ok().zip(tcp.peer_addr().ok()));
716 let permit =
736 match tokio::time::timeout(auth.deadline, auth.unauthenticated.clone().acquire_owned())
737 .await
738 {
739 Ok(Ok(permit)) => permit,
740 Ok(Err(_)) | Err(_) => {
741 let _ = stream.shutdown().await;
742 return Err(ConnectionError::UnauthenticatedCapacity);
743 }
744 };
745
746 let authenticated = authenticate_server(
747 &mut stream,
748 auth.key.as_ref(),
749 &auth.daemon_id,
750 auth.daemon_ver.as_ref(),
751 auth.deadline,
752 )
753 .await
754 .map_err(ConnectionError::Auth)?;
755 drop(permit);
756
757 let mut connection = router.begin_connection();
758 let connection_id = connection.id();
759 router.record_caller_endpoints(connection_id, endpoints);
760 if matches!(origin, LocalOrigin::Local) {
761 router.record_local_connection(connection_id);
762 }
763 if let Some(peer) = peer {
764 router.record_captured_peer(connection_id, peer);
765 }
766 let _connected_client = (authenticated.role != WATCHDOG_CLIENT_ROLE)
781 .then(|| auth.connected_clients.open(connection_id));
782 let close_receiver = connection.take_close_receiver();
783 debug!(
784 connection_id = connection_id.get(),
785 "subc authenticated connection opened"
786 );
787
788 let (read_half, write_half) = tokio::io::split(stream);
789 let mut read_half = BufReader::new(read_half);
793 let (egress, rx) = connection_egress();
794 let delivery_guard = ClientDeliveryGuard(egress.clone());
795 let writer = tokio::spawn(drain_writer(write_half, rx));
796
797 let ctx = RouteCtx {
798 connection_id,
799 egress: egress.clone(),
800 };
801
802 let mut route_open_tasks = JoinSet::new();
803 let mut privileged_tasks = JoinSet::new();
804 let loop_result = connection_loop(
805 &mut read_half,
806 Arc::clone(&router),
807 ctx.clone(),
808 close_receiver,
809 &mut route_open_tasks,
810 &mut privileged_tasks,
811 origin,
812 )
813 .await;
814
815 drop(delivery_guard);
816 route_open_tasks.shutdown().await;
824 privileged_tasks.shutdown().await;
827
828 drop(ctx);
829 drop(egress);
830 drop(connection);
831
832 let close_reason = match &loop_result {
833 Ok(ConnectionLoopExit::CloseRequested(reason)) => Some(reason.to_string()),
834 Ok(ConnectionLoopExit::PeerClosed) | Err(_) => None,
835 };
836 let writer_result = finish_connection_writer(writer, connection_id.get()).await;
837
838 let result = if let Some(reason) = close_reason.as_deref() {
839 match writer_result {
840 Some(Ok(Ok(()))) | None => Ok(()),
841 Some(Ok(Err(writer_err))) => {
842 debug!(
843 connection_id = connection_id.get(),
844 close_reason = reason,
845 writer_error = %writer_err,
846 "writer failed after requested connection close"
847 );
848 Ok(())
849 }
850 Some(Err(join_err)) => {
851 warn!(
852 connection_id = connection_id.get(),
853 close_reason = reason,
854 join_error = %join_err,
855 "writer task join failed after requested connection close"
856 );
857 Ok(())
858 }
859 }
860 } else {
861 match (loop_result, writer_result) {
862 (Err(loop_err), Some(Ok(Ok(()))) | None) => Err(loop_err),
863 (Err(loop_err), Some(Ok(Err(writer_err)))) => {
864 warn!(
865 connection_id = connection_id.get(),
866 writer_error = %writer_err,
867 "writer failed while closing after connection error"
868 );
869 Err(loop_err)
870 }
871 (Err(loop_err), Some(Err(join_err))) => {
872 warn!(
873 connection_id = connection_id.get(),
874 join_error = %join_err,
875 "writer task join failed while closing after connection error"
876 );
877 Err(loop_err)
878 }
879 (Ok(ConnectionLoopExit::PeerClosed), Some(Ok(Ok(()))) | None) => Ok(()),
880 (Ok(ConnectionLoopExit::PeerClosed), Some(Ok(Err(writer_err)))) => {
881 Err(ConnectionError::FrameIo(writer_err))
882 }
883 (Ok(ConnectionLoopExit::PeerClosed), Some(Err(join_err))) => Err(join_err),
884 (Ok(ConnectionLoopExit::CloseRequested(_)), _) => {
885 unreachable!("close requests are handled before normal writer result matching")
886 }
887 }
888 };
889
890 match &result {
891 Ok(()) => {
892 if let Some(reason) = close_reason.as_deref() {
893 debug!(
894 connection_id = connection_id.get(),
895 close_reason = reason,
896 "subc connection closed by request"
897 );
898 } else {
899 debug!(
900 connection_id = connection_id.get(),
901 "subc connection closed"
902 );
903 }
904 }
905 Err(err) => debug!(
906 connection_id = connection_id.get(),
907 error = %err,
908 "subc connection exited with error"
909 ),
910 }
911
912 result
913}
914
915async fn finish_connection_writer(
916 mut writer: JoinHandle<Result<(), FrameIoError>>,
917 connection_id: u64,
918) -> Option<Result<Result<(), FrameIoError>, ConnectionError>> {
919 match timeout(CLOSE_DRAIN_GRACE, &mut writer).await {
922 Ok(result) => Some(result.map_err(ConnectionError::WriterTask)),
923 Err(_) => {
924 warn!(
925 connection_id,
926 grace = ?CLOSE_DRAIN_GRACE,
927 "connection writer did not drain during teardown; aborting writer task"
928 );
929 writer.abort();
930 let _ = writer.await;
931 None
932 }
933 }
934}
935
936async fn connection_loop<R>(
937 read_half: &mut R,
938 router: Arc<Router>,
939 ctx: RouteCtx,
940 mut close_receiver: ConnectionCloseReceiver,
941 route_open_tasks: &mut JoinSet<Result<(), RouterError>>,
942 privileged_tasks: &mut JoinSet<Result<(), RouterError>>,
943 origin: LocalOrigin,
944) -> Result<ConnectionLoopExit, ConnectionError>
945where
946 R: AsyncRead + Unpin,
947{
948 let mut activity_tasks = JoinSet::new();
949 loop {
950 while let Some(result) = activity_tasks.try_join_next() {
951 finish_route_open_task(result)?;
952 }
953 while let Some(result) = privileged_tasks.try_join_next() {
954 finish_route_open_task(result)?;
955 }
956 while let Some(result) = route_open_tasks.try_join_next() {
957 finish_route_open_task(result)?;
958 }
959
960 let read = read_frame(&mut *read_half);
963 tokio::pin!(read);
964 let frame = loop {
965 tokio::select! {
966 close = &mut close_receiver => {
967 return Ok(ConnectionLoopExit::CloseRequested(close_reason(close)));
968 }
969 result = route_open_tasks.join_next(), if !route_open_tasks.is_empty() => {
970 finish_route_open_task(
971 result.expect("a non-empty route.open JoinSet has a next task")
972 )?;
973 }
974 result = activity_tasks.join_next(), if !activity_tasks.is_empty() => {
975 finish_route_open_task(result.expect("non-empty activity task set"))?;
976 }
977 result = privileged_tasks.join_next(), if !privileged_tasks.is_empty() => {
978 finish_route_open_task(result.expect("non-empty privileged task set"))?;
979 }
980 result = &mut read => {
981 break match result.map_err(ConnectionError::FrameIo)? {
982 Some(frame) => frame,
983 None => return Ok(ConnectionLoopExit::PeerClosed),
984 };
985 }
986 }
987 };
988
989 if origin == LocalOrigin::Local && frame.header.ty == subc_protocol::FrameType::Hello {
990 let refusal = crate::control::control_error_frame(
995 &frame,
996 crate::local_listener::LOCAL_ENDPOINT_CLIENTS_ONLY,
997 "the local endpoint serves clients only; modules register over TCP",
998 )
999 .map_err(ConnectionError::Router)?;
1000 let send_result = tokio::select! {
1004 close = &mut close_receiver => {
1005 return Ok(ConnectionLoopExit::CloseRequested(close_reason(close)));
1006 }
1007 result = router.observe_connection_wait(&ctx, "reply_egress", send_refusal(&ctx.egress, refusal)) => result,
1008 };
1009 send_result.map_err(ConnectionError::Router)?;
1010 return Ok(ConnectionLoopExit::PeerClosed);
1011 }
1012
1013 if frame.header.channel == 0
1014 && frame.header.ty == subc_protocol::FrameType::Request
1015 && serde_json::from_slice::<serde_json::Value>(&frame.body)
1016 .ok()
1017 .is_some_and(|body| {
1018 matches!(
1019 body["op"].as_str(),
1020 Some(
1021 subc_control::ops::ACTIVITY_RECORD
1022 | subc_control::ops::ACTIVITY_QUERY
1023 | subc_control::ops::ACTIVITY_SUBSCRIBE
1024 )
1025 )
1026 })
1027 {
1028 if activity_tasks.len() >= 128 {
1031 let response = crate::control::control_error_frame(
1032 &frame,
1033 subc_control::ACTIVITY_UNAVAILABLE,
1034 "activity queue full",
1035 )
1036 .map_err(ConnectionError::Router)?;
1037 let send_result = tokio::select! {
1038 close = &mut close_receiver => {
1039 return Ok(ConnectionLoopExit::CloseRequested(close_reason(close)));
1040 }
1041 result = router.observe_connection_wait(&ctx, "reply_egress", ctx.egress.send(response)) => result,
1042 };
1043 send_result.map_err(ConnectionError::Router)?;
1044 } else {
1045 let task_router = Arc::clone(&router);
1046 let task_ctx = ctx.clone();
1047 activity_tasks.spawn(async move {
1048 route_open_tail(task_router, task_ctx, frame, Instant::now()).await
1049 });
1050 }
1051 continue;
1052 }
1053
1054 if let Some(target_module_id) = router.route_open_target(&frame) {
1055 while let Some(result) = route_open_tasks.try_join_next() {
1059 finish_route_open_task(result)?;
1060 }
1061
1062 if route_open_tasks.len() >= MAX_PENDING_ROUTE_OPENS_PER_CONNECTION {
1063 let refusal = router
1066 .route_open_capacity_refusal(
1067 &ctx,
1068 &frame,
1069 &target_module_id,
1070 route_open_tasks.len(),
1071 MAX_PENDING_ROUTE_OPENS_PER_CONNECTION,
1072 )
1073 .map_err(ConnectionError::Router)?;
1074 let send_result = tokio::select! {
1075 close = &mut close_receiver => {
1076 return Ok(ConnectionLoopExit::CloseRequested(close_reason(close)));
1077 }
1078 result = router.observe_connection_wait(&ctx, "reply_egress", ctx.egress.send(refusal)) => result,
1079 };
1080 send_result.map_err(ConnectionError::Router)?;
1081 continue;
1082 }
1083
1084 let task_router = Arc::clone(&router);
1085 let task_ctx = ctx.clone();
1086 let dispatch_started_at = Instant::now();
1089 route_open_tasks.spawn(async move {
1090 route_open_tail(task_router, task_ctx, frame, dispatch_started_at).await
1091 });
1092 continue;
1093 }
1094
1095 if router.defers_privileged_control(&ctx, &frame) {
1096 let task_router = Arc::clone(&router);
1097 let task_ctx = ctx.clone();
1098 let started = Instant::now();
1099 privileged_tasks
1100 .spawn(async move { route_open_tail(task_router, task_ctx, frame, started).await });
1101 continue;
1102 }
1103
1104 let route_result = tokio::select! {
1107 close = &mut close_receiver => {
1108 return Ok(ConnectionLoopExit::CloseRequested(close_reason(close)));
1109 }
1110 result = router.route_for_connection(&ctx, frame) => result,
1111 };
1112
1113 if let Err(err) = route_result {
1114 if let Some(error_frame) = err.to_error_frame() {
1115 warn!(
1116 connection_id = ctx.connection_id.get(),
1117 error = %err,
1118 "routing failure recovered with ERROR frame"
1119 );
1120 let send_result = tokio::select! {
1121 close = &mut close_receiver => {
1122 return Ok(ConnectionLoopExit::CloseRequested(close_reason(close)));
1123 }
1124 result = router.observe_connection_wait(&ctx, "reply_egress", ctx.egress.send(error_frame)) => result,
1125 };
1126 send_result.map_err(ConnectionError::Router)?;
1127 } else {
1128 debug!(
1129 connection_id = ctx.connection_id.get(),
1130 error = %err,
1131 "fatal routing failure"
1132 );
1133 return Err(ConnectionError::Router(err));
1134 }
1135 }
1136 }
1137}
1138
1139async fn send_refusal(sink: &FrameSink, frame: crate::Frame) -> Result<(), RouterError> {
1143 #[cfg(unix)]
1147 {
1148 sink.send_flushed(frame).await
1149 }
1150 #[cfg(not(unix))]
1151 {
1152 sink.send(frame).await
1153 }
1154}
1155
1156async fn route_open_tail(
1157 router: Arc<Router>,
1158 ctx: RouteCtx,
1159 frame: crate::Frame,
1160 dispatch_started_at: Instant,
1161) -> Result<(), RouterError> {
1162 match router
1163 .route_for_connection_started(&ctx, frame, Some(dispatch_started_at))
1164 .await
1165 {
1166 Ok(()) => Ok(()),
1167 Err(err) => {
1168 let Some(error_frame) = err.to_error_frame() else {
1169 return Err(err);
1170 };
1171 warn!(
1172 connection_id = ctx.connection_id.get(),
1173 error = %err,
1174 "routing failure recovered with ERROR frame"
1175 );
1176 router
1177 .observe_connection_wait(&ctx, "reply_egress", ctx.egress.send(error_frame))
1178 .await
1179 }
1180 }
1181}
1182
1183fn finish_route_open_task(
1184 result: Result<Result<(), RouterError>, tokio::task::JoinError>,
1185) -> Result<(), ConnectionError> {
1186 match result {
1187 Ok(Ok(())) => Ok(()),
1188 Ok(Err(err)) => Err(ConnectionError::Router(err)),
1189 Err(err) => Err(ConnectionError::Router(RouterError::backend(
1190 0,
1191 0,
1192 format!("route.open task failed: {err}"),
1193 ))),
1194 }
1195}
1196
1197fn close_reason(
1198 result: Result<CloseReason, tokio::sync::oneshot::error::RecvError>,
1199) -> CloseReason {
1200 result.unwrap_or_else(|_| {
1201 CloseReason::new(
1202 "close_registry_dropped",
1203 "connection close registration was dropped without a reason",
1204 )
1205 })
1206}
1207
1208pub(crate) fn connection_egress() -> (FrameSink, mpsc::Receiver<crate::router::OutboundFrame>) {
1212 let (tx, rx) = mpsc::channel::<crate::router::OutboundFrame>(CONNECTION_EGRESS_FRAME_CAP);
1213 (
1214 FrameSink::with_byte_budget(tx, CONNECTION_EGRESS_BYTE_BUDGET),
1215 rx,
1216 )
1217}
1218
1219async fn drain_writer<W>(
1220 write_half: W,
1221 mut rx: mpsc::Receiver<crate::router::OutboundFrame>,
1222) -> Result<(), FrameIoError>
1223where
1224 W: AsyncWrite + Unpin,
1225{
1226 let mut writer = BufWriter::new(write_half);
1227 while let Some(outbound) = rx.recv().await {
1228 write_outbound(&mut writer, outbound).await?;
1229 while let Ok(outbound) = rx.try_recv() {
1230 write_outbound(&mut writer, outbound).await?;
1231 }
1232 writer.flush().await.map_err(FrameIoError::Io)?;
1233 }
1234 writer.flush().await.map_err(FrameIoError::Io)?;
1235 Ok(())
1236}
1237
1238async fn write_outbound<W>(
1245 writer: &mut BufWriter<W>,
1246 outbound: crate::router::OutboundFrame,
1247) -> Result<(), FrameIoError>
1248where
1249 W: AsyncWrite + Unpin,
1250{
1251 const SLOW_REPLY_QUEUE: Duration = Duration::from_millis(1000);
1252 let queued = outbound.enqueued_at.elapsed();
1253 let charge = outbound.charge;
1256 if let Some(charge) = &charge {
1257 charge.taken_by_writer();
1258 }
1259 let frame = outbound.frame;
1260 if frame.header.channel == 0 && queued >= SLOW_REPLY_QUEUE {
1261 let write_started = std::time::Instant::now();
1262 let result = write_frame(writer, &frame).await;
1263 tracing::warn!(
1264 corr = frame.header.corr,
1265 queued_ms = queued.as_millis() as u64,
1266 write_ms = write_started.elapsed().as_millis() as u64,
1267 "slow control reply write"
1268 );
1269 result?;
1270 } else {
1271 write_frame(writer, &frame).await?;
1272 }
1273 if let Some(flushed) = outbound.flushed {
1274 writer.flush().await.map_err(FrameIoError::Io)?;
1275 let _ = flushed.send(());
1276 }
1277 Ok(())
1278}
1279
1280#[cfg(all(test, unix))]
1281#[tokio::test]
1282async fn shutdown_notice_ack_waits_for_socket_flush() {
1283 let (socket, mut peer) = tokio::io::duplex(1);
1284 let (tx, rx) = mpsc::channel(1);
1285 let sink = FrameSink::new(tx);
1286 let writer = tokio::spawn(drain_writer(socket, rx));
1287 let frame = crate::Frame::build(
1288 subc_protocol::FrameType::Push,
1289 subc_protocol::Flags::new(false, subc_protocol::Priority::Interactive, false),
1290 0,
1291 0,
1292 0,
1293 b"notice".to_vec(),
1294 )
1295 .unwrap();
1296 let send = sink.send_flushed(frame);
1297 tokio::pin!(send);
1298 assert!(
1299 timeout(Duration::from_millis(20), &mut send).await.is_err(),
1300 "queueing bytes is not a socket flush acknowledgement"
1301 );
1302 let (sent, received) = timeout(Duration::from_secs(1), async {
1303 tokio::join!(&mut send, read_frame(&mut peer))
1304 })
1305 .await
1306 .unwrap();
1307 sent.unwrap();
1308 assert_eq!(received.unwrap().unwrap().body, b"notice");
1309 writer.abort();
1310}
1311
1312#[derive(Debug)]
1313pub enum ServerError {
1314 NoListeners,
1315 Accept {
1316 local_addr: Option<SocketAddr>,
1317 source: io::Error,
1318 },
1319 LocalAccept {
1322 endpoint: String,
1323 source: io::Error,
1324 },
1325}
1326
1327impl fmt::Display for ServerError {
1328 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1329 match self {
1330 Self::NoListeners => write!(f, "no TCP listeners were provided"),
1331 Self::Accept { local_addr, source } => match local_addr {
1332 Some(addr) => write!(f, "failed to accept TCP connection on {addr}: {source}"),
1333 None => write!(f, "failed to accept TCP connection: {source}"),
1334 },
1335 Self::LocalAccept { endpoint, source } => {
1336 write!(f, "local endpoint {endpoint} failed: {source}")
1337 }
1338 }
1339 }
1340}
1341
1342impl Error for ServerError {
1343 fn source(&self) -> Option<&(dyn Error + 'static)> {
1344 match self {
1345 Self::Accept { source, .. } | Self::LocalAccept { source, .. } => Some(source),
1346 Self::NoListeners => None,
1347 }
1348 }
1349}
1350
1351#[derive(Debug)]
1352pub enum ConnectionError {
1353 Auth(AuthError),
1354 UnauthenticatedCapacity,
1355 FrameIo(FrameIoError),
1356 Router(RouterError),
1357 WriterTask(tokio::task::JoinError),
1358}
1359
1360impl ConnectionError {
1361 fn is_quiet_reject(&self) -> bool {
1362 matches!(self, Self::Auth(_) | Self::UnauthenticatedCapacity)
1363 }
1364
1365 fn pre_auth_reject_stage(&self) -> Option<PreAuthRejectStage> {
1366 match self {
1367 Self::UnauthenticatedCapacity => Some(PreAuthRejectStage::WaitingForHandshakeSlot),
1368 Self::Auth(AuthError::Timeout { .. }) => Some(PreAuthRejectStage::HandshakeDeadline),
1369 Self::Auth(
1370 AuthError::InvalidClientAuth
1371 | AuthError::InvalidServerProof
1372 | AuthError::DaemonIdMismatch,
1373 ) => Some(PreAuthRejectStage::InvalidProof),
1374 Self::Auth(AuthError::MessageTooLarge { .. } | AuthError::JsonDecode { .. }) => {
1375 Some(PreAuthRejectStage::MalformedMessage)
1376 }
1377 Self::Auth(AuthError::UnexpectedEof { .. }) => Some(PreAuthRejectStage::PeerClosed),
1378 Self::Auth(AuthError::Io { source, .. })
1379 if matches!(
1380 source.kind(),
1381 io::ErrorKind::BrokenPipe
1382 | io::ErrorKind::ConnectionAborted
1383 | io::ErrorKind::ConnectionReset
1384 | io::ErrorKind::UnexpectedEof
1385 ) =>
1386 {
1387 Some(PreAuthRejectStage::PeerClosed)
1388 }
1389 Self::Auth(_) => Some(PreAuthRejectStage::OtherAuthenticationFailure),
1390 Self::FrameIo(_) | Self::Router(_) | Self::WriterTask(_) => None,
1391 }
1392 }
1393
1394 fn auth_stage(&self) -> Option<AuthStage> {
1395 match self {
1396 Self::Auth(
1397 AuthError::Io { stage, .. }
1398 | AuthError::Timeout { stage, .. }
1399 | AuthError::UnexpectedEof { stage, .. }
1400 | AuthError::MessageTooLarge { stage, .. }
1401 | AuthError::JsonEncode { stage, .. }
1402 | AuthError::JsonDecode { stage, .. },
1403 ) => Some(*stage),
1404 Self::Auth(
1405 AuthError::Random(_)
1406 | AuthError::KeyTooShort { .. }
1407 | AuthError::InvalidServerProof
1408 | AuthError::DaemonIdMismatch
1409 | AuthError::InvalidClientAuth,
1410 )
1411 | Self::UnauthenticatedCapacity
1412 | Self::FrameIo(_)
1413 | Self::Router(_)
1414 | Self::WriterTask(_) => None,
1415 }
1416 }
1417}
1418
1419impl fmt::Display for ConnectionError {
1420 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1421 match self {
1422 Self::Auth(err) => write!(f, "connection auth failed: {err}"),
1423 Self::UnauthenticatedCapacity => write!(
1424 f,
1425 "too many concurrent unauthenticated subc TCP connections"
1426 ),
1427 Self::FrameIo(err) => write!(f, "frame connection error: {err}"),
1428 Self::Router(err) => write!(f, "router connection error: {err}"),
1429 Self::WriterTask(err) => write!(f, "connection writer task failed: {err}"),
1430 }
1431 }
1432}
1433
1434impl Error for ConnectionError {
1435 fn source(&self) -> Option<&(dyn Error + 'static)> {
1436 match self {
1437 Self::Auth(err) => Some(err),
1438 Self::FrameIo(err) => Some(err),
1439 Self::Router(err) => Some(err),
1440 Self::WriterTask(err) => Some(err),
1441 Self::UnauthenticatedCapacity => None,
1442 }
1443 }
1444}
1445
1446#[cfg(test)]
1447mod tests {
1448 use std::{
1449 future::Future,
1450 pin::Pin,
1451 sync::atomic::{AtomicUsize, Ordering},
1452 task::{Context, Poll},
1453 };
1454
1455 use super::*;
1456 use serde::{de::DeserializeOwned, Serialize};
1457 use subc_protocol::{
1458 DecodeError, ErrorBody, Flags, FrameType, Priority, HEADER_LEN, PROTOCOL_VERSION,
1459 };
1460 use tokio::io::{duplex, AsyncReadExt, AsyncWriteExt, ReadBuf};
1461
1462 use subc_transport::{
1463 authenticate_client, compute_proof, ClientAuth, ClientHello, ConnectionInfo, Endpoint,
1464 ServerProof, CLIENT_AUTH_DOMAIN, NONCE_LEN, SCHEMA_VERSION,
1465 };
1466
1467 use crate::{ControlHandler, EchoBackend, Frame, ReadStage, Registry};
1468
1469 const TEST_DEADLINE: Duration = Duration::from_secs(2);
1470 const TEST_DAEMON_VER: &str = "test-subc-server";
1471
1472 #[test]
1476 fn egress_budget_is_the_published_delivered_frame_limit() {
1477 assert_eq!(
1478 CONNECTION_EGRESS_BYTE_BUDGET,
1479 subc_protocol::MAX_DELIVERED_FRAME_LEN
1480 );
1481 }
1482
1483 struct ObservedStream {
1484 inner: tokio::io::DuplexStream,
1485 write_blocked: Arc<tokio::sync::Notify>,
1486 dropped: Arc<AtomicUsize>,
1487 read_error: Option<tokio::sync::oneshot::Receiver<()>>,
1488 }
1489
1490 impl AsyncRead for ObservedStream {
1491 fn poll_read(
1492 mut self: Pin<&mut Self>,
1493 cx: &mut Context<'_>,
1494 buf: &mut ReadBuf<'_>,
1495 ) -> Poll<io::Result<()>> {
1496 if let Some(receiver) = self.read_error.as_mut() {
1497 match Pin::new(receiver).poll(cx) {
1498 Poll::Ready(Ok(())) => {
1499 self.read_error = None;
1500 return Poll::Ready(Err(io::ErrorKind::ConnectionReset.into()));
1501 }
1502 Poll::Ready(Err(_)) => self.read_error = None,
1503 Poll::Pending => {}
1504 }
1505 }
1506 Pin::new(&mut self.inner).poll_read(cx, buf)
1507 }
1508 }
1509
1510 impl AsyncWrite for ObservedStream {
1511 fn poll_write(
1512 mut self: Pin<&mut Self>,
1513 cx: &mut Context<'_>,
1514 buf: &[u8],
1515 ) -> Poll<io::Result<usize>> {
1516 let result = Pin::new(&mut self.inner).poll_write(cx, buf);
1517 if result.is_pending() {
1518 self.write_blocked.notify_one();
1519 }
1520 result
1521 }
1522
1523 fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
1524 Pin::new(&mut self.inner).poll_flush(cx)
1525 }
1526
1527 fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
1528 Pin::new(&mut self.inner).poll_shutdown(cx)
1529 }
1530 }
1531
1532 impl Drop for ObservedStream {
1533 fn drop(&mut self) {
1534 self.dropped.fetch_add(1, Ordering::SeqCst);
1535 }
1536 }
1537
1538 struct BlockedWriterConnection {
1539 client: tokio::io::DuplexStream,
1540 server: JoinHandle<Result<(), ConnectionError>>,
1541 stream_dropped: Arc<AtomicUsize>,
1542 read_error: Option<tokio::sync::oneshot::Sender<()>>,
1543 router: Arc<Router>,
1544 }
1545
1546 impl BlockedWriterConnection {
1547 async fn new() -> Self {
1548 let (mut client, inner) = duplex(256);
1549 let write_blocked = Arc::new(tokio::sync::Notify::new());
1550 let stream_dropped = Arc::new(AtomicUsize::new(0));
1551 let (read_error, receiver) = tokio::sync::oneshot::channel();
1552 let stream = ObservedStream {
1553 inner,
1554 write_blocked: Arc::clone(&write_blocked),
1555 dropped: Arc::clone(&stream_dropped),
1556 read_error: Some(receiver),
1557 };
1558 let (auth, conn) = test_auth();
1559 let router = echo_router();
1560 let server = tokio::spawn(handle_connection(stream, Arc::clone(&router), auth));
1561 authenticate(&mut client, &conn).await;
1562
1563 let _ = timeout(Duration::ZERO, write_blocked.notified()).await;
1566 write_frame(&mut client, &request(7, 1, &[b'x'; 1024]))
1567 .await
1568 .unwrap();
1569 timeout(TEST_DEADLINE, write_blocked.notified())
1570 .await
1571 .expect("writer should block on the unread echo reply");
1572 assert!(!server.is_finished());
1573 assert_eq!(stream_dropped.load(Ordering::SeqCst), 0);
1574 Self {
1575 client,
1576 server,
1577 stream_dropped,
1578 read_error: Some(read_error),
1579 router,
1580 }
1581 }
1582
1583 async fn finish(&mut self) -> Result<(), ConnectionError> {
1584 let started = tokio::time::Instant::now();
1585 let result = timeout(CLOSE_DRAIN_GRACE + Duration::from_secs(1), &mut self.server)
1586 .await
1587 .expect("connection teardown exceeded writer drain grace plus margin")
1588 .expect("connection task should not panic");
1589 assert!(started.elapsed() >= CLOSE_DRAIN_GRACE);
1590 assert_eq!(
1591 self.stream_dropped.load(Ordering::SeqCst),
1592 1,
1593 "writer must release its stream half before teardown completes"
1594 );
1595 result
1596 }
1597
1598 fn request_close(&self) {
1599 assert!(self.router.forwarding().request_connection_close(
1601 crate::registry::ConnectionId::new(1),
1602 CloseReason::new("test_close", "test requested close"),
1603 ));
1604 }
1605 }
1606
1607 #[tokio::test(start_paused = true)]
1608 async fn peer_eof_bounds_blocked_writer_teardown() {
1609 let mut connection = BlockedWriterConnection::new().await;
1610 connection.client.shutdown().await.unwrap();
1613 connection.finish().await.unwrap();
1614 }
1615
1616 #[tokio::test(start_paused = true)]
1617 async fn read_error_bounds_blocked_writer_teardown() {
1618 let mut connection = BlockedWriterConnection::new().await;
1619 connection.read_error.take().unwrap().send(()).unwrap();
1620 let error = connection.finish().await.unwrap_err();
1621 assert!(matches!(
1622 error,
1623 ConnectionError::FrameIo(FrameIoError::Io(error))
1624 if error.kind() == io::ErrorKind::ConnectionReset
1625 ));
1626 }
1627
1628 #[tokio::test(start_paused = true)]
1629 async fn close_requested_bounds_blocked_writer_teardown() {
1630 let mut connection = BlockedWriterConnection::new().await;
1631 connection.request_close();
1632 connection.finish().await.unwrap();
1633 }
1634
1635 #[tokio::test(start_paused = true)]
1636 async fn close_requested_flushes_queued_reply_before_grace() {
1637 let mut connection = BlockedWriterConnection::new().await;
1638 connection.request_close();
1639 let started = tokio::time::Instant::now();
1640 let reply = timeout(CLOSE_DRAIN_GRACE, read_frame(&mut connection.client))
1641 .await
1642 .expect("queued reply should flush before the grace expires")
1643 .unwrap()
1644 .unwrap();
1645 assert_eq!(reply.body, vec![b'x'; 1024]);
1646 timeout(CLOSE_DRAIN_GRACE, &mut connection.server)
1647 .await
1648 .unwrap()
1649 .unwrap()
1650 .unwrap();
1651 assert!(started.elapsed() < CLOSE_DRAIN_GRACE);
1652 assert_eq!(connection.stream_dropped.load(Ordering::SeqCst), 1);
1653 }
1654
1655 #[tokio::test]
1660 async fn stale_queued_control_reply_logs_slow_reply_write() {
1661 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
1662 let (tx, rx) = mpsc::channel::<crate::router::OutboundFrame>(4);
1663 let reply = Frame::build_with_version(
1664 PROTOCOL_VERSION,
1665 FrameType::Error,
1666 Flags::new(false, Priority::Interactive, false),
1667 0,
1668 0,
1669 7,
1670 serde_json::to_vec(&ErrorBody {
1671 code: "test".into(),
1672 message: "reply".into(),
1673 detail: None,
1674 })
1675 .expect("body encodes"),
1676 )
1677 .expect("frame builds");
1678 tx.send(crate::router::OutboundFrame {
1679 frame: reply,
1680 enqueued_at: std::time::Instant::now() - Duration::from_millis(1500),
1681 flushed: None,
1682 charge: None,
1683 })
1684 .await
1685 .expect("queued");
1686 drop(tx);
1687 let (write_half, mut read_half) = duplex(64 * 1024);
1688 drain_writer(write_half, rx).await.expect("writer drains");
1689 let mut sink = Vec::new();
1690 read_half.read_to_end(&mut sink).await.expect("read");
1691 assert!(!sink.is_empty(), "frame reached the socket");
1692 let captured = crate::router::test_log::captured_logs(&logs);
1693 assert!(
1694 captured.contains("slow control reply write") && captured.contains("corr=7"),
1695 "expected slow reply WARN naming corr, got: {captured}"
1696 );
1697 let queued_ms: u64 = captured
1698 .split("queued_ms=")
1699 .nth(1)
1700 .and_then(|s| s.split_whitespace().next())
1701 .and_then(|s| s.parse().ok())
1702 .expect("queued_ms present");
1703 assert!(
1704 queued_ms >= 1500,
1705 "queued_ms reflects residency: {queued_ms}"
1706 );
1707 }
1708
1709 #[tokio::test]
1713 async fn fresh_control_and_stale_data_frames_log_nothing() {
1714 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
1715 let (tx, rx) = mpsc::channel::<crate::router::OutboundFrame>(4);
1716 let control = Frame::build_with_version(
1717 PROTOCOL_VERSION,
1718 FrameType::Error,
1719 Flags::new(false, Priority::Interactive, false),
1720 0,
1721 0,
1722 8,
1723 serde_json::to_vec(&ErrorBody {
1724 code: "test".into(),
1725 message: "fresh".into(),
1726 detail: None,
1727 })
1728 .expect("body encodes"),
1729 )
1730 .expect("frame builds");
1731 tx.send(crate::router::OutboundFrame {
1732 frame: control,
1733 enqueued_at: std::time::Instant::now(),
1734 flushed: None,
1735 charge: None,
1736 })
1737 .await
1738 .expect("queued");
1739 let data = Frame::build_with_version(
1740 PROTOCOL_VERSION,
1741 FrameType::Error,
1742 Flags::new(false, Priority::Interactive, false),
1743 9,
1744 1,
1745 9,
1746 serde_json::to_vec(&ErrorBody {
1747 code: "test".into(),
1748 message: "data".into(),
1749 detail: None,
1750 })
1751 .expect("body encodes"),
1752 )
1753 .expect("frame builds");
1754 tx.send(crate::router::OutboundFrame {
1755 frame: data,
1756 enqueued_at: std::time::Instant::now() - Duration::from_millis(5000),
1757 flushed: None,
1758 charge: None,
1759 })
1760 .await
1761 .expect("queued");
1762 drop(tx);
1763 let (write_half, _read_half) = duplex(64 * 1024);
1764 drain_writer(write_half, rx).await.expect("writer drains");
1765 let captured = crate::router::test_log::captured_logs(&logs);
1766 assert!(
1767 !captured.contains("slow control reply write"),
1768 "no WARN for fresh control or stale data frames, got: {captured}"
1769 );
1770 }
1771
1772 struct CountingReader {
1773 bytes: Vec<u8>,
1774 offset: usize,
1775 first_read_end: Option<usize>,
1776 reads: Arc<AtomicUsize>,
1777 }
1778
1779 impl CountingReader {
1780 fn new(bytes: Vec<u8>, first_read_end: Option<usize>) -> (Self, Arc<AtomicUsize>) {
1781 let reads = Arc::new(AtomicUsize::new(0));
1782 (
1783 Self {
1784 bytes,
1785 offset: 0,
1786 first_read_end,
1787 reads: Arc::clone(&reads),
1788 },
1789 reads,
1790 )
1791 }
1792 }
1793
1794 impl AsyncRead for CountingReader {
1795 fn poll_read(
1796 mut self: Pin<&mut Self>,
1797 _cx: &mut Context<'_>,
1798 buf: &mut ReadBuf<'_>,
1799 ) -> Poll<io::Result<()>> {
1800 self.reads.fetch_add(1, Ordering::Relaxed);
1801 let available = self.bytes.len().saturating_sub(self.offset);
1802 let first_read_remaining = self
1803 .first_read_end
1804 .filter(|end| self.offset < *end)
1805 .map_or(available, |end| end - self.offset);
1806 let count = available.min(first_read_remaining).min(buf.remaining());
1807 let end = self.offset + count;
1808 buf.put_slice(&self.bytes[self.offset..end]);
1809 self.offset = end;
1810 Poll::Ready(Ok(()))
1811 }
1812 }
1813
1814 fn encode_frames(frames: &[Frame]) -> Vec<u8> {
1815 let mut bytes = Vec::new();
1816 for frame in frames {
1817 bytes.extend_from_slice(&frame.header.encode());
1818 bytes.extend_from_slice(&frame.body);
1819 }
1820 bytes
1821 }
1822
1823 async fn read_frames<R>(reader: &mut R, count: usize) -> Vec<Frame>
1824 where
1825 R: AsyncRead + Unpin,
1826 {
1827 let mut frames = Vec::with_capacity(count);
1828 for _ in 0..count {
1829 frames.push(read_frame(reader).await.unwrap().unwrap());
1830 }
1831 frames
1832 }
1833
1834 fn request(channel: u16, corr: u64, body: &[u8]) -> Frame {
1835 Frame::build(
1836 FrameType::Request,
1837 Flags::new(true, Priority::Interactive, false),
1838 channel,
1839 0,
1840 corr,
1841 body.to_vec(),
1842 )
1843 .unwrap()
1844 }
1845
1846 fn echo_router() -> Arc<Router> {
1847 let mut router = Router::with_default_self_handler();
1848 router.register_backend(7, EchoBackend).unwrap();
1849 router.register_backend(9, EchoBackend).unwrap();
1850 Arc::new(router)
1851 }
1852
1853 fn test_auth() -> (ServerAuth, ConnectionInfo) {
1854 test_auth_with_limit(4)
1855 }
1856
1857 fn test_auth_with_limit(max_unauthenticated: usize) -> (ServerAuth, ConnectionInfo) {
1858 test_auth_with_deadline_and_limit(TEST_DEADLINE, max_unauthenticated)
1859 }
1860
1861 fn test_auth_with_deadline_and_limit(
1862 deadline: Duration,
1863 max_unauthenticated: usize,
1864 ) -> (ServerAuth, ConnectionInfo) {
1865 let key = vec![0x42; 32];
1866 let daemon_id = [0x24; 16];
1867 let conn = ConnectionInfo {
1868 schema: SCHEMA_VERSION,
1869 wire_version: None,
1870 endpoints: vec![Endpoint {
1871 host: "127.0.0.1".to_owned(),
1872 port: 1,
1873 }],
1874 key: key.clone(),
1875 daemon_id,
1876 pid: std::process::id(),
1877 daemon_ver: TEST_DAEMON_VER.to_owned(),
1878 };
1879 (
1880 ServerAuth::with_limits(
1881 key,
1882 daemon_id,
1883 TEST_DAEMON_VER,
1884 deadline,
1885 max_unauthenticated,
1886 ),
1887 conn,
1888 )
1889 }
1890
1891 async fn authenticate<S>(stream: &mut S, conn: &ConnectionInfo)
1892 where
1893 S: AsyncRead + AsyncWrite + Unpin,
1894 {
1895 authenticate_client(stream, conn, TEST_DEADLINE)
1896 .await
1897 .expect("test client should authenticate")
1898 }
1899
1900 async fn write_auth_message<S, T>(stream: &mut S, value: &T)
1901 where
1902 S: AsyncWrite + Unpin,
1903 T: Serialize,
1904 {
1905 let body = serde_json::to_vec(value).expect("auth message encodes");
1906 stream
1907 .write_all(&(body.len() as u32).to_le_bytes())
1908 .await
1909 .expect("auth message length writes");
1910 stream
1911 .write_all(&body)
1912 .await
1913 .expect("auth message body writes");
1914 }
1915
1916 async fn read_auth_message<S, T>(stream: &mut S) -> T
1917 where
1918 S: AsyncRead + Unpin,
1919 T: DeserializeOwned,
1920 {
1921 let mut length = [0; 4];
1922 stream
1923 .read_exact(&mut length)
1924 .await
1925 .expect("auth message length reads");
1926 let mut body = vec![0; u32::from_le_bytes(length) as usize];
1927 stream
1928 .read_exact(&mut body)
1929 .await
1930 .expect("auth message body reads");
1931 serde_json::from_slice(&body).expect("auth message decodes")
1932 }
1933
1934 async fn send_wrong_key_client_auth<S>(stream: &mut S, conn: &ConnectionInfo)
1935 where
1936 S: AsyncRead + AsyncWrite + Unpin,
1937 {
1938 let hello = ClientHello {
1939 client_nonce: [0x57; NONCE_LEN],
1940 role: "client".to_owned(),
1941 };
1942 write_auth_message(stream, &hello).await;
1943 let proof: ServerProof = read_auth_message(stream).await;
1944 let mut wrong_key = conn.key.clone();
1945 wrong_key[0] ^= 0xff;
1946 let client_auth = ClientAuth {
1947 client_auth: compute_proof(
1948 &wrong_key,
1949 CLIENT_AUTH_DOMAIN,
1950 &hello.client_nonce,
1951 &proof.server_nonce,
1952 &conn.daemon_id,
1953 ),
1954 };
1955 write_auth_message(stream, &client_auth).await;
1956 }
1957
1958 fn assert_pre_auth_log(logs: &Arc<std::sync::Mutex<Vec<u8>>>, stage: &str, port: u16) {
1959 let captured = crate::router::test_log::captured_logs(logs);
1960 assert!(
1961 captured.contains(&format!("stage=\"{stage}\"")),
1962 "expected rejection stage {stage}, got: {captured}"
1963 );
1964 assert!(
1965 captured.contains(&format!("peer_addr=127.0.0.1:{port}")),
1966 "expected peer address, got: {captured}"
1967 );
1968 assert!(
1969 captured.contains("elapsed_ms=") && captured.contains("suppressed_since_last="),
1970 "expected elapsed time and suppression count, got: {captured}"
1971 );
1972 assert!(
1973 !captured.contains("WARN"),
1974 "pre-auth rejection must not be logged as a warning: {captured}"
1975 );
1976 }
1977
1978 #[tokio::test]
1979 async fn logs_handshake_deadline_pre_auth_rejection() {
1980 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
1981 let deadline = Duration::from_millis(60);
1982 let (auth, _) = test_auth_with_deadline_and_limit(deadline, 4);
1983 let (client, server_stream) = duplex(512);
1984 let port = 41001;
1985 let server = handle_accepted_connection(
1986 server_stream,
1987 echo_router(),
1988 auth,
1989 SocketAddr::from(([127, 0, 0, 1], port)),
1990 Instant::now(),
1991 );
1992
1993 let error = tokio::time::timeout(deadline * 3, server)
1994 .await
1995 .expect("idle handshake should end at its deadline")
1996 .expect_err("idle handshake must be rejected");
1997 drop(client);
1998 assert!(matches!(
1999 error,
2000 ConnectionError::Auth(AuthError::Timeout {
2001 stage: AuthStage::ClientHello,
2002 ..
2003 })
2004 ));
2005 let captured = crate::router::test_log::captured_logs(&logs);
2006 assert!(
2007 captured.contains("reason=\"handshake_deadline_expired\""),
2008 "{captured}"
2009 );
2010 assert_pre_auth_log(&logs, "handshake_deadline", port);
2011 }
2012
2013 #[tokio::test]
2014 async fn logs_invalid_client_proof_pre_auth_rejection() {
2015 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
2016 let (auth, conn) = test_auth_with_deadline_and_limit(Duration::from_millis(200), 4);
2017 let (mut client, server_stream) = duplex(512);
2018 let port = 41002;
2019 let server = handle_accepted_connection(
2020 server_stream,
2021 echo_router(),
2022 auth,
2023 SocketAddr::from(([127, 0, 0, 1], port)),
2024 Instant::now(),
2025 );
2026
2027 let (error, ()) = tokio::join!(server, send_wrong_key_client_auth(&mut client, &conn));
2028 assert!(matches!(
2029 error,
2030 Err(ConnectionError::Auth(AuthError::InvalidClientAuth))
2031 ));
2032 let captured = crate::router::test_log::captured_logs(&logs);
2033 assert!(
2034 captured.contains("reason=\"client_proof_mismatch\""),
2035 "{captured}"
2036 );
2037 assert_pre_auth_log(&logs, "client_authentication", port);
2038 }
2039
2040 #[tokio::test]
2041 async fn logs_malformed_pre_auth_message() {
2042 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
2043 let (auth, _) = test_auth_with_deadline_and_limit(Duration::from_millis(200), 4);
2044 let (mut client, server_stream) = duplex(512);
2045 let port = 41003;
2046 let server = handle_accepted_connection(
2047 server_stream,
2048 echo_router(),
2049 auth,
2050 SocketAddr::from(([127, 0, 0, 1], port)),
2051 Instant::now(),
2052 );
2053 let send_malformed_message = async {
2054 let body = b"{";
2055 client
2056 .write_all(&(body.len() as u32).to_le_bytes())
2057 .await
2058 .expect("message length writes");
2059 client.write_all(body).await.expect("message body writes");
2060 };
2061
2062 let (error, ()) = tokio::join!(server, send_malformed_message);
2063 assert!(matches!(
2064 error,
2065 Err(ConnectionError::Auth(AuthError::JsonDecode {
2066 stage: AuthStage::ClientHello,
2067 ..
2068 }))
2069 ));
2070 let captured = crate::router::test_log::captured_logs(&logs);
2071 assert!(
2072 captured.contains("reason=\"malformed_or_oversized_message\""),
2073 "{captured}"
2074 );
2075 assert_pre_auth_log(&logs, "malformed_message", port);
2076 }
2077
2078 #[tokio::test]
2079 async fn logs_peer_close_during_handshake() {
2080 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
2081 let (auth, _) = test_auth_with_deadline_and_limit(Duration::from_millis(200), 4);
2082 let (client, server_stream) = duplex(512);
2083 let port = 41004;
2084 drop(client);
2085
2086 let error = handle_accepted_connection(
2087 server_stream,
2088 echo_router(),
2089 auth,
2090 SocketAddr::from(([127, 0, 0, 1], port)),
2091 Instant::now(),
2092 )
2093 .await
2094 .expect_err("closed peer must be rejected");
2095 assert!(matches!(
2096 error,
2097 ConnectionError::Auth(AuthError::UnexpectedEof {
2098 stage: AuthStage::ClientHello,
2099 actual: 0,
2100 ..
2101 })
2102 ));
2103 let captured = crate::router::test_log::captured_logs(&logs);
2104 assert!(
2105 captured.contains("reason=\"peer_closed_before_authentication_completed\""),
2106 "{captured}"
2107 );
2108 assert_pre_auth_log(&logs, "peer_closed", port);
2109 }
2110
2111 #[tokio::test]
2115 async fn each_listener_rate_limits_its_own_rejection_logs() {
2116 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
2117 for port in [41011, 41012] {
2118 let (auth, _) = test_auth_with_deadline_and_limit(Duration::from_millis(200), 4);
2119 let (client, server_stream) = duplex(512);
2120 drop(client);
2121 handle_accepted_connection(
2122 server_stream,
2123 echo_router(),
2124 auth,
2125 SocketAddr::from(([127, 0, 0, 1], port)),
2126 Instant::now(),
2127 )
2128 .await
2129 .expect_err("closed peer must be rejected");
2130 }
2131 let captured = crate::router::test_log::captured_logs(&logs);
2132 for port in [41011, 41012] {
2133 assert!(
2134 captured.contains(&format!("127.0.0.1:{port}")),
2135 "listener on port {port} logged nothing: {captured}"
2136 );
2137 }
2138 }
2139
2140 #[tokio::test]
2141 async fn logs_unauthenticated_capacity_rejection() {
2142 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
2143 let deadline = Duration::from_millis(60);
2144 let (auth, _) = test_auth_with_deadline_and_limit(deadline, 1);
2145 let held_slot = auth
2146 .unauthenticated
2147 .clone()
2148 .try_acquire_owned()
2149 .expect("the only pre-auth slot is available");
2150 let (client, server_stream) = duplex(512);
2151 let port = 41005;
2152 let server = handle_accepted_connection(
2153 server_stream,
2154 echo_router(),
2155 auth,
2156 SocketAddr::from(([127, 0, 0, 1], port)),
2157 Instant::now(),
2158 );
2159
2160 let error = tokio::time::timeout(deadline * 3, server)
2161 .await
2162 .expect("queued connection should end at the capacity deadline")
2163 .expect_err("connection without a free slot must be rejected");
2164 drop(held_slot);
2165 drop(client);
2166 assert!(matches!(error, ConnectionError::UnauthenticatedCapacity));
2167 let captured = crate::router::test_log::captured_logs(&logs);
2168 assert!(
2169 captured.contains("reason=\"capacity_wait_timed_out\""),
2170 "{captured}"
2171 );
2172 assert_pre_auth_log(&logs, "waiting_for_handshake_slot", port);
2173 }
2174
2175 #[test]
2176 fn pre_auth_rejection_limiter_suppresses_bursts_and_reports_count() {
2177 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
2178 let limiter = PreAuthRejectLimiter::new();
2179 let capacity = ConnectionError::UnauthenticatedCapacity;
2180 let handshake_timeout = ConnectionError::Auth(AuthError::Timeout {
2181 stage: AuthStage::ClientHello,
2182 deadline: Duration::from_millis(200),
2183 });
2184 let peer_addr = SocketAddr::from(([127, 0, 0, 1], 41006));
2185
2186 log_pre_auth_rejection(
2187 &capacity,
2188 peer_addr,
2189 Duration::from_millis(1),
2190 Duration::ZERO,
2191 &limiter,
2192 );
2193 for _ in 0..4 {
2194 log_pre_auth_rejection(
2195 &capacity,
2196 peer_addr,
2197 Duration::from_millis(2),
2198 Duration::from_millis(100),
2199 &limiter,
2200 );
2201 }
2202 log_pre_auth_rejection(
2203 &handshake_timeout,
2204 peer_addr,
2205 Duration::from_millis(3),
2206 Duration::from_millis(100),
2207 &limiter,
2208 );
2209 log_pre_auth_rejection(
2210 &capacity,
2211 peer_addr,
2212 Duration::from_millis(4),
2213 Duration::from_secs(1),
2214 &limiter,
2215 );
2216
2217 let captured = crate::router::test_log::captured_logs(&logs);
2218 assert_eq!(
2219 captured
2220 .matches("subc TCP connection rejected before routing")
2221 .count(),
2222 3,
2223 "one line per stage in a burst, then a later capacity line: {captured}"
2224 );
2225 assert!(
2226 captured.contains("suppressed_since_last=4"),
2227 "later capacity line should report the four hidden rejects: {captured}"
2228 );
2229 assert!(
2230 captured.contains("stage=\"handshake_deadline\""),
2231 "{captured}"
2232 );
2233 }
2234
2235 #[tokio::test]
2236 async fn buffered_frame_reader_coalesces_reads_and_preserves_short_reads() {
2237 let frames = vec![
2238 request(7, 1, b"first"),
2239 request(9, 2, b"second"),
2240 request(7, 3, b"third"),
2241 request(9, 4, b"fourth"),
2242 ];
2243 let bytes = encode_frames(&frames);
2244
2245 let (mut direct, direct_reads) = CountingReader::new(bytes.clone(), None);
2246 assert_eq!(read_frames(&mut direct, frames.len()).await, frames);
2247 assert_eq!(direct_reads.load(Ordering::Relaxed), frames.len() * 3);
2248
2249 let (buffered_source, buffered_reads) = CountingReader::new(bytes, None);
2250 let mut buffered = BufReader::new(buffered_source);
2251 assert_eq!(read_frames(&mut buffered, frames.len()).await, frames);
2252 assert_eq!(buffered_reads.load(Ordering::Relaxed), 1);
2253
2254 let split_frame = request(7, 5, b"split-body");
2255 let split_bytes = encode_frames(std::slice::from_ref(&split_frame));
2256 let (split_source, split_reads) = CountingReader::new(split_bytes, Some(10));
2257 let mut split_reader = BufReader::new(split_source);
2258 assert_eq!(
2259 read_frame(&mut split_reader).await.unwrap(),
2260 Some(split_frame)
2261 );
2262 assert_eq!(split_reads.load(Ordering::Relaxed), 2);
2263 }
2264
2265 #[tokio::test]
2266 async fn interleaved_channels_on_one_stream_demux_byte_identically_after_auth() {
2267 let (mut client, server_stream) = duplex(4096);
2268 let (auth, conn) = test_auth();
2269 let server = tokio::spawn(handle_connection(server_stream, echo_router(), auth));
2270 authenticate(&mut client, &conn).await;
2271 let frames = [
2272 request(7, 1, b"chan7-first\0opaque"),
2273 request(9, 2, b"chan9-middle-{json?}"),
2274 request(7, 3, b"chan7-second\xffbytes"),
2275 ];
2276
2277 for frame in &frames {
2278 crate::write_frame(&mut client, frame).await.unwrap();
2279 }
2280
2281 for expected in &frames {
2282 let response = crate::read_frame(&mut client).await.unwrap().unwrap();
2283 assert_eq!(response.header.ty, FrameType::Response);
2284 assert_eq!(response.header.channel, expected.header.channel);
2285 assert_eq!(response.header.corr, expected.header.corr);
2286 assert_eq!(response.body, expected.body);
2287 }
2288
2289 drop(client);
2290 server.await.unwrap().unwrap();
2291 }
2292
2293 #[tokio::test]
2294 async fn channel_zero_goes_to_subc_self_handler_after_auth() {
2295 let (mut client, server_stream) = duplex(512);
2296 let (auth, conn) = test_auth();
2297 let server = tokio::spawn(handle_connection(
2298 server_stream,
2299 Arc::new(Router::with_default_self_handler()),
2300 auth,
2301 ));
2302 authenticate(&mut client, &conn).await;
2303 let ping = Frame::build(
2304 FrameType::Ping,
2305 Flags::new(false, Priority::Passive, false),
2306 0,
2307 0,
2308 55,
2309 Vec::new(),
2310 )
2311 .unwrap();
2312
2313 crate::write_frame(&mut client, &ping).await.unwrap();
2314 let response = crate::read_frame(&mut client).await.unwrap().unwrap();
2315
2316 assert_eq!(response.header.ty, FrameType::Pong);
2317 assert_eq!(response.header.channel, 0);
2318 assert_eq!(response.header.corr, 55);
2319 assert!(response.body.is_empty());
2320
2321 drop(client);
2322 server.await.unwrap().unwrap();
2323 }
2324
2325 #[tokio::test]
2326 async fn unauthenticated_connection_is_rejected_before_routing() {
2327 let (mut client, server_stream) = duplex(512);
2328 let (auth, _conn) = test_auth();
2329 let registry = Arc::new(Registry::default());
2330 let router = Arc::new(Router::with_control_handler(Arc::new(ControlHandler::new(
2331 Arc::clone(®istry),
2332 ))));
2333 let server = tokio::spawn(handle_connection(server_stream, router, auth));
2334 let ping = Frame::build(
2335 FrameType::Ping,
2336 Flags::new(false, Priority::Passive, false),
2337 0,
2338 0,
2339 66,
2340 Vec::new(),
2341 )
2342 .unwrap();
2343
2344 crate::write_frame(&mut client, &ping).await.unwrap();
2345 if let Ok(Ok(Some(frame))) =
2346 tokio::time::timeout(Duration::from_millis(200), crate::read_frame(&mut client)).await
2347 {
2348 panic!("unauthenticated frame reached router: {frame:?}");
2349 }
2350
2351 let err = server.await.unwrap().unwrap_err();
2352 assert!(matches!(err, ConnectionError::Auth(_)));
2353 assert_eq!(registry.active_registration_count().unwrap(), 0);
2354 }
2355
2356 #[tokio::test]
2357 async fn over_cap_peer_queues_for_a_slot_and_authenticates_when_one_frees() {
2358 let (mut first_client, first_server_stream) = duplex(2048);
2363 let (mut second_client, second_server_stream) = duplex(2048);
2364 let (auth, conn) = test_auth_with_limit(1);
2365 let registry = Arc::new(Registry::default());
2366 let router = Arc::new(Router::with_control_handler(Arc::new(ControlHandler::new(
2367 Arc::clone(®istry),
2368 ))));
2369
2370 let first_server = tokio::spawn(handle_connection(
2371 first_server_stream,
2372 Arc::clone(&router),
2373 auth.clone(),
2374 ));
2375
2376 let second_server = tokio::spawn(handle_connection(
2377 second_server_stream,
2378 Arc::clone(&router),
2379 auth.clone(),
2380 ));
2381
2382 tokio::time::sleep(Duration::from_millis(50)).await;
2384 assert!(
2385 !second_server.is_finished(),
2386 "queued peer must not be reset"
2387 );
2388
2389 authenticate(&mut first_client, &conn).await;
2392 authenticate(&mut second_client, &conn).await;
2393
2394 drop(first_client);
2395 drop(second_client);
2396 let _ = first_server.await;
2397 let _ = second_server.await;
2398 assert_eq!(registry.active_registration_count().unwrap(), 0);
2399 }
2400
2401 #[tokio::test]
2402 async fn over_cap_peer_is_rejected_when_no_slot_frees_within_deadline() {
2403 let (mut second_client, second_server_stream) = duplex(512);
2409 let (auth, conn) = test_auth_with_limit(1);
2410 let registry = Arc::new(Registry::default());
2411 let router = Arc::new(Router::with_control_handler(Arc::new(ControlHandler::new(
2412 Arc::clone(®istry),
2413 ))));
2414
2415 let held_slot = auth
2416 .unauthenticated
2417 .clone()
2418 .try_acquire_owned()
2419 .expect("sole pre-auth slot");
2420
2421 let second_server = tokio::spawn(handle_connection(
2422 second_server_stream,
2423 Arc::clone(&router),
2424 auth.clone(),
2425 ));
2426 let second_err = tokio::time::timeout(TEST_DEADLINE * 2, second_server)
2427 .await
2428 .expect("capacity reject should settle at the deadline")
2429 .expect("second connection task should not panic")
2430 .expect_err("queued peer must be rejected when no slot frees");
2431 drop(held_slot);
2432 assert!(matches!(
2433 second_err,
2434 ConnectionError::UnauthenticatedCapacity
2435 ));
2436 let mut closed = [0u8; 1];
2437 assert_eq!(
2438 second_client.read(&mut closed).await.unwrap(),
2439 0,
2440 "capacity-rejected peer should observe a closed stream"
2441 );
2442 assert_eq!(registry.active_registration_count().unwrap(), 0);
2443
2444 let (mut authed_client, authed_server_stream) = duplex(2048);
2445 let authed_server = tokio::spawn(handle_connection(
2446 authed_server_stream,
2447 Arc::clone(&router),
2448 auth,
2449 ));
2450 authenticate(&mut authed_client, &conn).await;
2451 let ping = Frame::build(
2452 FrameType::Ping,
2453 Flags::new(false, Priority::Passive, false),
2454 0,
2455 0,
2456 77,
2457 Vec::new(),
2458 )
2459 .unwrap();
2460 crate::write_frame(&mut authed_client, &ping).await.unwrap();
2461 let pong = crate::read_frame(&mut authed_client)
2462 .await
2463 .unwrap()
2464 .unwrap();
2465 assert_eq!(pong.header.ty, FrameType::Pong);
2466 assert_eq!(pong.header.channel, 0);
2467 assert_eq!(pong.header.corr, 77);
2468 assert_eq!(registry.active_registration_count().unwrap(), 0);
2469
2470 drop(authed_client);
2471 authed_server.await.unwrap().unwrap();
2472 }
2473
2474 #[tokio::test]
2475 async fn bind_ack_during_partial_frame_preserves_next_request() {
2476 use subc_control::ClientControlRequest;
2477 use subc_protocol::{
2478 manifest::{
2479 Concurrency, ExecutionMode, IdentityScope, ModuleManifest, ProviderRole, Tool,
2480 },
2481 session::{ModuleControlRequest, ModuleControlResponse},
2482 BindIdentity, ModuleHelloBody, RouteTarget,
2483 };
2484
2485 let mut configured_router = Router::with_default_self_handler();
2486 configured_router.register_backend(7, EchoBackend).unwrap();
2487 let router = Arc::new(configured_router);
2488 let (auth, conn) = test_auth();
2489 let (mut module, module_stream) = duplex(4096);
2490 let module_server = tokio::spawn(handle_connection(
2491 module_stream,
2492 Arc::clone(&router),
2493 auth.clone(),
2494 ));
2495 authenticate(&mut module, &conn).await;
2496 let manifest = ModuleManifest::builder("frame-test", "0.1.0")
2497 .protocol_ver(PROTOCOL_VERSION)
2498 .provides(vec![ProviderRole::ToolProvider {
2499 tools: vec![Tool {
2500 name: "read".into(),
2501 description: None,
2502 execution_mode: ExecutionMode::Pure,
2503 schema: serde_json::json!({"type": "object"}),
2504 }],
2505 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
2506 concurrency: Concurrency::ModuleManaged,
2507 emits_push: true,
2508 sub_supervises: true,
2509 }])
2510 .build();
2511 let hello = Frame::build(
2512 FrameType::Hello,
2513 Flags::new(false, Priority::Passive, false),
2514 0,
2515 0,
2516 1,
2517 serde_json::to_vec(&ModuleHelloBody {
2518 manifest,
2519 protocol_ver: PROTOCOL_VERSION,
2520 control_ops: None,
2521 launch_nonce: None,
2522 })
2523 .unwrap(),
2524 )
2525 .unwrap();
2526 crate::write_frame(&mut module, &hello).await.unwrap();
2527 assert_eq!(
2528 read_frame(&mut module).await.unwrap().unwrap().header.ty,
2529 FrameType::HelloAck
2530 );
2531
2532 let (mut client, client_stream) = duplex(4096);
2533 let client_server = tokio::spawn(handle_connection(client_stream, router, auth));
2534 authenticate(&mut client, &conn).await;
2535 let open = Frame::build(
2536 FrameType::Request,
2537 Flags::new(false, Priority::Passive, false),
2538 0,
2539 0,
2540 2,
2541 serde_json::to_vec(&ClientControlRequest::RouteOpen {
2542 busy_refusals: None,
2543 target: RouteTarget::ToolProvider {
2544 module_id: "frame-test".into(),
2545 },
2546 identity: BindIdentity::new("unit", "session"),
2547 consumer_identity: None,
2548 consumer_capabilities: None,
2549 role_versions: None,
2550 admission_facts: None,
2551 scope: None,
2552 })
2553 .unwrap(),
2554 )
2555 .unwrap();
2556 crate::write_frame(&mut client, &open).await.unwrap();
2557 let bind = timeout(TEST_DEADLINE, read_frame(&mut module))
2558 .await
2559 .unwrap()
2560 .unwrap()
2561 .unwrap();
2562 assert!(matches!(
2563 serde_json::from_slice::<ModuleControlRequest>(&bind.body).unwrap(),
2564 ModuleControlRequest::RouteBind { .. }
2565 ));
2566
2567 let ping = request(7, 3, b"partial-frame-body");
2568 client.write_all(&ping.header.encode()).await.unwrap();
2569 tokio::time::sleep(Duration::from_millis(30)).await;
2571 let ack = Frame::build(
2572 FrameType::Response,
2573 Flags::new(false, Priority::Passive, false),
2574 0,
2575 0,
2576 bind.header.corr,
2577 serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap(),
2578 )
2579 .unwrap();
2580 crate::write_frame(&mut module, &ack).await.unwrap();
2581 let opened = timeout(TEST_DEADLINE, read_frame(&mut client))
2582 .await
2583 .unwrap()
2584 .unwrap();
2585 if opened.is_none() {
2586 panic!("client closed: {:?}", client_server.await);
2587 }
2588 let opened = opened.unwrap();
2589 assert_eq!(opened.header.corr, 2);
2590 client.write_all(&ping.body).await.unwrap();
2591 let pong = timeout(TEST_DEADLINE, read_frame(&mut client))
2592 .await
2593 .expect("partial frame must reach the router after the bind ack")
2594 .expect("frame must decode")
2595 .expect("connection must remain open");
2596 assert_eq!(pong.header.ty, FrameType::Response);
2597 assert_eq!(pong.header.corr, 3);
2598 assert_eq!(pong.body, ping.body);
2599 drop(client);
2600 drop(module);
2601 client_server.await.unwrap().unwrap();
2602 let _ = module_server.await.unwrap();
2603 }
2604
2605 #[tokio::test]
2606 async fn full_module_egress_health_probe_does_not_block_same_client_other_route() {
2607 use crate::ConnectionId;
2608 use subc_control::{ClientControlRequest, ClientControlResponse};
2609 use subc_protocol::{
2610 manifest::{
2611 Concurrency, ExecutionMode, IdentityScope, ModuleManifest, ProviderRole, Tool,
2612 },
2613 session::ModuleControlResponse,
2614 BindIdentity, ModuleHelloBody, RouteTarget,
2615 };
2616
2617 let probe_timeout = Duration::from_millis(100);
2618 let handler = Arc::new(
2619 ControlHandler::new(Arc::new(Registry::default()))
2620 .with_health_probe_timeout(probe_timeout),
2621 );
2622 let router = Arc::new(Router::with_control_handler(Arc::clone(&handler)));
2623 let hello = |name: &str| {
2624 Frame::build(
2625 FrameType::Hello,
2626 Flags::new(false, Priority::Passive, false),
2627 0,
2628 0,
2629 1,
2630 serde_json::to_vec(&ModuleHelloBody {
2631 manifest: ModuleManifest::builder(name, "0.1.0")
2632 .protocol_ver(PROTOCOL_VERSION)
2633 .provides(vec![ProviderRole::ToolProvider {
2634 tools: vec![Tool {
2635 name: "read".into(),
2636 description: None,
2637 execution_mode: ExecutionMode::Pure,
2638 schema: serde_json::json!({"type": "object"}),
2639 }],
2640 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
2641 concurrency: Concurrency::ModuleManaged,
2642 emits_push: false,
2643 sub_supervises: false,
2644 }])
2645 .build(),
2646 protocol_ver: PROTOCOL_VERSION,
2647 control_ops: Some(vec!["health.check".to_owned()]),
2648 launch_nonce: None,
2649 })
2650 .unwrap(),
2651 )
2652 .unwrap()
2653 };
2654 let (blocked_tx, mut blocked_rx) = mpsc::channel(1);
2655 let blocked_ctx = RouteCtx {
2656 connection_id: ConnectionId::new(90001),
2657 egress: FrameSink::new(blocked_tx),
2658 };
2659 handler
2660 .handle_control_frame(&blocked_ctx, hello("blocked"))
2661 .await
2662 .unwrap();
2663 assert_eq!(
2664 blocked_rx.try_recv().unwrap().header.ty,
2665 FrameType::HelloAck
2666 );
2667 let filler = request(0, 99, b"queue filler");
2668 blocked_ctx.egress.try_send(filler.clone()).unwrap();
2669 assert!(
2670 blocked_ctx.egress.try_send(filler).is_err(),
2671 "module queue must actually be full"
2672 );
2673
2674 let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
2675 let healthy_ctx = RouteCtx {
2676 connection_id: ConnectionId::new(90002),
2677 egress: FrameSink::new(healthy_tx),
2678 };
2679 handler
2680 .handle_control_frame(&healthy_ctx, hello("healthy"))
2681 .await
2682 .unwrap();
2683 assert_eq!(
2684 healthy_rx.try_recv().unwrap().header.ty,
2685 FrameType::HelloAck
2686 );
2687 let (auth, conn) = test_auth();
2688 let (mut client, client_stream) = duplex(4096);
2689 let client_server =
2690 tokio::spawn(handle_connection(client_stream, Arc::clone(&router), auth));
2691 authenticate(&mut client, &conn).await;
2692 let control_request = |corr, body: ClientControlRequest| {
2693 Frame::build(
2694 FrameType::Request,
2695 Flags::new(false, Priority::Interactive, false),
2696 0,
2697 0,
2698 corr,
2699 serde_json::to_vec(&body).unwrap(),
2700 )
2701 .unwrap()
2702 };
2703 crate::write_frame(
2704 &mut client,
2705 &control_request(
2706 2,
2707 ClientControlRequest::RouteOpen {
2708 busy_refusals: None,
2709 target: RouteTarget::ToolProvider {
2710 module_id: "healthy".into(),
2711 },
2712 identity: BindIdentity::new("unit", "session"),
2713 consumer_identity: None,
2714 consumer_capabilities: None,
2715 role_versions: None,
2716 admission_facts: None,
2717 scope: None,
2718 },
2719 ),
2720 )
2721 .await
2722 .unwrap();
2723 let bind = timeout(TEST_DEADLINE, healthy_rx.recv())
2724 .await
2725 .unwrap()
2726 .unwrap();
2727 handler
2728 .handle_control_frame(
2729 &healthy_ctx,
2730 Frame::build(
2731 FrameType::Response,
2732 Flags::new(false, Priority::Passive, false),
2733 0,
2734 0,
2735 bind.header.corr,
2736 serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap(),
2737 )
2738 .unwrap(),
2739 )
2740 .await
2741 .unwrap();
2742 let opened = timeout(TEST_DEADLINE, read_frame(&mut client))
2743 .await
2744 .unwrap()
2745 .unwrap()
2746 .unwrap();
2747 let (channel, epoch) =
2748 match serde_json::from_slice::<ClientControlResponse>(&opened.body).unwrap() {
2749 ClientControlResponse::RouteOpen {
2750 route_channel,
2751 route_epoch,
2752 ..
2753 } => (route_channel, route_epoch),
2754 other => panic!("expected a healthy module route, got {other:?}"),
2755 };
2756
2757 let probe = control_request(
2758 3,
2759 ClientControlRequest::SupervisorHealthProbe {
2760 module_id: "blocked".into(),
2761 },
2762 );
2763 let mut ping = request(channel, 4, b"healthy request");
2764 ping.header.epoch = epoch;
2765 crate::write_frame(&mut client, &probe).await.unwrap();
2766 crate::write_frame(&mut client, &ping).await.unwrap();
2767 timeout(probe_timeout + Duration::from_secs(1), async {
2768 let timed_out = read_frame(&mut client).await.unwrap().unwrap();
2769 assert_eq!(timed_out.header.corr, 3);
2770 assert_eq!(timed_out.header.ty, FrameType::Error);
2771 let error: ErrorBody = serde_json::from_slice(&timed_out.body).unwrap();
2772 assert_eq!(error.code, "module_timeout");
2773 let forwarded = healthy_rx.recv().await.unwrap();
2774 assert_eq!(forwarded.header.corr, ping.header.corr);
2775 assert_eq!(forwarded.body, ping.body);
2776 let reply = Frame::build(
2777 FrameType::Response,
2778 ping.header.flags,
2779 forwarded.header.channel,
2780 forwarded.header.epoch,
2781 forwarded.header.corr,
2782 b"healthy response".to_vec(),
2783 )
2784 .unwrap();
2785 router
2786 .route_for_connection(&healthy_ctx, reply)
2787 .await
2788 .unwrap();
2789 let pong = read_frame(&mut client).await.unwrap().unwrap();
2790 assert_eq!(pong.header.ty, FrameType::Response);
2791 assert_eq!(pong.header.channel, channel);
2792 assert_eq!(pong.header.epoch, epoch);
2793 assert_eq!(pong.header.corr, 4);
2794 assert_eq!(pong.body, b"healthy response");
2795 })
2796 .await
2797 .expect(
2798 "the same client's healthy route must answer within the probe deadline plus margin",
2799 );
2800 assert_eq!(
2801 blocked_rx.len(),
2802 1,
2803 "the blocked module must remain undrained"
2804 );
2805 assert_eq!(blocked_rx.try_recv().unwrap().header.corr, 99);
2806 assert!(
2807 blocked_rx.try_recv().is_err(),
2808 "a timed-out probe must not be enqueued later"
2809 );
2810 drop(client);
2811 client_server.await.unwrap().unwrap();
2812 }
2813
2814 #[tokio::test]
2815 async fn aborted_accept_does_not_end_listener() {
2816 let listener = Arc::new(TcpListener::bind("127.0.0.1:0").await.unwrap());
2817 let addr = listener.local_addr().unwrap();
2818 let (auth, conn) = test_auth();
2819 let mut attempts = 0;
2820 let server = tokio::spawn(serve_listener_with_accept(
2821 Some(addr),
2822 echo_router(),
2823 auth,
2824 move || {
2825 attempts += 1;
2826 let result = if attempts == 1 {
2827 Some(io::Error::from(io::ErrorKind::ConnectionAborted))
2828 } else {
2829 None
2830 };
2831 let listener = Arc::clone(&listener);
2832 async move {
2833 match result {
2834 Some(err) => Err(err),
2835 None => listener.accept().await,
2836 }
2837 }
2838 },
2839 ));
2840 let mut client = tokio::net::TcpStream::connect(addr).await.unwrap();
2841 authenticate(&mut client, &conn).await;
2842 let ping = Frame::build(
2843 FrameType::Ping,
2844 Flags::new(false, Priority::Passive, false),
2845 0,
2846 0,
2847 77,
2848 Vec::new(),
2849 )
2850 .unwrap();
2851 crate::write_frame(&mut client, &ping).await.unwrap();
2852 let pong = timeout(TEST_DEADLINE, read_frame(&mut client))
2853 .await
2854 .unwrap()
2855 .unwrap()
2856 .unwrap();
2857 assert_eq!(pong.header.ty, FrameType::Pong);
2858 assert!(
2859 !server.is_finished(),
2860 "a temporary accept failure must not stop the listener"
2861 );
2862 server.abort();
2863 }
2864
2865 #[tokio::test]
2866 async fn exhausted_accept_backs_off_then_serves_connection() {
2867 let listener = Arc::new(TcpListener::bind("127.0.0.1:0").await.unwrap());
2868 let addr = listener.local_addr().unwrap();
2869 let (auth, conn) = test_auth();
2870 let mut attempts = 0;
2871 let server = tokio::spawn(serve_listener_with_accept(
2872 Some(addr),
2873 echo_router(),
2874 auth,
2875 move || {
2876 attempts += 1;
2877 #[cfg(unix)]
2880 let emfile = rustix::io::Errno::MFILE.raw_os_error();
2881 #[cfg(windows)]
2882 let emfile = 10024;
2883 let error = (attempts == 1).then(|| io::Error::from_raw_os_error(emfile));
2884 let listener = Arc::clone(&listener);
2885 async move {
2886 match error {
2887 Some(err) => Err(err),
2888 None => listener.accept().await,
2889 }
2890 }
2891 },
2892 ));
2893 let started = Instant::now();
2894 let mut client = tokio::net::TcpStream::connect(addr).await.unwrap();
2895 authenticate(&mut client, &conn).await;
2896 assert!(
2897 started.elapsed() >= Duration::from_millis(50),
2898 "fd exhaustion must back off before retrying"
2899 );
2900 assert!(!server.is_finished());
2901 server.abort();
2902 }
2903
2904 #[tokio::test]
2905 async fn fatal_accept_error_still_stops_listener() {
2906 let (auth, _) = test_auth();
2907 let err = serve_listener_with_accept(None, echo_router(), auth, || async {
2908 Err(io::Error::from(io::ErrorKind::PermissionDenied))
2909 })
2910 .await
2911 .unwrap_err();
2912 assert!(
2913 matches!(err, ServerError::Accept { source, .. } if source.kind() == io::ErrorKind::PermissionDenied)
2914 );
2915 }
2916
2917 #[tokio::test]
2918 async fn serve_listeners_with_no_listeners_returns_typed_error() {
2919 let (auth, _conn) = test_auth();
2920 let err = serve_listeners(Vec::new(), echo_router(), auth)
2921 .await
2922 .expect_err("empty listener set must fail loudly");
2923 assert!(matches!(err, ServerError::NoListeners));
2924 }
2925
2926 #[tokio::test]
2927 async fn malformed_header_returns_typed_error_no_panic() {
2928 let (mut client, server_stream) = duplex(128);
2929 let (auth, conn) = test_auth();
2930 let server = tokio::spawn(handle_connection(server_stream, echo_router(), auth));
2931 authenticate(&mut client, &conn).await;
2932 let mut header = [0u8; HEADER_LEN];
2933 header[4] = PROTOCOL_VERSION;
2934 header[5] = 250;
2935
2936 client.write_all(&header).await.unwrap();
2937 drop(client);
2938
2939 let err = server.await.unwrap().unwrap_err();
2940 assert!(matches!(
2941 err,
2942 ConnectionError::FrameIo(FrameIoError::DecodeHeader(DecodeError::UnknownFrameType {
2943 byte: 250
2944 }))
2945 ));
2946 }
2947
2948 #[tokio::test]
2949 async fn truncated_body_returns_typed_error_no_panic() {
2950 let (mut client, server_stream) = duplex(128);
2951 let (auth, conn) = test_auth();
2952 let server = tokio::spawn(handle_connection(server_stream, echo_router(), auth));
2953 authenticate(&mut client, &conn).await;
2954 let frame = request(7, 8, b"abcd");
2955
2956 client.write_all(&frame.header.encode()).await.unwrap();
2957 client.write_all(b"ab").await.unwrap();
2958 drop(client);
2959
2960 let err = server.await.unwrap().unwrap_err();
2961 assert!(matches!(
2962 err,
2963 ConnectionError::FrameIo(FrameIoError::UnexpectedEof {
2964 stage: ReadStage::Body,
2965 expected: 4,
2966 actual: 2
2967 })
2968 ));
2969 }
2970
2971 #[tokio::test]
2972 async fn unknown_channel_is_returned_as_error_frame_and_connection_continues() {
2973 let (mut client, server_stream) = duplex(1024);
2974 let (auth, conn) = test_auth();
2975 let server = tokio::spawn(handle_connection(server_stream, echo_router(), auth));
2976 authenticate(&mut client, &conn).await;
2977 let unknown = request(42, 10, b"lost");
2978 let known = request(7, 11, b"still-routes");
2979
2980 crate::write_frame(&mut client, &unknown).await.unwrap();
2981 crate::write_frame(&mut client, &known).await.unwrap();
2982
2983 let error = crate::read_frame(&mut client).await.unwrap().unwrap();
2984 assert_eq!(error.header.ty, FrameType::Error);
2985 assert_eq!(error.header.channel, 42);
2986 assert_eq!(error.header.corr, 10);
2987 let error_body: ErrorBody = serde_json::from_slice(&error.body).unwrap();
2988 assert_eq!(error_body.code, "unknown_channel");
2989
2990 let response = crate::read_frame(&mut client).await.unwrap().unwrap();
2991 assert_eq!(response.header.ty, FrameType::Response);
2992 assert_eq!(response.header.channel, 7);
2993 assert_eq!(response.header.corr, 11);
2994 assert_eq!(response.body, b"still-routes");
2995
2996 drop(client);
2997 server.await.unwrap().unwrap();
2998 }
2999}